From 443fed796affe6f03d8e9a41f27296de0e892848 Mon Sep 17 00:00:00 2001 From: Ruslan Garnov Date: Tue, 30 Oct 2018 21:12:36 +0300 Subject: [PATCH] Merge pull request #12990 from rgarnov:gapi_fluid_reshape_support G-API: Introduce new `reshape()` API (#12990) * Moved initFluidUnits, initLineConsumption, calcLatency, calcSkew to separate functions * Added Fluid::View::allocate method (moved allocation logic from constructor) * Changed util::zip to util::indexed, utilized collectInputMeta in GFluidExecutable constructor * Added makeReshape method to FluidExecutable * Removed m_outputRoi from GFluidExecutable * Added reshape feature * Added switch of resize mapper if agent ratio was changed * Added more TODOs and renamed a function * G-API reshape(): add missing `override` specifiers Fix warnings on all platforms --- .../include/opencv2/gapi/fluid/gfluidbuffer.hpp | 2 +- modules/gapi/include/opencv2/gapi/gcompiled.hpp | 3 + modules/gapi/src/api/gcomputation.cpp | 37 +- modules/gapi/src/backends/cpu/gcpubackend.hpp | 9 + modules/gapi/src/backends/fluid/gfluidbackend.cpp | 615 ++++++++++++--------- modules/gapi/src/backends/fluid/gfluidbackend.hpp | 30 +- modules/gapi/src/backends/fluid/gfluidbuffer.cpp | 58 +- .../gapi/src/backends/fluid/gfluidbuffer_priv.hpp | 16 +- modules/gapi/src/compiler/gcompiled.cpp | 25 +- modules/gapi/src/compiler/gcompiled_priv.hpp | 3 + modules/gapi/src/compiler/gcompiler.cpp | 2 +- modules/gapi/src/compiler/gislandmodel.hpp | 3 + modules/gapi/src/compiler/passes/meta.cpp | 6 +- modules/gapi/src/compiler/passes/passes.hpp | 2 +- modules/gapi/src/executor/gexecutor.cpp | 20 +- modules/gapi/src/executor/gexecutor.hpp | 3 + modules/gapi/test/gapi_fluid_test.cpp | 4 +- .../test/internal/gapi_int_recompilation_test.cpp | 159 ++++++ 18 files changed, 691 insertions(+), 306 deletions(-) diff --git a/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp b/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp index 1b94bba..3a0f02f 100644 --- a/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp +++ b/modules/gapi/include/opencv2/gapi/fluid/gfluidbuffer.hpp @@ -105,7 +105,7 @@ public: const GMatDesc& meta() const; - View mkView(int lineConsumption, int borderSize, BorderOpt border, bool ownStorage); + View mkView(int borderSize, bool ownStorage); class GAPI_EXPORTS Priv; // internal use only Priv& priv(); // internal use only diff --git a/modules/gapi/include/opencv2/gapi/gcompiled.hpp b/modules/gapi/include/opencv2/gapi/gcompiled.hpp index e0f95b5..70011a8 100644 --- a/modules/gapi/include/opencv2/gapi/gcompiled.hpp +++ b/modules/gapi/include/opencv2/gapi/gcompiled.hpp @@ -50,6 +50,9 @@ public: const GMetaArgs& metas() const; // Meta passed to compile() const GMetaArgs& outMetas() const; // Inferred output metadata + bool canReshape() const; // is reshape mechanism supported by GCompiled + void reshape(const GMetaArgs& inMetas, const GCompileArgs& args); // run reshape procedure + protected: std::shared_ptr m_priv; }; diff --git a/modules/gapi/src/api/gcomputation.cpp b/modules/gapi/src/api/gcomputation.cpp index 96640e2..ab761ed 100644 --- a/modules/gapi/src/api/gcomputation.cpp +++ b/modules/gapi/src/api/gcomputation.cpp @@ -76,15 +76,46 @@ cv::GCompiled cv::GComputation::compile(GMetaArgs &&metas, GCompileArgs &&args) return comp.compile(); } +// FIXME: Introduce similar query/test method for GMetaArgs as a building block +// for functions like this? +static bool formats_are_same(const cv::GMetaArgs& metas1, const cv::GMetaArgs& metas2) +{ + return std::equal(metas1.cbegin(), metas1.cend(), metas2.cbegin(), + [](const cv::GMetaArg& meta1, const cv::GMetaArg& meta2) { + if (meta1.index() == meta2.index() && meta1.index() == cv::GMetaArg::index_of()) + { + const auto& desc1 = cv::util::get(meta1); + const auto& desc2 = cv::util::get(meta2); + + // comparison by size is omitted + return (desc1.chan == desc2.chan && + desc1.depth == desc2.depth); + } + else + { + return meta1 == meta2; + } + }); +} + void cv::GComputation::apply(GRunArgs &&ins, GRunArgsP &&outs, GCompileArgs &&args) { const auto in_metas = descr_of(ins); // FIXME Graph should be recompiled when GCompileArgs have changed if (m_priv->m_lastMetas != in_metas) { - // FIXME: Had to construct temporary object as compile() takes && (r-value) - m_priv->m_lastCompiled = compile(GMetaArgs(in_metas), std::move(args)); - m_priv->m_lastMetas = in_metas; // Update only here, if compile() was ok + if (m_priv->m_lastCompiled && + m_priv->m_lastCompiled.canReshape() && + formats_are_same(m_priv->m_lastMetas, in_metas)) + { + m_priv->m_lastCompiled.reshape(in_metas, args); + } + else + { + // FIXME: Had to construct temporary object as compile() takes && (r-value) + m_priv->m_lastCompiled = compile(GMetaArgs(in_metas), std::move(args)); + } + m_priv->m_lastMetas = in_metas; } m_priv->m_lastCompiled(std::move(ins), std::move(outs)); } diff --git a/modules/gapi/src/backends/cpu/gcpubackend.hpp b/modules/gapi/src/backends/cpu/gcpubackend.hpp index c5ba79e..6ce8c48 100644 --- a/modules/gapi/src/backends/cpu/gcpubackend.hpp +++ b/modules/gapi/src/backends/cpu/gcpubackend.hpp @@ -54,6 +54,15 @@ public: GCPUExecutable(const ade::Graph &graph, const std::vector &nodes); + virtual inline bool canReshape() const override { return false; } + virtual inline void reshape(ade::Graph&, const GCompileArgs&) override + { + // FIXME: CPU plugin is in fact reshapeable (as it was initially, + // even before outMeta() has been introduced), so this limitation + // should be dropped. + util::throw_error(std::logic_error("GCPUExecutable::reshape() should never be called")); + } + virtual void run(std::vector &&input_objs, std::vector &&output_objs) override; }; diff --git a/modules/gapi/src/backends/fluid/gfluidbackend.cpp b/modules/gapi/src/backends/fluid/gfluidbackend.cpp index 39aba08..282766a 100644 --- a/modules/gapi/src/backends/fluid/gfluidbackend.cpp +++ b/modules/gapi/src/backends/fluid/gfluidbackend.cpp @@ -109,39 +109,46 @@ cv::gapi::GBackend cv::gapi::fluid::backend() // FluidAgent implementation /////////////////////////////////////////////////// namespace cv { namespace gimpl { -struct FluidFilterAgent : public FluidAgent +struct FluidDownscaleMapper : public FluidMapper +{ + virtual int firstWindow(int outCoord, int lpi) const override; + virtual int nextWindow(int outCoord, int lpi) const override; + virtual int linesRead(int outCoord) const override; + using FluidMapper::FluidMapper; +}; + +struct FluidUpscaleMapper : public FluidMapper { + virtual int firstWindow(int outCoord, int lpi) const override; + virtual int nextWindow(int outCoord, int lpi) const override; + virtual int linesRead(int outCoord) const override; + FluidUpscaleMapper(double ratio, int lpi, int inHeight) : FluidMapper(ratio, lpi), m_inHeight(inHeight) {} private: - virtual int firstWindow() const override; - virtual int nextWindow() const override; - virtual int linesRead() const override; -public: - using FluidAgent::FluidAgent; - virtual void setInHeight(int) override { /* nothing */ } + int m_inHeight = 0; }; -struct FluidResizeAgent : public FluidAgent +struct FluidFilterAgent : public FluidAgent { private: virtual int firstWindow() const override; virtual int nextWindow() const override; virtual int linesRead() const override; + virtual void setRatio(double) override { /* nothing */ } public: using FluidAgent::FluidAgent; - virtual void setInHeight(int) override { /* nothing */ } }; -struct FluidUpscaleAgent : public FluidAgent +struct FluidResizeAgent : public FluidAgent { private: virtual int firstWindow() const override; virtual int nextWindow() const override; virtual int linesRead() const override; + virtual void setRatio(double ratio) override; - int m_inH; + std::unique_ptr m_mapper; public: using FluidAgent::FluidAgent; - virtual void setInHeight(int h) override { m_inH = h; } }; }} // namespace cv::gimpl @@ -301,6 +308,40 @@ int upscaleWindowEnd(int outCoord, double ratio, int inSz) } } // anonymous namespace +int cv::gimpl::FluidDownscaleMapper::firstWindow(int outCoord, int lpi) const +{ + return windowEnd(outCoord + lpi - 1, m_ratio) - windowStart(outCoord, m_ratio); +} + +int cv::gimpl::FluidDownscaleMapper::nextWindow(int outCoord, int lpi) const +{ + auto nextStartIdx = outCoord + 1 + m_lpi - 1; + auto nextEndIdx = nextStartIdx + lpi - 1; + return windowEnd(nextEndIdx, m_ratio) - windowStart(nextStartIdx, m_ratio); +} + +int cv::gimpl::FluidDownscaleMapper::linesRead(int outCoord) const +{ + return windowStart(outCoord + 1 + m_lpi - 1, m_ratio) - windowStart(outCoord, m_ratio); +} + +int cv::gimpl::FluidUpscaleMapper::firstWindow(int outCoord, int lpi) const +{ + return upscaleWindowEnd(outCoord + lpi - 1, m_ratio, m_inHeight) - upscaleWindowStart(outCoord, m_ratio); +} + +int cv::gimpl::FluidUpscaleMapper::nextWindow(int outCoord, int lpi) const +{ + auto nextStartIdx = outCoord + 1 + m_lpi - 1; + auto nextEndIdx = nextStartIdx + lpi - 1; + return upscaleWindowEnd(nextEndIdx, m_ratio, m_inHeight) - upscaleWindowStart(nextStartIdx, m_ratio); +} + +int cv::gimpl::FluidUpscaleMapper::linesRead(int outCoord) const +{ + return upscaleWindowStart(outCoord + 1 + m_lpi - 1, m_ratio) - upscaleWindowStart(outCoord, m_ratio); +} + int cv::gimpl::FluidFilterAgent::firstWindow() const { return k.m_window + k.m_lpi - 1; @@ -321,44 +362,32 @@ int cv::gimpl::FluidResizeAgent::firstWindow() const { auto outIdx = out_buffers[0]->priv().y(); auto lpi = std::min(m_outputLines - m_producedLines, k.m_lpi); - return windowEnd(outIdx + lpi - 1, m_ratio) - windowStart(outIdx, m_ratio); + return m_mapper->firstWindow(outIdx, lpi); } int cv::gimpl::FluidResizeAgent::nextWindow() const { auto outIdx = out_buffers[0]->priv().y(); auto lpi = std::min(m_outputLines - m_producedLines - k.m_lpi, k.m_lpi); - auto nextStartIdx = outIdx + 1 + k.m_lpi - 1; - auto nextEndIdx = nextStartIdx + lpi - 1; - return windowEnd(nextEndIdx, m_ratio) - windowStart(nextStartIdx, m_ratio); + return m_mapper->nextWindow(outIdx, lpi); } int cv::gimpl::FluidResizeAgent::linesRead() const { auto outIdx = out_buffers[0]->priv().y(); - return windowStart(outIdx + 1 + k.m_lpi - 1, m_ratio) - windowStart(outIdx, m_ratio); -} - -int cv::gimpl::FluidUpscaleAgent::firstWindow() const -{ - auto outIdx = out_buffers[0]->priv().y(); - auto lpi = std::min(m_outputLines - m_producedLines, k.m_lpi); - return upscaleWindowEnd(outIdx + lpi - 1, m_ratio, m_inH) - upscaleWindowStart(outIdx, m_ratio); -} - -int cv::gimpl::FluidUpscaleAgent::nextWindow() const -{ - auto outIdx = out_buffers[0]->priv().y(); - auto lpi = std::min(m_outputLines - m_producedLines - k.m_lpi, k.m_lpi); - auto nextStartIdx = outIdx + 1 + k.m_lpi - 1; - auto nextEndIdx = nextStartIdx + lpi - 1; - return upscaleWindowEnd(nextEndIdx, m_ratio, m_inH) - upscaleWindowStart(nextStartIdx, m_ratio); + return m_mapper->linesRead(outIdx); } -int cv::gimpl::FluidUpscaleAgent::linesRead() const +void cv::gimpl::FluidResizeAgent::setRatio(double ratio) { - auto outIdx = out_buffers[0]->priv().y(); - return upscaleWindowStart(outIdx + 1 + k.m_lpi - 1, m_ratio) - upscaleWindowStart(outIdx, m_ratio); + if (ratio >= 1.0) + { + m_mapper.reset(new FluidDownscaleMapper(ratio, k.m_lpi)); + } + else + { + m_mapper.reset(new FluidUpscaleMapper(ratio, k.m_lpi, in_views[0].meta().size.height)); + } } bool cv::gimpl::FluidAgent::canRead() const @@ -448,15 +477,21 @@ void cv::gimpl::FluidAgent::debug(std::ostream &os) // GCPUExcecutable implementation ////////////////////////////////////////////// -void cv::gimpl::GFluidExecutable::initBufferRois(std::vector& readStarts, std::vector& rois) +void cv::gimpl::GFluidExecutable::initBufferRois(std::vector& readStarts, + std::vector& rois, + const std::vector& out_rois) { GConstFluidModel fg(m_g); auto proto = m_gm.metadata().get(); std::stack nodesToVisit; - if (proto.outputs.size() != m_outputRois.size()) + // FIXME? + // There is possible case when user pass the vector full of default Rect{}-s, + // Can be diagnosed and handled appropriately + if (proto.outputs.size() != out_rois.size()) { - GAPI_Assert(m_outputRois.size() == 0); + GAPI_Assert(out_rois.size() == 0); + // No inference required, buffers will obtain roi from meta return; } @@ -477,18 +512,21 @@ void cv::gimpl::GFluidExecutable::initBufferRois(std::vector& readStarts, s if (d.shape == GShape::GMAT) { auto desc = util::get(d.meta); - if (m_outputRois[idx] == cv::gapi::own::Rect{}) + auto id = m_id_map.at(d.rc); + readStarts[id] = 0; + + if (out_rois[idx] == gapi::own::Rect{}) { - m_outputRois[idx] = cv::gapi::own::Rect{0, 0, desc.size.width, desc.size.height}; + rois[id] = gapi::own::Rect{ 0, 0, desc.size.width, desc.size.height }; + } + else + { + // Only slices are supported at the moment + GAPI_Assert(out_rois[idx].x == 0); + GAPI_Assert(out_rois[idx].width == desc.size.width); + rois[id] = out_rois[idx]; } - // Only slices are supported at the moment - GAPI_Assert(m_outputRois[idx].x == 0); - GAPI_Assert(m_outputRois[idx].width == desc.size.width); - - auto id = m_id_map.at(d.rc); - readStarts[id] = 0; - rois[id] = m_outputRois[idx]; nodesToVisit.push(nh); } } @@ -591,7 +629,7 @@ void cv::gimpl::GFluidExecutable::initBufferRois(std::vector& readStarts, s cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, const std::vector &nodes, const std::vector &outputRois) - : m_g(g), m_gm(m_g), m_nodes(nodes), m_outputRois(outputRois) + : m_g(g), m_gm(m_g) { GConstFluidModel fg(m_g); @@ -599,18 +637,17 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, // FIXME: There _must_ be a better way to [query] count number of DATA nodes std::size_t mat_count = 0; std::size_t last_agent = 0; - std::map all_gmat_ids; auto grab_mat_nh = [&](ade::NodeHandle nh) { auto rc = m_gm.metadata(nh).get().rc; if (m_id_map.count(rc) == 0) { - all_gmat_ids[mat_count] = nh; + m_all_gmat_ids[mat_count] = nh; m_id_map[rc] = mat_count++; } }; - for (const auto &nh : m_nodes) + for (const auto &nh : nodes) { switch (m_gm.metadata(nh).get().t) { @@ -625,17 +662,7 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, switch (fu.k.m_kind) { case GFluidKernel::Kind::Filter: m_agents.emplace_back(new FluidFilterAgent(m_g, nh)); break; - case GFluidKernel::Kind::Resize: - { - if (fu.ratio >= 1.0) - { - m_agents.emplace_back(new FluidResizeAgent(m_g, nh)); - } - else - { - m_agents.emplace_back(new FluidUpscaleAgent(m_g, nh)); - } - } break; + case GFluidKernel::Kind::Resize: m_agents.emplace_back(new FluidResizeAgent(m_g, nh)); break; default: GAPI_Assert(false); } // NB.: in_buffer_ids size is equal to Arguments size, not Edges size!!! @@ -681,35 +708,7 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, GAPI_LOG_INFO(NULL, "Initializing " << mat_count << " fluid buffer(s)" << std::endl); m_num_int_buffers = mat_count; const std::size_t num_scratch = m_scratch_users.size(); - - std::vector readStarts(mat_count); - std::vector rois(mat_count); - - initBufferRois(readStarts, rois); - - // NB: Allocate ALL buffer object at once, and avoid any further reallocations - // (since raw pointers-to-elements are taken) m_buffers.resize(m_num_int_buffers + num_scratch); - for (const auto &it : all_gmat_ids) - { - auto id = it.first; - auto nh = it.second; - const auto & d = m_gm.metadata(nh).get(); - const auto &fd = fg.metadata(nh).get(); - const auto meta = cv::util::get(d.meta); - - m_buffers[id].priv().init(meta, fd.lpi_write, readStarts[id], rois[id]); - - // TODO: - // Introduce Storage::INTERNAL_GRAPH and Storage::INTERNAL_ISLAND? - if (fd.internal == true) - { - m_buffers[id].priv().allocate(fd.border, fd.border_size, fd.max_consumption, fd.skew); - std::stringstream stream; - m_buffers[id].debug(stream); - GAPI_LOG_INFO(NULL, stream.str()); - } - } // After buffers are allocated, repack: ... for (auto &agent : m_agents) @@ -719,11 +718,10 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, const auto &fu = fg.metadata(agent->op_handle).get(); agent->in_args.resize(op.args.size()); agent->in_views.resize(op.args.size()); - for (auto it : ade::util::zip(ade::util::iota(op.args.size()), - ade::util::toRange(agent->in_buffer_ids))) + for (auto it : ade::util::indexed(ade::util::toRange(agent->in_buffer_ids))) { - auto in_idx = std::get<0>(it); - auto buf_idx = std::get<1>(it); + auto in_idx = ade::util::index(it); + auto buf_idx = ade::util::value(it); if (buf_idx >= 0) { @@ -734,11 +732,10 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, auto inEdge = GModel::getInEdgeByPort(m_g, agent->op_handle, in_idx); auto ownStorage = fg.metadata(inEdge).get().use; - gapi::fluid::View view = buffer.mkView(fu.line_consumption, fu.border_size, fu.border, ownStorage); + gapi::fluid::View view = buffer.mkView(fu.border_size, ownStorage); // NB: It is safe to keep ptr as view lifetime is buffer lifetime agent->in_views[in_idx] = view; agent->in_args[in_idx] = GArg(view); - agent->m_ratio = fu.ratio; } else { @@ -747,22 +744,13 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, } } - // cache input height to avoid costly meta() call - // (actually cached and used only in upscale) - if (agent->in_views[0]) - { - agent->setInHeight(agent->in_views[0].meta().size.height); - } - // b. Agent output parameters with Buffer pointers. agent->out_buffers.resize(agent->op_handle->outEdges().size(), nullptr); - for (auto it : ade::util::zip(ade::util::iota(agent->out_buffers.size()), - ade::util::toRange(agent->out_buffer_ids))) + for (auto it : ade::util::indexed(ade::util::toRange(agent->out_buffer_ids))) { - auto out_idx = std::get<0>(it); - auto buf_idx = m_id_map.at(std::get<1>(it)); + auto out_idx = ade::util::index(it); + auto buf_idx = m_id_map.at(ade::util::value(it)); agent->out_buffers.at(out_idx) = &m_buffers.at(buf_idx); - agent->m_outputLines = m_buffers.at(buf_idx).priv().outputLines(); } } @@ -776,28 +764,14 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, { auto &agent = m_agents.at(i); GAPI_Assert(agent->k.m_scratch); - - // Collect input metas to trigger scratch buffer initialization - // Array is sparse (num of elements == num of GArgs, not edges) - GMetaArgs in_metas(agent->in_args.size()); - for (auto eh : agent->op_handle->inEdges()) - { - const auto& in_data = m_gm.metadata(eh->srcNode()).get(); - in_metas[m_gm.metadata(eh).get().port] = in_data.meta; - } - - // Trigger Scratch buffer initialization method const std::size_t new_scratch_idx = m_num_int_buffers + last_scratch_id; - - agent->k.m_is(in_metas, agent->in_args, m_buffers.at(new_scratch_idx)); - std::stringstream stream; - m_buffers[new_scratch_idx].debug(stream); - GAPI_LOG_INFO(NULL, stream.str()); agent->out_buffers.emplace_back(&m_buffers[new_scratch_idx]); last_scratch_id++; } } + makeReshape(outputRois); + std::size_t total_size = 0; for (const auto &i : ade::util::indexed(m_buffers)) { @@ -805,7 +779,7 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, const auto idx = ade::util::index(i); const auto b = ade::util::value(i); if (idx >= m_num_int_buffers || - fg.metadata(all_gmat_ids[idx]).get().internal == true) + fg.metadata(m_all_gmat_ids[idx]).get().internal == true) { GAPI_Assert(b.priv().size() > 0); } @@ -817,6 +791,264 @@ cv::gimpl::GFluidExecutable::GFluidExecutable(const ade::Graph &g, GAPI_LOG_INFO(NULL, "Internal buffers: " << std::fixed << std::setprecision(2) << static_cast(total_size)/1024 << " KB\n"); } +namespace +{ + void resetFluidData(ade::Graph& graph) + { + using namespace cv::gimpl; + GModel::Graph g(graph); + GFluidModel fg(graph); + for (const auto node : g.nodes()) + { + if (g.metadata(node).get().t == NodeType::DATA) + { + auto& fd = fg.metadata(node).get(); + fd.latency = 0; + fd.skew = 0; + fd.max_consumption = 0; + } + } + } + + void initFluidUnits(ade::Graph& graph) + { + using namespace cv::gimpl; + GModel::Graph g(graph); + GFluidModel fg(graph); + + auto sorted = g.metadata().get().nodes(); + for (auto node : sorted) + { + if (fg.metadata(node).contains()) + { + std::set in_hs, out_ws, out_hs; + + for (const auto& in : node->inNodes()) + { + const auto& d = g.metadata(in).get(); + if (d.shape == cv::GShape::GMAT) + { + const auto& meta = cv::util::get(d.meta); + in_hs.insert(meta.size.height); + } + } + + for (const auto& out : node->outNodes()) + { + const auto& d = g.metadata(out).get(); + if (d.shape == cv::GShape::GMAT) + { + const auto& meta = cv::util::get(d.meta); + out_ws.insert(meta.size.width); + out_hs.insert(meta.size.height); + } + } + + GAPI_Assert(in_hs.size() == 1 && out_ws.size() == 1 && out_hs.size() == 1); + + auto in_h = *in_hs .cbegin(); + auto out_h = *out_hs.cbegin(); + + auto &fu = fg.metadata(node).get(); + fu.ratio = (double)in_h / out_h; + + int line_consumption = maxLineConsumption(fu.k, in_h, out_h, fu.k.m_lpi); + int border_size = borderSize(fu.k); + + fu.border_size = border_size; + fu.line_consumption = line_consumption; + + GModel::log(g, node, "Line consumption: " + std::to_string(fu.line_consumption)); + GModel::log(g, node, "Border size: " + std::to_string(fu.border_size)); + } + } + } + + // FIXME! + // Split into initLineConsumption and initBorderSizes, + // call only consumption related stuff during reshape + void initLineConsumption(ade::Graph& graph) + { + using namespace cv::gimpl; + GModel::Graph g(graph); + GFluidModel fg(graph); + + for (const auto &node : g.nodes()) + { + if (fg.metadata(node).contains()) + { + const auto &fu = fg.metadata(node).get(); + + for (const auto &in_data_node : node->inNodes()) + { + auto &fd = fg.metadata(in_data_node).get(); + + // Update (not Set) fields here since a single data node may be + // accessed by multiple consumers + fd.max_consumption = std::max(fu.line_consumption, fd.max_consumption); + fd.border_size = std::max(fu.border_size, fd.border_size); + + GModel::log(g, in_data_node, "Line consumption: " + std::to_string(fd.max_consumption) + + " (upd by " + std::to_string(fu.line_consumption) + ")", node); + GModel::log(g, in_data_node, "Border size: " + std::to_string(fd.border_size), node); + } + } + } + } + + void calcLatency(ade::Graph& graph) + { + using namespace cv::gimpl; + GModel::Graph g(graph); + GFluidModel fg(graph); + + auto sorted = g.metadata().get().nodes(); + for (const auto &node : sorted) + { + if (fg.metadata(node).contains()) + { + const auto &fu = fg.metadata(node).get(); + + const int own_latency = fu.line_consumption - fu.border_size; + GModel::log(g, node, "LPI: " + std::to_string(fu.k.m_lpi)); + + // Output latency is max(input_latency) + own_latency + int in_latency = 0; + for (const auto &in_data_node : node->inNodes()) + { + // FIXME: ASSERT(DATA), ASSERT(FLUIDDATA) + in_latency = std::max(in_latency, fg.metadata(in_data_node).get().latency); + } + const int out_latency = in_latency + own_latency; + + for (const auto &out_data_node : node->outNodes()) + { + // FIXME: ASSERT(DATA), ASSERT(FLUIDDATA) + auto &fd = fg.metadata(out_data_node).get(); + fd.latency = out_latency; + fd.lpi_write = fu.k.m_lpi; + GModel::log(g, out_data_node, "Latency: " + std::to_string(out_latency)); + } + } + } + } + + void calcSkew(ade::Graph& graph) + { + using namespace cv::gimpl; + GModel::Graph g(graph); + GFluidModel fg(graph); + + auto sorted = g.metadata().get().nodes(); + for (const auto &node : sorted) + { + if (fg.metadata(node).contains()) + { + int max_latency = 0; + for (const auto &in_data_node : node->inNodes()) + { + // FIXME: ASSERT(DATA), ASSERT(FLUIDDATA) + max_latency = std::max(max_latency, fg.metadata(in_data_node).get().latency); + } + for (const auto &in_data_node : node->inNodes()) + { + // FIXME: ASSERT(DATA), ASSERT(FLUIDDATA) + auto &fd = fg.metadata(in_data_node).get(); + + // Update (not Set) fields here since a single data node may be + // accessed by multiple consumers + fd.skew = std::max(fd.skew, max_latency - fd.latency); + + GModel::log(g, in_data_node, "Skew: " + std::to_string(fd.skew), node); + } + } + } + } +} + +void cv::gimpl::GFluidExecutable::makeReshape(const std::vector &out_rois) +{ + GConstFluidModel fg(m_g); + + // Calculate rois for each fluid buffer + std::vector readStarts(m_num_int_buffers); + std::vector rois(m_num_int_buffers); + initBufferRois(readStarts, rois, out_rois); + + // NB: Allocate ALL buffer object at once, and avoid any further reallocations + // (since raw pointers-to-elements are taken) + for (const auto &it : m_all_gmat_ids) + { + auto id = it.first; + auto nh = it.second; + const auto & d = m_gm.metadata(nh).get(); + const auto &fd = fg.metadata(nh).get(); + const auto meta = cv::util::get(d.meta); + + m_buffers[id].priv().init(meta, fd.lpi_write, readStarts[id], rois[id]); + + // TODO: + // Introduce Storage::INTERNAL_GRAPH and Storage::INTERNAL_ISLAND? + if (fd.internal == true) + { + m_buffers[id].priv().allocate(fd.border, fd.border_size, fd.max_consumption, fd.skew); + std::stringstream stream; + m_buffers[id].debug(stream); + GAPI_LOG_INFO(NULL, stream.str()); + } + } + + // Allocate views, initialize agents + for (auto &agent : m_agents) + { + const auto &fu = fg.metadata(agent->op_handle).get(); + for (auto it : ade::util::indexed(ade::util::toRange(agent->in_buffer_ids))) + { + auto in_idx = ade::util::index(it); + auto buf_idx = ade::util::value(it); + + if (buf_idx >= 0) + { + agent->in_views[in_idx].priv().allocate(fu.line_consumption, fu.border); + } + } + + agent->setRatio(fu.ratio); + agent->m_outputLines = agent->out_buffers.front()->priv().outputLines(); + } + + // Initialize scratch buffers + if (m_scratch_users.size()) + { + for (auto i : m_scratch_users) + { + auto &agent = m_agents.at(i); + GAPI_Assert(agent->k.m_scratch); + + // Trigger Scratch buffer initialization method + agent->k.m_is(GModel::collectInputMeta(m_gm, agent->op_handle), agent->in_args, *agent->out_buffers.back()); + std::stringstream stream; + agent->out_buffers.back()->debug(stream); + GAPI_LOG_INFO(NULL, stream.str()); + } + } +} + +void cv::gimpl::GFluidExecutable::reshape(ade::Graph &g, const GCompileArgs &args) +{ + // FIXME: Probably this needs to be integrated into common pass re-run routine + // Backends may want to mark with passes to re-run on reshape and framework could + // do it system-wide (without need in every backend handling reshape() directly). + // This design needs to be analyzed for implementation. + resetFluidData(g); + initFluidUnits(g); + initLineConsumption(g); + calcLatency(g); + calcSkew(g); + const auto out_rois = cv::gimpl::getCompileArg(args).value_or(cv::GFluidOutputRois()); + makeReshape(out_rois.rois); +} + // FIXME: Document what it does void cv::gimpl::GFluidExecutable::bindInArg(const cv::gimpl::RcDesc &rc, const GRunArg &arg) { @@ -1016,54 +1248,7 @@ void GFluidBackendImpl::addBackendPasses(ade::ExecutionEngineSetupContext &ectx) if (!GModel::isActive(g, cv::gapi::fluid::backend())) // FIXME: Rearchitect this! return; - GFluidModel fg(ctx.graph); - - auto sorted = g.metadata().get().nodes(); - for (auto node : sorted) - { - if (fg.metadata(node).contains()) - { - std::set in_hs, out_ws, out_hs; - - for (const auto& in : node->inNodes()) - { - const auto& d = g.metadata(in).get(); - if (d.shape == cv::GShape::GMAT) - { - const auto& meta = cv::util::get(d.meta); - in_hs.insert(meta.size.height); - } - } - - for (const auto& out : node->outNodes()) - { - const auto& d = g.metadata(out).get(); - if (d.shape == cv::GShape::GMAT) - { - const auto& meta = cv::util::get(d.meta); - out_ws.insert(meta.size.width); - out_hs.insert(meta.size.height); - } - } - - GAPI_Assert(in_hs.size() == 1 && out_ws.size() == 1 && out_hs.size() == 1); - - auto in_h = *in_hs .cbegin(); - auto out_h = *out_hs.cbegin(); - - auto &fu = fg.metadata(node).get(); - fu.ratio = (double)in_h / out_h; - - int line_consumption = maxLineConsumption(fu.k, in_h, out_h, fu.k.m_lpi); - int border_size = borderSize(fu.k); - - fu.border_size = border_size; - fu.line_consumption = line_consumption; - - GModel::log(g, node, "Line consumption: " + std::to_string(fu.line_consumption)); - GModel::log(g, node, "Border size: " + std::to_string(fu.border_size)); - } - } + initFluidUnits(ctx.graph); }); ectx.addPass("exec", "init_line_consumption", [](ade::passes::PassContext &ctx) { @@ -1071,28 +1256,7 @@ void GFluidBackendImpl::addBackendPasses(ade::ExecutionEngineSetupContext &ectx) if (!GModel::isActive(g, cv::gapi::fluid::backend())) // FIXME: Rearchitect this! return; - GFluidModel fg(ctx.graph); - for (const auto node : g.nodes()) - { - if (fg.metadata(node).contains()) - { - const auto &fu = fg.metadata(node).get(); - - for (auto in_data_node : node->inNodes()) - { - auto &fd = fg.metadata(in_data_node).get(); - - // Update (not Set) fields here since a single data node may be - // accessed by multiple consumers - fd.max_consumption = std::max(fu.line_consumption, fd.max_consumption); - fd.border_size = std::max(fu.border_size, fd.border_size); - - GModel::log(g, in_data_node, "Line consumption: " + std::to_string(fd.max_consumption) - + " (upd by " + std::to_string(fu.line_consumption) + ")", node); - GModel::log(g, in_data_node, "Border size: " + std::to_string(fd.border_size), node); - } - } - } + initLineConsumption(ctx.graph); }); ectx.addPass("exec", "calc_latency", [](ade::passes::PassContext &ctx) { @@ -1100,37 +1264,7 @@ void GFluidBackendImpl::addBackendPasses(ade::ExecutionEngineSetupContext &ectx) if (!GModel::isActive(g, cv::gapi::fluid::backend())) // FIXME: Rearchitect this! return; - GFluidModel fg(ctx.graph); - - auto sorted = g.metadata().get().nodes(); - for (auto node : sorted) - { - if (fg.metadata(node).contains()) - { - const auto &fu = fg.metadata(node).get(); - - const int own_latency = fu.line_consumption - fu.border_size; - GModel::log(g, node, "LPI: " + std::to_string(fu.k.m_lpi)); - - // Output latency is max(input_latency) + own_latency - int in_latency = 0; - for (auto in_data_node : node->inNodes()) - { - // FIXME: ASSERT(DATA), ASSERT(FLUIDDATA) - in_latency = std::max(in_latency, fg.metadata(in_data_node).get().latency); - } - const int out_latency = in_latency + own_latency; - - for (auto out_data_node : node->outNodes()) - { - // FIXME: ASSERT(DATA), ASSERT(FLUIDDATA) - auto &fd = fg.metadata(out_data_node).get(); - fd.latency = out_latency; - fd.lpi_write = fu.k.m_lpi; - GModel::log(g, out_data_node, "Latency: " + std::to_string(out_latency)); - } - } - } + calcLatency(ctx.graph); }); ectx.addPass("exec", "calc_skew", [](ade::passes::PassContext &ctx) { @@ -1138,32 +1272,7 @@ void GFluidBackendImpl::addBackendPasses(ade::ExecutionEngineSetupContext &ectx) if (!GModel::isActive(g, cv::gapi::fluid::backend())) // FIXME: Rearchitect this! return; - GFluidModel fg(ctx.graph); - - auto sorted = g.metadata().get().nodes(); - for (auto node : sorted) - { - if (fg.metadata(node).contains()) - { - int max_latency = 0; - for (auto in_data_node : node->inNodes()) - { - // FIXME: ASSERT(DATA), ASSERT(FLUIDDATA) - max_latency = std::max(max_latency, fg.metadata(in_data_node).get().latency); - } - for (auto in_data_node : node->inNodes()) - { - // FIXME: ASSERT(DATA), ASSERT(FLUIDDATA) - auto &fd = fg.metadata(in_data_node).get(); - - // Update (not Set) fields here since a single data node may be - // accessed by multiple consumers - fd.skew = std::max(fd.skew, max_latency - fd.latency); - - GModel::log(g, in_data_node, "Skew: " + std::to_string(fd.skew), node); - } - } - } + calcSkew(ctx.graph); }); ectx.addPass("exec", "init_buffer_borders", [](ade::passes::PassContext &ctx) diff --git a/modules/gapi/src/backends/fluid/gfluidbackend.hpp b/modules/gapi/src/backends/fluid/gfluidbackend.hpp index e91a41b..4e921f2 100644 --- a/modules/gapi/src/backends/fluid/gfluidbackend.hpp +++ b/modules/gapi/src/backends/fluid/gfluidbackend.hpp @@ -49,6 +49,19 @@ struct FluidData gapi::fluid::BorderOpt border; }; +struct FluidMapper +{ + FluidMapper(double ratio, int lpi) : m_ratio(ratio), m_lpi(lpi) {} + virtual ~FluidMapper() = default; + virtual int firstWindow(int outCoord, int lpi) const = 0; + virtual int nextWindow(int outCoord, int lpi) const = 0; + virtual int linesRead(int outCoord) const = 0; + +protected: + double m_ratio = 0.0; + int m_lpi = 0; +}; + struct FluidAgent { public: @@ -72,8 +85,6 @@ public: int m_outputLines = 0; int m_producedLines = 0; - double m_ratio = 0.0f; - // Execution methods void reset(); bool canWork() const; @@ -83,10 +94,12 @@ public: bool done() const; void debug(std::ostream& os); + // FIXME: // refactor (implement a more solid replacement or // drop this method completely) - virtual void setInHeight(int h) = 0; + virtual void setRatio(double ratio) = 0; + private: // FIXME!!! // move to another class @@ -99,7 +112,6 @@ class GFluidExecutable final: public GIslandExecutable { const ade::Graph &m_g; GModel::ConstGraph m_gm; - const std::vector m_nodes; std::vector> m_agents; std::vector m_buffers; @@ -109,23 +121,25 @@ class GFluidExecutable final: public GIslandExecutable std::size_t m_num_int_buffers; // internal buffers counter (m_buffers - num_scratch) std::vector m_scratch_users; - std::vector m_views; - - std::vector m_outputRois; std::unordered_map m_id_map; // GMat id -> buffer idx map + std::map m_all_gmat_ids; void bindInArg (const RcDesc &rc, const GRunArg &arg); void bindOutArg(const RcDesc &rc, const GRunArgP &arg); void packArg (GArg &in_arg, const GArg &op_arg); - void initBufferRois(std::vector& readStarts, std::vector& rois); + void initBufferRois(std::vector& readStarts, std::vector& rois, const std::vector &out_rois); + void makeReshape(const std::vector& out_rois); public: GFluidExecutable(const ade::Graph &g, const std::vector &nodes, const std::vector &outputRois); + virtual inline bool canReshape() const override { return true; } + virtual void reshape(ade::Graph& g, const GCompileArgs& args) override; + virtual void run(std::vector &&input_objs, std::vector &&output_objs) override; }; diff --git a/modules/gapi/src/backends/fluid/gfluidbuffer.cpp b/modules/gapi/src/backends/fluid/gfluidbuffer.cpp index b945cc4..741943e 100644 --- a/modules/gapi/src/backends/fluid/gfluidbuffer.cpp +++ b/modules/gapi/src/backends/fluid/gfluidbuffer.cpp @@ -115,10 +115,10 @@ template fluid::BorderHandlerT::BorderHandlerT(int border_size, int data_type) : BorderHandler(border_size) { - auto getFillBorderRowFunc = [&](int border, int dataType) { + auto getFillBorderRowFunc = [&](int border, int depth) { if (border == cv::BORDER_REPLICATE) { - switch(dataType) + switch(depth) { case CV_8U: return &fillBorderReplicateRow< uint8_t>; break; case CV_16S: return &fillBorderReplicateRow< int16_t>; break; @@ -129,7 +129,7 @@ fluid::BorderHandlerT::BorderHandlerT(int border_size, int data_type } else if (border == cv::BORDER_REFLECT_101) { - switch(dataType) + switch(depth) { case CV_8U: return &fillBorderReflectRow< uint8_t>; break; case CV_16S: return &fillBorderReflectRow< int16_t>; break; @@ -145,7 +145,7 @@ fluid::BorderHandlerT::BorderHandlerT(int border_size, int data_type } }; - m_fill_border_row = getFillBorderRowFunc(BorderType, data_type); + m_fill_border_row = getFillBorderRowFunc(BorderType, CV_MAT_DEPTH(data_type)); } namespace { @@ -169,20 +169,20 @@ const uint8_t* fluid::BorderHandlerT::inLineB(int log_idx, const Buf return data.ptr(idx); } -fluid::BorderHandlerT::BorderHandlerT(int border_size, cv::gapi::own::Scalar border_value, int data_type, int desc_width) +fluid::BorderHandlerT::BorderHandlerT(int border_size, cv::gapi::own::Scalar border_value) : BorderHandler(border_size), m_border_value(border_value) -{ - m_const_border.create(1, desc_width + 2*m_border_size, data_type); - m_const_border = border_value; -} +{ /* nothing */ } const uint8_t* fluid::BorderHandlerT::inLineB(int /*log_idx*/, const BufferStorageWithBorder& /*data*/, int /*desc_height*/) const { return m_const_border.ptr(0, m_border_size); } -void fluid::BorderHandlerT::fillCompileTimeBorder(BufferStorageWithBorder& data) const +void fluid::BorderHandlerT::fillCompileTimeBorder(BufferStorageWithBorder& data) { + m_const_border.create(1, data.cols(), data.data().type()); + m_const_border = m_border_value; + cv::gapi::fillBorderConstant(m_border_size, m_border_value, data.data()); } @@ -206,15 +206,12 @@ std::size_t fluid::BorderHandlerT::size() const } // Fluid BufferStorage implementation ////////////////////////////////////////// -void fluid::BufferStorageWithBorder::create(int capacity, int desc_width, int dtype, int border_size, Border border) +void fluid::BufferStorageWithBorder::init(int dtype, int border_size, Border border) { - auto width = (desc_width + 2*border_size); - m_data.create(capacity, width, dtype); - switch(border.type) { case cv::BORDER_CONSTANT: - m_borderHandler.reset(new BorderHandlerT(border_size, border.value, dtype, desc_width)); break; + m_borderHandler.reset(new BorderHandlerT(border_size, border.value)); break; case cv::BORDER_REPLICATE: m_borderHandler.reset(new BorderHandlerT(border_size, dtype)); break; case cv::BORDER_REFLECT_101: @@ -222,6 +219,13 @@ void fluid::BufferStorageWithBorder::create(int capacity, int desc_width, int dt default: GAPI_Assert(false); } +} + +void fluid::BufferStorageWithBorder::create(int capacity, int desc_width, int dtype) +{ + auto borderSize = m_borderHandler->borderSize(); + auto width = (desc_width + 2*borderSize); + m_data.create(capacity, width, dtype); m_borderHandler->fillCompileTimeBorder(*this); } @@ -329,7 +333,8 @@ std::unique_ptr createStorage(int capacity, int desc_width if (border) { std::unique_ptr storage(new BufferStorageWithBorder); - storage->create(capacity, desc_width, type, border_size, border.value()); + storage->init(type, border_size, border.value()); + storage->create(capacity, desc_width, type); return std::move(storage); } @@ -400,15 +405,19 @@ const uint8_t* fluid::ViewPrivWithoutOwnBorder::InLineB(int index) const return p_priv.storage().inLineB(log_idx, m_p->meta().size.height); } -fluid::ViewPrivWithOwnBorder::ViewPrivWithOwnBorder(const Buffer *parent, int lineConsumption, int borderSize, Border border) +fluid::ViewPrivWithOwnBorder::ViewPrivWithOwnBorder(const Buffer *parent, int borderSize) { GAPI_Assert(parent); m_p = parent; m_border_size = borderSize; +} +void fluid::ViewPrivWithOwnBorder::allocate(int lineConsumption, BorderOpt border) +{ auto desc = m_p->meta(); int type = CV_MAKETYPE(desc.depth, desc.chan); - m_own_storage.create(lineConsumption, desc.size.width, type, borderSize, border); + m_own_storage.init(type, m_border_size, border.value()); + m_own_storage.create(lineConsumption, desc.size.width, type); } void fluid::ViewPrivWithOwnBorder::prepareToRead() @@ -497,15 +506,15 @@ fluid::Buffer::Priv::Priv(int read_start, cv::gapi::own::Rect roi) {} void fluid::Buffer::Priv::init(const cv::GMatDesc &desc, - int wlpi, + int writer_lpi, int readStartPos, cv::gapi::own::Rect roi) { - m_writer_lpi = wlpi; + m_writer_lpi = writer_lpi; m_desc = desc; m_readStart = readStartPos; - m_roi = roi == cv::Rect{} ? cv::Rect{0, 0, desc.size.width, desc.size.height} - : roi; + m_roi = roi == own::Rect{} ? own::Rect{ 0, 0, desc.size.width, desc.size.height } + : roi; } void fluid::Buffer::Priv::allocate(BorderOpt border, @@ -513,7 +522,6 @@ void fluid::Buffer::Priv::allocate(BorderOpt border, int line_consumption, int skew) { - GAPI_Assert(!m_storage); GAPI_Assert(line_consumption > 0); // Init physical buffer @@ -690,10 +698,10 @@ fluid::View::View(Priv* p) : m_priv(p) { /* nothing */ } -fluid::View fluid::Buffer::mkView(int lineConsumption, int borderSize, BorderOpt border, bool ownStorage) +fluid::View fluid::Buffer::mkView(int borderSize, bool ownStorage) { // FIXME: logic outside of Priv (because View takes pointer to Buffer) - auto view = ownStorage ? View(new ViewPrivWithOwnBorder(this, lineConsumption, borderSize, border.value())) + auto view = ownStorage ? View(new ViewPrivWithOwnBorder(this, borderSize)) : View(new ViewPrivWithoutOwnBorder(this, borderSize)); m_priv->addView(view); return view; diff --git a/modules/gapi/src/backends/fluid/gfluidbuffer_priv.hpp b/modules/gapi/src/backends/fluid/gfluidbuffer_priv.hpp index 7057cba..f21b11c 100644 --- a/modules/gapi/src/backends/fluid/gfluidbuffer_priv.hpp +++ b/modules/gapi/src/backends/fluid/gfluidbuffer_priv.hpp @@ -30,7 +30,7 @@ public: virtual const uint8_t* inLineB(int log_idx, const BufferStorageWithBorder &data, int desc_height) const = 0; // Fills border pixels after buffer allocation (if possible (for const border)) - inline virtual void fillCompileTimeBorder(BufferStorageWithBorder &) const { /* nothing */ } + inline virtual void fillCompileTimeBorder(BufferStorageWithBorder &) { /* nothing */ } // Fills required border lines inline virtual void updateBorderPixels(BufferStorageWithBorder& /*data*/, int /*startLine*/, int /*lpi*/) const { /* nothing */ } @@ -56,9 +56,9 @@ class BorderHandlerT : public BorderHandler cv::gapi::own::Mat m_const_border; public: - BorderHandlerT(int border_size, cv::gapi::own::Scalar border_value, int data_type, int desc_width); + BorderHandlerT(int border_size, cv::gapi::own::Scalar border_value); virtual const uint8_t* inLineB(int log_idx, const BufferStorageWithBorder &data, int desc_height) const override; - virtual void fillCompileTimeBorder(BufferStorageWithBorder &) const override; + virtual void fillCompileTimeBorder(BufferStorageWithBorder &) override; virtual std::size_t size() const override; }; @@ -151,7 +151,8 @@ public: return m_data.ptr(physIdx(idx), borderSize()); } - void create(int capacity, int desc_width, int type, int border_size, Border border); + void init(int depth, int border_size, Border border); + void create(int capacity, int desc_width, int dtype); virtual const uint8_t* inLineB(int log_idx, int desc_height) const override; @@ -178,6 +179,7 @@ public: virtual ~Priv() = default; // API used by actors/backend + virtual void allocate(int lineConsumption, BorderOpt border) = 0; virtual void prepareToRead() = 0; void readDone(int linesRead, int linesForNextIteration); @@ -198,6 +200,7 @@ public: // API used by actors/backend ViewPrivWithoutOwnBorder(const Buffer *p, int borderSize); + inline virtual void allocate(int, BorderOpt) override { /* nothing */ } inline virtual void prepareToRead() override { /* nothing */ } inline virtual std::size_t size() const override { return 0; } @@ -212,8 +215,9 @@ class ViewPrivWithOwnBorder final : public View::Priv public: // API used by actors/backend - ViewPrivWithOwnBorder(const Buffer *p, int lineCapacity, int borderSize, Border border); + ViewPrivWithOwnBorder(const Buffer *p, int borderSize); + inline virtual void allocate(int lineConsumption, BorderOpt border) override; virtual void prepareToRead() override; virtual std::size_t size() const override; @@ -253,7 +257,7 @@ public: // API used by actors/backend void init(const cv::GMatDesc &desc, - int wlpi, + int writer_lpi, int readStart, cv::gapi::own::Rect roi); diff --git a/modules/gapi/src/compiler/gcompiled.cpp b/modules/gapi/src/compiler/gcompiled.cpp index 4bcca9f..876575d 100644 --- a/modules/gapi/src/compiler/gcompiled.cpp +++ b/modules/gapi/src/compiler/gcompiled.cpp @@ -59,6 +59,19 @@ void cv::GCompiled::Priv::checkArgs(const cv::gimpl::GRuntimeArgs &args) const } } +bool cv::GCompiled::Priv::canReshape() const +{ + GAPI_Assert(m_exec); + return m_exec->canReshape(); +} + +void cv::GCompiled::Priv::reshape(const GMetaArgs& inMetas, const GCompileArgs& args) +{ + GAPI_Assert(m_exec); + m_exec->reshape(inMetas, args); + m_metas = inMetas; +} + const cv::gimpl::GModel::Graph& cv::GCompiled::Priv::model() const { GAPI_Assert(nullptr != m_exec); @@ -118,7 +131,6 @@ void cv::GCompiled::operator ()(const std::vector &ins, } #endif // !defined(GAPI_STANDALONE) - const cv::GMetaArgs& cv::GCompiled::metas() const { return m_priv->metas(); @@ -129,8 +141,17 @@ const cv::GMetaArgs& cv::GCompiled::outMetas() const return m_priv->outMetas(); } - cv::GCompiled::Priv& cv::GCompiled::priv() { return *m_priv; } + +bool cv::GCompiled::canReshape() const +{ + return m_priv->canReshape(); +} + +void cv::GCompiled::reshape(const GMetaArgs& inMetas, const GCompileArgs& args) +{ + m_priv->reshape(inMetas, args); +} diff --git a/modules/gapi/src/compiler/gcompiled_priv.hpp b/modules/gapi/src/compiler/gcompiled_priv.hpp index 1bb3bbf..e616b2b 100644 --- a/modules/gapi/src/compiler/gcompiled_priv.hpp +++ b/modules/gapi/src/compiler/gcompiled_priv.hpp @@ -46,6 +46,9 @@ public: std::unique_ptr &&pE); bool isEmpty() const; + bool canReshape() const; + void reshape(const GMetaArgs& inMetas, const GCompileArgs &args); + void run(cv::gimpl::GRuntimeArgs &&args); const GMetaArgs& metas() const; const GMetaArgs& outMetas() const; diff --git a/modules/gapi/src/compiler/gcompiler.cpp b/modules/gapi/src/compiler/gcompiler.cpp index e37301a..32ce8e3 100644 --- a/modules/gapi/src/compiler/gcompiler.cpp +++ b/modules/gapi/src/compiler/gcompiler.cpp @@ -111,7 +111,7 @@ cv::gimpl::GCompiler::GCompiler(const cv::GComputation &c, m_e.addPass("init", "check_islands_content", passes::checkIslandsContent); m_e.addPassStage("meta"); m_e.addPass("meta", "initialize", std::bind(passes::initMeta, _1, std::ref(m_metas))); - m_e.addPass("meta", "propagate", passes::inferMeta); + m_e.addPass("meta", "propagate", std::bind(passes::inferMeta, _1, false)); m_e.addPass("meta", "finalize", passes::storeResultingMeta); // moved to another stage, FIXME: two dumps? // m_e.addPass("meta", "dump_dot", passes::dumpDotStdout); diff --git a/modules/gapi/src/compiler/gislandmodel.hpp b/modules/gapi/src/compiler/gislandmodel.hpp index a0e44c9..03b42ff 100644 --- a/modules/gapi/src/compiler/gislandmodel.hpp +++ b/modules/gapi/src/compiler/gislandmodel.hpp @@ -109,6 +109,9 @@ public: virtual void run(std::vector &&input_objs, std::vector &&output_objs) = 0; + virtual bool canReshape() const = 0; + virtual void reshape(ade::Graph& g, const GCompileArgs& args) = 0; + virtual ~GIslandExecutable() = default; }; diff --git a/modules/gapi/src/compiler/passes/meta.cpp b/modules/gapi/src/compiler/passes/meta.cpp index 0a5bb5a..528d84c 100644 --- a/modules/gapi/src/compiler/passes/meta.cpp +++ b/modules/gapi/src/compiler/passes/meta.cpp @@ -33,7 +33,7 @@ void cv::gimpl::passes::initMeta(ade::passes::PassContext &ctx, const GMetaArgs // Iterate over all operations in the topological order, trigger kernels // validate() function, update output objects metadata. -void cv::gimpl::passes::inferMeta(ade::passes::PassContext &ctx) +void cv::gimpl::passes::inferMeta(ade::passes::PassContext &ctx, bool meta_is_initialized) { // FIXME: ADE pass dependency on topo_sort? // FIXME: ADE pass dependency on initMeta? @@ -75,7 +75,7 @@ void cv::gimpl::passes::inferMeta(ade::passes::PassContext &ctx) // Now ask kernel for it's output meta. // Resulting out_args may have a larger size than op.outs, since some // outputs could stay unused (unconnected) - const GMetaArgs out_metas = op.k.outMeta(input_meta_args, op.args); + const auto& out_metas = op.k.outMeta(input_meta_args, op.args); // Walk through operation's outputs, update meta of output objects // appropriately @@ -87,7 +87,7 @@ void cv::gimpl::passes::inferMeta(ade::passes::PassContext &ctx) GAPI_Assert(gr.metadata(output_nh).get().t == NodeType::DATA); auto &output_meta = gr.metadata(output_nh).get().meta; - if (!util::holds_alternative(output_meta)) + if (!meta_is_initialized && !util::holds_alternative(output_meta)) { GAPI_LOG_INFO(NULL, "!!! Output object has an initialized meta - " diff --git a/modules/gapi/src/compiler/passes/passes.hpp b/modules/gapi/src/compiler/passes/passes.hpp index d73653b..14f6acd 100644 --- a/modules/gapi/src/compiler/passes/passes.hpp +++ b/modules/gapi/src/compiler/passes/passes.hpp @@ -38,7 +38,7 @@ void checkIslands(ade::passes::PassContext &ctx); void checkIslandsContent(ade::passes::PassContext &ctx); void initMeta(ade::passes::PassContext &ctx, const GMetaArgs &metas); -void inferMeta(ade::passes::PassContext &ctx); +void inferMeta(ade::passes::PassContext &ctx, bool meta_is_initialized); void storeResultingMeta(ade::passes::PassContext &ctx); void expandKernels(ade::passes::PassContext &ctx, diff --git a/modules/gapi/src/executor/gexecutor.cpp b/modules/gapi/src/executor/gexecutor.cpp index 3885820..f117c06 100644 --- a/modules/gapi/src/executor/gexecutor.cpp +++ b/modules/gapi/src/executor/gexecutor.cpp @@ -13,6 +13,7 @@ #include "opencv2/gapi/opencv_includes.hpp" #include "executor/gexecutor.hpp" +#include "compiler/passes/passes.hpp" cv::gimpl::GExecutor::GExecutor(std::unique_ptr &&g_model) : m_orig_graph(std::move(g_model)) @@ -34,7 +35,6 @@ cv::gimpl::GExecutor::GExecutor(std::unique_ptr &&g_model) // 6. Run GIslandExecutable // 7. writeBack - m_ops.reserve(m_gim.nodes().size()); auto sorted = m_gim.metadata().get(); for (auto nh : sorted.nodes()) { @@ -59,6 +59,7 @@ cv::gimpl::GExecutor::GExecutor(std::unique_ptr &&g_model) // (3) for (auto in_slot_nh : nh->inNodes()) xtract(in_slot_nh, input_rcs); for (auto out_slot_nh : nh->outNodes()) xtract(out_slot_nh, output_rcs); + m_ops.emplace_back(OpDesc{ std::move(input_rcs) , std::move(output_rcs) , m_gim.metadata(nh).get().object}); @@ -224,3 +225,20 @@ const cv::gimpl::GModel::Graph& cv::gimpl::GExecutor::model() const { return m_gm; } + +bool cv::gimpl::GExecutor::canReshape() const +{ + // FIXME: Introduce proper reshaping support on GExecutor level + // for all cases! + return (m_ops.size() == 1) && m_ops[0].isl_exec->canReshape(); +} + +void cv::gimpl::GExecutor::reshape(const GMetaArgs& inMetas, const GCompileArgs& args) +{ + GAPI_Assert(canReshape()); + auto& g = *m_orig_graph.get(); + ade::passes::PassContext ctx{g}; + passes::initMeta(ctx, inMetas); + passes::inferMeta(ctx, true); + m_ops[0].isl_exec->reshape(g, args); +} diff --git a/modules/gapi/src/executor/gexecutor.hpp b/modules/gapi/src/executor/gexecutor.hpp index 47c5582..e4128ba 100644 --- a/modules/gapi/src/executor/gexecutor.hpp +++ b/modules/gapi/src/executor/gexecutor.hpp @@ -85,6 +85,9 @@ public: explicit GExecutor(std::unique_ptr &&g_model); void run(cv::gimpl::GRuntimeArgs &&args); + bool canReshape() const; + void reshape(const GMetaArgs& inMetas, const GCompileArgs& args); + const GModel::Graph& model() const; // FIXME: make it ConstGraph? }; diff --git a/modules/gapi/test/gapi_fluid_test.cpp b/modules/gapi/test/gapi_fluid_test.cpp index 57d2085..0951081 100644 --- a/modules/gapi/test/gapi_fluid_test.cpp +++ b/modules/gapi/test/gapi_fluid_test.cpp @@ -51,7 +51,7 @@ TEST(FluidBuffer, InputTest) cv::Mat in_mat = cv::Mat::eye(buffer_size, CV_8U); cv::gapi::fluid::Buffer buffer(to_own(in_mat), true); - cv::gapi::fluid::View view = buffer.mkView(1, 0, {}, false); + cv::gapi::fluid::View view = buffer.mkView(0, {}); view.priv().reset(1); int this_y = 0; @@ -74,7 +74,7 @@ TEST(FluidBuffer, CircularTest) cv::gapi::fluid::Buffer buffer(cv::GMatDesc{CV_8U,1,buffer_size}, 3, 1, 0, 1, util::make_optional(cv::gapi::fluid::Border{cv::BORDER_CONSTANT, cv::gapi::own::Scalar(255)})); - cv::gapi::fluid::View view = buffer.mkView(3, 1, {}, false); + cv::gapi::fluid::View view = buffer.mkView(1, {}); view.priv().reset(3); buffer.debug(std::cout); diff --git a/modules/gapi/test/internal/gapi_int_recompilation_test.cpp b/modules/gapi/test/internal/gapi_int_recompilation_test.cpp index de8d46e..ced7324 100644 --- a/modules/gapi/test/internal/gapi_int_recompilation_test.cpp +++ b/modules/gapi/test/internal/gapi_int_recompilation_test.cpp @@ -8,6 +8,9 @@ #include "test_precomp.hpp" #include "api/gcomputation_priv.hpp" +#include +#include + namespace opencv_test { @@ -68,4 +71,160 @@ TEST(GComputationCompile, RecompileWithDifferentMeta) EXPECT_NE(&comp1.priv(), &comp2.priv()); } +TEST(GComputationCompile, FluidReshapeWithDifferentDims) +{ + cv::GMat in; + cv::GComputation cc(in, in+in); + + cv::Mat in_mat1 = cv::Mat::eye (32, 32, CV_8UC1); + cv::Mat in_mat2 = cv::Mat::zeros(64, 64, CV_8UC1); + cv::Mat out_mat; + + cc.apply(in_mat1, out_mat, cv::compile_args(cv::gapi::core::fluid::kernels())); + auto comp1 = cc.priv().m_lastCompiled; + + cc.apply(in_mat2, out_mat); + auto comp2 = cc.priv().m_lastCompiled; + + // Both compiled objects are actually the same unique executable + EXPECT_EQ(&comp1.priv(), &comp2.priv()); +} + +TEST(GComputationCompile, FluidReshapeResizeDownScale) +{ + cv::Size szOut(4, 4); + cv::GMat in; + cv::GComputation cc(in, cv::gapi::resize(in, szOut)); + + cv::Mat in_mat1( 8, 8, CV_8UC3); + cv::Mat in_mat2(16, 16, CV_8UC3); + cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::Mat out_mat1, out_mat2; + + cc.apply(in_mat1, out_mat1, cv::compile_args(cv::gapi::core::fluid::kernels())); + auto comp1 = cc.priv().m_lastCompiled; + + cc.apply(in_mat2, out_mat2); + auto comp2 = cc.priv().m_lastCompiled; + + // Both compiled objects are actually the same unique executable + EXPECT_EQ(&comp1.priv(), &comp2.priv()); + + cv::Mat cv_out_mat1, cv_out_mat2; + cv::resize(in_mat1, cv_out_mat1, szOut); + cv::resize(in_mat2, cv_out_mat2, szOut); + + EXPECT_EQ(0, cv::countNonZero(out_mat1 != cv_out_mat1)); + EXPECT_EQ(0, cv::countNonZero(out_mat2 != cv_out_mat2)); +} + +TEST(GComputationCompile, FluidReshapeSwitchToUpscaleFromDownscale) +{ + cv::Size szOut(4, 4); + cv::GMat in; + cv::GComputation cc(in, cv::gapi::resize(in, szOut)); + + cv::Mat in_mat1( 8, 8, CV_8UC3); + cv::Mat in_mat2( 2, 2, CV_8UC3); + cv::Mat in_mat3(16, 16, CV_8UC3); + cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::randu(in_mat3, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::Mat out_mat1, out_mat2, out_mat3; + + cc.apply(in_mat1, out_mat1, cv::compile_args(cv::gapi::core::fluid::kernels())); + auto comp1 = cc.priv().m_lastCompiled; + + cc.apply(in_mat2, out_mat2); + auto comp2 = cc.priv().m_lastCompiled; + + cc.apply(in_mat3, out_mat3); + auto comp3 = cc.priv().m_lastCompiled; + + EXPECT_EQ(&comp1.priv(), &comp2.priv()); + EXPECT_EQ(&comp1.priv(), &comp3.priv()); + + cv::Mat cv_out_mat1, cv_out_mat2, cv_out_mat3; + cv::resize(in_mat1, cv_out_mat1, szOut); + cv::resize(in_mat2, cv_out_mat2, szOut); + cv::resize(in_mat3, cv_out_mat3, szOut); + + EXPECT_EQ(0, cv::countNonZero(out_mat1 != cv_out_mat1)); + EXPECT_EQ(0, cv::countNonZero(out_mat2 != cv_out_mat2)); + EXPECT_EQ(0, cv::countNonZero(out_mat3 != cv_out_mat3)); +} + +TEST(GComputationCompile, ReshapeBlur) +{ + cv::Size kernelSize{3, 3}; + cv::GMat in; + cv::GComputation cc(in, cv::gapi::blur(in, kernelSize)); + + cv::Mat in_mat1( 8, 8, CV_8UC1); + cv::Mat in_mat2(16, 16, CV_8UC1); + cv::randu(in_mat1, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255)); + cv::Mat out_mat1, out_mat2; + + cc.apply(in_mat1, out_mat1, cv::compile_args(cv::gapi::imgproc::fluid::kernels())); + auto comp1 = cc.priv().m_lastCompiled; + + cc.apply(in_mat2, out_mat2); + auto comp2 = cc.priv().m_lastCompiled; + + // Both compiled objects are actually the same unique executable + EXPECT_EQ(&comp1.priv(), &comp2.priv()); + + cv::Mat cv_out_mat1, cv_out_mat2; + cv::blur(in_mat1, cv_out_mat1, kernelSize); + cv::blur(in_mat2, cv_out_mat2, kernelSize); + + EXPECT_EQ(0, cv::countNonZero(out_mat1 != cv_out_mat1)); + EXPECT_EQ(0, cv::countNonZero(out_mat2 != cv_out_mat2)); +} + +TEST(GComputationCompile, ReshapeRois) +{ + cv::Size kernelSize{3, 3}; + cv::Size szOut(8, 8); + cv::GMat in; + auto blurred = cv::gapi::blur(in, kernelSize); + cv::GComputation cc(in, cv::gapi::resize(blurred, szOut)); + + cv::Mat first_in_mat(8, 8, CV_8UC3); + cv::Mat first_out_mat; + auto fluidKernels = cv::gapi::combine(gapi::imgproc::fluid::kernels(), + gapi::core::fluid::kernels(), + cv::unite_policy::REPLACE); + cc.apply(first_in_mat, first_out_mat, cv::compile_args(fluidKernels)); + auto first_comp = cc.priv().m_lastCompiled; + + constexpr int niter = 4; + for (int i = 0; i < niter; i++) + { + int width = 4 + 2*i; + int height = width; + cv::Mat in_mat(width, height, CV_8UC3); + cv::Mat out_mat = cv::Mat::zeros(szOut, CV_8UC3); + + int x = 0; + int y = szOut.height * i / niter; + int roiW = szOut.width; + int roiH = szOut.height / niter; + cv::Rect roi{x, y, roiW, roiH}; + + cc.apply(in_mat, out_mat, cv::compile_args(cv::GFluidOutputRois{{to_own(roi)}})); + auto comp = cc.priv().m_lastCompiled; + + EXPECT_EQ(&first_comp.priv(), &comp.priv()); + + cv::Mat blur_mat, cv_out_mat; + cv::blur(in_mat, blur_mat, kernelSize); + cv::resize(blur_mat, cv_out_mat, szOut); + + EXPECT_EQ(0, cv::countNonZero(out_mat(roi) != cv_out_mat(roi))); + } +} + } // opencv_test -- 2.7.4