From: Maxim Pashchenkov Date: Wed, 10 Mar 2021 16:06:46 +0000 (+0300) Subject: Merge pull request #18240 from mpashchenkov:mp/ocv-gapi-input-cnn-reshape X-Git-Tag: accepted/tizen/unified/20220125.121719~1^2~157 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=12fa8d8444ec93af80ca5681837ce0a46202c959;p=platform%2Fupstream%2Fopencv.git Merge pull request #18240 from mpashchenkov:mp/ocv-gapi-input-cnn-reshape [G-API]: Adding reshape for CNN input. * Added CNN input IE reshape * rbs * Added unordered_set instead vector * Alignment --- diff --git a/modules/gapi/include/opencv2/gapi/infer/ie.hpp b/modules/gapi/include/opencv2/gapi/infer/ie.hpp index e1df80f..3f5c57f 100644 --- a/modules/gapi/include/opencv2/gapi/infer/ie.hpp +++ b/modules/gapi/include/opencv2/gapi/infer/ie.hpp @@ -8,6 +8,7 @@ #define OPENCV_GAPI_INFER_IE_HPP #include +#include #include #include #include // tuple, tuple_size @@ -68,6 +69,9 @@ namespace detail { bool is_generic; IEConfig config; + std::map> reshape_table; + std::unordered_set layer_names_to_reshape; + // NB: Number of asyncrhonious infer requests size_t nireq; }; @@ -95,6 +99,8 @@ public: , detail::ParamDesc::Kind::Load , false , {} + , {} + , {} , 1u} { }; @@ -106,6 +112,8 @@ public: , detail::ParamDesc::Kind::Import , false , {} + , {} + , {} , 1u} { }; @@ -148,6 +156,36 @@ public: return *this; } + Params& cfgInputReshape(std::map>&& reshape_table) { + desc.reshape_table = std::move(reshape_table); + return *this; + } + + Params& cfgInputReshape(const std::map>& reshape_table) { + desc.reshape_table = reshape_table; + return *this; + } + + Params& cfgInputReshape(std::string&& layer_name, std::vector&& layer_dims) { + desc.reshape_table.emplace(layer_name, layer_dims); + return *this; + } + + Params& cfgInputReshape(const std::string& layer_name, const std::vector& layer_dims) { + desc.reshape_table.emplace(layer_name, layer_dims); + return *this; + } + + Params& cfgInputReshape(std::unordered_set&& layer_names) { + desc.layer_names_to_reshape = std::move(layer_names); + return *this; + } + + Params& cfgInputReshape(const std::unordered_set& layer_names) { + desc.layer_names_to_reshape = layer_names; + return *this; + } + // BEGIN(G-API's network parametrization API) GBackend backend() const { return cv::gapi::ie::backend(); } std::string tag() const { return Net::tag(); } @@ -165,13 +203,13 @@ public: const std::string &model, const std::string &weights, const std::string &device) - : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, 1u}, m_tag(tag) { + : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u}, m_tag(tag) { }; Params(const std::string &tag, const std::string &model, const std::string &device) - : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, 1u}, m_tag(tag) { + : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u}, m_tag(tag) { }; Params& pluginConfig(IEConfig&& cfg) { diff --git a/modules/gapi/src/backends/ie/giebackend.cpp b/modules/gapi/src/backends/ie/giebackend.cpp index 1e573d8..2a26f0c 100644 --- a/modules/gapi/src/backends/ie/giebackend.cpp +++ b/modules/gapi/src/backends/ie/giebackend.cpp @@ -45,6 +45,7 @@ #include "backends/ie/giebackend/giewrapper.hpp" #include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK! +#include "logger.hpp" #if INF_ENGINE_RELEASE < 2021010000 #include "ie_compound_blob.h" @@ -224,6 +225,9 @@ struct IEUnit { // but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo()); outputs = cv::gimpl::ie::wrap::toOutputsDataMap(this_network.GetOutputsInfo()); + if (!params.reshape_table.empty() || !params.layer_names_to_reshape.empty()) { + GAPI_LOG_WARNING(NULL, "Reshape isn't supported for imported network"); + } } else { cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind")); } @@ -249,6 +253,11 @@ struct IEUnit { if (params.num_out == 1u && params.output_names.empty()) { params.output_names = { outputs.begin()->first }; } + if (!params.reshape_table.empty()) { + GAPI_Assert((params.reshape_table.size() + params.layer_names_to_reshape.size()) <= + params.num_in && + "Number of layers to reshape must be less than or equal to number of inputs"); + } } // This method is [supposed to be] called at Island compilation stage @@ -669,6 +678,46 @@ void cv::gimpl::ie::GIEExecutable::run(cv::gimpl::GIslandExecutable::IInput &in namespace cv { namespace gimpl { namespace ie { +static void configureInputReshapeByImage(const IE::InputInfo::Ptr& ii, + const cv::GMetaArg mm, + IE::ICNNNetwork::InputShapes& input_reshape_table) { + const auto& layer_name = ii->name(); + // Finding name in reshape table + const auto name_pos_in_table = input_reshape_table.find(layer_name); + // If contains then reshape for this layer already configured by shapes + // otherwise create a new element of reshape table with name and dimension + // which based on input image size. + if (name_pos_in_table != input_reshape_table.end()) { + GAPI_Assert(false && + "Names of layers for reshape with specified dimensions shouldn't intersect with names for reshape by image"); + } + cv::Size image_sz; + switch (mm.index()) { + case cv::GMetaArg::index_of(): + { + const auto &meta = util::get(mm); + image_sz = meta.size; + break; + } + case cv::GMetaArg::index_of(): + { + const auto &meta = util::get(mm); + image_sz = meta.size; + break; + } + default: + util::throw_error(std::runtime_error("Unsupported input meta for IE backend")); + } + auto input_dims = ii->getTensorDesc().getDims(); + const auto size = input_dims.size(); + if (size <= 1) { + GAPI_Assert(false && "Unsupported number of dimensions for reshape by image"); + } + input_dims.at(size - 2) = static_cast(image_sz.height); + input_dims.at(size - 1) = static_cast(image_sz.width); + // Adding new element to reshape table + input_reshape_table.emplace(layer_name, input_dims); +} static void configureInputInfo(const IE::InputInfo::Ptr& ii, const cv::GMetaArg mm) { switch (mm.index()) { @@ -732,22 +781,34 @@ struct Infer: public cv::detail::KernelTag { GConstGIEModel gm(gr); const auto &uu = gm.metadata(nh).get(); + IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table; // Initialize input information // Note our input layers list order matches the API order and so // meta order. GAPI_Assert(uu.params.input_names.size() == in_metas.size() && "Known input layers count doesn't match input meta count"); - for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names), ade::util::toRange(in_metas))) { - auto &&ii = uu.inputs.at(std::get<0>(it)); - const auto & mm = std::get<1>(it); + const auto &input_name = std::get<0>(it); + auto &&ii = uu.inputs.at(input_name); + const auto & mm = std::get<1>(it); configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } + // FIXME: It would be nice here to have an exact number of network's // input/output parameters. Probably GCall should store it here for us. // It doesn't, as far as I know.. @@ -802,6 +863,7 @@ struct InferROI: public cv::detail::KernelTag { GConstGIEModel gm(gr); const auto &uu = gm.metadata(nh).get(); + IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table; // Initialize input information // FIXME: So far it is pretty limited @@ -809,11 +871,23 @@ struct InferROI: public cv::detail::KernelTag { GAPI_Assert(2u == in_metas.size()); // 0th is ROI, 1st is input image - auto &&ii = uu.inputs.at(uu.params.input_names.at(0)); + const auto &input_name = uu.params.input_names.at(0); + auto &&ii = uu.inputs.at(input_name); auto &&mm = in_metas.at(1u); configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } + // FIXME: It would be nice here to have an exact number of network's // input/output parameters. Probably GCall should store it here for us. // It doesn't, as far as I know.. @@ -870,6 +944,7 @@ struct InferList: public cv::detail::KernelTag { GConstGIEModel gm(gr); const auto &uu = gm.metadata(nh).get(); + IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table; // Initialize input information // Note our input layers list order matches the API order and so @@ -882,9 +957,20 @@ struct InferList: public cv::detail::KernelTag { auto &&ii = uu.inputs.at(input_name); const auto & mm = in_metas[idx++]; configureInputInfo(ii, mm); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm, input_reshape_table); + } ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); } + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } + // roi-list version is much easier at the moment. // All our outputs are vectors which don't have // metadata at the moment - so just create a vector of @@ -973,6 +1059,7 @@ struct InferList2: public cv::detail::KernelTag { GConstGIEModel gm(gr); const auto &uu = gm.metadata(nh).get(); + IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table; // Initialize input information // Note our input layers list order matches the API order and so @@ -1023,7 +1110,18 @@ struct InferList2: public cv::detail::KernelTag { if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) { // This is a cv::Rect -- configure the IE preprocessing configureInputInfo(ii, mm_0); + if (uu.params.layer_names_to_reshape.find(input_name) != + uu.params.layer_names_to_reshape.end()) { + configureInputReshapeByImage(ii, mm_0, input_reshape_table); + } ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR); + + // FIXME: This isn't the best place to call reshape function. + // Сorrect solution would be to do this in compile() method of network, + // but now input meta isn't passed to compile() method. + if (!input_reshape_table.empty()) { + const_cast(&uu.net)->reshape(input_reshape_table); + } } else { // This is a cv::GMat (equals to: cv::Mat) // Just validate that it is really the type diff --git a/modules/gapi/test/infer/gapi_infer_ie_test.cpp b/modules/gapi/test/infer/gapi_infer_ie_test.cpp index a996430..b950a9a 100644 --- a/modules/gapi/test/infer/gapi_infer_ie_test.cpp +++ b/modules/gapi/test/infer/gapi_infer_ie_test.cpp @@ -233,6 +233,115 @@ TEST(TestAgeGenderIE, InferBasicImage) normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output"); } +struct InferWithReshape: public ::testing::Test { + cv::gapi::ie::detail::ParamDesc params; + cv::Mat m_in_mat; + std::vector m_roi_list; + std::vector reshape_dims; + std::vector m_out_ie_ages; + std::vector m_out_ie_genders; + std::vector m_out_gapi_ages; + std::vector m_out_gapi_genders; + using AGInfo = std::tuple; + G_API_NET(AgeGender, , "test-age-gender"); + + InferenceEngine::CNNNetwork net; + InferenceEngine::Core plugin; + + InferWithReshape() { + // FIXME: it must be cv::imread(findDataFile("../dnn/grace_hopper_227.png", false)); + m_in_mat = cv::Mat(cv::Size(320, 240), CV_8UC3); + cv::randu(m_in_mat, 0, 255); + + m_out_gapi_ages.resize(1); + m_out_gapi_genders.resize(1); + + // both ROIs point to the same face, with a slightly changed geometry + m_roi_list = { + cv::Rect(cv::Point{64, 60}, cv::Size{ 96, 96}), + cv::Rect(cv::Point{50, 32}, cv::Size{128, 160}), + }; + + // New dimensions for "data" input + reshape_dims = {1, 3, 70, 70}; + + initDLDTDataPath(); + params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml"); + params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin"); + + params.device_id = "CPU"; + + plugin = cv::gimpl::ie::wrap::getPlugin(params); + net = cv::gimpl::ie::wrap::readNetwork(params); + setNetParameters(net); + net.reshape({{"data", reshape_dims}}); + } + + void inferROIs(IE::Blob::Ptr blob) { + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + for (auto &&rc : m_roi_list) { + const auto ie_rc = IE::ROI { + 0u + , static_cast(rc.x) + , static_cast(rc.y) + , static_cast(rc.width) + , static_cast(rc.height) + }; + infer_request.SetBlob("data", IE::make_shared_blob(blob, ie_rc)); + infer_request.Infer(); + using namespace cv::gapi::ie::util; + m_out_ie_ages.push_back(to_ocv(infer_request.GetBlob("age_conv3")).clone()); + m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone()); + } + } + + void infer(cv::Mat& in, const bool with_roi = false) { + if (!with_roi) { + auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params); + auto infer_request = this_network.CreateInferRequest(); + infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in)); + infer_request.Infer(); + using namespace cv::gapi::ie::util; + m_out_ie_ages.push_back(to_ocv(infer_request.GetBlob("age_conv3")).clone()); + m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone()); + } else { + auto frame_blob = cv::gapi::ie::util::to_ie(in); + inferROIs(frame_blob); + } + } + + void validate() { + // Validate with IE itself (avoid DNN module dependency here) + GAPI_Assert(!m_out_gapi_ages.empty()); + ASSERT_EQ(m_out_gapi_genders.size(), m_out_gapi_ages.size()); + ASSERT_EQ(m_out_gapi_ages.size(), m_out_ie_ages.size()); + ASSERT_EQ(m_out_gapi_genders.size(), m_out_ie_genders.size()); + + const size_t size = m_out_gapi_ages.size(); + for (size_t i = 0; i < size; ++i) { + normAssert(m_out_ie_ages [i], m_out_gapi_ages [i], "Test age output"); + normAssert(m_out_ie_genders[i], m_out_gapi_genders[i], "Test gender output"); + } + } +}; // InferWithReshape + +struct InferWithReshapeNV12: public InferWithReshape { + cv::Mat m_in_uv; + cv::Mat m_in_y; + void SetUp() { + cv::Size sz{320, 240}; + m_in_y = cv::Mat{sz, CV_8UC1}; + cv::randu(m_in_y, 0, 255); + m_in_uv = cv::Mat{sz / 2, CV_8UC2}; + cv::randu(m_in_uv, 0, 255); + setNetParameters(net, true); + net.reshape({{"data", reshape_dims}}); + auto frame_blob = cv::gapi::ie::util::to_ie(m_in_y, m_in_uv); + inferROIs(frame_blob); + } +}; + struct ROIList: public ::testing::Test { cv::gapi::ie::detail::ParamDesc params; @@ -1403,6 +1512,153 @@ TEST(Infer2EmptyList, TestStreamingInfer) } } +TEST_F(InferWithReshape, TestInfer) +{ + // IE code + infer(m_in_mat); + // G-API code + cv::GMat in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender)); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}}); + comp.apply(cv::gin(m_in_mat), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()), + cv::compile_args(cv::gapi::networks(pp))); + // Validate + validate(); +} + +TEST_F(InferWithReshape, TestInferInImage) +{ + // Input image already has 70x70 size + cv::Mat rsz; + cv::resize(m_in_mat, rsz, cv::Size(70, 70)); + // IE code + infer(rsz); + // G-API code + cv::GMat in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender)); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({"data"}); + // Reshape CNN input by input image size + comp.apply(cv::gin(rsz), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()), + cv::compile_args(cv::gapi::networks(pp))); + // Validate + validate(); +} + +TEST_F(InferWithReshape, TestInferForSingleLayer) +{ + // IE code + infer(m_in_mat); + // G-API code + cv::GMat in; + cv::GMat age, gender; + std::tie(age, gender) = cv::gapi::infer(in); + cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender)); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }) + .cfgInputReshape("data", reshape_dims); + comp.apply(cv::gin(m_in_mat), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()), + cv::compile_args(cv::gapi::networks(pp))); + // Validate + validate(); +} + +TEST_F(InferWithReshape, TestInferList) +{ + // IE code + infer(m_in_mat, true); + // G-API code + cv::GArray rr; + cv::GMat in; + cv::GArray age, gender; + std::tie(age, gender) = cv::gapi::infer(rr, in); + cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender)); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}}); + comp.apply(cv::gin(m_in_mat, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + // Validate + validate(); +} + +TEST_F(InferWithReshape, TestInferList2) +{ + // IE code + infer(m_in_mat, true); + // G-API code + cv::GArray rr; + cv::GMat in; + cv::GArray age, gender; + std::tie(age, gender) = cv::gapi::infer2(in, rr); + cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender)); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}}); + comp.apply(cv::gin(m_in_mat, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + // Validate + validate(); +} + +TEST_F(InferWithReshape, TestInferListBGR) +{ + // IE code + infer(m_in_mat, true); + // G-API code + cv::GArray rr; + cv::GFrame in; + cv::GArray age, gender; + std::tie(age, gender) = cv::gapi::infer(rr, in); + cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(m_in_mat); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}}); + comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + // Validate + validate(); +} + +TEST_F(InferWithReshapeNV12, TestInferListYUV) +{ + // G-API code + cv::GFrame in; + cv::GArray rr; + cv::GArray age, gender; + std::tie(age, gender) = cv::gapi::infer(rr, in); + cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender)); + + auto frame = MediaFrame::Create(m_in_y, m_in_uv); + + auto pp = cv::gapi::ie::Params { + params.model_path, params.weights_path, params.device_id + }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}}); + comp.apply(cv::gin(frame, m_roi_list), + cv::gout(m_out_gapi_ages, m_out_gapi_genders), + cv::compile_args(cv::gapi::networks(pp))); + // Validate + validate(); +} } // namespace opencv_test #endif // HAVE_INF_ENGINE