#define OPENCV_GAPI_INFER_IE_HPP
#include <unordered_map>
+#include <unordered_set>
#include <string>
#include <array>
#include <tuple> // tuple, tuple_size
bool is_generic;
IEConfig config;
+ std::map<std::string, std::vector<std::size_t>> reshape_table;
+ std::unordered_set<std::string> layer_names_to_reshape;
+
// NB: Number of asyncrhonious infer requests
size_t nireq;
};
, detail::ParamDesc::Kind::Load
, false
, {}
+ , {}
+ , {}
, 1u} {
};
, detail::ParamDesc::Kind::Import
, false
, {}
+ , {}
+ , {}
, 1u} {
};
return *this;
}
+ Params<Net>& cfgInputReshape(std::map<std::string, std::vector<std::size_t>>&& reshape_table) {
+ desc.reshape_table = std::move(reshape_table);
+ return *this;
+ }
+
+ Params<Net>& cfgInputReshape(const std::map<std::string, std::vector<std::size_t>>& reshape_table) {
+ desc.reshape_table = reshape_table;
+ return *this;
+ }
+
+ Params<Net>& cfgInputReshape(std::string&& layer_name, std::vector<size_t>&& layer_dims) {
+ desc.reshape_table.emplace(layer_name, layer_dims);
+ return *this;
+ }
+
+ Params<Net>& cfgInputReshape(const std::string& layer_name, const std::vector<size_t>& layer_dims) {
+ desc.reshape_table.emplace(layer_name, layer_dims);
+ return *this;
+ }
+
+ Params<Net>& cfgInputReshape(std::unordered_set<std::string>&& layer_names) {
+ desc.layer_names_to_reshape = std::move(layer_names);
+ return *this;
+ }
+
+ Params<Net>& cfgInputReshape(const std::unordered_set<std::string>& layer_names) {
+ desc.layer_names_to_reshape = layer_names;
+ return *this;
+ }
+
// BEGIN(G-API's network parametrization API)
GBackend backend() const { return cv::gapi::ie::backend(); }
std::string tag() const { return Net::tag(); }
const std::string &model,
const std::string &weights,
const std::string &device)
- : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, 1u}, m_tag(tag) {
+ : desc{ model, weights, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Load, true, {}, {}, {}, 1u}, m_tag(tag) {
};
Params(const std::string &tag,
const std::string &model,
const std::string &device)
- : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, 1u}, m_tag(tag) {
+ : desc{ model, {}, device, {}, {}, {}, 0u, 0u, detail::ParamDesc::Kind::Import, true, {}, {}, {}, 1u}, m_tag(tag) {
};
Params& pluginConfig(IEConfig&& cfg) {
#include "backends/ie/giebackend/giewrapper.hpp"
#include "api/gbackend_priv.hpp" // FIXME: Make it part of Backend SDK!
+#include "logger.hpp"
#if INF_ENGINE_RELEASE < 2021010000
#include "ie_compound_blob.h"
// but ExecutableNetwork returns ConstInputsDataMap/ConstOutputsDataMap
inputs = cv::gimpl::ie::wrap::toInputsDataMap(this_network.GetInputsInfo());
outputs = cv::gimpl::ie::wrap::toOutputsDataMap(this_network.GetOutputsInfo());
+ if (!params.reshape_table.empty() || !params.layer_names_to_reshape.empty()) {
+ GAPI_LOG_WARNING(NULL, "Reshape isn't supported for imported network");
+ }
} else {
cv::util::throw_error(std::logic_error("Unsupported ParamDesc::Kind"));
}
if (params.num_out == 1u && params.output_names.empty()) {
params.output_names = { outputs.begin()->first };
}
+ if (!params.reshape_table.empty()) {
+ GAPI_Assert((params.reshape_table.size() + params.layer_names_to_reshape.size()) <=
+ params.num_in &&
+ "Number of layers to reshape must be less than or equal to number of inputs");
+ }
}
// This method is [supposed to be] called at Island compilation stage
namespace cv {
namespace gimpl {
namespace ie {
+static void configureInputReshapeByImage(const IE::InputInfo::Ptr& ii,
+ const cv::GMetaArg mm,
+ IE::ICNNNetwork::InputShapes& input_reshape_table) {
+ const auto& layer_name = ii->name();
+ // Finding name in reshape table
+ const auto name_pos_in_table = input_reshape_table.find(layer_name);
+ // If contains then reshape for this layer already configured by shapes
+ // otherwise create a new element of reshape table with name and dimension
+ // which based on input image size.
+ if (name_pos_in_table != input_reshape_table.end()) {
+ GAPI_Assert(false &&
+ "Names of layers for reshape with specified dimensions shouldn't intersect with names for reshape by image");
+ }
+ cv::Size image_sz;
+ switch (mm.index()) {
+ case cv::GMetaArg::index_of<cv::GMatDesc>():
+ {
+ const auto &meta = util::get<cv::GMatDesc>(mm);
+ image_sz = meta.size;
+ break;
+ }
+ case cv::GMetaArg::index_of<cv::GFrameDesc>():
+ {
+ const auto &meta = util::get<cv::GFrameDesc>(mm);
+ image_sz = meta.size;
+ break;
+ }
+ default:
+ util::throw_error(std::runtime_error("Unsupported input meta for IE backend"));
+ }
+ auto input_dims = ii->getTensorDesc().getDims();
+ const auto size = input_dims.size();
+ if (size <= 1) {
+ GAPI_Assert(false && "Unsupported number of dimensions for reshape by image");
+ }
+ input_dims.at(size - 2) = static_cast<size_t>(image_sz.height);
+ input_dims.at(size - 1) = static_cast<size_t>(image_sz.width);
+ // Adding new element to reshape table
+ input_reshape_table.emplace(layer_name, input_dims);
+}
static void configureInputInfo(const IE::InputInfo::Ptr& ii, const cv::GMetaArg mm) {
switch (mm.index()) {
GConstGIEModel gm(gr);
const auto &uu = gm.metadata(nh).get<IEUnit>();
+ IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
// Initialize input information
// Note our input layers list order matches the API order and so
// meta order.
GAPI_Assert(uu.params.input_names.size() == in_metas.size()
&& "Known input layers count doesn't match input meta count");
-
for (auto &&it : ade::util::zip(ade::util::toRange(uu.params.input_names),
ade::util::toRange(in_metas))) {
- auto &&ii = uu.inputs.at(std::get<0>(it));
- const auto & mm = std::get<1>(it);
+ const auto &input_name = std::get<0>(it);
+ auto &&ii = uu.inputs.at(input_name);
+ const auto & mm = std::get<1>(it);
configureInputInfo(ii, mm);
+ if (uu.params.layer_names_to_reshape.find(input_name) !=
+ uu.params.layer_names_to_reshape.end()) {
+ configureInputReshapeByImage(ii, mm, input_reshape_table);
+ }
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
}
+ // FIXME: This isn't the best place to call reshape function.
+ // Сorrect solution would be to do this in compile() method of network,
+ // but now input meta isn't passed to compile() method.
+ if (!input_reshape_table.empty()) {
+ const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
+ }
+
// FIXME: It would be nice here to have an exact number of network's
// input/output parameters. Probably GCall should store it here for us.
// It doesn't, as far as I know..
GConstGIEModel gm(gr);
const auto &uu = gm.metadata(nh).get<IEUnit>();
+ IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
// Initialize input information
// FIXME: So far it is pretty limited
GAPI_Assert(2u == in_metas.size());
// 0th is ROI, 1st is input image
- auto &&ii = uu.inputs.at(uu.params.input_names.at(0));
+ const auto &input_name = uu.params.input_names.at(0);
+ auto &&ii = uu.inputs.at(input_name);
auto &&mm = in_metas.at(1u);
configureInputInfo(ii, mm);
+ if (uu.params.layer_names_to_reshape.find(input_name) !=
+ uu.params.layer_names_to_reshape.end()) {
+ configureInputReshapeByImage(ii, mm, input_reshape_table);
+ }
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
+ // FIXME: This isn't the best place to call reshape function.
+ // Сorrect solution would be to do this in compile() method of network,
+ // but now input meta isn't passed to compile() method.
+ if (!input_reshape_table.empty()) {
+ const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
+ }
+
// FIXME: It would be nice here to have an exact number of network's
// input/output parameters. Probably GCall should store it here for us.
// It doesn't, as far as I know..
GConstGIEModel gm(gr);
const auto &uu = gm.metadata(nh).get<IEUnit>();
+ IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
// Initialize input information
// Note our input layers list order matches the API order and so
auto &&ii = uu.inputs.at(input_name);
const auto & mm = in_metas[idx++];
configureInputInfo(ii, mm);
+ if (uu.params.layer_names_to_reshape.find(input_name) !=
+ uu.params.layer_names_to_reshape.end()) {
+ configureInputReshapeByImage(ii, mm, input_reshape_table);
+ }
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
}
+ // FIXME: This isn't the best place to call reshape function.
+ // Сorrect solution would be to do this in compile() method of network,
+ // but now input meta isn't passed to compile() method.
+ if (!input_reshape_table.empty()) {
+ const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
+ }
+
// roi-list version is much easier at the moment.
// All our outputs are vectors which don't have
// metadata at the moment - so just create a vector of
GConstGIEModel gm(gr);
const auto &uu = gm.metadata(nh).get<IEUnit>();
+ IE::ICNNNetwork::InputShapes input_reshape_table = uu.params.reshape_table;
// Initialize input information
// Note our input layers list order matches the API order and so
if (op.k.inKinds[idx] == cv::detail::OpaqueKind::CV_RECT) {
// This is a cv::Rect -- configure the IE preprocessing
configureInputInfo(ii, mm_0);
+ if (uu.params.layer_names_to_reshape.find(input_name) !=
+ uu.params.layer_names_to_reshape.end()) {
+ configureInputReshapeByImage(ii, mm_0, input_reshape_table);
+ }
ii->getPreProcess().setResizeAlgorithm(IE::RESIZE_BILINEAR);
+
+ // FIXME: This isn't the best place to call reshape function.
+ // Сorrect solution would be to do this in compile() method of network,
+ // but now input meta isn't passed to compile() method.
+ if (!input_reshape_table.empty()) {
+ const_cast<IE::CNNNetwork *>(&uu.net)->reshape(input_reshape_table);
+ }
} else {
// This is a cv::GMat (equals to: cv::Mat)
// Just validate that it is really the type
normAssert(cv::gapi::ie::util::to_ocv(ie_gender), gapi_gender, "Test gender output");
}
+struct InferWithReshape: public ::testing::Test {
+ cv::gapi::ie::detail::ParamDesc params;
+ cv::Mat m_in_mat;
+ std::vector<cv::Rect> m_roi_list;
+ std::vector<size_t> reshape_dims;
+ std::vector<cv::Mat> m_out_ie_ages;
+ std::vector<cv::Mat> m_out_ie_genders;
+ std::vector<cv::Mat> m_out_gapi_ages;
+ std::vector<cv::Mat> m_out_gapi_genders;
+ using AGInfo = std::tuple<cv::GMat, cv::GMat>;
+ G_API_NET(AgeGender, <AGInfo(cv::GMat)>, "test-age-gender");
+
+ InferenceEngine::CNNNetwork net;
+ InferenceEngine::Core plugin;
+
+ InferWithReshape() {
+ // FIXME: it must be cv::imread(findDataFile("../dnn/grace_hopper_227.png", false));
+ m_in_mat = cv::Mat(cv::Size(320, 240), CV_8UC3);
+ cv::randu(m_in_mat, 0, 255);
+
+ m_out_gapi_ages.resize(1);
+ m_out_gapi_genders.resize(1);
+
+ // both ROIs point to the same face, with a slightly changed geometry
+ m_roi_list = {
+ cv::Rect(cv::Point{64, 60}, cv::Size{ 96, 96}),
+ cv::Rect(cv::Point{50, 32}, cv::Size{128, 160}),
+ };
+
+ // New dimensions for "data" input
+ reshape_dims = {1, 3, 70, 70};
+
+ initDLDTDataPath();
+ params.model_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.xml");
+ params.weights_path = findDataFile(SUBDIR + "age-gender-recognition-retail-0013.bin");
+
+ params.device_id = "CPU";
+
+ plugin = cv::gimpl::ie::wrap::getPlugin(params);
+ net = cv::gimpl::ie::wrap::readNetwork(params);
+ setNetParameters(net);
+ net.reshape({{"data", reshape_dims}});
+ }
+
+ void inferROIs(IE::Blob::Ptr blob) {
+ auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+ auto infer_request = this_network.CreateInferRequest();
+ for (auto &&rc : m_roi_list) {
+ const auto ie_rc = IE::ROI {
+ 0u
+ , static_cast<std::size_t>(rc.x)
+ , static_cast<std::size_t>(rc.y)
+ , static_cast<std::size_t>(rc.width)
+ , static_cast<std::size_t>(rc.height)
+ };
+ infer_request.SetBlob("data", IE::make_shared_blob(blob, ie_rc));
+ infer_request.Infer();
+ using namespace cv::gapi::ie::util;
+ m_out_ie_ages.push_back(to_ocv(infer_request.GetBlob("age_conv3")).clone());
+ m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone());
+ }
+ }
+
+ void infer(cv::Mat& in, const bool with_roi = false) {
+ if (!with_roi) {
+ auto this_network = cv::gimpl::ie::wrap::loadNetwork(plugin, net, params);
+ auto infer_request = this_network.CreateInferRequest();
+ infer_request.SetBlob("data", cv::gapi::ie::util::to_ie(in));
+ infer_request.Infer();
+ using namespace cv::gapi::ie::util;
+ m_out_ie_ages.push_back(to_ocv(infer_request.GetBlob("age_conv3")).clone());
+ m_out_ie_genders.push_back(to_ocv(infer_request.GetBlob("prob")).clone());
+ } else {
+ auto frame_blob = cv::gapi::ie::util::to_ie(in);
+ inferROIs(frame_blob);
+ }
+ }
+
+ void validate() {
+ // Validate with IE itself (avoid DNN module dependency here)
+ GAPI_Assert(!m_out_gapi_ages.empty());
+ ASSERT_EQ(m_out_gapi_genders.size(), m_out_gapi_ages.size());
+ ASSERT_EQ(m_out_gapi_ages.size(), m_out_ie_ages.size());
+ ASSERT_EQ(m_out_gapi_genders.size(), m_out_ie_genders.size());
+
+ const size_t size = m_out_gapi_ages.size();
+ for (size_t i = 0; i < size; ++i) {
+ normAssert(m_out_ie_ages [i], m_out_gapi_ages [i], "Test age output");
+ normAssert(m_out_ie_genders[i], m_out_gapi_genders[i], "Test gender output");
+ }
+ }
+}; // InferWithReshape
+
+struct InferWithReshapeNV12: public InferWithReshape {
+ cv::Mat m_in_uv;
+ cv::Mat m_in_y;
+ void SetUp() {
+ cv::Size sz{320, 240};
+ m_in_y = cv::Mat{sz, CV_8UC1};
+ cv::randu(m_in_y, 0, 255);
+ m_in_uv = cv::Mat{sz / 2, CV_8UC2};
+ cv::randu(m_in_uv, 0, 255);
+ setNetParameters(net, true);
+ net.reshape({{"data", reshape_dims}});
+ auto frame_blob = cv::gapi::ie::util::to_ie(m_in_y, m_in_uv);
+ inferROIs(frame_blob);
+ }
+};
+
struct ROIList: public ::testing::Test {
cv::gapi::ie::detail::ParamDesc params;
}
}
+TEST_F(InferWithReshape, TestInfer)
+{
+ // IE code
+ infer(m_in_mat);
+ // G-API code
+ cv::GMat in;
+ cv::GMat age, gender;
+ std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
+ cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
+
+ auto pp = cv::gapi::ie::Params<AgeGender> {
+ params.model_path, params.weights_path, params.device_id
+ }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
+ comp.apply(cv::gin(m_in_mat), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()),
+ cv::compile_args(cv::gapi::networks(pp)));
+ // Validate
+ validate();
+}
+
+TEST_F(InferWithReshape, TestInferInImage)
+{
+ // Input image already has 70x70 size
+ cv::Mat rsz;
+ cv::resize(m_in_mat, rsz, cv::Size(70, 70));
+ // IE code
+ infer(rsz);
+ // G-API code
+ cv::GMat in;
+ cv::GMat age, gender;
+ std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
+ cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
+
+ auto pp = cv::gapi::ie::Params<AgeGender> {
+ params.model_path, params.weights_path, params.device_id
+ }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({"data"});
+ // Reshape CNN input by input image size
+ comp.apply(cv::gin(rsz), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()),
+ cv::compile_args(cv::gapi::networks(pp)));
+ // Validate
+ validate();
+}
+
+TEST_F(InferWithReshape, TestInferForSingleLayer)
+{
+ // IE code
+ infer(m_in_mat);
+ // G-API code
+ cv::GMat in;
+ cv::GMat age, gender;
+ std::tie(age, gender) = cv::gapi::infer<AgeGender>(in);
+ cv::GComputation comp(cv::GIn(in), cv::GOut(age, gender));
+
+ auto pp = cv::gapi::ie::Params<AgeGender> {
+ params.model_path, params.weights_path, params.device_id
+ }.cfgOutputLayers({ "age_conv3", "prob" })
+ .cfgInputReshape("data", reshape_dims);
+ comp.apply(cv::gin(m_in_mat), cv::gout(m_out_gapi_ages.front(), m_out_gapi_genders.front()),
+ cv::compile_args(cv::gapi::networks(pp)));
+ // Validate
+ validate();
+}
+
+TEST_F(InferWithReshape, TestInferList)
+{
+ // IE code
+ infer(m_in_mat, true);
+ // G-API code
+ cv::GArray<cv::Rect> rr;
+ cv::GMat in;
+ cv::GArray<cv::GMat> age, gender;
+ std::tie(age, gender) = cv::gapi::infer<AgeGender>(rr, in);
+ cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
+
+ auto pp = cv::gapi::ie::Params<AgeGender> {
+ params.model_path, params.weights_path, params.device_id
+ }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
+ comp.apply(cv::gin(m_in_mat, m_roi_list),
+ cv::gout(m_out_gapi_ages, m_out_gapi_genders),
+ cv::compile_args(cv::gapi::networks(pp)));
+ // Validate
+ validate();
+}
+
+TEST_F(InferWithReshape, TestInferList2)
+{
+ // IE code
+ infer(m_in_mat, true);
+ // G-API code
+ cv::GArray<cv::Rect> rr;
+ cv::GMat in;
+ cv::GArray<cv::GMat> age, gender;
+ std::tie(age, gender) = cv::gapi::infer2<AgeGender>(in, rr);
+ cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
+
+ auto pp = cv::gapi::ie::Params<AgeGender> {
+ params.model_path, params.weights_path, params.device_id
+ }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
+ comp.apply(cv::gin(m_in_mat, m_roi_list),
+ cv::gout(m_out_gapi_ages, m_out_gapi_genders),
+ cv::compile_args(cv::gapi::networks(pp)));
+ // Validate
+ validate();
+}
+
+TEST_F(InferWithReshape, TestInferListBGR)
+{
+ // IE code
+ infer(m_in_mat, true);
+ // G-API code
+ cv::GArray<cv::Rect> rr;
+ cv::GFrame in;
+ cv::GArray<cv::GMat> age, gender;
+ std::tie(age, gender) = cv::gapi::infer<AgeGender>(rr, in);
+ cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
+
+ auto frame = MediaFrame::Create<TestMediaBGR>(m_in_mat);
+
+ auto pp = cv::gapi::ie::Params<AgeGender> {
+ params.model_path, params.weights_path, params.device_id
+ }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
+ comp.apply(cv::gin(frame, m_roi_list),
+ cv::gout(m_out_gapi_ages, m_out_gapi_genders),
+ cv::compile_args(cv::gapi::networks(pp)));
+ // Validate
+ validate();
+}
+
+TEST_F(InferWithReshapeNV12, TestInferListYUV)
+{
+ // G-API code
+ cv::GFrame in;
+ cv::GArray<cv::Rect> rr;
+ cv::GArray<cv::GMat> age, gender;
+ std::tie(age, gender) = cv::gapi::infer<AgeGender>(rr, in);
+ cv::GComputation comp(cv::GIn(in, rr), cv::GOut(age, gender));
+
+ auto frame = MediaFrame::Create<TestMediaNV12>(m_in_y, m_in_uv);
+
+ auto pp = cv::gapi::ie::Params<AgeGender> {
+ params.model_path, params.weights_path, params.device_id
+ }.cfgOutputLayers({ "age_conv3", "prob" }).cfgInputReshape({{"data", reshape_dims}});
+ comp.apply(cv::gin(frame, m_roi_list),
+ cv::gout(m_out_gapi_ages, m_out_gapi_genders),
+ cv::compile_args(cv::gapi::networks(pp)));
+ // Validate
+ validate();
+}
} // namespace opencv_test
#endif // HAVE_INF_ENGINE
projects / platform / upstream / opencv.git / commitdiff