From d8dea7896b41cc9afdfa3738e7904ea34d2fc698 Mon Sep 17 00:00:00 2001 From: Dmitry Kurtaev Date: Wed, 26 Feb 2020 17:51:18 +0300 Subject: [PATCH] Merge pull request #16628 from dkurt:dnn_ngraph_custom_layers * Custom layers with nGraph * nGraph: multiple outputs from nodes --- modules/dnn/src/dnn.cpp | 87 ++++++++++++++++++----- modules/dnn/src/ie_ngraph.cpp | 118 ++++++++++++++++++++++++++++++- modules/dnn/src/ie_ngraph.hpp | 5 ++ modules/dnn/src/layers/const_layer.cpp | 17 ++++- modules/dnn/test/test_backends.cpp | 2 +- modules/dnn/test/test_tf_importer.cpp | 12 ++-- modules/dnn/test/test_torch_importer.cpp | 7 +- 7 files changed, 221 insertions(+), 27 deletions(-) diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp index b0c52b1..e6baa53 100644 --- a/modules/dnn/src/dnn.cpp +++ b/modules/dnn/src/dnn.cpp @@ -1897,7 +1897,9 @@ struct Net::Impl for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) { InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); - dataPtr->setName(netInputLayer->outNames.empty() ? ld.name : netInputLayer->outNames[i]); + std::string outputName = netInputLayer->outNames.empty() ? ld.name : netInputLayer->outNames[i]; + outputName = ld.outputBlobsWrappers.size() > 1 ? (outputName + "." + std::to_string(i)) : outputName; + dataPtr->setName(outputName); } } else @@ -1905,7 +1907,8 @@ struct Net::Impl for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) { InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); - dataPtr->setName(ld.name); + std::string outputName = ld.outputBlobsWrappers.size() > 1 ? (ld.name + "." + std::to_string(i)) : ld.name; + dataPtr->setName(outputName); } } } @@ -1946,6 +1949,9 @@ struct Net::Impl return; } + bool supportsCPUFallback = preferableTarget == DNN_TARGET_CPU || + BackendRegistry::checkIETarget(DNN_TARGET_CPU); + // Build Inference Engine networks from sets of layers that support this // backend. Split a whole model on several Inference Engine networks if // some of layers are not implemented. @@ -1960,20 +1966,47 @@ struct Net::Impl Ptr layer = ld.layerInstance; if (!fused && !layer->supportBackend(preferableBackend)) { - addNgraphOutputs(ld); - net = Ptr(); - layer->preferableTarget = DNN_TARGET_CPU; + bool customizable = ld.id != 0 && supportsCPUFallback; - for (int i = 0; i < ld.inputBlobsId.size(); ++i) + // TODO: there is a bug in Myriad plugin with custom layers shape infer. + if (preferableTarget == DNN_TARGET_MYRIAD) { - LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; - Ptr inpNode = inpLd.backendNodes[preferableBackend]; - if (!inpNode.empty()) { - Ptr ieNode = inpNode.dynamicCast(); - ieNode->net->setUnconnectedNodes(ieNode); + for (int i = 0; customizable && i < ld.inputBlobs.size(); ++i) + { + customizable = ld.inputBlobs[i]->size[0] == 1; } } - continue; + + // TODO: fix these workarounds + if (preferableTarget == DNN_TARGET_MYRIAD || + preferableTarget == DNN_TARGET_OPENCL || + preferableTarget == DNN_TARGET_OPENCL_FP16) + customizable &= ld.type != "Concat"; + + if (preferableTarget == DNN_TARGET_OPENCL || + preferableTarget == DNN_TARGET_OPENCL_FP16) + customizable &= ld.type != "Power"; + + if (preferableTarget == DNN_TARGET_OPENCL) + customizable &= ld.type != "Eltwise"; + + if (!customizable) + { + addNgraphOutputs(ld); + net = Ptr(); + layer->preferableTarget = DNN_TARGET_CPU; + + for (int i = 0; i < ld.inputBlobsId.size(); ++i) + { + LayerData &inpLd = layers[ld.inputBlobsId[i].lid]; + Ptr inpNode = inpLd.backendNodes[preferableBackend]; + if (!inpNode.empty()) { + Ptr ieNode = inpNode.dynamicCast(); + ieNode->net->setUnconnectedNodes(ieNode); + } + } + continue; + } } ld.skip = true; // Initially skip all Inference Engine supported layers. @@ -2047,12 +2080,32 @@ struct Net::Impl if (!fused) { - CV_Assert(!inputNodes.empty()); - node = layer->initNgraph(ld.inputBlobsWrappers, inputNodes); - for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + CV_Assert(ld.inputBlobsId.size() == inputNodes.size()); + for (int i = 0; i < ld.inputBlobsId.size(); ++i) { - InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); - node.dynamicCast()->setName(dataPtr->getName()); + int lid = ld.inputBlobsId[i].lid; + int oid = ld.inputBlobsId[i].oid; + if (oid == 0 || lid == 0) + continue; + + auto ieInpNode = inputNodes[i].dynamicCast(); + CV_Assert(oid < ieInpNode->node->get_output_size()); + inputNodes[i] = Ptr(new InfEngineNgraphNode(ieInpNode->node->get_output_as_single_output_node(oid, false))); + } + + if (layer->supportBackend(preferableBackend)) + { + node = layer->initNgraph(ld.inputBlobsWrappers, inputNodes); + for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i) + { + InferenceEngine::DataPtr dataPtr = ngraphDataNode(ld.outputBlobsWrappers[i]); + node.dynamicCast()->setName(dataPtr->getName()); + } + } + else + { + node = Ptr(new InfEngineNgraphNode(inputNodes, + ld.layerInstance, ld.inputBlobs, ld.outputBlobs, ld.internals)); } } else if (node.empty()) diff --git a/modules/dnn/src/ie_ngraph.cpp b/modules/dnn/src/ie_ngraph.cpp index d7df547..e8cfd12 100644 --- a/modules/dnn/src/ie_ngraph.cpp +++ b/modules/dnn/src/ie_ngraph.cpp @@ -26,6 +26,35 @@ namespace cv { namespace dnn { // OpenCV lets users use an empty input name and to prevent unexpected naming, // we can use some predefined name. static std::string kDefaultInpLayerName = "empty_inp_layer_name"; +static constexpr const char* kOpenCVLayersType = "OpenCVLayer"; + +static std::string shapesToStr(const std::vector& mats) +{ + std::ostringstream shapes; + shapes << mats.size() << " "; + for (const Mat& m : mats) + { + shapes << m.dims << " "; + for (int i = 0; i < m.dims; ++i) + shapes << m.size[i] << " "; + } + return shapes.str(); +} + +static void strToShapes(const std::string& str, std::vector >& shapes) +{ + std::istringstream ss(str); + int num, dims; + ss >> num; + shapes.resize(num); + for (int i = 0; i < num; ++i) + { + ss >> dims; + shapes[i].resize(dims); + for (int j = 0; j < dims; ++j) + ss >> shapes[i][j]; + } +} static std::vector > ngraphWrappers(const std::vector >& ptrs) @@ -40,12 +69,82 @@ ngraphWrappers(const std::vector >& ptrs) return wrappers; } +class NgraphCustomOp: public ngraph::op::Op { +public: + const ngraph::NodeTypeInfo& get_type_info() const override + { + static constexpr ngraph::NodeTypeInfo type_info{kOpenCVLayersType, 0}; + return type_info; + } + + NgraphCustomOp() {}; + NgraphCustomOp(const ngraph::NodeVector& inputs, + const std::map& params = {}): + Op(inputs), params(params) + { + constructor_validate_and_infer_types(); + } + + void validate_and_infer_types() override + { + std::vector > shapes; + strToShapes(params["outputs"], shapes); + set_output_size(shapes.size()); + for (size_t i = 0; i < shapes.size(); ++i) + { + ngraph::Shape output_shape(shapes[i]); + set_output_type(i, get_input_element_type(0), output_shape); + } + } + + std::shared_ptr copy_with_new_args(const ngraph::NodeVector& new_args) const override + { + return std::make_shared(new_args, params); + } + + bool visit_attributes(ngraph::AttributeVisitor& visitor) override + { + for (auto& attr : params) + { + if (attr.second.is()) + visitor.on_attribute(attr.first, attr.second.as()); + } + return true; + } + +private: + std::map params; +}; + InfEngineNgraphNode::InfEngineNgraphNode(std::shared_ptr&& _node) : BackendNode(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH), node(std::move(_node)) {} InfEngineNgraphNode::InfEngineNgraphNode(std::shared_ptr& _node) : BackendNode(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH), node(_node) {} +InfEngineNgraphNode::InfEngineNgraphNode(const std::vector >& nodes, + Ptr& cvLayer_, std::vector& inputs, + std::vector& outputs, std::vector& internals) + : BackendNode(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH), cvLayer(cvLayer_) +{ + std::ostringstream oss; + oss << (size_t)cvLayer.get(); + + std::map params = { + {"impl", oss.str()}, + {"outputs", shapesToStr(outputs)}, + {"internals", shapesToStr(internals)} + }; + + ngraph::NodeVector inp_nodes; + for (const auto& node : nodes) + inp_nodes.emplace_back(node.dynamicCast()->node); + node = std::make_shared(inp_nodes, params); + + CV_Assert(!cvLayer->name.empty()); + setName(cvLayer->name); +} + void InfEngineNgraphNode::setName(const std::string& name) { node->set_friendly_name(name); } @@ -342,7 +441,24 @@ void InfEngineNgraphNet::initPlugin(InferenceEngine::CNNNetwork& net) if (device_name == "MYRIAD") { config.emplace("VPU_DETECT_NETWORK_BATCH", CONFIG_VALUE(NO)); } - netExec = ie.LoadNetwork(net, device_name, config); + + bool isHetero = false; + if (device_name != "CPU") + { + isHetero = device_name == "FPGA"; + for (auto& layer : net) + { + if (layer->type == kOpenCVLayersType) + { + isHetero = true; + break; + } + } + } + if (isHetero) + netExec = ie.LoadNetwork(net, "HETERO:" + device_name + ",CPU", config); + else + netExec = ie.LoadNetwork(net, device_name, config); } catch (const std::exception& ex) { diff --git a/modules/dnn/src/ie_ngraph.hpp b/modules/dnn/src/ie_ngraph.hpp index c24839d..3058178 100644 --- a/modules/dnn/src/ie_ngraph.hpp +++ b/modules/dnn/src/ie_ngraph.hpp @@ -90,6 +90,10 @@ private: class InfEngineNgraphNode : public BackendNode { public: + InfEngineNgraphNode(const std::vector >& nodes, Ptr& layer, + std::vector& inputs, std::vector& outputs, + std::vector& internals); + InfEngineNgraphNode(std::shared_ptr&& _node); InfEngineNgraphNode(std::shared_ptr& _node); @@ -98,6 +102,7 @@ public: // Inference Engine network object that allows to obtain the outputs of this layer. std::shared_ptr node; Ptr net; + Ptr cvLayer; }; class NgraphBackendWrapper : public BackendWrapper diff --git a/modules/dnn/src/layers/const_layer.cpp b/modules/dnn/src/layers/const_layer.cpp index 2c6b51e..5de4525 100644 --- a/modules/dnn/src/layers/const_layer.cpp +++ b/modules/dnn/src/layers/const_layer.cpp @@ -8,6 +8,7 @@ #include "../precomp.hpp" #include "../op_inf_engine.hpp" #include "layers_common.hpp" +#include "../ie_ngraph.hpp" #ifdef HAVE_OPENCL #include "opencl_kernels_dnn.hpp" @@ -26,7 +27,9 @@ public: virtual bool supportBackend(int backendId) CV_OVERRIDE { - return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019; + return backendId == DNN_BACKEND_OPENCV || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || + backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH; } virtual bool getMemoryShapes(const std::vector &inputs, @@ -73,6 +76,18 @@ public: return Ptr(new InfEngineBackendNode(ieLayer)); } #endif // HAVE_INF_ENGINE + + +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector >& inputs, + const std::vector >& nodes) CV_OVERRIDE + { + auto node = std::make_shared(ngraph::element::f32, + getShape(blobs[0]), + blobs[0].data); + return Ptr(new InfEngineNgraphNode(node)); + } +#endif // HAVE_INF_ENGINE }; Ptr ConstLayer::create(const LayerParams& params) diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp index 5223987..f1cb266 100644 --- a/modules/dnn/test/test_backends.cpp +++ b/modules/dnn/test/test_backends.cpp @@ -234,7 +234,7 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow_Different_Width_Height) Mat sample = imread(findDataFile("dnn/street.png")); Mat inp = blobFromImage(sample, 1.0f, Size(300, 560), Scalar(), false); - float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 0.0; + float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.013 : 0.0; float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.06 : 0.0; processNet("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", "dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt", inp, "detection_out", "", l1, lInf); diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp index ecbf776..e8064f1 100644 --- a/modules/dnn/test/test_tf_importer.cpp +++ b/modules/dnn/test/test_tf_importer.cpp @@ -158,12 +158,12 @@ TEST_P(Test_TensorFlow_layers, padding) runTensorFlowNet("spatial_padding"); runTensorFlowNet("mirror_pad"); #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000) - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) + if (target == DNN_TARGET_MYRIAD) { - if (target == DNN_TARGET_MYRIAD) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); - if (target == DNN_TARGET_OPENCL_FP16) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER, CV_TEST_TAG_DNN_SKIP_IE_VERSION); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); } #endif runTensorFlowNet("keras_pad_concat"); @@ -784,6 +784,8 @@ TEST_P(Test_TensorFlow_layers, split) { if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); runTensorFlowNet("split"); } @@ -922,7 +924,7 @@ TEST_P(Test_TensorFlow_nets, Mask_RCNN) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER); if (target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X) - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG); Mat img = imread(findDataFile("dnn/street.png")); diff --git a/modules/dnn/test/test_torch_importer.cpp b/modules/dnn/test/test_torch_importer.cpp index 55ce803..4b89afc 100644 --- a/modules/dnn/test/test_torch_importer.cpp +++ b/modules/dnn/test/test_torch_importer.cpp @@ -360,9 +360,12 @@ TEST_P(Test_Torch_nets, ENet_accuracy) throw SkipTestException(""); } #endif - if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target != DNN_TARGET_CPU) { - applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH); + throw SkipTestException(""); } Net net; -- 2.7.4