OpenCV face detection network using Inference Engine backend

diff --git a/cmake/OpenCVDetectInferenceEngine.cmake b/cmake/OpenCVDetectInferenceEngine.cmake
index f1bccb4..923a4a7 100644 (file)
--- a/cmake/OpenCVDetectInferenceEngine.cmake
+++ b/cmake/OpenCVDetectInferenceEngine.cmake
@@ -20,6 +20,9 @@ if(NOT INF_ENGINE_ROOT_DIR OR NOT EXISTS "${INF_ENGINE_ROOT_DIR}/inference_engin
     if(DEFINED ENV{INTEL_CVSDK_DIR})
         list(APPEND ie_root_paths "$ENV{INTEL_CVSDK_DIR}")
     endif()
+    if(DEFINED INTEL_CVSDK_DIR)
+        list(APPEND ie_root_paths "${INTEL_CVSDK_DIR}")
+    endif()
 
     if(WITH_INF_ENGINE AND NOT ie_root_paths)
         list(APPEND ie_root_paths "/opt/intel/deeplearning_deploymenttoolkit/deployment_tools")

diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp
index dd9e1e1..1d7c4f7 100644 (file)
--- a/modules/dnn/perf/perf_net.cpp
+++ b/modules/dnn/perf/perf_net.cpp
@@ -150,6 +150,7 @@ PERF_TEST_P_(DNNTestNetwork, SSD)
 
 PERF_TEST_P_(DNNTestNetwork, OpenFace)
 {
+    if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
     processNet("dnn/openface_nn4.small2.v1.t7", "", "",
             Mat(cv::Size(96, 96), CV_32FC3), "", "torch");
 }
@@ -197,6 +198,15 @@ PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
                Mat(cv::Size(368, 368), CV_32FC3), "", "caffe");
 }
 
+PERF_TEST_P_(DNNTestNetwork, opencv_face_detector)
+{
+    if (backend == DNN_BACKEND_HALIDE ||
+        backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL)
+        throw SkipTestException("");
+    processNet("dnn/opencv_face_detector.caffemodel", "dnn/opencv_face_detector.prototxt", "",
+               Mat(cv::Size(300, 300), CV_32FC3), "", "caffe");
+}
+
 const tuple<DNNBackend, DNNTarget> testCases[] = {
 #ifdef HAVE_HALIDE
     tuple<DNNBackend, DNNTarget>(DNN_BACKEND_HALIDE, DNN_TARGET_CPU),

diff --git a/modules/dnn/src/dnn.cpp b/modules/dnn/src/dnn.cpp
index 1bc53b5..7cc44da 100644 (file)
--- a/modules/dnn/src/dnn.cpp
+++ b/modules/dnn/src/dnn.cpp
@@ -1077,35 +1077,72 @@ struct Net::Impl
         }
     }
 
+#ifdef HAVE_INF_ENGINE
+    // Before launching Inference Engine graph we need to specify output blobs.
+    // This function requests output blobs based on inputs references of
+    // layers from default backend or layers from different graphs.
+    void addInfEngineNetOutputs(LayerData &ld)
+    {
+        Ptr<InfEngineBackendNet> layerNet;
+        if (ld.backendNodes.find(preferableBackend) != ld.backendNodes.end())
+        {
+            Ptr<BackendNode> node = ld.backendNodes[preferableBackend];
+            if (!node.empty())
+            {
+                Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
+                CV_Assert(!ieNode.empty(), !ieNode->net.empty());
+                layerNet = ieNode->net;
+            }
+        }
+        // For an every input reference we check that it belongs to one of
+        // the Inference Engine backend graphs. Request an output blob if it is.
+        // Do nothing if layer's input is from the same graph.
+        for (int i = 0; i < ld.inputBlobsId.size(); ++i)
+        {
+            LayerData &inpLd = layers[ld.inputBlobsId[i].lid];
+            Ptr<BackendNode> inpNode = inpLd.backendNodes[preferableBackend];
+            if (!inpNode.empty())
+            {
+                Ptr<InfEngineBackendNode> ieInpNode = inpNode.dynamicCast<InfEngineBackendNode>();
+                CV_Assert(!ieInpNode.empty(), !ieInpNode->net.empty());
+                if (layerNet != ieInpNode->net)
+                {
+                    // layerNet is empty or nodes are from different graphs.
+                    ieInpNode->net->addOutput(inpLd.name);
+                }
+            }
+        }
+    }
+#endif  // HAVE_INF_ENGINE
+
     void initInfEngineBackend()
     {
         // Build Inference Engine networks from sets of layers that support this
-        // backend. If an internal layer isn't supported we'll use default
-        // implementation of it but build a new network after it.
+        // backend. Split a whole model on several Inference Engine networks if
+        // some of layers is not implemented.
         CV_TRACE_FUNCTION();
         CV_Assert(preferableBackend == DNN_BACKEND_INFERENCE_ENGINE, haveInfEngine());
 #ifdef HAVE_INF_ENGINE
         MapIdToLayerData::iterator it;
         Ptr<InfEngineBackendNet> net;
+        // Set of all input and output blobs wrappers for current network.
+        std::map<int, Ptr<BackendWrapper> > netBlobsWrappers;
         for (it = layers.begin(); it != layers.end(); ++it)
         {
             LayerData &ld = it->second;
-            ld.skip = true;
+            ld.skip = true;  // Initially skip all Inference Engine supported layers.
             Ptr<Layer> layer = ld.layerInstance;
 
             if (!layer->supportBackend(preferableBackend))
             {
-                for (int i = 0; i < ld.outputBlobsWrappers.size(); ++i)
-                {
-                    auto dataPtr = infEngineDataNode(ld.outputBlobsWrappers[i]);
-                    dataPtr->name = ld.name;
-                }
+                addInfEngineNetOutputs(ld);
                 ld.skip = false;
                 net = Ptr<InfEngineBackendNet>();
+                netBlobsWrappers.clear();
                 continue;
             }
 
-            // Check what all inputs are from the same network or from default backend.
+            // Create a new network if one of inputs from different Inference Engine graph.
             for (int i = 0; i < ld.inputBlobsId.size(); ++i)
             {
                 LayerData &inpLd = layers[ld.inputBlobsId[i].lid];
@@ -1113,10 +1150,36 @@ struct Net::Impl
                 if (!inpNode.empty())
                 {
                     Ptr<InfEngineBackendNode> ieInpNode = inpNode.dynamicCast<InfEngineBackendNode>();
-                    CV_Assert(!ieInpNode.empty(), net.empty() || net == ieInpNode->net);
+                    CV_Assert(!ieInpNode.empty(), !ieInpNode->net.empty());
+                    if (ieInpNode->net != net)
+                    {
+                        net = Ptr<InfEngineBackendNet>();
+                        netBlobsWrappers.clear();
+                        break;
+                    }
                 }
             }
 
+            // The same blobs wrappers cannot be shared between two Inference Engine
+            // networks because of explicit references between layers and blobs.
+            // So we need to rewrap all the external blobs.
+            for (int i = 0; i < ld.inputBlobsId.size(); ++i)
+            {
+                int lid = ld.inputBlobsId[i].lid;
+                LayerData &inpLd = layers[lid];
+                auto it = netBlobsWrappers.find(lid);
+                if (it == netBlobsWrappers.end())
+                {
+                    ld.inputBlobsWrappers[i] = wrap(*ld.inputBlobs[i]);
+                    auto dataPtr = infEngineDataNode(ld.inputBlobsWrappers[i]);
+                    dataPtr->name = inpLd.name;
+                    netBlobsWrappers[lid] = ld.inputBlobsWrappers[i];
+                }
+                else
+                    ld.inputBlobsWrappers[i] = it->second;
+            }
+            netBlobsWrappers[ld.id] = ld.outputBlobsWrappers[0];
+
             bool fused = false;
             Ptr<BackendNode> node;
             if (!net.empty())
@@ -1153,6 +1216,7 @@ struct Net::Impl
 
             if (!fused)
                 net->addLayer(ieNode->layer);
+            addInfEngineNetOutputs(ld);
         }
 
         // Initialize all networks.

diff --git a/modules/dnn/src/layers/batch_norm_layer.cpp b/modules/dnn/src/layers/batch_norm_layer.cpp
index d5f0df6..cab9bb7 100644 (file)
--- a/modules/dnn/src/layers/batch_norm_layer.cpp
+++ b/modules/dnn/src/layers/batch_norm_layer.cpp
@@ -277,14 +277,12 @@ public:
 #ifdef HAVE_INF_ENGINE
         InferenceEngine::LayerParams lp;
         lp.name = name;
-        lp.type = "BatchNormalization";
+        lp.type = "ScaleShift";
         lp.precision = InferenceEngine::Precision::FP32;
-        std::shared_ptr<InferenceEngine::BatchNormalizationLayer> ieLayer(new InferenceEngine::BatchNormalizationLayer(lp));
+        std::shared_ptr<InferenceEngine::ScaleShiftLayer> ieLayer(new InferenceEngine::ScaleShiftLayer(lp));
 
-        size_t numChannels = weights_.total();
-        ieLayer->epsilon = epsilon;
-        ieLayer->_weights = wrapToInfEngineBlob(blobs[1], {numChannels});
-        ieLayer->_biases = wrapToInfEngineBlob(blobs[0], {numChannels});
+        ieLayer->_weights = wrapToInfEngineBlob(weights_);
+        ieLayer->_biases = wrapToInfEngineBlob(bias_);
 
         return Ptr<BackendNode>(new InfEngineBackendNode(ieLayer));
 #endif  // HAVE_INF_ENGINE

diff --git a/modules/dnn/src/layers/prior_box_layer.cpp b/modules/dnn/src/layers/prior_box_layer.cpp
index cacc279..96f1c74 100644 (file)
--- a/modules/dnn/src/layers/prior_box_layer.cpp
+++ b/modules/dnn/src/layers/prior_box_layer.cpp
@@ -550,7 +550,7 @@ public:
         for (int i = 1; i < _variance.size(); ++i)
             ieLayer->params["variance"] += format(",%f", _variance[i]);
 
-        ieLayer->params["step"] = "0";
+        ieLayer->params["step"] = _stepX == _stepY ? format("%f", _stepX) : "0";
         ieLayer->params["step_h"] = _stepY;
         ieLayer->params["step_w"] = _stepX;
 

diff --git a/modules/dnn/src/op_inf_engine.cpp b/modules/dnn/src/op_inf_engine.cpp
index 7bda0b8..6c245fa 100644 (file)
--- a/modules/dnn/src/op_inf_engine.cpp
+++ b/modules/dnn/src/op_inf_engine.cpp
@@ -116,31 +116,6 @@ InferenceEngine::Precision InfEngineBackendNet::getPrecision() noexcept
 // Assume that outputs of network is unconnected blobs.
 void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) noexcept
 {
-    if (outputs.empty())
-    {
-        for (const auto& l : layers)
-        {
-            // Add all outputs.
-            for (const InferenceEngine::DataPtr& out : l->outData)
-            {
-                // TODO: Replace to uniquness assertion.
-                if (outputs.find(out->name) == outputs.end())
-                    outputs[out->name] = out;
-            }
-            // Remove internally connected outputs.
-            for (const InferenceEngine::DataWeakPtr& inp : l->insData)
-            {
-                outputs.erase(InferenceEngine::DataPtr(inp)->name);
-            }
-        }
-        CV_Assert(layers.empty() || !outputs.empty());
-    }
-    outBlobs.clear();
-    for (const auto& it : outputs)
-    {
-        CV_Assert(allBlobs.find(it.first) != allBlobs.end());
-        outBlobs[it.first] = allBlobs[it.first];
-    }
     outputs_ = outputs;
 }
 
@@ -216,7 +191,18 @@ InferenceEngine::StatusCode
 InfEngineBackendNet::addOutput(const std::string &layerName, size_t outputIndex,
                                InferenceEngine::ResponseDesc *resp) noexcept
 {
-    CV_Error(Error::StsNotImplemented, "");
+    for (const auto& l : layers)
+    {
+        for (const InferenceEngine::DataPtr& out : l->outData)
+        {
+            if (out->name == layerName)
+            {
+                outputs[out->name] = out;
+                return InferenceEngine::StatusCode::OK;
+            }
+        }
+    }
+    CV_Error(Error::StsObjectNotFound, "Cannot find a layer " + layerName);
     return InferenceEngine::StatusCode::OK;
 }
 
@@ -254,6 +240,39 @@ size_t InfEngineBackendNet::getBatchSize() const noexcept
 void InfEngineBackendNet::initEngine()
 {
     CV_Assert(!isInitialized());
+
+    // Add all unconnected blobs to output blobs.
+    InferenceEngine::OutputsDataMap unconnectedOuts;
+    for (const auto& l : layers)
+    {
+        // Add all outputs.
+        for (const InferenceEngine::DataPtr& out : l->outData)
+        {
+            // TODO: Replace to uniquness assertion.
+            if (unconnectedOuts.find(out->name) == unconnectedOuts.end())
+                unconnectedOuts[out->name] = out;
+        }
+        // Remove internally connected outputs.
+        for (const InferenceEngine::DataWeakPtr& inp : l->insData)
+        {
+            unconnectedOuts.erase(InferenceEngine::DataPtr(inp)->name);
+        }
+    }
+    CV_Assert(layers.empty() || !unconnectedOuts.empty());
+
+    for (auto it = unconnectedOuts.begin(); it != unconnectedOuts.end(); ++it)
+    {
+        outputs[it->first] = it->second;
+    }
+
+    // Set up output blobs.
+    outBlobs.clear();
+    for (const auto& it : outputs)
+    {
+        CV_Assert(allBlobs.find(it.first) != allBlobs.end());
+        outBlobs[it.first] = allBlobs[it.first];
+    }
+
     engine = InferenceEngine::InferenceEnginePluginPtr("libMKLDNNPlugin.so");
     InferenceEngine::ResponseDesc resp;
     InferenceEngine::StatusCode status = engine->LoadNetwork(*this, &resp);

diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp
index 0a76000..58b25f3 100644 (file)
--- a/modules/dnn/test/test_backends.cpp
+++ b/modules/dnn/test/test_backends.cpp
@@ -206,9 +206,21 @@ TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
 
 TEST_P(DNNTestNetwork, OpenFace)
 {
+    if (backend == DNN_BACKEND_HALIDE) throw SkipTestException("");
     processNet("dnn/openface_nn4.small2.v1.t7", "", Size(96, 96), "", "torch");
 }
 
+TEST_P(DNNTestNetwork, opencv_face_detector)
+{
+    if (backend == DNN_BACKEND_HALIDE ||
+        backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL)
+        throw SkipTestException("");
+    Mat img = imread(findDataFile("gpu/lbpcascade/er.png", false));
+    Mat inp = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
+    processNet("dnn/opencv_face_detector.caffemodel", "dnn/opencv_face_detector.prototxt",
+               inp, "detection_out", "caffe");
+}
+
 const tuple<DNNBackend, DNNTarget> testCases[] = {
 #ifdef HAVE_HALIDE
     tuple<DNNBackend, DNNTarget>(DNN_BACKEND_HALIDE, DNN_TARGET_CPU),

diff --git a/modules/dnn/test/test_tf_importer.cpp b/modules/dnn/test/test_tf_importer.cpp
index 5bddd71..9cebd23 100644 (file)
--- a/modules/dnn/test/test_tf_importer.cpp
+++ b/modules/dnn/test/test_tf_importer.cpp
@@ -279,9 +279,8 @@ TEST(Test_TensorFlow, Inception_v2_SSD)
     normAssert(detections, ref);
 }
 
-OCL_TEST(Test_TensorFlow, MobileNet_SSD)
+OCL_TEST(Test_TensorFlow, DISABLED_MobileNet_SSD)
 {
-    throw SkipTestException("TODO: test is failed");
     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt", false);
     std::string imgPath = findDataFile("dnn/street.png", false);