Enable SSD models from TensorFlow with OpenCL plugin of Intel's Inference Engine

diff --git a/modules/dnn/perf/perf_net.cpp b/modules/dnn/perf/perf_net.cpp
index 8e777f8..e8569dc 100644 (file)
--- a/modules/dnn/perf/perf_net.cpp
+++ b/modules/dnn/perf/perf_net.cpp
@@ -158,13 +158,19 @@ PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_Caffe)
             Mat(cv::Size(300, 300), CV_32FC3));
 }
 
-// TODO: update MobileNet model.
-PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_TensorFlow)
+PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow)
 {
-    if (backend == DNN_BACKEND_HALIDE ||
-        backend == DNN_BACKEND_INFERENCE_ENGINE)
+    if (backend == DNN_BACKEND_HALIDE)
+        throw SkipTestException("");
+    processNet("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", "ssd_mobilenet_v1_coco_2017_11_17.pbtxt", "",
+            Mat(cv::Size(300, 300), CV_32FC3));
+}
+
+PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
+{
+    if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
-    processNet("dnn/ssd_mobilenet_v1_coco.pb", "ssd_mobilenet_v1_coco.pbtxt", "",
+    processNet("dnn/ssd_mobilenet_v2_coco_2018_03_29.pb", "ssd_mobilenet_v2_coco_2018_03_29.pbtxt", "",
             Mat(cv::Size(300, 300), CV_32FC3));
 }
 
@@ -217,9 +223,7 @@ PERF_TEST_P_(DNNTestNetwork, opencv_face_detector)
 
 PERF_TEST_P_(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
 {
-    if (backend == DNN_BACKEND_HALIDE ||
-        (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL) ||
-        (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16))
+    if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
     processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "ssd_inception_v2_coco_2017_11_17.pbtxt", "",
             Mat(cv::Size(300, 300), CV_32FC3));

diff --git a/modules/dnn/test/test_backends.cpp b/modules/dnn/test/test_backends.cpp
index 6f41610..f6563cb 100644 (file)
--- a/modules/dnn/test/test_backends.cpp
+++ b/modules/dnn/test/test_backends.cpp
@@ -38,7 +38,7 @@ public:
     void processNet(std::string weights, std::string proto,
                     Mat inp, const std::string& outputLayer = "",
                     std::string halideScheduler = "",
-                    double l1 = 0.0, double lInf = 0.0)
+                    double l1 = 0.0, double lInf = 0.0, double detectionConfThresh = 0.2)
     {
         if (backend == DNN_BACKEND_OPENCV && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
         {
@@ -87,7 +87,7 @@ public:
         }
         Mat out = net.forward(outputLayer).clone();
 
-        check(outDefault, out, outputLayer, l1, lInf, "First run");
+        check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "First run");
 
         // Test 2: change input.
         float* inpData = (float*)inp.data;
@@ -101,10 +101,11 @@ public:
         net.setInput(inp);
         outDefault = netDefault.forward(outputLayer).clone();
         out = net.forward(outputLayer).clone();
-        check(outDefault, out, outputLayer, l1, lInf, "Second run");
+        check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "Second run");
     }
 
-    void check(Mat& ref, Mat& out, const std::string& outputLayer, double l1, double lInf, const char* msg)
+    void check(Mat& ref, Mat& out, const std::string& outputLayer, double l1, double lInf,
+               double detectionConfThresh, const char* msg)
     {
         if (outputLayer == "detection_out")
         {
@@ -119,7 +120,7 @@ public:
                 }
                 out = out.rowRange(0, numDetections);
             }
-            normAssertDetections(ref, out, msg, 0.2, l1, lInf);
+            normAssertDetections(ref, out, msg, detectionConfThresh, l1, lInf);
         }
         else
             normAssert(ref, out, msg, l1, lInf);
@@ -188,20 +189,30 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe)
                inp, "detection_out", "", l1, lInf);
 }
 
-// TODO: update MobileNet model.
-TEST_P(DNNTestNetwork, MobileNet_SSD_TensorFlow)
+TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow)
 {
-    if (backend == DNN_BACKEND_HALIDE ||
-        backend == DNN_BACKEND_INFERENCE_ENGINE)
+    if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
     Mat sample = imread(findDataFile("dnn/street.png", false));
     Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false);
-    float l1 = (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) ? 0.008 : 0.0;
-    float lInf = (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16) ? 0.06 : 0.0;
-    processNet("dnn/ssd_mobilenet_v1_coco.pb", "dnn/ssd_mobilenet_v1_coco.pbtxt",
+    float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 0.0;
+    float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.06 : 0.0;
+    processNet("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", "dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt",
                inp, "detection_out", "", l1, lInf);
 }
 
+TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
+{
+    if (backend == DNN_BACKEND_HALIDE)
+        throw SkipTestException("");
+    Mat sample = imread(findDataFile("dnn/street.png", false));
+    Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false);
+    float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 0.0;
+    float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.06 : 0.0;
+    processNet("dnn/ssd_mobilenet_v2_coco_2018_03_29.pb", "dnn/ssd_mobilenet_v2_coco_2018_03_29.pbtxt",
+               inp, "detection_out", "", l1, lInf, 0.25);
+}
+
 TEST_P(DNNTestNetwork, SSD_VGG16)
 {
     if (backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU)
@@ -265,9 +276,7 @@ TEST_P(DNNTestNetwork, opencv_face_detector)
 
 TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
 {
-    if (backend == DNN_BACKEND_HALIDE ||
-        (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL) ||
-        (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16))
+    if (backend == DNN_BACKEND_HALIDE)
         throw SkipTestException("");
     Mat sample = imread(findDataFile("dnn/street.png", false));
     Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false);

diff --git a/modules/dnn/test/test_layers.cpp b/modules/dnn/test/test_layers.cpp
index da7dd77..111f354 100644 (file)
--- a/modules/dnn/test/test_layers.cpp
+++ b/modules/dnn/test/test_layers.cpp
@@ -877,6 +877,7 @@ TEST_P(Layer_Test_DWconv_Prelu, Accuracy)
     int shape[] = {1, num_input, 16, 16};
     Mat in_blob(4, &shape[0], CV_32FC1, Scalar(1));
 
+    net.setPreferableBackend(DNN_BACKEND_OPENCV);
     net.setInput(in_blob);
     Mat out = net.forward();
 

diff --git a/samples/dnn/tf_text_graph_ssd.py b/samples/dnn/tf_text_graph_ssd.py
index f571047..851e0d8 100644 (file)
--- a/samples/dnn/tf_text_graph_ssd.py
+++ b/samples/dnn/tf_text_graph_ssd.py
@@ -160,27 +160,40 @@ graph_def.node[1].input.append(weights)
 # Create SSD postprocessing head ###############################################
 
 # Concatenate predictions of classes, predictions of bounding boxes and proposals.
+def tensorMsg(values):
+    if all([isinstance(v, float) for v in values]):
+        dtype = 'DT_FLOAT'
+        field = 'float_val'
+    elif all([isinstance(v, int) for v in values]):
+        dtype = 'DT_INT32'
+        field = 'int_val'
+    else:
+        raise Exception('Wrong values types')
 
-concatAxis = NodeDef()
-concatAxis.name = 'concat/axis_flatten'
-concatAxis.op = 'Const'
-text_format.Merge(
-'tensor {'
-'  dtype: DT_INT32'
-'  tensor_shape { }'
-'  int_val: -1'
-'}', concatAxis.attr["value"])
-graph_def.node.extend([concatAxis])
-
-def addConcatNode(name, inputs):
+    msg = 'tensor { dtype: ' + dtype + ' tensor_shape { dim { size: %d } }' % len(values)
+    for value in values:
+        msg += '%s: %s ' % (field, str(value))
+    return msg + '}'
+
+def addConstNode(name, values):
+    node = NodeDef()
+    node.name = name
+    node.op = 'Const'
+    text_format.Merge(tensorMsg(values), node.attr["value"])
+    graph_def.node.extend([node])
+
+def addConcatNode(name, inputs, axisNodeName):
     concat = NodeDef()
     concat.name = name
     concat.op = 'ConcatV2'
     for inp in inputs:
         concat.input.append(inp)
-    concat.input.append(concatAxis.name)
+    concat.input.append(axisNodeName)
     graph_def.node.extend([concat])
 
+addConstNode('concat/axis_flatten', [-1])
+addConstNode('PriorBox/concat/axis', [-2])
+
 for label in ['ClassPredictor', 'BoxEncodingPredictor']:
     concatInputs = []
     for i in range(args.num_layers):
@@ -193,19 +206,14 @@ for label in ['ClassPredictor', 'BoxEncodingPredictor']:
 
         concatInputs.append(flatten.name)
         graph_def.node.extend([flatten])
-    addConcatNode('%s/concat' % label, concatInputs)
+    addConcatNode('%s/concat' % label, concatInputs, 'concat/axis_flatten')
 
 # Add layers that generate anchors (bounding boxes proposals).
 scales = [args.min_scale + (args.max_scale - args.min_scale) * i / (args.num_layers - 1)
           for i in range(args.num_layers)] + [1.0]
 
-def tensorMsg(values):
-    msg = 'tensor { dtype: DT_FLOAT tensor_shape { dim { size: %d } }' % len(values)
-    for value in values:
-        msg += 'float_val: %f ' % value
-    return msg + '}'
-
 priorBoxes = []
+addConstNode('reshape_prior_boxes_to_4d', [1, 2, -1, 1])
 for i in range(args.num_layers):
     priorBox = NodeDef()
     priorBox.name = 'PriorBox_%d' % i
@@ -232,9 +240,18 @@ for i in range(args.num_layers):
     text_format.Merge(tensorMsg([0.1, 0.1, 0.2, 0.2]), priorBox.attr["variance"])
 
     graph_def.node.extend([priorBox])
-    priorBoxes.append(priorBox.name)
 
-addConcatNode('PriorBox/concat', priorBoxes)
+    # Reshape from 1x2xN to 1x2xNx1
+    reshape = NodeDef()
+    reshape.name = priorBox.name + '/4d'
+    reshape.op = 'Reshape'
+    reshape.input.append(priorBox.name)
+    reshape.input.append('reshape_prior_boxes_to_4d')
+    graph_def.node.extend([reshape])
+
+    priorBoxes.append(reshape.name)
+
+addConcatNode('PriorBox/concat', priorBoxes, 'PriorBox/concat/axis')
 
 # Sigmoid for classes predictions and DetectionOutput layer
 sigmoid = NodeDef()