1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
9 Test for Tensorflow models loading
12 #include "test_precomp.hpp"
13 #include "npy_blob.hpp"
15 #include <opencv2/dnn/layer.details.hpp> // CV_DNN_REGISTER_LAYER_CLASS
21 using namespace cv::dnn;
23 template<typename TString>
24 static std::string _tf(TString filename)
26 return (getOpenCVExtraDir() + "/dnn/") + filename;
29 TEST(Test_TensorFlow, read_inception)
33 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
34 net = readNetFromTensorflow(model);
35 ASSERT_FALSE(net.empty());
37 net.setPreferableBackend(DNN_BACKEND_OPENCV);
39 Mat sample = imread(_tf("grace_hopper_227.png"));
40 ASSERT_TRUE(!sample.empty());
42 resize(sample, input, Size(224, 224));
43 input -= Scalar::all(117); // mean sub
45 Mat inputBlob = blobFromImage(input);
47 net.setInput(inputBlob, "input");
48 Mat out = net.forward("softmax2");
50 std::cout << out.dims << std::endl;
53 TEST(Test_TensorFlow, inception_accuracy)
57 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
58 net = readNetFromTensorflow(model);
59 ASSERT_FALSE(net.empty());
61 net.setPreferableBackend(DNN_BACKEND_OPENCV);
63 Mat sample = imread(_tf("grace_hopper_227.png"));
64 ASSERT_TRUE(!sample.empty());
65 Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
67 net.setInput(inputBlob, "input");
68 Mat out = net.forward("softmax2");
70 Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
75 static std::string path(const std::string& file)
77 return findDataFile("dnn/tensorflow/" + file);
80 class Test_TensorFlow_layers : public DNNTestLayer
83 void runTensorFlowNet(const std::string& prefix, bool hasText = false,
84 double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
86 std::string netPath = path(prefix + "_net.pb");
87 std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
88 std::string inpPath = path(prefix + "_in.npy");
89 std::string outPath = path(prefix + "_out.npy");
91 cv::Mat input = blobFromNPY(inpPath);
92 cv::Mat ref = blobFromNPY(outPath);
93 checkBackend(&input, &ref);
98 // Load files into a memory buffers
99 std::vector<char> dataModel;
100 readFileContent(netPath, dataModel);
102 std::vector<char> dataConfig;
105 readFileContent(netConfig, dataConfig);
108 net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
109 dataConfig.data(), dataConfig.size());
112 net = readNetFromTensorflow(netPath, netConfig);
114 ASSERT_FALSE(net.empty());
116 net.setPreferableBackend(backend);
117 net.setPreferableTarget(target);
119 cv::Mat output = net.forward();
120 normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
124 TEST_P(Test_TensorFlow_layers, conv)
126 runTensorFlowNet("single_conv");
127 runTensorFlowNet("atrous_conv2d_valid");
128 runTensorFlowNet("atrous_conv2d_same");
129 runTensorFlowNet("depthwise_conv2d");
130 runTensorFlowNet("keras_atrous_conv2d_same");
131 runTensorFlowNet("conv_pool_nchw");
134 TEST_P(Test_TensorFlow_layers, Convolution3D)
136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
137 throw SkipTestException("Test is enabled starts from 2019R1");
139 if (target != DNN_TARGET_CPU)
140 throw SkipTestException("Only CPU is supported");
141 runTensorFlowNet("conv3d");
144 TEST_P(Test_TensorFlow_layers, padding)
146 runTensorFlowNet("padding_valid");
147 runTensorFlowNet("spatial_padding");
148 runTensorFlowNet("mirror_pad");
149 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000)
150 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
152 if (target == DNN_TARGET_MYRIAD)
153 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
154 if (target == DNN_TARGET_OPENCL_FP16)
155 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
158 runTensorFlowNet("keras_pad_concat");
161 TEST_P(Test_TensorFlow_layers, padding_same)
163 // Reference output values are in range [0.0006, 2.798]
164 runTensorFlowNet("padding_same");
167 TEST_P(Test_TensorFlow_layers, eltwise)
169 runTensorFlowNet("eltwise_add_mul");
170 runTensorFlowNet("eltwise_sub");
173 TEST_P(Test_TensorFlow_layers, pad_and_concat)
175 runTensorFlowNet("pad_and_concat");
178 TEST_P(Test_TensorFlow_layers, concat_axis_1)
180 runTensorFlowNet("concat_axis_1");
183 TEST_P(Test_TensorFlow_layers, batch_norm)
185 runTensorFlowNet("batch_norm");
186 runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
187 runTensorFlowNet("fused_batch_norm");
188 runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
189 runTensorFlowNet("batch_norm_text", true);
190 runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
191 runTensorFlowNet("unfused_batch_norm");
192 runTensorFlowNet("fused_batch_norm_no_gamma");
193 runTensorFlowNet("unfused_batch_norm_no_gamma");
194 runTensorFlowNet("mvn_batch_norm");
195 runTensorFlowNet("mvn_batch_norm_1x1");
196 runTensorFlowNet("switch_identity");
197 runTensorFlowNet("keras_batch_norm_training");
200 TEST_P(Test_TensorFlow_layers, batch_norm3D)
202 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
204 if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
205 if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
206 if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
207 throw SkipTestException("");
209 runTensorFlowNet("batch_norm3d");
212 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
214 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
215 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
216 // Output values range: [-40.0597, 207.827]
217 double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
218 double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
219 runTensorFlowNet("slim_batch_norm", false, l1, lInf);
222 TEST_P(Test_TensorFlow_layers, pooling)
224 runTensorFlowNet("max_pool_even");
225 runTensorFlowNet("max_pool_odd_valid");
226 runTensorFlowNet("max_pool_odd_same");
227 runTensorFlowNet("reduce_mean"); // an average pooling over all spatial dimensions.
230 TEST_P(Test_TensorFlow_layers, max_pool_grad)
232 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
233 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
234 runTensorFlowNet("max_pool_grad");
237 // TODO: fix tests and replace to pooling
238 TEST_P(Test_TensorFlow_layers, ave_pool_same)
240 // Reference output values are in range [-0.519531, 0.112976]
241 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
242 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
243 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
245 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
247 runTensorFlowNet("ave_pool_same");
250 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
252 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
253 throw SkipTestException("Test is enabled starts from 2019R1");
255 if (target != DNN_TARGET_CPU)
256 throw SkipTestException("Only CPU is supported");
257 runTensorFlowNet("max_pool3d");
260 TEST_P(Test_TensorFlow_layers, AvePooling3D)
262 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
263 throw SkipTestException("Test is enabled starts from 2019R1");
265 if (target != DNN_TARGET_CPU)
266 throw SkipTestException("Only CPU is supported");
267 runTensorFlowNet("ave_pool3d");
270 TEST_P(Test_TensorFlow_layers, deconvolution)
272 runTensorFlowNet("deconvolution");
273 runTensorFlowNet("deconvolution_same");
274 runTensorFlowNet("deconvolution_stride_2_same");
275 runTensorFlowNet("deconvolution_adj_pad_valid");
276 runTensorFlowNet("deconvolution_adj_pad_same");
277 runTensorFlowNet("keras_deconv_valid");
278 runTensorFlowNet("keras_deconv_same");
279 runTensorFlowNet("keras_deconv_same_v2");
282 TEST_P(Test_TensorFlow_layers, matmul)
284 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
285 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
286 runTensorFlowNet("matmul");
287 runTensorFlowNet("nhwc_transpose_reshape_matmul");
288 // Reference output values are in range [-5.688, 4.484]
289 double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
290 runTensorFlowNet("nhwc_reshape_matmul", false, l1);
291 runTensorFlowNet("matmul_layout");
294 TEST_P(Test_TensorFlow_layers, reshape)
296 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
297 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
298 runTensorFlowNet("shift_reshape_no_reorder");
299 runTensorFlowNet("reshape_no_reorder");
300 runTensorFlowNet("reshape_reduce");
301 runTensorFlowNet("reshape_as_shape");
304 TEST_P(Test_TensorFlow_layers, flatten)
306 #if defined(INF_ENGINE_RELEASE)
307 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
308 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
310 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
313 runTensorFlowNet("flatten", true);
316 TEST_P(Test_TensorFlow_layers, unfused_flatten)
318 runTensorFlowNet("unfused_flatten");
319 runTensorFlowNet("unfused_flatten_unknown_batch");
322 TEST_P(Test_TensorFlow_layers, leaky_relu)
324 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
325 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
326 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
328 runTensorFlowNet("leaky_relu_order1");
329 runTensorFlowNet("leaky_relu_order2");
330 runTensorFlowNet("leaky_relu_order3");
333 TEST_P(Test_TensorFlow_layers, l2_normalize)
335 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
336 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
337 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
339 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
342 runTensorFlowNet("l2_normalize");
345 // TODO: fix it and add to l2_normalize
346 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
348 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
349 if (backend == DNN_BACKEND_INFERENCE_ENGINE
350 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
352 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
354 #if defined(INF_ENGINE_RELEASE)
355 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
356 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
359 runTensorFlowNet("l2_normalize_3d");
362 class Test_TensorFlow_nets : public DNNTestLayer {};
364 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
366 #if defined(INF_ENGINE_RELEASE)
367 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
369 #if INF_ENGINE_VER_MAJOR_GE(2019020000)
370 if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
371 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
377 std::string imgPath = findDataFile("dnn/street.png");
378 std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
379 std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
382 resize(imread(imgPath), inp, Size(300, 300));
383 inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
385 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
387 Net net = readNetFromTensorflow(netPath, netConfig);
388 net.setPreferableBackend(backend);
389 net.setPreferableTarget(target);
392 Mat out = net.forward();
394 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
395 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
396 normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
397 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
398 expectNoFallbacksFromIE(net);
402 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
404 applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
405 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
406 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
407 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
408 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
412 Mat img = imread(findDataFile("dnn/street.png"));
413 std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
414 std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
416 Net net = readNetFromTensorflow(model, proto);
417 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
419 net.setPreferableBackend(backend);
420 net.setPreferableTarget(target);
423 // Output has shape 1x1xNx7 where N - number of detections.
424 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
425 Mat out = net.forward();
426 Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
427 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
428 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
429 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
430 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
432 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
433 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
434 normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
435 expectNoFallbacksFromIE(net);
438 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
441 std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
442 std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
444 Net net = readNetFromTensorflow(model, proto);
445 Mat img = imread(findDataFile("dnn/dog416.png"));
446 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
448 net.setPreferableBackend(backend);
449 net.setPreferableTarget(target);
452 Mat out = net.forward();
454 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
455 float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 1.5e-5;
456 float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
457 float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
459 #if defined(INF_ENGINE_RELEASE)
460 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
461 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
465 detectionConfThresh = 0.36;
468 normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
469 expectNoFallbacksFromIE(net);
472 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
476 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
478 CV_TEST_TAG_DEBUG_VERYLONG
480 static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
481 "faster_rcnn_resnet50_coco_2018_01_28"};
484 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
485 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
486 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
487 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
489 double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
490 for (int i = 0; i < 2; ++i)
492 std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
493 std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
495 Net net = readNetFromTensorflow(model, proto);
496 net.setPreferableBackend(backend);
497 net.setPreferableTarget(target);
498 Mat img = imread(findDataFile("dnn/dog416.png"));
499 Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
502 Mat out = net.forward();
504 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
505 normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
509 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
511 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
512 if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
513 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
516 std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
517 std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
519 Net net = readNetFromTensorflow(model, proto);
520 Mat img = imread(findDataFile("dnn/dog416.png"));
521 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
522 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
524 net.setPreferableBackend(backend);
525 net.setPreferableTarget(target);
528 Mat out = net.forward();
530 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
531 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
532 normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
533 expectNoFallbacksFromIE(net);
536 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
539 std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
540 std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
542 Net net = readNetFromTensorflow(model, proto);
543 Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
544 Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
546 net.setPreferableBackend(backend);
547 net.setPreferableTarget(target);
549 // Output has shape 1x1xNx7 where N - number of detections.
550 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
551 Mat out = net.forward();
553 // References are from test for Caffe model.
554 Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
555 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
556 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
557 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
558 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
559 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
560 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
561 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
562 normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
563 expectNoFallbacksFromIE(net);
566 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
567 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
568 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
569 // sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
570 // feed_dict={'input_images:0': inp})
571 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
572 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
573 // np.save('east_text_detection.scores.npy', scores)
574 // np.save('east_text_detection.geometry.npy', geometry)
575 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
578 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
579 CV_TEST_TAG_DEBUG_LONG
582 #if defined(INF_ENGINE_RELEASE)
583 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
584 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
586 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16 &&
587 INF_ENGINE_VER_MAJOR_EQ(2019020000))
588 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
593 std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
594 std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
595 std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
596 std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
598 Net net = readNet(netPath);
600 net.setPreferableBackend(backend);
601 net.setPreferableTarget(target);
603 Mat img = imread(imgPath);
604 Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
607 std::vector<Mat> outs;
608 std::vector<String> outNames(2);
609 outNames[0] = "feature_fusion/Conv_7/Sigmoid";
610 outNames[1] = "feature_fusion/concat_3";
611 net.forward(outs, outNames);
613 Mat scores = outs[0];
614 Mat geometry = outs[1];
616 // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
617 double l1_scores = default_l1, lInf_scores = default_lInf;
618 double l1_geometry = default_l1, lInf_geometry = default_lInf;
619 if (target == DNN_TARGET_OPENCL_FP16)
621 lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
622 l1_geometry = 0.28; lInf_geometry = 5.94;
624 else if (target == DNN_TARGET_MYRIAD)
627 l1_geometry = 0.28; lInf_geometry = 5.94;
631 l1_geometry = 1e-4, lInf_geometry = 3e-3;
633 normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
634 normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
635 expectNoFallbacksFromIE(net);
638 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
640 TEST_P(Test_TensorFlow_layers, fp16_weights)
644 runTensorFlowNet("fp16_single_conv", false, l1, lInf);
645 runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
646 runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
647 runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
648 runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
649 // Reference output values are in range [0.0889, 1.651]
650 runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
651 if (target == DNN_TARGET_MYRIAD) {
655 // Reference output values are in range [0, 10.75]
656 runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
657 // Reference output values are in range [0.418, 2.297]
658 runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
661 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
663 // Reference output values are in range [-3.504, -0.002]
664 runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
667 TEST_P(Test_TensorFlow_layers, defun)
669 runTensorFlowNet("defun_dropout");
672 TEST_P(Test_TensorFlow_layers, quantized)
674 runTensorFlowNet("uint8_single_conv");
677 TEST_P(Test_TensorFlow_layers, lstm)
679 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
680 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
681 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
682 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
683 runTensorFlowNet("lstm", true);
684 runTensorFlowNet("lstm", true, 0.0, 0.0, true);
687 TEST_P(Test_TensorFlow_layers, split)
689 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
690 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2)
691 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
692 runTensorFlowNet("split");
693 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
694 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
695 runTensorFlowNet("split_equals");
698 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
700 runTensorFlowNet("resize_nearest_neighbor");
701 runTensorFlowNet("keras_upsampling2d");
704 TEST_P(Test_TensorFlow_layers, slice)
706 if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
707 (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
708 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
709 runTensorFlowNet("slice_4d");
710 runTensorFlowNet("strided_slice");
713 TEST_P(Test_TensorFlow_layers, softmax)
715 runTensorFlowNet("keras_softmax");
716 runTensorFlowNet("slim_softmax");
719 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
721 #if defined(INF_ENGINE_RELEASE)
722 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
723 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
725 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
727 runTensorFlowNet("slim_softmax_v2");
730 TEST_P(Test_TensorFlow_layers, relu6)
732 runTensorFlowNet("keras_relu6");
733 runTensorFlowNet("keras_relu6", /*hasText*/ true);
736 TEST_P(Test_TensorFlow_layers, subpixel)
738 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
739 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
740 runTensorFlowNet("subpixel");
743 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
745 runTensorFlowNet("keras_mobilenet_head");
746 runTensorFlowNet("keras_learning_phase");
749 TEST_P(Test_TensorFlow_layers, resize_bilinear)
751 runTensorFlowNet("resize_bilinear");
752 runTensorFlowNet("resize_bilinear_factor");
755 TEST_P(Test_TensorFlow_layers, squeeze)
757 #if defined(INF_ENGINE_RELEASE)
758 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
759 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
761 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
763 int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}}; // TensorFlow's shape (NHWC)
764 int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
765 int squeeze_dims[] = {0, 2, 3, -1};
766 for (int i = 0; i < 4; ++i)
768 SCOPED_TRACE(format("i=%d", i));
770 "node { name: \"input\" op: \"Placeholder\""
771 "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
772 "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
773 "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
774 Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
775 net.setPreferableBackend(backend);
776 net.setPreferableTarget(target);
777 Mat tfInp(4, &inpShapes[i][0], CV_32F);
781 CV_Assert(inpShapes[i][0] == 1);
782 std::swap(inpShapes[i][2], inpShapes[i][3]);
783 std::swap(inpShapes[i][1], inpShapes[i][2]);
784 Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
785 cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
788 Mat out = net.forward();
789 normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
793 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
795 TEST(Test_TensorFlow, two_inputs)
797 Net net = readNet(path("two_inputs_net.pbtxt"));
798 net.setPreferableBackend(DNN_BACKEND_OPENCV);
800 Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
801 randu(firstInput, -1, 1);
802 randu(secondInput, -1, 1);
804 net.setInput(firstInput, "first_input");
805 net.setInput(secondInput, "second_input");
806 Mat out = net.forward();
808 normAssert(out, firstInput + secondInput);
811 TEST(Test_TensorFlow, Mask_RCNN)
813 applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
814 Mat img = imread(findDataFile("dnn/street.png"));
815 std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
816 std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
818 Net net = readNetFromTensorflow(model, proto);
819 Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
820 Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
821 Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
823 net.setPreferableBackend(DNN_BACKEND_OPENCV);
827 // Mask-RCNN predicts bounding boxes and segmentation masks.
828 std::vector<String> outNames(2);
829 outNames[0] = "detection_out_final";
830 outNames[1] = "detection_masks";
832 std::vector<Mat> outs;
833 net.forward(outs, outNames);
835 Mat outDetections = outs[0];
836 Mat outMasks = outs[1];
837 normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
839 // Output size of masks is NxCxHxW where
840 // N - number of detected boxes
841 // C - number of classes (excluding background)
842 // HxW - segmentation shape
843 const int numDetections = outDetections.size[2];
845 int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
846 Mat masks(4, &masksSize[0], CV_32F);
848 std::vector<cv::Range> srcRanges(4, cv::Range::all());
849 std::vector<cv::Range> dstRanges(4, cv::Range::all());
851 outDetections = outDetections.reshape(1, outDetections.total() / 7);
852 for (int i = 0; i < numDetections; ++i)
854 // Get a class id for this bounding box and copy mask only for that class.
855 int classId = static_cast<int>(outDetections.at<float>(i, 1));
856 srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
857 srcRanges[1] = cv::Range(classId, classId + 1);
858 outMasks(srcRanges).copyTo(masks(dstRanges));
860 cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
861 normAssert(masks, refMasks(&topRefMasks[0]));