1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
9 Test for Tensorflow models loading
12 #include "test_precomp.hpp"
13 #include "npy_blob.hpp"
15 #include <opencv2/dnn/layer.details.hpp> // CV_DNN_REGISTER_LAYER_CLASS
21 using namespace cv::dnn;
23 template<typename TString>
24 static std::string _tf(TString filename)
26 return (getOpenCVExtraDir() + "/dnn/") + filename;
29 TEST(Test_TensorFlow, read_inception)
33 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
34 net = readNetFromTensorflow(model);
35 ASSERT_FALSE(net.empty());
37 net.setPreferableBackend(DNN_BACKEND_OPENCV);
39 Mat sample = imread(_tf("grace_hopper_227.png"));
40 ASSERT_TRUE(!sample.empty());
42 resize(sample, input, Size(224, 224));
43 input -= Scalar::all(117); // mean sub
45 Mat inputBlob = blobFromImage(input);
47 net.setInput(inputBlob, "input");
48 Mat out = net.forward("softmax2");
50 std::cout << out.dims << std::endl;
53 TEST(Test_TensorFlow, inception_accuracy)
57 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
58 net = readNetFromTensorflow(model);
59 ASSERT_FALSE(net.empty());
61 net.setPreferableBackend(DNN_BACKEND_OPENCV);
63 Mat sample = imread(_tf("grace_hopper_227.png"));
64 ASSERT_TRUE(!sample.empty());
65 Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
67 net.setInput(inputBlob, "input");
68 Mat out = net.forward("softmax2");
70 Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
75 static std::string path(const std::string& file)
77 return findDataFile("dnn/tensorflow/" + file);
80 class Test_TensorFlow_layers : public DNNTestLayer
83 void runTensorFlowNet(const std::string& prefix, bool hasText = false,
84 double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
86 std::string netPath = path(prefix + "_net.pb");
87 std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
88 std::string inpPath = path(prefix + "_in.npy");
89 std::string outPath = path(prefix + "_out.npy");
91 cv::Mat input = blobFromNPY(inpPath);
92 cv::Mat ref = blobFromNPY(outPath);
93 checkBackend(&input, &ref);
98 // Load files into a memory buffers
99 std::vector<char> dataModel;
100 readFileContent(netPath, dataModel);
102 std::vector<char> dataConfig;
105 readFileContent(netConfig, dataConfig);
108 net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
109 dataConfig.data(), dataConfig.size());
112 net = readNetFromTensorflow(netPath, netConfig);
114 ASSERT_FALSE(net.empty());
116 net.setPreferableBackend(backend);
117 net.setPreferableTarget(target);
119 cv::Mat output = net.forward();
120 normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
124 TEST_P(Test_TensorFlow_layers, conv)
126 runTensorFlowNet("single_conv");
127 runTensorFlowNet("atrous_conv2d_valid");
128 runTensorFlowNet("atrous_conv2d_same");
129 runTensorFlowNet("depthwise_conv2d");
130 runTensorFlowNet("keras_atrous_conv2d_same");
131 runTensorFlowNet("conv_pool_nchw");
134 TEST_P(Test_TensorFlow_layers, Convolution3D)
136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
137 throw SkipTestException("Test is enabled starts from 2019R1");
139 if (target != DNN_TARGET_CPU)
140 throw SkipTestException("Only CPU is supported");
141 runTensorFlowNet("conv3d");
144 TEST_P(Test_TensorFlow_layers, padding)
146 runTensorFlowNet("padding_valid");
147 runTensorFlowNet("spatial_padding");
148 runTensorFlowNet("mirror_pad");
149 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000)
150 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
152 if (target == DNN_TARGET_MYRIAD)
153 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_2019R3, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
154 if (target == DNN_TARGET_OPENCL_FP16)
155 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R3, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
158 runTensorFlowNet("keras_pad_concat");
161 TEST_P(Test_TensorFlow_layers, padding_same)
163 // Reference output values are in range [0.0006, 2.798]
164 runTensorFlowNet("padding_same");
167 TEST_P(Test_TensorFlow_layers, eltwise)
169 runTensorFlowNet("eltwise_add_mul");
170 runTensorFlowNet("eltwise_sub");
173 TEST_P(Test_TensorFlow_layers, pad_and_concat)
175 runTensorFlowNet("pad_and_concat");
178 TEST_P(Test_TensorFlow_layers, concat_axis_1)
180 runTensorFlowNet("concat_axis_1");
183 TEST_P(Test_TensorFlow_layers, batch_norm)
185 runTensorFlowNet("batch_norm");
186 runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
187 runTensorFlowNet("fused_batch_norm");
188 runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
189 runTensorFlowNet("batch_norm_text", true);
190 runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
191 runTensorFlowNet("unfused_batch_norm");
192 runTensorFlowNet("fused_batch_norm_no_gamma");
193 runTensorFlowNet("unfused_batch_norm_no_gamma");
194 runTensorFlowNet("mvn_batch_norm");
195 runTensorFlowNet("mvn_batch_norm_1x1");
196 runTensorFlowNet("switch_identity");
197 runTensorFlowNet("keras_batch_norm_training");
200 TEST_P(Test_TensorFlow_layers, batch_norm3D)
202 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
204 if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
205 if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
206 if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
207 throw SkipTestException("");
209 runTensorFlowNet("batch_norm3d");
212 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
214 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
215 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
216 // Output values range: [-40.0597, 207.827]
217 double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
218 double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
219 runTensorFlowNet("slim_batch_norm", false, l1, lInf);
222 TEST_P(Test_TensorFlow_layers, pooling)
224 runTensorFlowNet("max_pool_even");
225 runTensorFlowNet("max_pool_odd_valid");
226 runTensorFlowNet("max_pool_odd_same");
227 runTensorFlowNet("reduce_mean"); // an average pooling over all spatial dimensions.
230 TEST_P(Test_TensorFlow_layers, max_pool_grad)
232 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
233 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
234 runTensorFlowNet("max_pool_grad");
237 // TODO: fix tests and replace to pooling
238 TEST_P(Test_TensorFlow_layers, ave_pool_same)
240 // Reference output values are in range [-0.519531, 0.112976]
241 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
242 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
243 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
245 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
247 runTensorFlowNet("ave_pool_same");
250 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
252 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
253 throw SkipTestException("Test is enabled starts from 2019R1");
255 if (target != DNN_TARGET_CPU)
256 throw SkipTestException("Only CPU is supported");
257 runTensorFlowNet("max_pool3d");
260 TEST_P(Test_TensorFlow_layers, AvePooling3D)
262 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
263 throw SkipTestException("Test is enabled starts from 2019R1");
265 if (target != DNN_TARGET_CPU)
266 throw SkipTestException("Only CPU is supported");
267 runTensorFlowNet("ave_pool3d");
270 TEST_P(Test_TensorFlow_layers, deconvolution)
272 runTensorFlowNet("deconvolution");
273 runTensorFlowNet("deconvolution_same");
274 runTensorFlowNet("deconvolution_stride_2_same");
275 runTensorFlowNet("deconvolution_adj_pad_valid");
276 runTensorFlowNet("deconvolution_adj_pad_same");
277 runTensorFlowNet("keras_deconv_valid");
278 runTensorFlowNet("keras_deconv_same");
279 runTensorFlowNet("keras_deconv_same_v2");
282 TEST_P(Test_TensorFlow_layers, matmul)
284 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
285 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
286 runTensorFlowNet("matmul");
287 runTensorFlowNet("nhwc_transpose_reshape_matmul");
288 // Reference output values are in range [-5.688, 4.484]
289 double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
290 runTensorFlowNet("nhwc_reshape_matmul", false, l1);
291 runTensorFlowNet("matmul_layout");
294 TEST_P(Test_TensorFlow_layers, reshape)
296 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
297 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
298 runTensorFlowNet("shift_reshape_no_reorder");
299 runTensorFlowNet("reshape_no_reorder");
300 runTensorFlowNet("reshape_reduce");
301 runTensorFlowNet("reshape_as_shape");
304 TEST_P(Test_TensorFlow_layers, flatten)
306 #if defined(INF_ENGINE_RELEASE)
307 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
308 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
310 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
313 runTensorFlowNet("flatten", true);
316 TEST_P(Test_TensorFlow_layers, unfused_flatten)
318 runTensorFlowNet("unfused_flatten");
319 runTensorFlowNet("unfused_flatten_unknown_batch");
322 TEST_P(Test_TensorFlow_layers, leaky_relu)
324 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
325 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
326 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
328 runTensorFlowNet("leaky_relu_order1");
329 runTensorFlowNet("leaky_relu_order2");
330 runTensorFlowNet("leaky_relu_order3");
333 TEST_P(Test_TensorFlow_layers, l2_normalize)
335 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
336 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
337 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
339 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
342 runTensorFlowNet("l2_normalize");
345 // TODO: fix it and add to l2_normalize
346 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
348 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
349 if (backend == DNN_BACKEND_INFERENCE_ENGINE
350 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
352 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
354 #if defined(INF_ENGINE_RELEASE)
355 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
356 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
359 runTensorFlowNet("l2_normalize_3d");
362 class Test_TensorFlow_nets : public DNNTestLayer {};
364 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
366 #if defined(INF_ENGINE_RELEASE)
367 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
369 #if INF_ENGINE_VER_MAJOR_GE(2019020000)
370 if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
371 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
377 std::string imgPath = findDataFile("dnn/street.png");
378 std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
379 std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
382 resize(imread(imgPath), inp, Size(300, 300));
383 inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
385 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
387 Net net = readNetFromTensorflow(netPath, netConfig);
388 net.setPreferableBackend(backend);
389 net.setPreferableTarget(target);
392 Mat out = net.forward();
394 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
395 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
396 normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
397 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
398 expectNoFallbacksFromIE(net);
402 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
404 applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
405 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
406 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
407 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
408 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
412 Mat img = imread(findDataFile("dnn/street.png"));
413 std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
414 std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
416 Net net = readNetFromTensorflow(model, proto);
417 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
419 net.setPreferableBackend(backend);
420 net.setPreferableTarget(target);
423 // Output has shape 1x1xNx7 where N - number of detections.
424 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
425 Mat out = net.forward();
426 Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
427 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
428 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
429 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
430 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
432 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
433 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
434 normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
435 expectNoFallbacksFromIE(net);
438 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
441 std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
442 std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
444 Net net = readNetFromTensorflow(model, proto);
445 Mat img = imread(findDataFile("dnn/dog416.png"));
446 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
448 net.setPreferableBackend(backend);
449 net.setPreferableTarget(target);
452 Mat out = net.forward();
454 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
455 float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 1.5e-5;
456 float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
457 float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
459 #if defined(INF_ENGINE_RELEASE)
460 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
461 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
465 detectionConfThresh = 0.36;
468 normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
469 expectNoFallbacksFromIE(net);
472 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
476 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
478 CV_TEST_TAG_DEBUG_VERYLONG
480 static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
481 "faster_rcnn_resnet50_coco_2018_01_28"};
484 #ifdef INF_ENGINE_RELEASE
485 if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
486 (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU))
487 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
489 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
490 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
492 double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
493 for (int i = 0; i < 2; ++i)
495 std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
496 std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
498 Net net = readNetFromTensorflow(model, proto);
499 net.setPreferableBackend(backend);
500 net.setPreferableTarget(target);
501 Mat img = imread(findDataFile("dnn/dog416.png"));
502 Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
505 Mat out = net.forward();
507 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
508 normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
512 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
514 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
515 if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
516 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
519 std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
520 std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
522 Net net = readNetFromTensorflow(model, proto);
523 Mat img = imread(findDataFile("dnn/dog416.png"));
524 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
525 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
527 net.setPreferableBackend(backend);
528 net.setPreferableTarget(target);
531 Mat out = net.forward();
533 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
534 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
535 normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
536 expectNoFallbacksFromIE(net);
539 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
542 std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
543 std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
545 Net net = readNetFromTensorflow(model, proto);
546 Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
547 Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
549 net.setPreferableBackend(backend);
550 net.setPreferableTarget(target);
552 // Output has shape 1x1xNx7 where N - number of detections.
553 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
554 Mat out = net.forward();
556 // References are from test for Caffe model.
557 Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
558 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
559 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
560 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
561 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
562 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
563 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
564 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
565 normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
566 expectNoFallbacksFromIE(net);
569 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
570 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
571 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
572 // sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
573 // feed_dict={'input_images:0': inp})
574 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
575 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
576 // np.save('east_text_detection.scores.npy', scores)
577 // np.save('east_text_detection.geometry.npy', geometry)
578 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
581 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
582 CV_TEST_TAG_DEBUG_LONG
585 #if defined(INF_ENGINE_RELEASE)
586 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
587 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
589 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16 &&
590 INF_ENGINE_VER_MAJOR_EQ(2019020000))
591 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
596 std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
597 std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
598 std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
599 std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
601 Net net = readNet(netPath);
603 net.setPreferableBackend(backend);
604 net.setPreferableTarget(target);
606 Mat img = imread(imgPath);
607 Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
610 std::vector<Mat> outs;
611 std::vector<String> outNames(2);
612 outNames[0] = "feature_fusion/Conv_7/Sigmoid";
613 outNames[1] = "feature_fusion/concat_3";
614 net.forward(outs, outNames);
616 Mat scores = outs[0];
617 Mat geometry = outs[1];
619 // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
620 double l1_scores = default_l1, lInf_scores = default_lInf;
621 double l1_geometry = default_l1, lInf_geometry = default_lInf;
622 if (target == DNN_TARGET_OPENCL_FP16)
624 lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
625 l1_geometry = 0.28; lInf_geometry = 5.94;
627 else if (target == DNN_TARGET_MYRIAD)
630 l1_geometry = 0.28; lInf_geometry = 5.94;
634 l1_geometry = 1e-4, lInf_geometry = 3e-3;
636 normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
637 normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
638 expectNoFallbacksFromIE(net);
641 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
643 TEST_P(Test_TensorFlow_layers, fp16_weights)
647 runTensorFlowNet("fp16_single_conv", false, l1, lInf);
648 runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
649 runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
650 runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
651 runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
652 // Reference output values are in range [0.0889, 1.651]
653 runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
654 if (target == DNN_TARGET_MYRIAD) {
658 // Reference output values are in range [0, 10.75]
659 runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
660 // Reference output values are in range [0.418, 2.297]
661 runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
664 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
666 // Reference output values are in range [-3.504, -0.002]
667 runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
670 TEST_P(Test_TensorFlow_layers, defun)
672 runTensorFlowNet("defun_dropout");
675 TEST_P(Test_TensorFlow_layers, quantized)
677 runTensorFlowNet("uint8_single_conv");
680 TEST_P(Test_TensorFlow_layers, lstm)
682 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
683 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
684 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
685 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
686 runTensorFlowNet("lstm", true);
687 runTensorFlowNet("lstm", true, 0.0, 0.0, true);
690 TEST_P(Test_TensorFlow_layers, split)
692 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
693 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2)
694 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
695 runTensorFlowNet("split");
696 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
697 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
698 runTensorFlowNet("split_equals");
701 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
703 runTensorFlowNet("resize_nearest_neighbor");
704 runTensorFlowNet("keras_upsampling2d");
707 TEST_P(Test_TensorFlow_layers, slice)
709 if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
710 (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
711 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
712 runTensorFlowNet("slice_4d");
713 runTensorFlowNet("strided_slice");
716 TEST_P(Test_TensorFlow_layers, softmax)
718 runTensorFlowNet("keras_softmax");
719 runTensorFlowNet("slim_softmax");
722 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
724 #if defined(INF_ENGINE_RELEASE)
725 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
726 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
728 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
730 runTensorFlowNet("slim_softmax_v2");
733 TEST_P(Test_TensorFlow_layers, relu6)
735 runTensorFlowNet("keras_relu6");
736 runTensorFlowNet("keras_relu6", /*hasText*/ true);
739 TEST_P(Test_TensorFlow_layers, subpixel)
741 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
742 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
743 runTensorFlowNet("subpixel");
746 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
748 runTensorFlowNet("keras_mobilenet_head");
749 runTensorFlowNet("keras_learning_phase");
752 TEST_P(Test_TensorFlow_layers, resize_bilinear)
754 runTensorFlowNet("resize_bilinear");
755 runTensorFlowNet("resize_bilinear_factor");
758 TEST_P(Test_TensorFlow_layers, squeeze)
760 #if defined(INF_ENGINE_RELEASE)
761 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
762 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
764 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
766 int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}}; // TensorFlow's shape (NHWC)
767 int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
768 int squeeze_dims[] = {0, 2, 3, -1};
769 for (int i = 0; i < 4; ++i)
771 SCOPED_TRACE(format("i=%d", i));
773 "node { name: \"input\" op: \"Placeholder\""
774 "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
775 "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
776 "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
777 Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
778 net.setPreferableBackend(backend);
779 net.setPreferableTarget(target);
780 Mat tfInp(4, &inpShapes[i][0], CV_32F);
784 CV_Assert(inpShapes[i][0] == 1);
785 std::swap(inpShapes[i][2], inpShapes[i][3]);
786 std::swap(inpShapes[i][1], inpShapes[i][2]);
787 Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
788 cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
791 Mat out = net.forward();
792 normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
796 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
798 TEST(Test_TensorFlow, two_inputs)
800 Net net = readNet(path("two_inputs_net.pbtxt"));
801 net.setPreferableBackend(DNN_BACKEND_OPENCV);
803 Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
804 randu(firstInput, -1, 1);
805 randu(secondInput, -1, 1);
807 net.setInput(firstInput, "first_input");
808 net.setInput(secondInput, "second_input");
809 Mat out = net.forward();
811 normAssert(out, firstInput + secondInput);
814 TEST(Test_TensorFlow, Mask_RCNN)
816 applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
817 Mat img = imread(findDataFile("dnn/street.png"));
818 std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
819 std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
821 Net net = readNetFromTensorflow(model, proto);
822 Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
823 Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
824 Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
826 net.setPreferableBackend(DNN_BACKEND_OPENCV);
830 // Mask-RCNN predicts bounding boxes and segmentation masks.
831 std::vector<String> outNames(2);
832 outNames[0] = "detection_out_final";
833 outNames[1] = "detection_masks";
835 std::vector<Mat> outs;
836 net.forward(outs, outNames);
838 Mat outDetections = outs[0];
839 Mat outMasks = outs[1];
840 normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
842 // Output size of masks is NxCxHxW where
843 // N - number of detected boxes
844 // C - number of classes (excluding background)
845 // HxW - segmentation shape
846 const int numDetections = outDetections.size[2];
848 int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
849 Mat masks(4, &masksSize[0], CV_32F);
851 std::vector<cv::Range> srcRanges(4, cv::Range::all());
852 std::vector<cv::Range> dstRanges(4, cv::Range::all());
854 outDetections = outDetections.reshape(1, outDetections.total() / 7);
855 for (int i = 0; i < numDetections; ++i)
857 // Get a class id for this bounding box and copy mask only for that class.
858 int classId = static_cast<int>(outDetections.at<float>(i, 1));
859 srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
860 srcRanges[1] = cv::Range(classId, classId + 1);
861 outMasks(srcRanges).copyTo(masks(dstRanges));
863 cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
864 normAssert(masks, refMasks(&topRefMasks[0]));