1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
9 Test for Tensorflow models loading
12 #include "test_precomp.hpp"
13 #include "npy_blob.hpp"
15 #include <opencv2/dnn/layer.details.hpp> // CV_DNN_REGISTER_LAYER_CLASS
21 using namespace cv::dnn;
23 template<typename TString>
24 static std::string _tf(TString filename)
26 return (getOpenCVExtraDir() + "/dnn/") + filename;
29 TEST(Test_TensorFlow, read_inception)
33 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
34 net = readNetFromTensorflow(model);
35 ASSERT_FALSE(net.empty());
37 net.setPreferableBackend(DNN_BACKEND_OPENCV);
39 Mat sample = imread(_tf("grace_hopper_227.png"));
40 ASSERT_TRUE(!sample.empty());
42 resize(sample, input, Size(224, 224));
43 input -= Scalar::all(117); // mean sub
45 Mat inputBlob = blobFromImage(input);
47 net.setInput(inputBlob, "input");
48 Mat out = net.forward("softmax2");
50 std::cout << out.dims << std::endl;
53 TEST(Test_TensorFlow, inception_accuracy)
57 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
58 net = readNetFromTensorflow(model);
59 ASSERT_FALSE(net.empty());
61 net.setPreferableBackend(DNN_BACKEND_OPENCV);
63 Mat sample = imread(_tf("grace_hopper_227.png"));
64 ASSERT_TRUE(!sample.empty());
65 Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
67 net.setInput(inputBlob, "input");
68 Mat out = net.forward("softmax2");
70 Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
75 static std::string path(const std::string& file)
77 return findDataFile("dnn/tensorflow/" + file);
80 class Test_TensorFlow_layers : public DNNTestLayer
83 void runTensorFlowNet(const std::string& prefix, bool hasText = false,
84 double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
86 std::string netPath = path(prefix + "_net.pb");
87 std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
88 std::string inpPath = path(prefix + "_in.npy");
89 std::string outPath = path(prefix + "_out.npy");
91 cv::Mat input = blobFromNPY(inpPath);
92 cv::Mat ref = blobFromNPY(outPath);
93 checkBackend(&input, &ref);
98 // Load files into a memory buffers
99 std::vector<char> dataModel;
100 readFileContent(netPath, dataModel);
102 std::vector<char> dataConfig;
105 readFileContent(netConfig, dataConfig);
108 net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
109 dataConfig.data(), dataConfig.size());
112 net = readNetFromTensorflow(netPath, netConfig);
114 ASSERT_FALSE(net.empty());
116 net.setPreferableBackend(backend);
117 net.setPreferableTarget(target);
119 cv::Mat output = net.forward();
120 normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
124 TEST_P(Test_TensorFlow_layers, conv)
126 runTensorFlowNet("single_conv");
127 runTensorFlowNet("atrous_conv2d_valid");
128 runTensorFlowNet("atrous_conv2d_same");
129 runTensorFlowNet("depthwise_conv2d");
130 runTensorFlowNet("keras_atrous_conv2d_same");
131 runTensorFlowNet("conv_pool_nchw");
134 TEST_P(Test_TensorFlow_layers, Convolution3D)
136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
137 throw SkipTestException("Test is enabled starts from 2019R1");
139 if (target != DNN_TARGET_CPU && backend != DNN_BACKEND_CUDA)
140 throw SkipTestException("Only CPU and CUDA is supported");
141 runTensorFlowNet("conv3d");
144 TEST_P(Test_TensorFlow_layers, padding)
146 runTensorFlowNet("padding_valid");
147 runTensorFlowNet("spatial_padding");
148 runTensorFlowNet("mirror_pad");
149 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000)
150 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
152 if (target == DNN_TARGET_MYRIAD)
153 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_2019R3, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
154 if (target == DNN_TARGET_OPENCL_FP16)
155 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R3, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
158 runTensorFlowNet("keras_pad_concat");
161 TEST_P(Test_TensorFlow_layers, padding_same)
163 // Reference output values are in range [0.0006, 2.798]
164 runTensorFlowNet("padding_same");
167 TEST_P(Test_TensorFlow_layers, eltwise)
169 runTensorFlowNet("eltwise_add_mul");
170 runTensorFlowNet("eltwise_sub");
173 TEST_P(Test_TensorFlow_layers, pad_and_concat)
175 runTensorFlowNet("pad_and_concat");
178 TEST_P(Test_TensorFlow_layers, concat_axis_1)
180 runTensorFlowNet("concat_axis_1");
183 TEST_P(Test_TensorFlow_layers, batch_norm)
185 runTensorFlowNet("batch_norm");
186 runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
187 runTensorFlowNet("fused_batch_norm");
188 runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
189 runTensorFlowNet("batch_norm_text", true);
190 runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
191 runTensorFlowNet("unfused_batch_norm");
192 runTensorFlowNet("fused_batch_norm_no_gamma");
193 runTensorFlowNet("unfused_batch_norm_no_gamma");
194 runTensorFlowNet("mvn_batch_norm");
195 runTensorFlowNet("mvn_batch_norm_1x1");
196 runTensorFlowNet("switch_identity");
197 runTensorFlowNet("keras_batch_norm_training");
200 TEST_P(Test_TensorFlow_layers, batch_norm3D)
202 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
204 if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
205 if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
206 if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
207 throw SkipTestException("");
209 runTensorFlowNet("batch_norm3d");
212 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
214 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
215 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
216 // Output values range: [-40.0597, 207.827]
217 double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
218 double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
219 runTensorFlowNet("slim_batch_norm", false, l1, lInf);
222 TEST_P(Test_TensorFlow_layers, pooling)
224 runTensorFlowNet("max_pool_even");
225 runTensorFlowNet("max_pool_odd_valid");
226 runTensorFlowNet("max_pool_odd_same");
227 runTensorFlowNet("reduce_mean"); // an average pooling over all spatial dimensions.
230 TEST_P(Test_TensorFlow_layers, max_pool_grad)
232 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
233 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
234 runTensorFlowNet("max_pool_grad");
237 // TODO: fix tests and replace to pooling
238 TEST_P(Test_TensorFlow_layers, ave_pool_same)
240 // Reference output values are in range [-0.519531, 0.112976]
241 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
242 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
243 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
245 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
247 runTensorFlowNet("ave_pool_same");
250 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
252 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
253 throw SkipTestException("Test is enabled starts from 2019R1");
255 if (target != DNN_TARGET_CPU && backend != DNN_BACKEND_CUDA)
256 throw SkipTestException("Only CPU and CUDA is supported");
257 runTensorFlowNet("max_pool3d");
260 TEST_P(Test_TensorFlow_layers, AvePooling3D)
262 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
263 throw SkipTestException("Test is enabled starts from 2019R1");
265 if (target != DNN_TARGET_CPU && backend != DNN_BACKEND_CUDA)
266 throw SkipTestException("Only CPU and CUDA is supported");
267 runTensorFlowNet("ave_pool3d");
270 TEST_P(Test_TensorFlow_layers, deconvolution)
272 if(backend == DNN_BACKEND_CUDA)
273 applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); /* bugged */
275 runTensorFlowNet("deconvolution");
276 runTensorFlowNet("deconvolution_same");
277 runTensorFlowNet("deconvolution_stride_2_same");
278 runTensorFlowNet("deconvolution_adj_pad_valid");
279 runTensorFlowNet("deconvolution_adj_pad_same");
280 runTensorFlowNet("keras_deconv_valid");
281 runTensorFlowNet("keras_deconv_same");
282 runTensorFlowNet("keras_deconv_same_v2");
285 TEST_P(Test_TensorFlow_layers, matmul)
287 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
288 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
289 runTensorFlowNet("matmul");
290 runTensorFlowNet("nhwc_transpose_reshape_matmul");
291 // Reference output values are in range [-5.688, 4.484]
292 double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
293 runTensorFlowNet("nhwc_reshape_matmul", false, l1);
294 runTensorFlowNet("matmul_layout");
297 TEST_P(Test_TensorFlow_layers, reshape)
299 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
300 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
301 runTensorFlowNet("shift_reshape_no_reorder");
302 runTensorFlowNet("reshape_no_reorder");
303 runTensorFlowNet("reshape_reduce");
304 runTensorFlowNet("reshape_as_shape");
307 TEST_P(Test_TensorFlow_layers, flatten)
309 #if defined(INF_ENGINE_RELEASE)
310 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
311 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
313 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
316 runTensorFlowNet("flatten", true);
319 TEST_P(Test_TensorFlow_layers, unfused_flatten)
321 runTensorFlowNet("unfused_flatten");
322 runTensorFlowNet("unfused_flatten_unknown_batch");
325 TEST_P(Test_TensorFlow_layers, leaky_relu)
327 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
328 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
329 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
331 runTensorFlowNet("leaky_relu_order1");
332 runTensorFlowNet("leaky_relu_order2");
333 runTensorFlowNet("leaky_relu_order3");
336 TEST_P(Test_TensorFlow_layers, l2_normalize)
338 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
339 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
340 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
342 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
345 runTensorFlowNet("l2_normalize");
348 // TODO: fix it and add to l2_normalize
349 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
351 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
352 if (backend == DNN_BACKEND_INFERENCE_ENGINE
353 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
355 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
357 #if defined(INF_ENGINE_RELEASE)
358 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
359 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
362 runTensorFlowNet("l2_normalize_3d");
365 class Test_TensorFlow_nets : public DNNTestLayer {};
367 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
369 #if defined(INF_ENGINE_RELEASE)
370 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
372 #if INF_ENGINE_VER_MAJOR_GE(2019020000)
373 if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
374 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
380 std::string imgPath = findDataFile("dnn/street.png");
381 std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
382 std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
385 resize(imread(imgPath), inp, Size(300, 300));
386 inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
388 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
390 Net net = readNetFromTensorflow(netPath, netConfig);
391 net.setPreferableBackend(backend);
392 net.setPreferableTarget(target);
395 Mat out = net.forward();
397 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
398 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
399 normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
400 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
401 expectNoFallbacksFromIE(net);
405 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
407 applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
408 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
409 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
410 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
411 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
415 Mat img = imread(findDataFile("dnn/street.png"));
416 std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
417 std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
419 Net net = readNetFromTensorflow(model, proto);
420 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
422 net.setPreferableBackend(backend);
423 net.setPreferableTarget(target);
426 // Output has shape 1x1xNx7 where N - number of detections.
427 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
428 Mat out = net.forward();
429 Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
430 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
431 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
432 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
433 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
435 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
436 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
437 normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
438 expectNoFallbacksFromIE(net);
441 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
444 std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
445 std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
447 Net net = readNetFromTensorflow(model, proto);
448 Mat img = imread(findDataFile("dnn/dog416.png"));
449 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
451 net.setPreferableBackend(backend);
452 net.setPreferableTarget(target);
455 Mat out = net.forward();
457 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
458 float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 1.5e-5;
459 float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
460 float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
462 #if defined(INF_ENGINE_RELEASE)
463 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
464 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
468 detectionConfThresh = 0.36;
471 normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
472 expectNoFallbacksFromIE(net);
475 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
479 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
481 CV_TEST_TAG_DEBUG_VERYLONG
483 static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
484 "faster_rcnn_resnet50_coco_2018_01_28"};
487 #ifdef INF_ENGINE_RELEASE
488 if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
489 (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU))
490 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
492 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
493 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
495 double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
496 for (int i = 0; i < 2; ++i)
498 std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
499 std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
501 Net net = readNetFromTensorflow(model, proto);
502 net.setPreferableBackend(backend);
503 net.setPreferableTarget(target);
504 Mat img = imread(findDataFile("dnn/dog416.png"));
505 Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
508 Mat out = net.forward();
510 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
511 normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
515 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
517 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
518 if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
519 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
522 std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
523 std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
525 Net net = readNetFromTensorflow(model, proto);
526 Mat img = imread(findDataFile("dnn/dog416.png"));
527 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
528 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
530 net.setPreferableBackend(backend);
531 net.setPreferableTarget(target);
534 Mat out = net.forward();
536 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
537 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
538 normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
539 expectNoFallbacksFromIE(net);
542 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
545 std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
546 std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
548 Net net = readNetFromTensorflow(model, proto);
549 Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
550 Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
552 net.setPreferableBackend(backend);
553 net.setPreferableTarget(target);
555 // Output has shape 1x1xNx7 where N - number of detections.
556 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
557 Mat out = net.forward();
559 // References are from test for Caffe model.
560 Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
561 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
562 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
563 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
564 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
565 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
566 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
567 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
568 normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
569 expectNoFallbacksFromIE(net);
572 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
573 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
574 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
575 // sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
576 // feed_dict={'input_images:0': inp})
577 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
578 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
579 // np.save('east_text_detection.scores.npy', scores)
580 // np.save('east_text_detection.geometry.npy', geometry)
581 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
584 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
585 CV_TEST_TAG_DEBUG_LONG
588 #if defined(INF_ENGINE_RELEASE)
589 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
590 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
592 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16 &&
593 INF_ENGINE_VER_MAJOR_EQ(2019020000))
594 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
599 std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
600 std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
601 std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
602 std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
604 Net net = readNet(netPath);
606 net.setPreferableBackend(backend);
607 net.setPreferableTarget(target);
609 Mat img = imread(imgPath);
610 Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
613 std::vector<Mat> outs;
614 std::vector<String> outNames(2);
615 outNames[0] = "feature_fusion/Conv_7/Sigmoid";
616 outNames[1] = "feature_fusion/concat_3";
617 net.forward(outs, outNames);
619 Mat scores = outs[0];
620 Mat geometry = outs[1];
622 // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
623 double l1_scores = default_l1, lInf_scores = default_lInf;
624 double l1_geometry = default_l1, lInf_geometry = default_lInf;
625 if (target == DNN_TARGET_OPENCL_FP16)
627 lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
628 l1_geometry = 0.28; lInf_geometry = 5.94;
630 else if (target == DNN_TARGET_MYRIAD)
633 l1_geometry = 0.28; lInf_geometry = 5.94;
637 l1_geometry = 1e-4, lInf_geometry = 3e-3;
639 normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
640 normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
641 expectNoFallbacksFromIE(net);
644 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
646 TEST_P(Test_TensorFlow_layers, fp16_weights)
650 runTensorFlowNet("fp16_single_conv", false, l1, lInf);
651 runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
652 runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
653 runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
654 runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
655 // Reference output values are in range [0.0889, 1.651]
656 runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
657 if (target == DNN_TARGET_MYRIAD) {
661 // Reference output values are in range [0, 10.75]
662 runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
663 // Reference output values are in range [0.418, 2.297]
664 runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
667 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
669 // Reference output values are in range [-3.504, -0.002]
670 runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
673 TEST_P(Test_TensorFlow_layers, defun)
675 runTensorFlowNet("defun_dropout");
678 TEST_P(Test_TensorFlow_layers, quantized)
680 runTensorFlowNet("uint8_single_conv");
683 TEST_P(Test_TensorFlow_layers, lstm)
685 if(backend == DNN_BACKEND_CUDA)
686 applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); /* not supported */
687 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
688 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
689 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
690 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
691 runTensorFlowNet("lstm", true);
692 runTensorFlowNet("lstm", true, 0.0, 0.0, true);
695 TEST_P(Test_TensorFlow_layers, split)
697 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
698 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2)
699 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
700 runTensorFlowNet("split");
701 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
702 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
703 runTensorFlowNet("split_equals");
706 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
708 runTensorFlowNet("resize_nearest_neighbor");
709 runTensorFlowNet("keras_upsampling2d");
712 TEST_P(Test_TensorFlow_layers, slice)
714 if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
715 (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
716 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
717 runTensorFlowNet("slice_4d");
718 runTensorFlowNet("strided_slice");
721 TEST_P(Test_TensorFlow_layers, softmax)
723 runTensorFlowNet("keras_softmax");
724 runTensorFlowNet("slim_softmax");
727 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
729 #if defined(INF_ENGINE_RELEASE)
730 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
731 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
733 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
735 runTensorFlowNet("slim_softmax_v2");
738 TEST_P(Test_TensorFlow_layers, relu6)
740 runTensorFlowNet("keras_relu6");
741 runTensorFlowNet("keras_relu6", /*hasText*/ true);
744 TEST_P(Test_TensorFlow_layers, subpixel)
746 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
747 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
748 runTensorFlowNet("subpixel");
751 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
753 runTensorFlowNet("keras_mobilenet_head");
754 runTensorFlowNet("keras_learning_phase");
757 TEST_P(Test_TensorFlow_layers, resize_bilinear)
759 runTensorFlowNet("resize_bilinear");
760 runTensorFlowNet("resize_bilinear_factor");
763 TEST_P(Test_TensorFlow_layers, squeeze)
765 #if defined(INF_ENGINE_RELEASE)
766 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
767 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
769 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
771 int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}}; // TensorFlow's shape (NHWC)
772 int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
773 int squeeze_dims[] = {0, 2, 3, -1};
774 for (int i = 0; i < 4; ++i)
776 SCOPED_TRACE(format("i=%d", i));
778 "node { name: \"input\" op: \"Placeholder\""
779 "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
780 "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
781 "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
782 Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
783 net.setPreferableBackend(backend);
784 net.setPreferableTarget(target);
785 Mat tfInp(4, &inpShapes[i][0], CV_32F);
789 CV_Assert(inpShapes[i][0] == 1);
790 std::swap(inpShapes[i][2], inpShapes[i][3]);
791 std::swap(inpShapes[i][1], inpShapes[i][2]);
792 Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
793 cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
796 Mat out = net.forward();
797 normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
801 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
803 TEST(Test_TensorFlow, two_inputs)
805 Net net = readNet(path("two_inputs_net.pbtxt"));
806 net.setPreferableBackend(DNN_BACKEND_OPENCV);
808 Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
809 randu(firstInput, -1, 1);
810 randu(secondInput, -1, 1);
812 net.setInput(firstInput, "first_input");
813 net.setInput(secondInput, "second_input");
814 Mat out = net.forward();
816 normAssert(out, firstInput + secondInput);
819 TEST(Test_TensorFlow, Mask_RCNN)
821 applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
822 Mat img = imread(findDataFile("dnn/street.png"));
823 std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
824 std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
826 Net net = readNetFromTensorflow(model, proto);
827 Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
828 Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
829 Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
831 net.setPreferableBackend(DNN_BACKEND_OPENCV);
835 // Mask-RCNN predicts bounding boxes and segmentation masks.
836 std::vector<String> outNames(2);
837 outNames[0] = "detection_out_final";
838 outNames[1] = "detection_masks";
840 std::vector<Mat> outs;
841 net.forward(outs, outNames);
843 Mat outDetections = outs[0];
844 Mat outMasks = outs[1];
845 normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
847 // Output size of masks is NxCxHxW where
848 // N - number of detected boxes
849 // C - number of classes (excluding background)
850 // HxW - segmentation shape
851 const int numDetections = outDetections.size[2];
853 int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
854 Mat masks(4, &masksSize[0], CV_32F);
856 std::vector<cv::Range> srcRanges(4, cv::Range::all());
857 std::vector<cv::Range> dstRanges(4, cv::Range::all());
859 outDetections = outDetections.reshape(1, outDetections.total() / 7);
860 for (int i = 0; i < numDetections; ++i)
862 // Get a class id for this bounding box and copy mask only for that class.
863 int classId = static_cast<int>(outDetections.at<float>(i, 1));
864 srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
865 srcRanges[1] = cv::Range(classId, classId + 1);
866 outMasks(srcRanges).copyTo(masks(dstRanges));
868 cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
869 normAssert(masks, refMasks(&topRefMasks[0]));