1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
9 Test for Tensorflow models loading
12 #include "test_precomp.hpp"
13 #include "npy_blob.hpp"
15 #include <opencv2/dnn/layer.details.hpp> // CV_DNN_REGISTER_LAYER_CLASS
21 using namespace cv::dnn;
23 template<typename TString>
24 static std::string _tf(TString filename)
26 return (getOpenCVExtraDir() + "/dnn/") + filename;
29 TEST(Test_TensorFlow, read_inception)
33 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
34 net = readNetFromTensorflow(model);
35 ASSERT_FALSE(net.empty());
37 net.setPreferableBackend(DNN_BACKEND_OPENCV);
39 Mat sample = imread(_tf("grace_hopper_227.png"));
40 ASSERT_TRUE(!sample.empty());
42 resize(sample, input, Size(224, 224));
43 input -= Scalar::all(117); // mean sub
45 Mat inputBlob = blobFromImage(input);
47 net.setInput(inputBlob, "input");
48 Mat out = net.forward("softmax2");
50 std::cout << out.dims << std::endl;
53 TEST(Test_TensorFlow, inception_accuracy)
57 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
58 net = readNetFromTensorflow(model);
59 ASSERT_FALSE(net.empty());
61 net.setPreferableBackend(DNN_BACKEND_OPENCV);
63 Mat sample = imread(_tf("grace_hopper_227.png"));
64 ASSERT_TRUE(!sample.empty());
65 Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
67 net.setInput(inputBlob, "input");
68 Mat out = net.forward("softmax2");
70 Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
75 static std::string path(const std::string& file)
77 return findDataFile("dnn/tensorflow/" + file);
80 class Test_TensorFlow_layers : public DNNTestLayer
83 void runTensorFlowNet(const std::string& prefix, bool hasText = false,
84 double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
86 std::string netPath = path(prefix + "_net.pb");
87 std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
88 std::string inpPath = path(prefix + "_in.npy");
89 std::string outPath = path(prefix + "_out.npy");
91 cv::Mat input = blobFromNPY(inpPath);
92 cv::Mat ref = blobFromNPY(outPath);
93 checkBackend(&input, &ref);
98 // Load files into a memory buffers
99 std::vector<char> dataModel;
100 readFileContent(netPath, dataModel);
102 std::vector<char> dataConfig;
105 readFileContent(netConfig, dataConfig);
108 net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
109 dataConfig.data(), dataConfig.size());
112 net = readNetFromTensorflow(netPath, netConfig);
114 ASSERT_FALSE(net.empty());
116 net.setPreferableBackend(backend);
117 net.setPreferableTarget(target);
119 cv::Mat output = net.forward();
120 normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
124 TEST_P(Test_TensorFlow_layers, conv)
126 runTensorFlowNet("single_conv");
127 runTensorFlowNet("atrous_conv2d_valid");
128 runTensorFlowNet("atrous_conv2d_same");
129 runTensorFlowNet("depthwise_conv2d");
130 runTensorFlowNet("keras_atrous_conv2d_same");
131 runTensorFlowNet("conv_pool_nchw");
134 TEST_P(Test_TensorFlow_layers, Convolution3D)
136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
137 throw SkipTestException("Test is enabled starts from 2019R1");
139 if (target != DNN_TARGET_CPU)
140 throw SkipTestException("Only CPU is supported");
141 runTensorFlowNet("conv3d");
144 TEST_P(Test_TensorFlow_layers, padding)
146 runTensorFlowNet("padding_valid");
147 runTensorFlowNet("spatial_padding");
148 runTensorFlowNet("keras_pad_concat");
149 runTensorFlowNet("mirror_pad");
152 TEST_P(Test_TensorFlow_layers, padding_same)
154 // Reference output values are in range [0.0006, 2.798]
155 runTensorFlowNet("padding_same");
158 TEST_P(Test_TensorFlow_layers, eltwise)
160 runTensorFlowNet("eltwise_add_mul");
161 runTensorFlowNet("eltwise_sub");
164 TEST_P(Test_TensorFlow_layers, pad_and_concat)
166 runTensorFlowNet("pad_and_concat");
169 TEST_P(Test_TensorFlow_layers, concat_axis_1)
171 runTensorFlowNet("concat_axis_1");
174 TEST_P(Test_TensorFlow_layers, batch_norm)
176 runTensorFlowNet("batch_norm");
177 runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
178 runTensorFlowNet("fused_batch_norm");
179 runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
180 runTensorFlowNet("batch_norm_text", true);
181 runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
182 runTensorFlowNet("unfused_batch_norm");
183 runTensorFlowNet("fused_batch_norm_no_gamma");
184 runTensorFlowNet("unfused_batch_norm_no_gamma");
185 runTensorFlowNet("mvn_batch_norm");
186 runTensorFlowNet("mvn_batch_norm_1x1");
187 runTensorFlowNet("switch_identity");
188 runTensorFlowNet("keras_batch_norm_training");
191 TEST_P(Test_TensorFlow_layers, batch_norm3D)
193 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
195 if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
196 if (target == DNN_TARGET_OPENCL) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
197 if (target == DNN_TARGET_MYRIAD) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
198 throw SkipTestException("");
200 runTensorFlowNet("batch_norm3d");
203 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
205 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
206 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
207 // Output values range: [-40.0597, 207.827]
208 double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
209 double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
210 runTensorFlowNet("slim_batch_norm", false, l1, lInf);
213 TEST_P(Test_TensorFlow_layers, pooling)
215 runTensorFlowNet("max_pool_even");
216 runTensorFlowNet("max_pool_odd_valid");
217 runTensorFlowNet("max_pool_odd_same");
218 runTensorFlowNet("reduce_mean"); // an average pooling over all spatial dimensions.
221 TEST_P(Test_TensorFlow_layers, max_pool_grad)
223 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
224 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
225 runTensorFlowNet("max_pool_grad");
228 // TODO: fix tests and replace to pooling
229 TEST_P(Test_TensorFlow_layers, ave_pool_same)
231 // Reference output values are in range [-0.519531, 0.112976]
232 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
233 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
234 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
236 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
238 runTensorFlowNet("ave_pool_same");
241 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
243 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
244 throw SkipTestException("Test is enabled starts from 2019R1");
246 if (target != DNN_TARGET_CPU)
247 throw SkipTestException("Only CPU is supported");
248 runTensorFlowNet("max_pool3d");
251 TEST_P(Test_TensorFlow_layers, AvePooling3D)
253 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
254 throw SkipTestException("Test is enabled starts from 2019R1");
256 if (target != DNN_TARGET_CPU)
257 throw SkipTestException("Only CPU is supported");
258 runTensorFlowNet("ave_pool3d");
261 TEST_P(Test_TensorFlow_layers, deconvolution)
263 runTensorFlowNet("deconvolution");
264 runTensorFlowNet("deconvolution_same");
265 runTensorFlowNet("deconvolution_stride_2_same");
266 runTensorFlowNet("deconvolution_adj_pad_valid");
267 runTensorFlowNet("deconvolution_adj_pad_same");
268 runTensorFlowNet("keras_deconv_valid");
269 runTensorFlowNet("keras_deconv_same");
270 runTensorFlowNet("keras_deconv_same_v2");
273 TEST_P(Test_TensorFlow_layers, matmul)
275 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
276 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
277 runTensorFlowNet("matmul");
278 runTensorFlowNet("nhwc_transpose_reshape_matmul");
279 // Reference output values are in range [-5.688, 4.484]
280 double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
281 runTensorFlowNet("nhwc_reshape_matmul", false, l1);
285 TEST_P(Test_TensorFlow_layers, reshape)
287 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
288 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
289 runTensorFlowNet("shift_reshape_no_reorder");
290 runTensorFlowNet("reshape_no_reorder");
291 runTensorFlowNet("reshape_reduce");
292 runTensorFlowNet("reshape_as_shape");
295 TEST_P(Test_TensorFlow_layers, flatten)
297 #if defined(INF_ENGINE_RELEASE)
298 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
299 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
301 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
304 runTensorFlowNet("flatten", true);
307 TEST_P(Test_TensorFlow_layers, unfused_flatten)
309 runTensorFlowNet("unfused_flatten");
310 runTensorFlowNet("unfused_flatten_unknown_batch");
313 TEST_P(Test_TensorFlow_layers, leaky_relu)
315 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
316 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
317 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
319 runTensorFlowNet("leaky_relu_order1");
320 runTensorFlowNet("leaky_relu_order2");
321 runTensorFlowNet("leaky_relu_order3");
324 TEST_P(Test_TensorFlow_layers, l2_normalize)
326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
327 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
328 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
330 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
333 runTensorFlowNet("l2_normalize");
336 // TODO: fix it and add to l2_normalize
337 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
339 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
340 if (backend == DNN_BACKEND_INFERENCE_ENGINE
341 && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
343 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
345 #if defined(INF_ENGINE_RELEASE)
346 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
347 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
350 runTensorFlowNet("l2_normalize_3d");
353 class Test_TensorFlow_nets : public DNNTestLayer {};
355 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
357 #if defined(INF_ENGINE_RELEASE)
358 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
360 #if INF_ENGINE_VER_MAJOR_EQ(2019010000)
361 if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
362 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
364 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
370 std::string imgPath = findDataFile("dnn/street.png");
371 std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
372 std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
375 resize(imread(imgPath), inp, Size(300, 300));
376 inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
378 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
380 Net net = readNetFromTensorflow(netPath, netConfig);
381 net.setPreferableBackend(backend);
382 net.setPreferableTarget(target);
385 Mat out = net.forward();
387 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
388 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
389 normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
390 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
391 expectNoFallbacksFromIE(net);
395 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
397 applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
398 #if defined(INF_ENGINE_RELEASE)
399 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
401 #if INF_ENGINE_VER_MAJOR_LE(2019010000)
402 if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
403 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
405 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
411 Mat img = imread(findDataFile("dnn/street.png"));
412 std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
413 std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
415 Net net = readNetFromTensorflow(model, proto);
416 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
418 net.setPreferableBackend(backend);
419 net.setPreferableTarget(target);
422 // Output has shape 1x1xNx7 where N - number of detections.
423 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
424 Mat out = net.forward();
425 Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
426 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
427 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
428 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
429 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
431 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
432 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
433 normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
434 expectNoFallbacksFromIE(net);
437 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
439 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000)
440 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
441 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
445 std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
446 std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
448 Net net = readNetFromTensorflow(model, proto);
449 Mat img = imread(findDataFile("dnn/dog416.png"));
450 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
452 net.setPreferableBackend(backend);
453 net.setPreferableTarget(target);
456 Mat out = net.forward();
458 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
459 float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 7e-3 : 1.5e-5;
460 float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
461 float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
463 #if defined(INF_ENGINE_RELEASE)
464 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
465 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
469 detectionConfThresh = 0.36;
471 normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
472 expectNoFallbacksFromIE(net);
475 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
479 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
481 CV_TEST_TAG_DEBUG_VERYLONG
483 static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
484 "faster_rcnn_resnet50_coco_2018_01_28"};
487 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
488 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
489 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
490 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
492 double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
493 for (int i = 0; i < 2; ++i)
495 std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
496 std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
498 Net net = readNetFromTensorflow(model, proto);
499 net.setPreferableBackend(backend);
500 net.setPreferableTarget(target);
501 Mat img = imread(findDataFile("dnn/dog416.png"));
502 Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
505 Mat out = net.forward();
507 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
508 normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
512 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
514 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
515 if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
516 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
518 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000)
519 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
520 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
524 std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
525 std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
527 Net net = readNetFromTensorflow(model, proto);
528 Mat img = imread(findDataFile("dnn/dog416.png"));
529 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
530 Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
532 net.setPreferableBackend(backend);
533 net.setPreferableTarget(target);
536 Mat out = net.forward();
538 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
539 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
540 normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
541 expectNoFallbacksFromIE(net);
544 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
547 std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
548 std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
550 Net net = readNetFromTensorflow(model, proto);
551 Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
552 Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
554 net.setPreferableBackend(backend);
555 net.setPreferableTarget(target);
557 // Output has shape 1x1xNx7 where N - number of detections.
558 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
559 Mat out = net.forward();
561 // References are from test for Caffe model.
562 Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
563 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
564 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
565 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
566 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
567 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
568 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
569 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
570 normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
571 expectNoFallbacksFromIE(net);
574 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
575 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
576 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
577 // sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
578 // feed_dict={'input_images:0': inp})
579 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
580 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
581 // np.save('east_text_detection.scores.npy', scores)
582 // np.save('east_text_detection.geometry.npy', geometry)
583 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
586 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
587 CV_TEST_TAG_DEBUG_LONG
590 #if defined(INF_ENGINE_RELEASE)
591 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
592 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
597 std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
598 std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
599 std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
600 std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
602 Net net = readNet(netPath);
604 net.setPreferableBackend(backend);
605 net.setPreferableTarget(target);
607 Mat img = imread(imgPath);
608 Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
611 std::vector<Mat> outs;
612 std::vector<String> outNames(2);
613 outNames[0] = "feature_fusion/Conv_7/Sigmoid";
614 outNames[1] = "feature_fusion/concat_3";
615 net.forward(outs, outNames);
617 Mat scores = outs[0];
618 Mat geometry = outs[1];
620 // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
621 double l1_scores = default_l1, lInf_scores = default_lInf;
622 double l1_geometry = default_l1, lInf_geometry = default_lInf;
623 if (target == DNN_TARGET_OPENCL_FP16)
625 lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
626 l1_geometry = 0.28; lInf_geometry = 5.94;
628 else if (target == DNN_TARGET_MYRIAD)
631 l1_geometry = 0.28; lInf_geometry = 5.94;
635 l1_geometry = 1e-4, lInf_geometry = 3e-3;
637 normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
638 normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
639 expectNoFallbacksFromIE(net);
642 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
644 TEST_P(Test_TensorFlow_layers, fp16_weights)
648 runTensorFlowNet("fp16_single_conv", false, l1, lInf);
649 runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
650 runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
651 runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
652 runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
653 // Reference output values are in range [0.0889, 1.651]
654 runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
655 if (target == DNN_TARGET_MYRIAD) {
659 // Reference output values are in range [0, 10.75]
660 runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
661 // Reference output values are in range [0.418, 2.297]
662 runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
665 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
667 // Reference output values are in range [-3.504, -0.002]
668 runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
671 TEST_P(Test_TensorFlow_layers, defun)
673 runTensorFlowNet("defun_dropout");
676 TEST_P(Test_TensorFlow_layers, quantized)
678 runTensorFlowNet("uint8_single_conv");
681 TEST_P(Test_TensorFlow_layers, lstm)
683 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
684 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
685 if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
686 applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
687 runTensorFlowNet("lstm", true);
688 runTensorFlowNet("lstm", true, 0.0, 0.0, true);
691 TEST_P(Test_TensorFlow_layers, split)
693 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
694 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
695 runTensorFlowNet("split");
696 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
697 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
698 runTensorFlowNet("split_equals");
701 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
703 runTensorFlowNet("resize_nearest_neighbor");
704 runTensorFlowNet("keras_upsampling2d");
707 TEST_P(Test_TensorFlow_layers, slice)
709 if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
710 (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
711 applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
712 runTensorFlowNet("slice_4d");
713 runTensorFlowNet("strided_slice");
716 TEST_P(Test_TensorFlow_layers, softmax)
718 runTensorFlowNet("keras_softmax");
719 runTensorFlowNet("slim_softmax");
722 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
724 #if defined(INF_ENGINE_RELEASE)
725 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
726 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
728 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
730 runTensorFlowNet("slim_softmax_v2");
733 TEST_P(Test_TensorFlow_layers, relu6)
735 runTensorFlowNet("keras_relu6");
736 runTensorFlowNet("keras_relu6", /*hasText*/ true);
739 TEST_P(Test_TensorFlow_layers, subpixel)
741 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
742 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
743 runTensorFlowNet("subpixel");
746 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
748 runTensorFlowNet("keras_mobilenet_head");
751 TEST_P(Test_TensorFlow_layers, resize_bilinear)
753 runTensorFlowNet("resize_bilinear");
754 runTensorFlowNet("resize_bilinear_factor");
757 TEST_P(Test_TensorFlow_layers, squeeze)
759 #if defined(INF_ENGINE_RELEASE)
760 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
761 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
763 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
765 int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}}; // TensorFlow's shape (NHWC)
766 int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
767 int squeeze_dims[] = {0, 2, 3, -1};
768 for (int i = 0; i < 4; ++i)
770 SCOPED_TRACE(format("i=%d", i));
772 "node { name: \"input\" op: \"Placeholder\""
773 "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
774 "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
775 "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
776 Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
777 net.setPreferableBackend(backend);
778 net.setPreferableTarget(target);
779 Mat tfInp(4, &inpShapes[i][0], CV_32F);
783 CV_Assert(inpShapes[i][0] == 1);
784 std::swap(inpShapes[i][2], inpShapes[i][3]);
785 std::swap(inpShapes[i][1], inpShapes[i][2]);
786 Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
787 cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
790 Mat out = net.forward();
791 normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
795 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
797 TEST(Test_TensorFlow, two_inputs)
799 Net net = readNet(path("two_inputs_net.pbtxt"));
800 net.setPreferableBackend(DNN_BACKEND_OPENCV);
802 Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
803 randu(firstInput, -1, 1);
804 randu(secondInput, -1, 1);
806 net.setInput(firstInput, "first_input");
807 net.setInput(secondInput, "second_input");
808 Mat out = net.forward();
810 normAssert(out, firstInput + secondInput);
813 TEST(Test_TensorFlow, Mask_RCNN)
815 applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
816 Mat img = imread(findDataFile("dnn/street.png"));
817 std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
818 std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
820 Net net = readNetFromTensorflow(model, proto);
821 Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
822 Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
823 Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
825 net.setPreferableBackend(DNN_BACKEND_OPENCV);
829 // Mask-RCNN predicts bounding boxes and segmentation masks.
830 std::vector<String> outNames(2);
831 outNames[0] = "detection_out_final";
832 outNames[1] = "detection_masks";
834 std::vector<Mat> outs;
835 net.forward(outs, outNames);
837 Mat outDetections = outs[0];
838 Mat outMasks = outs[1];
839 normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
841 // Output size of masks is NxCxHxW where
842 // N - number of detected boxes
843 // C - number of classes (excluding background)
844 // HxW - segmentation shape
845 const int numDetections = outDetections.size[2];
847 int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
848 Mat masks(4, &masksSize[0], CV_32F);
850 std::vector<cv::Range> srcRanges(4, cv::Range::all());
851 std::vector<cv::Range> dstRanges(4, cv::Range::all());
853 outDetections = outDetections.reshape(1, outDetections.total() / 7);
854 for (int i = 0; i < numDetections; ++i)
856 // Get a class id for this bounding box and copy mask only for that class.
857 int classId = static_cast<int>(outDetections.at<float>(i, 1));
858 srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
859 srcRanges[1] = cv::Range(classId, classId + 1);
860 outMasks(srcRanges).copyTo(masks(dstRanges));
862 cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
863 normAssert(masks, refMasks(&topRefMasks[0]));