1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2018-2019, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
8 #include "test_precomp.hpp"
9 #include "opencv2/core/ocl.hpp"
11 namespace opencv_test { namespace {
13 class DNNTestNetwork : public DNNTestLayer
16 void processNet(const std::string& weights, const std::string& proto,
17 Size inpSize, const std::string& outputLayer = "",
18 const std::string& halideScheduler = "",
19 double l1 = 0.0, double lInf = 0.0)
21 // Create a common input blob.
22 int blobSize[] = {1, 3, inpSize.height, inpSize.width};
23 Mat inp(4, blobSize, CV_32FC1);
24 randu(inp, 0.0f, 1.0f);
26 processNet(weights, proto, inp, outputLayer, halideScheduler, l1, lInf);
29 void processNet(std::string weights, std::string proto,
30 Mat inp, const std::string& outputLayer = "",
31 std::string halideScheduler = "",
32 double l1 = 0.0, double lInf = 0.0, double detectionConfThresh = 0.2)
35 l1 = l1 ? l1 : default_l1;
36 lInf = lInf ? lInf : default_lInf;
38 weights = findDataFile(weights, false);
40 proto = findDataFile(proto);
42 // Create two networks - with default backend and target and a tested one.
43 Net netDefault = readNet(weights, proto);
44 netDefault.setPreferableBackend(DNN_BACKEND_OPENCV);
45 netDefault.setInput(inp);
46 Mat outDefault = netDefault.forward(outputLayer).clone();
48 net = readNet(weights, proto);
50 net.setPreferableBackend(backend);
51 net.setPreferableTarget(target);
52 if (backend == DNN_BACKEND_HALIDE && !halideScheduler.empty())
54 halideScheduler = findDataFile(halideScheduler);
55 net.setHalideScheduler(halideScheduler);
57 Mat out = net.forward(outputLayer).clone();
59 check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "First run");
61 // Test 2: change input.
62 float* inpData = (float*)inp.data;
63 for (int i = 0; i < inp.size[0] * inp.size[1]; ++i)
65 Mat slice(inp.size[2], inp.size[3], CV_32F, inpData);
66 cv::flip(slice, slice, 1);
67 inpData += slice.total();
69 netDefault.setInput(inp);
71 outDefault = netDefault.forward(outputLayer).clone();
72 out = net.forward(outputLayer).clone();
73 check(outDefault, out, outputLayer, l1, lInf, detectionConfThresh, "Second run");
76 void check(Mat& ref, Mat& out, const std::string& outputLayer, double l1, double lInf,
77 double detectionConfThresh, const char* msg)
79 if (outputLayer == "detection_out")
81 if (backend == DNN_BACKEND_INFERENCE_ENGINE)
83 // Inference Engine produces detections terminated by a row which starts from -1.
84 out = out.reshape(1, out.total() / 7);
85 int numDetections = 0;
86 while (numDetections < out.rows && out.at<float>(numDetections, 0) != -1)
90 out = out.rowRange(0, numDetections);
92 normAssertDetections(ref, out, msg, detectionConfThresh, l1, lInf);
95 normAssert(ref, out, msg, l1, lInf);
101 TEST_P(DNNTestNetwork, AlexNet)
103 applyTestTag(CV_TEST_TAG_MEMORY_1GB);
104 processNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt",
105 Size(227, 227), "prob",
106 target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_alexnet.yml" :
107 "dnn/halide_scheduler_alexnet.yml");
108 expectNoFallbacksFromIE(net);
111 TEST_P(DNNTestNetwork, ResNet_50)
114 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
115 CV_TEST_TAG_DEBUG_LONG
117 processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt",
118 Size(224, 224), "prob",
119 target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_resnet_50.yml" :
120 "dnn/halide_scheduler_resnet_50.yml");
121 expectNoFallbacksFromIE(net);
124 TEST_P(DNNTestNetwork, SqueezeNet_v1_1)
126 processNet("dnn/squeezenet_v1.1.caffemodel", "dnn/squeezenet_v1.1.prototxt",
127 Size(227, 227), "prob",
128 target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_squeezenet_v1_1.yml" :
129 "dnn/halide_scheduler_squeezenet_v1_1.yml");
130 expectNoFallbacksFromIE(net);
133 TEST_P(DNNTestNetwork, GoogLeNet)
135 applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
136 processNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt",
137 Size(224, 224), "prob");
138 expectNoFallbacksFromIE(net);
141 TEST_P(DNNTestNetwork, Inception_5h)
143 applyTestTag(CV_TEST_TAG_MEMORY_512MB);
144 double l1 = default_l1, lInf = default_lInf;
145 if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_CPU || target == DNN_TARGET_OPENCL))
150 processNet("dnn/tensorflow_inception_graph.pb", "", Size(224, 224), "softmax2",
151 target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_inception_5h.yml" :
152 "dnn/halide_scheduler_inception_5h.yml",
154 expectNoFallbacksFromIE(net);
157 TEST_P(DNNTestNetwork, ENet)
159 applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
160 if ((backend == DNN_BACKEND_INFERENCE_ENGINE) ||
161 (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16))
162 throw SkipTestException("");
163 processNet("dnn/Enet-model-best.net", "", Size(512, 512), "l367_Deconvolution",
164 target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_enet.yml" :
165 "dnn/halide_scheduler_enet.yml",
169 TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe)
171 applyTestTag(CV_TEST_TAG_MEMORY_512MB);
172 if (backend == DNN_BACKEND_HALIDE)
173 throw SkipTestException("");
174 Mat sample = imread(findDataFile("dnn/street.png"));
175 Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false);
176 float diffScores = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1.5e-2 : 0.0;
177 float diffSquares = (target == DNN_TARGET_MYRIAD) ? 0.063 : 0.0;
178 float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.252 : FLT_MIN;
179 processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt",
180 inp, "detection_out", "", diffScores, diffSquares, detectionConfThresh);
181 expectNoFallbacksFromIE(net);
184 TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe_Different_Width_Height)
186 if (backend == DNN_BACKEND_HALIDE)
187 throw SkipTestException("");
188 #if defined(INF_ENGINE_RELEASE)
189 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
190 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
191 throw SkipTestException("Test is disabled for MyriadX");
193 Mat sample = imread(findDataFile("dnn/street.png"));
194 Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 560), Scalar(127.5, 127.5, 127.5), false);
195 float diffScores = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.029 : 0.0;
196 float diffSquares = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : 0.0;
197 processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt",
198 inp, "detection_out", "", diffScores, diffSquares);
199 expectNoFallbacksFromIE(net);
202 TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow)
204 applyTestTag(target == DNN_TARGET_CPU ? "" : CV_TEST_TAG_MEMORY_512MB);
205 if (backend == DNN_BACKEND_HALIDE)
206 throw SkipTestException("");
207 Mat sample = imread(findDataFile("dnn/street.png"));
208 Mat inp = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false);
209 float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.095 : 0.0;
210 float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : 0.0;
211 float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.216 : 0.2;
212 processNet("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", "dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt",
213 inp, "detection_out", "", l1, lInf, detectionConfThresh);
214 expectNoFallbacksFromIE(net);
217 TEST_P(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow_Different_Width_Height)
219 if (backend == DNN_BACKEND_HALIDE)
220 throw SkipTestException("");
221 #if defined(INF_ENGINE_RELEASE)
222 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
223 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
224 throw SkipTestException("Test is disabled for MyriadX");
226 Mat sample = imread(findDataFile("dnn/street.png"));
227 Mat inp = blobFromImage(sample, 1.0f, Size(300, 560), Scalar(), false);
228 float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 0.0;
229 float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.06 : 0.0;
230 processNet("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", "dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt",
231 inp, "detection_out", "", l1, lInf);
232 expectNoFallbacksFromIE(net);
235 TEST_P(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
237 applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
238 if (backend == DNN_BACKEND_HALIDE)
239 throw SkipTestException("");
240 Mat sample = imread(findDataFile("dnn/street.png"));
241 Mat inp = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false);
242 float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.013 : 2e-5;
243 float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.062 : 0.0;
244 processNet("dnn/ssd_mobilenet_v2_coco_2018_03_29.pb", "dnn/ssd_mobilenet_v2_coco_2018_03_29.pbtxt",
245 inp, "detection_out", "", l1, lInf, 0.25);
246 expectNoFallbacksFromIE(net);
249 TEST_P(DNNTestNetwork, SSD_VGG16)
251 applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
252 CV_TEST_TAG_DEBUG_VERYLONG);
253 if (backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU)
254 throw SkipTestException("");
255 double scoreThreshold = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0325 : 0.0;
256 const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.032 : 0.0;
257 Mat sample = imread(findDataFile("dnn/street.png"));
258 Mat inp = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false);
259 processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel",
260 "dnn/ssd_vgg16.prototxt", inp, "detection_out", "", scoreThreshold, lInf);
261 expectNoFallbacksFromIE(net);
264 TEST_P(DNNTestNetwork, OpenPose_pose_coco)
266 applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
267 CV_TEST_TAG_DEBUG_VERYLONG);
268 if (backend == DNN_BACKEND_HALIDE)
269 throw SkipTestException("");
270 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
271 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
272 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
273 throw SkipTestException("Test is disabled for OpenVINO <= 2018R5 + MyriadX target");
276 const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.0056 : 0.0;
277 const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.072 : 0.0;
278 processNet("dnn/openpose_pose_coco.caffemodel", "dnn/openpose_pose_coco.prototxt",
279 Size(46, 46), "", "", l1, lInf);
280 expectNoFallbacksFromIE(net);
283 TEST_P(DNNTestNetwork, OpenPose_pose_mpi)
285 applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
286 CV_TEST_TAG_DEBUG_VERYLONG);
287 if (backend == DNN_BACKEND_HALIDE)
288 throw SkipTestException("");
289 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
290 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
291 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
292 throw SkipTestException("Test is disabled for OpenVINO <= 2018R5 + MyriadX target");
295 // output range: [-0.001, 0.97]
296 const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.012 : 0.0;
297 const float lInf = (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_OPENCL_FP16) ? 0.16 : 0.0;
298 processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi.prototxt",
299 Size(46, 46), "", "", l1, lInf);
300 expectNoFallbacksFromIE(net);
303 TEST_P(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
305 applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB);
306 if (backend == DNN_BACKEND_HALIDE)
307 throw SkipTestException("");
308 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2018050000)
309 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
310 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
311 throw SkipTestException("Test is disabled for OpenVINO <= 2018R5 + MyriadX target");
314 // The same .caffemodel but modified .prototxt
315 // See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp
316 processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt",
318 expectNoFallbacksFromIE(net);
321 TEST_P(DNNTestNetwork, OpenFace)
323 #if defined(INF_ENGINE_RELEASE)
324 #if INF_ENGINE_VER_MAJOR_EQ(2018050000)
325 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
326 throw SkipTestException("Test is disabled for Myriad targets");
329 if (backend == DNN_BACKEND_HALIDE)
330 throw SkipTestException("");
331 const float l1 = (target == DNN_TARGET_MYRIAD) ? 0.0024 : 0.0;
332 const float lInf = (target == DNN_TARGET_MYRIAD) ? 0.0071 : 0.0;
333 processNet("dnn/openface_nn4.small2.v1.t7", "", Size(96, 96), "", "", l1, lInf);
336 TEST_P(DNNTestNetwork, opencv_face_detector)
338 if (backend == DNN_BACKEND_HALIDE)
339 throw SkipTestException("");
340 Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
341 Mat inp = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
342 processNet("dnn/opencv_face_detector.caffemodel", "dnn/opencv_face_detector.prototxt",
343 inp, "detection_out");
344 expectNoFallbacksFromIE(net);
347 TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
350 (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
351 CV_TEST_TAG_DEBUG_LONG
353 #if defined(INF_ENGINE_RELEASE)
354 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
355 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
356 throw SkipTestException("Test is disabled for MyriadX");
358 if (backend == DNN_BACKEND_HALIDE)
359 throw SkipTestException("");
360 Mat sample = imread(findDataFile("dnn/street.png"));
361 Mat inp = blobFromImage(sample, 1.0f, Size(300, 300), Scalar(), false);
362 float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.015 : 0.0;
363 float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0731 : 0.0;
364 processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "dnn/ssd_inception_v2_coco_2017_11_17.pbtxt",
365 inp, "detection_out", "", l1, lInf);
366 expectNoFallbacksFromIE(net);
369 TEST_P(DNNTestNetwork, DenseNet_121)
371 applyTestTag(CV_TEST_TAG_MEMORY_512MB);
372 if (backend == DNN_BACKEND_HALIDE)
373 throw SkipTestException("");
374 // Reference output values are in range [-3.807, 4.605]
375 float l1 = 0.0, lInf = 0.0;
376 if (target == DNN_TARGET_OPENCL_FP16)
378 l1 = 9e-3; lInf = 5e-2;
380 else if (target == DNN_TARGET_MYRIAD)
382 l1 = 0.1; lInf = 0.6;
384 processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", Size(224, 224), "", "", l1, lInf);
385 expectNoFallbacksFromIE(net);
388 TEST_P(DNNTestNetwork, FastNeuralStyle_eccv16)
390 applyTestTag(CV_TEST_TAG_MEMORY_512MB, CV_TEST_TAG_DEBUG_VERYLONG);
392 if (backend == DNN_BACKEND_HALIDE ||
393 (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD))
394 throw SkipTestException("");
396 #if defined(INF_ENGINE_RELEASE)
397 #if INF_ENGINE_RELEASE <= 2018050000
398 if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
399 throw SkipTestException("");
403 Mat img = imread(findDataFile("dnn/googlenet_1.png"));
404 Mat inp = blobFromImage(img, 1.0, Size(320, 240), Scalar(103.939, 116.779, 123.68), false, false);
405 // Output image has values in range [-143.526, 148.539].
406 float l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.4 : 4e-5;
407 float lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 7.45 : 2e-3;
408 processNet("dnn/fast_neural_style_eccv16_starry_night.t7", "", inp, "", "", l1, lInf);
409 #if defined(HAVE_INF_ENGINE) && INF_ENGINE_RELEASE >= 2019010000
410 expectNoFallbacksFromIE(net);
414 INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, dnnBackendsAndTargets(true, true, false));