1 /*M///////////////////////////////////////////////////////////////////////////////////////
3 // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
5 // By downloading, copying, installing or using the software you agree to this license.
6 // If you do not agree to this license, do not download, install,
7 // copy or use the software.
11 // For Open Source Computer Vision Library
12 // (3-clause BSD License)
14 // Copyright (C) 2017, Intel Corporation, all rights reserved.
15 // Third party copyrights are property of their respective owners.
17 // Redistribution and use in source and binary forms, with or without modification,
18 // are permitted provided that the following conditions are met:
20 // * Redistributions of source code must retain the above copyright notice,
21 // this list of conditions and the following disclaimer.
23 // * Redistributions in binary form must reproduce the above copyright notice,
24 // this list of conditions and the following disclaimer in the documentation
25 // and/or other materials provided with the distribution.
27 // * Neither the names of the copyright holders nor the names of the contributors
28 // may be used to endorse or promote products derived from this software
29 // without specific prior written permission.
31 // This software is provided by the copyright holders and contributors "as is" and
32 // any express or implied warranties, including, but not limited to, the implied
33 // warranties of merchantability and fitness for a particular purpose are disclaimed.
34 // In no event shall copyright holders or contributors be liable for any direct,
35 // indirect, incidental, special, exemplary, or consequential damages
36 // (including, but not limited to, procurement of substitute goods or services;
37 // loss of use, data, or profits; or business interruption) however caused
38 // and on any theory of liability, whether in contract, strict liability,
39 // or tort (including negligence or otherwise) arising in any way out of
40 // the use of this software, even if advised of the possibility of such damage.
44 #include "test_precomp.hpp"
45 #include "npy_blob.hpp"
46 #include <opencv2/dnn/shape_utils.hpp>
48 namespace opencv_test { namespace {
50 template<typename TString>
51 static std::string _tf(TString filename)
53 return (getOpenCVExtraDir() + "/dnn/") + filename;
56 TEST(Test_Darknet, read_tiny_yolo_voc)
58 Net net = readNetFromDarknet(_tf("tiny-yolo-voc.cfg"));
59 ASSERT_FALSE(net.empty());
62 TEST(Test_Darknet, read_yolo_voc)
64 Net net = readNetFromDarknet(_tf("yolo-voc.cfg"));
65 ASSERT_FALSE(net.empty());
68 TEST(Test_Darknet, read_yolo_voc_stream)
70 applyTestTag(CV_TEST_TAG_MEMORY_1GB);
72 Mat sample = imread(_tf("dog416.png"));
73 Mat inp = blobFromImage(sample, 1.0/255, Size(416, 416), Scalar(), true, false);
74 const std::string cfgFile = findDataFile("dnn/yolo-voc.cfg");
75 const std::string weightsFile = findDataFile("dnn/yolo-voc.weights", false);
78 Net net = readNetFromDarknet(cfgFile, weightsFile);
80 net.setPreferableBackend(DNN_BACKEND_OPENCV);
83 // Import from bytes array.
85 std::vector<char> cfg, weights;
86 readFileContent(cfgFile, cfg);
87 readFileContent(weightsFile, weights);
89 Net net = readNetFromDarknet(cfg.data(), cfg.size(), weights.data(), weights.size());
91 net.setPreferableBackend(DNN_BACKEND_OPENCV);
92 Mat out = net.forward();
97 class Test_Darknet_layers : public DNNTestLayer
100 void testDarknetLayer(const std::string& name, bool hasWeights = false)
103 Mat inp = blobFromNPY(findDataFile("dnn/darknet/" + name + "_in.npy"));
104 Mat ref = blobFromNPY(findDataFile("dnn/darknet/" + name + "_out.npy"));
106 std::string cfg = findDataFile("dnn/darknet/" + name + ".cfg");
107 std::string model = "";
109 model = findDataFile("dnn/darknet/" + name + ".weights", false);
111 checkBackend(&inp, &ref);
113 Net net = readNet(cfg, model);
114 net.setPreferableBackend(backend);
115 net.setPreferableTarget(target);
117 Mat out = net.forward();
118 normAssert(out, ref, "", default_l1, default_lInf);
120 if (inp.size[0] == 1) // test handling of batch size
122 SCOPED_TRACE("batch size 2");
124 #if defined(INF_ENGINE_RELEASE)
125 if (target == DNN_TARGET_MYRIAD && name == "shortcut")
126 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
129 std::vector<int> sz2 = shape(inp);
132 Net net2 = readNet(cfg, model);
133 net2.setPreferableBackend(backend);
134 net2.setPreferableTarget(target);
135 Range ranges0[4] = { Range(0, 1), Range::all(), Range::all(), Range::all() };
136 Range ranges1[4] = { Range(1, 2), Range::all(), Range::all(), Range::all() };
137 Mat inp2(sz2, inp.type(), Scalar::all(0));
138 inp.copyTo(inp2(ranges0));
139 inp.copyTo(inp2(ranges1));
141 Mat out2 = net2.forward();
142 EXPECT_EQ(0, cv::norm(out2(ranges0), out2(ranges1), NORM_INF)) << "Batch result is not equal: " << name;
145 if (ref.dims == 2 && out2.dims == 3)
147 int ref_3d_sizes[3] = {1, ref.rows, ref.cols};
148 ref2 = Mat(3, ref_3d_sizes, ref.type(), (void*)ref.data);
150 /*else if (ref.dims == 3 && out2.dims == 4)
152 int ref_4d_sizes[4] = {1, ref.size[0], ref.size[1], ref.size[2]};
153 ref2 = Mat(4, ref_4d_sizes, ref.type(), (void*)ref.data);
155 ASSERT_EQ(out2.dims, ref2.dims) << ref.dims;
157 normAssert(out2(ranges0), ref2, "", default_l1, default_lInf);
158 normAssert(out2(ranges1), ref2, "", default_l1, default_lInf);
163 class Test_Darknet_nets : public DNNTestLayer
166 // Test object detection network from Darknet framework.
167 void testDarknetModel(const std::string& cfg, const std::string& weights,
168 const std::vector<std::vector<int> >& refClassIds,
169 const std::vector<std::vector<float> >& refConfidences,
170 const std::vector<std::vector<Rect2d> >& refBoxes,
171 double scoreDiff, double iouDiff, float confThreshold = 0.24, float nmsThreshold = 0.4)
175 Mat img1 = imread(_tf("dog416.png"));
176 Mat img2 = imread(_tf("street.png"));
177 std::vector<Mat> samples(2);
178 samples[0] = img1; samples[1] = img2;
180 // determine test type, whether batch or single img
181 int batch_size = refClassIds.size();
182 CV_Assert(batch_size == 1 || batch_size == 2);
183 samples.resize(batch_size);
185 Mat inp = blobFromImages(samples, 1.0/255, Size(416, 416), Scalar(), true, false);
187 Net net = readNet(findDataFile("dnn/" + cfg),
188 findDataFile("dnn/" + weights, false));
189 net.setPreferableBackend(backend);
190 net.setPreferableTarget(target);
192 std::vector<Mat> outs;
193 net.forward(outs, net.getUnconnectedOutLayersNames());
195 for (int b = 0; b < batch_size; ++b)
197 std::vector<int> classIds;
198 std::vector<float> confidences;
199 std::vector<Rect2d> boxes;
200 for (int i = 0; i < outs.size(); ++i)
204 // get the sample slice from 3D matrix (batch, box, classes+5)
205 Range ranges[3] = {Range(b, b+1), Range::all(), Range::all()};
206 out = outs[i](ranges).reshape(1, outs[i].size[1]);
210 for (int j = 0; j < out.rows; ++j)
212 Mat scores = out.row(j).colRange(5, out.cols);
215 minMaxLoc(scores, 0, &confidence, 0, &maxLoc);
217 if (confidence > confThreshold) {
218 float* detection = out.ptr<float>(j);
219 double centerX = detection[0];
220 double centerY = detection[1];
221 double width = detection[2];
222 double height = detection[3];
223 boxes.push_back(Rect2d(centerX - 0.5 * width, centerY - 0.5 * height,
225 confidences.push_back(confidence);
226 classIds.push_back(maxLoc.x);
231 // here we need NMS of boxes
232 std::vector<int> indices;
233 NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
235 std::vector<int> nms_classIds;
236 std::vector<float> nms_confidences;
237 std::vector<Rect2d> nms_boxes;
239 for (size_t i = 0; i < indices.size(); ++i)
241 int idx = indices[i];
242 Rect2d box = boxes[idx];
243 float conf = confidences[idx];
244 int class_id = classIds[idx];
245 nms_boxes.push_back(box);
246 nms_confidences.push_back(conf);
247 nms_classIds.push_back(class_id);
250 normAssertDetections(refClassIds[b], refConfidences[b], refBoxes[b], nms_classIds,
251 nms_confidences, nms_boxes, format("batch size %d, sample %d\n", batch_size, b).c_str(), confThreshold, scoreDiff, iouDiff);
255 void testDarknetModel(const std::string& cfg, const std::string& weights,
256 const std::vector<int>& refClassIds,
257 const std::vector<float>& refConfidences,
258 const std::vector<Rect2d>& refBoxes,
259 double scoreDiff, double iouDiff, float confThreshold = 0.24, float nmsThreshold = 0.4)
261 testDarknetModel(cfg, weights,
262 std::vector<std::vector<int> >(1, refClassIds),
263 std::vector<std::vector<float> >(1, refConfidences),
264 std::vector<std::vector<Rect2d> >(1, refBoxes),
265 scoreDiff, iouDiff, confThreshold, nmsThreshold);
268 void testDarknetModel(const std::string& cfg, const std::string& weights,
269 const cv::Mat& ref, double scoreDiff, double iouDiff,
270 float confThreshold = 0.24, float nmsThreshold = 0.4)
272 CV_Assert(ref.cols == 7);
273 std::vector<std::vector<int> > refClassIds;
274 std::vector<std::vector<float> > refScores;
275 std::vector<std::vector<Rect2d> > refBoxes;
276 for (int i = 0; i < ref.rows; ++i)
278 int batchId = static_cast<int>(ref.at<float>(i, 0));
279 int classId = static_cast<int>(ref.at<float>(i, 1));
280 float score = ref.at<float>(i, 2);
281 float left = ref.at<float>(i, 3);
282 float top = ref.at<float>(i, 4);
283 float right = ref.at<float>(i, 5);
284 float bottom = ref.at<float>(i, 6);
285 Rect2d box(left, top, right - left, bottom - top);
286 if (batchId >= refClassIds.size())
288 refClassIds.resize(batchId + 1);
289 refScores.resize(batchId + 1);
290 refBoxes.resize(batchId + 1);
292 refClassIds[batchId].push_back(classId);
293 refScores[batchId].push_back(score);
294 refBoxes[batchId].push_back(box);
296 testDarknetModel(cfg, weights, refClassIds, refScores, refBoxes,
297 scoreDiff, iouDiff, confThreshold, nmsThreshold);
301 TEST_P(Test_Darknet_nets, YoloVoc)
303 applyTestTag(CV_TEST_TAG_LONG, CV_TEST_TAG_MEMORY_1GB);
305 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
306 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL_FP16)
307 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
309 #if defined(INF_ENGINE_RELEASE)
310 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD
311 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
312 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); // need to update check function
315 // batchId, classId, confidence, left, top, right, bottom
316 Mat ref = (Mat_<float>(6, 7) << 0, 6, 0.750469f, 0.577374f, 0.127391f, 0.902949f, 0.300809f, // a car
317 0, 1, 0.780879f, 0.270762f, 0.264102f, 0.732475f, 0.745412f, // a bicycle
318 0, 11, 0.901615f, 0.1386f, 0.338509f, 0.421337f, 0.938789f, // a dog
319 1, 14, 0.623813f, 0.183179f, 0.381921f, 0.247726f, 0.625847f, // a person
320 1, 6, 0.667770f, 0.446555f, 0.453578f, 0.499986f, 0.519167f, // a car
321 1, 6, 0.844947f, 0.637058f, 0.460398f, 0.828508f, 0.66427f); // a car
323 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 1e-2 : 8e-5;
324 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.018 : 3e-4;
325 double nmsThreshold = (target == DNN_TARGET_MYRIAD) ? 0.397 : 0.4;
327 std::string config_file = "yolo-voc.cfg";
328 std::string weights_file = "yolo-voc.weights";
331 SCOPED_TRACE("batch size 1");
332 testDarknetModel(config_file, weights_file, ref.rowRange(0, 3), scoreDiff, iouDiff);
336 SCOPED_TRACE("batch size 2");
337 testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, nmsThreshold);
341 TEST_P(Test_Darknet_nets, TinyYoloVoc)
343 applyTestTag(CV_TEST_TAG_MEMORY_512MB);
345 #if defined(INF_ENGINE_RELEASE)
346 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD
347 && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
348 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X); // need to update check function
350 // batchId, classId, confidence, left, top, right, bottom
351 Mat ref = (Mat_<float>(4, 7) << 0, 6, 0.761967f, 0.579042f, 0.159161f, 0.894482f, 0.31994f, // a car
352 0, 11, 0.780595f, 0.129696f, 0.386467f, 0.445275f, 0.920994f, // a dog
353 1, 6, 0.651450f, 0.460526f, 0.458019f, 0.522527f, 0.5341f, // a car
354 1, 6, 0.928758f, 0.651024f, 0.463539f, 0.823784f, 0.654998f); // a car
356 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 8e-3 : 8e-5;
357 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.018 : 3e-4;
359 std::string config_file = "tiny-yolo-voc.cfg";
360 std::string weights_file = "tiny-yolo-voc.weights";
363 SCOPED_TRACE("batch size 1");
364 testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff);
368 SCOPED_TRACE("batch size 2");
369 testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff);
373 #ifdef HAVE_INF_ENGINE
374 static const std::chrono::milliseconds async_timeout(10000);
376 typedef testing::TestWithParam<tuple<std::string, tuple<Backend, Target> > > Test_Darknet_nets_async;
377 TEST_P(Test_Darknet_nets_async, Accuracy)
379 Backend backendId = get<0>(get<1>(GetParam()));
380 Target targetId = get<1>(get<1>(GetParam()));
382 if (INF_ENGINE_VER_MAJOR_LT(2019020000) && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
383 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER);
384 applyTestTag(CV_TEST_TAG_MEMORY_512MB);
386 if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
387 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);
389 std::string prefix = get<0>(GetParam());
391 if (backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
392 throw SkipTestException("No support for async forward");
394 const int numInputs = 2;
395 std::vector<Mat> inputs(numInputs);
396 int blobSize[] = {1, 3, 416, 416};
397 for (int i = 0; i < numInputs; ++i)
399 inputs[i].create(4, &blobSize[0], CV_32F);
400 randu(inputs[i], 0, 1);
403 Net netSync = readNet(findDataFile("dnn/" + prefix + ".cfg"),
404 findDataFile("dnn/" + prefix + ".weights", false));
405 netSync.setPreferableBackend(backendId);
406 netSync.setPreferableTarget(targetId);
408 // Run synchronously.
409 std::vector<Mat> refs(numInputs);
410 for (int i = 0; i < numInputs; ++i)
412 netSync.setInput(inputs[i]);
413 refs[i] = netSync.forward().clone();
416 Net netAsync = readNet(findDataFile("dnn/" + prefix + ".cfg"),
417 findDataFile("dnn/" + prefix + ".weights", false));
418 netAsync.setPreferableBackend(backendId);
419 netAsync.setPreferableTarget(targetId);
421 // Run asynchronously. To make test more robust, process inputs in the reversed order.
422 for (int i = numInputs - 1; i >= 0; --i)
424 netAsync.setInput(inputs[i]);
426 AsyncArray out = netAsync.forwardAsync();
427 ASSERT_TRUE(out.valid());
429 EXPECT_TRUE(out.get(result, async_timeout));
430 normAssert(refs[i], result, format("Index: %d", i).c_str(), 0, 0);
434 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets_async, Combine(
435 Values("yolo-voc", "tiny-yolo-voc", "yolov3"),
436 dnnBackendsAndTargets()
441 TEST_P(Test_Darknet_nets, YOLOv3)
443 applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
445 // batchId, classId, confidence, left, top, right, bottom
446 Mat ref = (Mat_<float>(9, 7) << 0, 7, 0.952983f, 0.614622f, 0.150257f, 0.901369f, 0.289251f, // a truck
447 0, 1, 0.987908f, 0.150913f, 0.221933f, 0.742255f, 0.74626f, // a bicycle
448 0, 16, 0.998836f, 0.160024f, 0.389964f, 0.417885f, 0.943716f, // a dog (COCO)
449 1, 9, 0.384801f, 0.659824f, 0.372389f, 0.673926f, 0.429412f, // a traffic light
450 1, 9, 0.733283f, 0.376029f, 0.315694f, 0.401776f, 0.395165f, // a traffic light
451 1, 9, 0.785352f, 0.665503f, 0.373543f, 0.688893f, 0.439245f, // a traffic light
452 1, 0, 0.980052f, 0.195856f, 0.378454f, 0.258626f, 0.629258f, // a person
453 1, 2, 0.989633f, 0.450719f, 0.463353f, 0.496305f, 0.522258f, // a car
454 1, 2, 0.997412f, 0.647584f, 0.459939f, 0.821038f, 0.663947f); // a car
456 double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.006 : 8e-5;
457 double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.018 : 3e-4;
459 std::string config_file = "yolov3.cfg";
460 std::string weights_file = "yolov3.weights";
462 #if defined(INF_ENGINE_RELEASE)
463 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD &&
464 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
472 SCOPED_TRACE("batch size 1");
473 testDarknetModel(config_file, weights_file, ref.rowRange(0, 3), scoreDiff, iouDiff);
476 #if defined(INF_ENGINE_RELEASE)
477 if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
479 if (INF_ENGINE_VER_MAJOR_LE(2018050000) && target == DNN_TARGET_OPENCL)
480 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
481 else if (INF_ENGINE_VER_MAJOR_EQ(2019020000))
483 if (target == DNN_TARGET_OPENCL)
484 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
485 if (target == DNN_TARGET_OPENCL_FP16)
486 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
488 else if (target == DNN_TARGET_MYRIAD &&
489 getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
490 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
495 SCOPED_TRACE("batch size 2");
496 testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff);
500 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets, dnnBackendsAndTargets());
502 TEST_P(Test_Darknet_layers, shortcut)
504 testDarknetLayer("shortcut");
505 testDarknetLayer("shortcut_leaky");
506 testDarknetLayer("shortcut_unequal");
507 testDarknetLayer("shortcut_unequal_2");
510 TEST_P(Test_Darknet_layers, upsample)
512 testDarknetLayer("upsample");
515 TEST_P(Test_Darknet_layers, avgpool_softmax)
517 testDarknetLayer("avgpool_softmax");
520 TEST_P(Test_Darknet_layers, region)
522 testDarknetLayer("region");
525 TEST_P(Test_Darknet_layers, reorg)
527 testDarknetLayer("reorg");
530 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_layers, dnnBackendsAndTargets());