modules/dnn/test/test_darknet_importer.cpp

   1 /*M///////////////////////////////////////////////////////////////////////////////////////
   2 //
   3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
   4 //
   5 //  By downloading, copying, installing or using the software you agree to this license.
   6 //  If you do not agree to this license, do not download, install,
   7 //  copy or use the software.
   8 //
   9 //
  10 //                           License Agreement
  11 //                For Open Source Computer Vision Library
  12 //                        (3-clause BSD License)
  13 //
  14 // Copyright (C) 2017, Intel Corporation, all rights reserved.
  15 // Third party copyrights are property of their respective owners.
  16 //
  17 // Redistribution and use in source and binary forms, with or without modification,
  18 // are permitted provided that the following conditions are met:
  19 //
  20 // * Redistributions of source code must retain the above copyright notice,
  21 // this list of conditions and the following disclaimer.
  22 //
  23 // * Redistributions in binary form must reproduce the above copyright notice,
  24 // this list of conditions and the following disclaimer in the documentation
  25 // and/or other materials provided with the distribution.
  26 //
  27 // * Neither the names of the copyright holders nor the names of the contributors
  28 // may be used to endorse or promote products derived from this software
  29 // without specific prior written permission.
  30 //
  31 // This software is provided by the copyright holders and contributors "as is" and
  32 // any express or implied warranties, including, but not limited to, the implied
  33 // warranties of merchantability and fitness for a particular purpose are disclaimed.
  34 // In no event shall copyright holders or contributors be liable for any direct,
  35 // indirect, incidental, special, exemplary, or consequential damages
  36 // (including, but not limited to, procurement of substitute goods or services;
  37 // loss of use, data, or profits; or business interruption) however caused
  38 // and on any theory of liability, whether in contract, strict liability,
  39 // or tort (including negligence or otherwise) arising in any way out of
  40 // the use of this software, even if advised of the possibility of such damage.
  41 //
  42 //M*/
  43
  44 #include "test_precomp.hpp"
  45 #include "npy_blob.hpp"
  46 #include <opencv2/dnn/shape_utils.hpp>
  47
  48 namespace opencv_test { namespace {
  49
  50 template<typename TString>
  51 static std::string _tf(TString filename)
  52 {
  53     return (getOpenCVExtraDir() + "/dnn/") + filename;
  54 }
  55
  56 TEST(Test_Darknet, read_tiny_yolo_voc)
  57 {
  58     Net net = readNetFromDarknet(_tf("tiny-yolo-voc.cfg"));
  59     ASSERT_FALSE(net.empty());
  60 }
  61
  62 TEST(Test_Darknet, read_yolo_voc)
  63 {
  64     Net net = readNetFromDarknet(_tf("yolo-voc.cfg"));
  65     ASSERT_FALSE(net.empty());
  66 }
  67
  68 TEST(Test_Darknet, read_yolo_voc_stream)
  69 {
  70     applyTestTag(
  71             CV_TEST_TAG_MEMORY_1GB,
  72             CV_TEST_TAG_DEBUG_VERYLONG
  73             );
  74     Mat ref;
  75     Mat sample = imread(_tf("dog416.png"));
  76     Mat inp = blobFromImage(sample, 1.0/255, Size(416, 416), Scalar(), true, false);
  77     const std::string cfgFile = findDataFile("dnn/yolo-voc.cfg");
  78     const std::string weightsFile = findDataFile("dnn/yolo-voc.weights", false);
  79     // Import by paths.
  80     {
  81         Net net = readNetFromDarknet(cfgFile, weightsFile);
  82         net.setInput(inp);
  83         net.setPreferableBackend(DNN_BACKEND_OPENCV);
  84         net.enableWinograd(false);
  85         ref = net.forward();
  86     }
  87     // Import from bytes array.
  88     {
  89         std::vector<char> cfg, weights;
  90         readFileContent(cfgFile, cfg);
  91         readFileContent(weightsFile, weights);
  92
  93         Net net = readNetFromDarknet(cfg.data(), cfg.size(), weights.data(), weights.size());
  94         net.setInput(inp);
  95         net.setPreferableBackend(DNN_BACKEND_OPENCV);
  96         net.enableWinograd(false);
  97         Mat out = net.forward();
  98         normAssert(ref, out);
  99     }
 100 }
 101
 102 class Test_Darknet_layers : public DNNTestLayer
 103 {
 104 public:
 105     void testDarknetLayer(const std::string& name, bool hasWeights = false, bool testBatchProcessing = true)
 106     {
 107         SCOPED_TRACE(name);
 108         Mat inp = blobFromNPY(findDataFile("dnn/darknet/" + name + "_in.npy"));
 109         Mat ref = blobFromNPY(findDataFile("dnn/darknet/" + name + "_out.npy"));
 110
 111         std::string cfg = findDataFile("dnn/darknet/" + name + ".cfg");
 112         std::string model = "";
 113         if (hasWeights)
 114             model = findDataFile("dnn/darknet/" + name + ".weights");
 115
 116         checkBackend(&inp, &ref);
 117
 118         Net net = readNet(cfg, model);
 119         net.setPreferableBackend(backend);
 120         net.setPreferableTarget(target);
 121         net.setInput(inp);
 122         Mat out = net.forward();
 123         normAssert(out, ref, "", default_l1, default_lInf);
 124
 125         if (inp.size[0] == 1 && testBatchProcessing)  // test handling of batch size
 126         {
 127             SCOPED_TRACE("batch size 2");
 128
 129 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000)
 130             if (target == DNN_TARGET_MYRIAD && name == "shortcut")
 131                 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 132 #endif
 133
 134             std::vector<int> sz2 = shape(inp);
 135             sz2[0] = 2;
 136
 137             Net net2 = readNet(cfg, model);
 138             net2.setPreferableBackend(backend);
 139             net2.setPreferableTarget(target);
 140             Range ranges0[4] = { Range(0, 1), Range::all(), Range::all(), Range::all() };
 141             Range ranges1[4] = { Range(1, 2), Range::all(), Range::all(), Range::all() };
 142             Mat inp2(sz2, inp.type(), Scalar::all(0));
 143             inp.copyTo(inp2(ranges0));
 144             inp.copyTo(inp2(ranges1));
 145             net2.setInput(inp2);
 146             Mat out2 = net2.forward();
 147             if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
 148             {
 149                 EXPECT_LT(cv::norm(out2(ranges0), out2(ranges1), NORM_INF), 1e-4) << "Batch result is not similar: " << name;
 150             }
 151             else
 152             {
 153                 EXPECT_EQ(0, cv::norm(out2(ranges0), out2(ranges1), NORM_INF)) << "Batch result is not equal: " << name;
 154             }
 155
 156             Mat ref2 = ref;
 157             if (ref.dims == 2 && out2.dims == 3)
 158             {
 159                 int ref_3d_sizes[3] = {1, ref.rows, ref.cols};
 160                 ref2 = Mat(3, ref_3d_sizes, ref.type(), (void*)ref.data);
 161             }
 162             /*else if (ref.dims == 3 && out2.dims == 4)
 163             {
 164                 int ref_4d_sizes[4] = {1, ref.size[0], ref.size[1], ref.size[2]};
 165                 ref2 = Mat(4, ref_4d_sizes, ref.type(), (void*)ref.data);
 166             }*/
 167             ASSERT_EQ(out2.dims, ref2.dims) << ref.dims;
 168
 169             normAssert(out2(ranges0), ref2, "", default_l1, default_lInf);
 170             normAssert(out2(ranges1), ref2, "", default_l1, default_lInf);
 171         }
 172     }
 173 };
 174
 175 class Test_Darknet_nets : public DNNTestLayer
 176 {
 177 public:
 178     // Test object detection network from Darknet framework.
 179     void testDarknetModel(const std::string& cfg, const std::string& weights,
 180                           const std::vector<std::vector<int> >& refClassIds,
 181                           const std::vector<std::vector<float> >& refConfidences,
 182                           const std::vector<std::vector<Rect2d> >& refBoxes,
 183                           double scoreDiff, double iouDiff, float confThreshold = 0.24,
 184                           float nmsThreshold = 0.4, bool useWinograd = true)
 185     {
 186         checkBackend();
 187
 188         Mat img1 = imread(_tf("dog416.png"));
 189         Mat img2 = imread(_tf("street.png"));
 190         std::vector<Mat> samples(2);
 191         samples[0] = img1; samples[1] = img2;
 192
 193         // determine test type, whether batch or single img
 194         int batch_size = refClassIds.size();
 195         CV_Assert(batch_size == 1 || batch_size == 2);
 196         samples.resize(batch_size);
 197
 198         Mat inp = blobFromImages(samples, 1.0/255, Size(416, 416), Scalar(), true, false);
 199
 200         Net net = readNet(findDataFile("dnn/" + cfg),
 201                           findDataFile("dnn/" + weights, false));
 202         net.setPreferableBackend(backend);
 203         net.setPreferableTarget(target);
 204         net.enableWinograd(useWinograd);
 205         net.setInput(inp);
 206         std::vector<Mat> outs;
 207         net.forward(outs, net.getUnconnectedOutLayersNames());
 208
 209         for (int b = 0; b < batch_size; ++b)
 210         {
 211             std::vector<int> classIds;
 212             std::vector<float> confidences;
 213             std::vector<Rect2d> boxes;
 214             for (int i = 0; i < outs.size(); ++i)
 215             {
 216                 Mat out;
 217                 if (batch_size > 1){
 218                     // get the sample slice from 3D matrix (batch, box, classes+5)
 219                     Range ranges[3] = {Range(b, b+1), Range::all(), Range::all()};
 220                     out = outs[i](ranges).reshape(1, outs[i].size[1]);
 221                 }else{
 222                     out = outs[i];
 223                 }
 224                 for (int j = 0; j < out.rows; ++j)
 225                 {
 226                     Mat scores = out.row(j).colRange(5, out.cols);
 227                     double confidence;
 228                     Point maxLoc;
 229                     minMaxLoc(scores, 0, &confidence, 0, &maxLoc);
 230
 231                     if (confidence > confThreshold) {
 232                         float* detection = out.ptr<float>(j);
 233                         double centerX = detection[0];
 234                         double centerY = detection[1];
 235                         double width = detection[2];
 236                         double height = detection[3];
 237                         boxes.push_back(Rect2d(centerX - 0.5 * width, centerY - 0.5 * height,
 238                                             width, height));
 239                         confidences.push_back(confidence);
 240                         classIds.push_back(maxLoc.x);
 241                     }
 242                 }
 243             }
 244
 245             // here we need NMS of boxes
 246             std::vector<int> indices;
 247             NMSBoxes(boxes, confidences, confThreshold, nmsThreshold, indices);
 248
 249             std::vector<int> nms_classIds;
 250             std::vector<float> nms_confidences;
 251             std::vector<Rect2d> nms_boxes;
 252
 253             for (size_t i = 0; i < indices.size(); ++i)
 254             {
 255                 int idx = indices[i];
 256                 Rect2d box = boxes[idx];
 257                 float conf = confidences[idx];
 258                 int class_id = classIds[idx];
 259                 nms_boxes.push_back(box);
 260                 nms_confidences.push_back(conf);
 261                 nms_classIds.push_back(class_id);
 262                 if (cvtest::debugLevel > 0)
 263                 {
 264                     std::cout << b << ", " << class_id << ", " << conf << "f, "
 265                               << box.x << "f, " << box.y << "f, "
 266                               << box.x + box.width << "f, " << box.y + box.height << "f,"
 267                               << std::endl;
 268                 }
 269             }
 270
 271             if (cvIsNaN(iouDiff))
 272             {
 273                 if (b == 0)
 274                     std::cout << "Skip accuracy checks" << std::endl;
 275                 continue;
 276             }
 277
 278             normAssertDetections(refClassIds[b], refConfidences[b], refBoxes[b], nms_classIds,
 279                              nms_confidences, nms_boxes, format("batch size %d, sample %d\n", batch_size, b).c_str(), confThreshold, scoreDiff, iouDiff);
 280         }
 281     }
 282
 283     void testDarknetModel(const std::string& cfg, const std::string& weights,
 284                           const std::vector<int>& refClassIds,
 285                           const std::vector<float>& refConfidences,
 286                           const std::vector<Rect2d>& refBoxes,
 287                           double scoreDiff, double iouDiff, float confThreshold = 0.24,
 288                           float nmsThreshold = 0.4, bool useWinograd = true)
 289     {
 290         testDarknetModel(cfg, weights,
 291                          std::vector<std::vector<int> >(1, refClassIds),
 292                          std::vector<std::vector<float> >(1, refConfidences),
 293                          std::vector<std::vector<Rect2d> >(1, refBoxes),
 294                          scoreDiff, iouDiff, confThreshold, nmsThreshold, useWinograd);
 295     }
 296
 297     void testDarknetModel(const std::string& cfg, const std::string& weights,
 298                           const cv::Mat& ref, double scoreDiff, double iouDiff,
 299                           float confThreshold = 0.24, float nmsThreshold = 0.4, bool useWinograd = true)
 300     {
 301         CV_Assert(ref.cols == 7);
 302         std::vector<std::vector<int> > refClassIds;
 303         std::vector<std::vector<float> > refScores;
 304         std::vector<std::vector<Rect2d> > refBoxes;
 305         for (int i = 0; i < ref.rows; ++i)
 306         {
 307             int batchId = static_cast<int>(ref.at<float>(i, 0));
 308             int classId = static_cast<int>(ref.at<float>(i, 1));
 309             float score = ref.at<float>(i, 2);
 310             float left  = ref.at<float>(i, 3);
 311             float top   = ref.at<float>(i, 4);
 312             float right  = ref.at<float>(i, 5);
 313             float bottom = ref.at<float>(i, 6);
 314             Rect2d box(left, top, right - left, bottom - top);
 315             if (batchId >= refClassIds.size())
 316             {
 317                 refClassIds.resize(batchId + 1);
 318                 refScores.resize(batchId + 1);
 319                 refBoxes.resize(batchId + 1);
 320             }
 321             refClassIds[batchId].push_back(classId);
 322             refScores[batchId].push_back(score);
 323             refBoxes[batchId].push_back(box);
 324         }
 325         testDarknetModel(cfg, weights, refClassIds, refScores, refBoxes,
 326                          scoreDiff, iouDiff, confThreshold, nmsThreshold, useWinograd);
 327     }
 328 };
 329
 330 TEST_P(Test_Darknet_nets, YoloVoc)
 331 {
 332     applyTestTag(
 333 #if defined(OPENCV_32BIT_CONFIGURATION) && defined(HAVE_OPENCL)
 334         CV_TEST_TAG_MEMORY_2GB,
 335 #else
 336         CV_TEST_TAG_MEMORY_1GB,
 337 #endif
 338         CV_TEST_TAG_LONG
 339     );
 340
 341 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
 342     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
 343         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 344     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 345         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 346 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 347     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL_FP16)
 348         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 349 #elif defined(INF_ENGINE_RELEASE)
 350     if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) &&
 351         target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 352         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);  // need to update check function
 353 #endif
 354
 355     // batchId, classId, confidence, left, top, right, bottom
 356     Mat ref = (Mat_<float>(6, 7) << 0, 6,  0.750469f, 0.577374f, 0.127391f, 0.902949f, 0.300809f,  // a car
 357                                     0, 1,  0.780879f, 0.270762f, 0.264102f, 0.732475f, 0.745412f,  // a bicycle
 358                                     0, 11, 0.901615f, 0.1386f,   0.338509f, 0.421337f, 0.938789f,  // a dog
 359                                     1, 14, 0.623813f, 0.183179f, 0.381921f, 0.247726f, 0.625847f,  // a person
 360                                     1, 6,  0.667770f, 0.446555f, 0.453578f, 0.499986f, 0.519167f,  // a car
 361                                     1, 6,  0.844947f, 0.637058f, 0.460398f, 0.828508f, 0.66427f);  // a car
 362
 363     double nmsThreshold = (target == DNN_TARGET_MYRIAD) ? 0.397 : 0.4;
 364     double scoreDiff = 8e-5, iouDiff = 3e-4;
 365     if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
 366     {
 367         scoreDiff = 1e-2;
 368         iouDiff = 0.018;
 369     }
 370     else if (target == DNN_TARGET_CUDA_FP16)
 371     {
 372         scoreDiff = 0.03;
 373         iouDiff = 0.018;
 374     }
 375 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
 376     // accuracy
 377     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 378     {
 379         iouDiff = std::numeric_limits<double>::quiet_NaN();
 380     }
 381     // accuracy
 382     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 383     {
 384         iouDiff = std::numeric_limits<double>::quiet_NaN();
 385     }
 386 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
 387     // accuracy
 388     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 389     {
 390         iouDiff = std::numeric_limits<double>::quiet_NaN();
 391     }
 392     // accuracy
 393     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 394     {
 395         iouDiff = std::numeric_limits<double>::quiet_NaN();
 396     }
 397 #endif
 398
 399     std::string config_file = "yolo-voc.cfg";
 400     std::string weights_file = "yolo-voc.weights";
 401
 402     {
 403     SCOPED_TRACE("batch size 1");
 404     testDarknetModel(config_file, weights_file, ref.rowRange(0, 3), scoreDiff, iouDiff, 0.24, 0.4, false);
 405     }
 406
 407 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
 408     // Exception: input != output
 409     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 410         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 411 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
 412     // [ GENERAL_ERROR ]  AssertionFailed: input != output
 413     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 414         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 415 #endif
 416     {
 417     SCOPED_TRACE("batch size 2");
 418     testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, nmsThreshold, false);
 419     }
 420
 421 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
 422     // accuracy
 423     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 424         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 425 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
 426     // accuracy
 427     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 428         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 429 #endif
 430 }
 431
 432 TEST_P(Test_Darknet_nets, TinyYoloVoc)
 433 {
 434     applyTestTag(CV_TEST_TAG_MEMORY_512MB);
 435
 436 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
 437     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
 438         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 439     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 440         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 441 #endif
 442 #if defined(INF_ENGINE_RELEASE)
 443     if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) &&
 444         target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 445         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);  // need to update check function
 446 #endif
 447     // batchId, classId, confidence, left, top, right, bottom
 448     Mat ref = (Mat_<float>(4, 7) << 0, 6,  0.761967f, 0.579042f, 0.159161f, 0.894482f, 0.31994f,   // a car
 449                                     0, 11, 0.780595f, 0.129696f, 0.386467f, 0.445275f, 0.920994f,  // a dog
 450                                     1, 6,  0.651450f, 0.460526f, 0.458019f, 0.522527f, 0.5341f,    // a car
 451                                     1, 6,  0.928758f, 0.651024f, 0.463539f, 0.823784f, 0.654998f); // a car
 452
 453     double scoreDiff = 8e-5, iouDiff = 3e-4;
 454     if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
 455     {
 456         scoreDiff = 8e-3;
 457         iouDiff = 0.018;
 458     }
 459     else if(target == DNN_TARGET_CUDA_FP16)
 460     {
 461         scoreDiff = 0.008;
 462         iouDiff = 0.02;
 463     }
 464
 465     std::string config_file = "tiny-yolo-voc.cfg";
 466     std::string weights_file = "tiny-yolo-voc.weights";
 467
 468     {
 469     SCOPED_TRACE("batch size 1");
 470     testDarknetModel(config_file, weights_file, ref.rowRange(0, 2), scoreDiff, iouDiff);
 471     }
 472
 473     {
 474     SCOPED_TRACE("batch size 2");
 475     testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff);
 476     }
 477 }
 478
 479 #ifdef HAVE_INF_ENGINE
 480 static const std::chrono::milliseconds async_timeout(10000);
 481
 482 typedef testing::TestWithParam<tuple<std::string, tuple<Backend, Target> > > Test_Darknet_nets_async;
 483 TEST_P(Test_Darknet_nets_async, Accuracy)
 484 {
 485     Backend backendId = get<0>(get<1>(GetParam()));
 486     Target targetId = get<1>(get<1>(GetParam()));
 487     std::string prefix = get<0>(GetParam());
 488
 489     applyTestTag(CV_TEST_TAG_MEMORY_512MB);
 490
 491 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000)
 492     if (INF_ENGINE_VER_MAJOR_LT(2019020000) && backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
 493         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER);
 494
 495     if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
 496         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);
 497 #endif
 498
 499     if (backendId != DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && backendId != DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
 500         throw SkipTestException("No support for async forward");
 501
 502 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
 503     if (targetId == DNN_TARGET_MYRIAD && prefix == "yolov3")  // NC_OUT_OF_MEMORY
 504         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 505 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
 506     if (targetId == DNN_TARGET_MYRIAD && prefix == "yolov3")  // NC_OUT_OF_MEMORY
 507         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 508 #elif defined(INF_ENGINE_RELEASE)
 509     if (targetId == DNN_TARGET_MYRIAD && prefix == "yolov4")  // NC_OUT_OF_MEMORY
 510         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 511 #endif
 512
 513     const int numInputs = 2;
 514     std::vector<Mat> inputs(numInputs);
 515     int blobSize[] = {1, 3, 416, 416};
 516     for (int i = 0; i < numInputs; ++i)
 517     {
 518         inputs[i].create(4, &blobSize[0], CV_32F);
 519         randu(inputs[i], 0, 1);
 520     }
 521
 522     Net netSync = readNet(findDataFile("dnn/" + prefix + ".cfg"),
 523                           findDataFile("dnn/" + prefix + ".weights", false));
 524     netSync.setPreferableBackend(backendId);
 525     netSync.setPreferableTarget(targetId);
 526
 527     // Run synchronously.
 528     std::vector<Mat> refs(numInputs);
 529     for (int i = 0; i < numInputs; ++i)
 530     {
 531         netSync.setInput(inputs[i]);
 532         refs[i] = netSync.forward().clone();
 533     }
 534
 535     Net netAsync = readNet(findDataFile("dnn/" + prefix + ".cfg"),
 536                            findDataFile("dnn/" + prefix + ".weights", false));
 537     netAsync.setPreferableBackend(backendId);
 538     netAsync.setPreferableTarget(targetId);
 539
 540     double l1 = 0.0;
 541     double lInf = 0.0;
 542 #if defined(INF_ENGINE_RELEASE)
 543     if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
 544     {
 545         if (targetId == DNN_TARGET_MYRIAD && prefix == "yolo-voc")
 546         {
 547             l1 = 0.02;
 548             lInf = 0.15;
 549         }
 550         if (targetId == DNN_TARGET_OPENCL_FP16 && prefix == "yolo-voc")
 551         {
 552             l1 = 0.02;
 553             lInf = 0.1;
 554         }
 555         if (targetId == DNN_TARGET_OPENCL_FP16 && prefix == "yolov3")
 556         {
 557             l1 = 0.001;
 558             lInf = 0.007;
 559         }
 560         if (targetId == DNN_TARGET_OPENCL_FP16 && prefix == "yolov4")
 561         {
 562             l1 = 0.001;
 563             lInf = 0.005;
 564         }
 565         if (INF_ENGINE_VER_MAJOR_EQ(2021040000) && targetId == DNN_TARGET_OPENCL_FP16 && prefix == "yolov4-tiny")  // FIXIT: 4.x only, 3.4 branch works well
 566         {
 567             l1 = 0.001;
 568             lInf = 0.005;
 569         }
 570         if (INF_ENGINE_VER_MAJOR_EQ(2022010000) && targetId == DNN_TARGET_OPENCL_FP16 && prefix == "yolov4-tiny")  // FIXIT: 4.x only, 3.4 branch works well
 571         {
 572             l1 = 0.001;
 573             lInf = 0.005;
 574         }
 575         if (targetId == DNN_TARGET_MYRIAD && prefix == "yolov4")
 576         {
 577             l1 = 0.005;
 578             lInf = 1.6f;  // |ref| = 0.95431125164031982
 579         }
 580     }
 581 #endif
 582
 583     // Run asynchronously. To make test more robust, process inputs in the reversed order.
 584     for (int i = numInputs - 1; i >= 0; --i)
 585     {
 586         netAsync.setInput(inputs[i]);
 587
 588         AsyncArray out = netAsync.forwardAsync();
 589         ASSERT_TRUE(out.valid());
 590         Mat result;
 591         EXPECT_TRUE(out.get(result, async_timeout));
 592         normAssert(refs[i], result, format("Index: %d", i).c_str(), l1, lInf);
 593     }
 594 }
 595
 596 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets_async, Combine(
 597     Values("yolo-voc", "tiny-yolo-voc", "yolov3", "yolov4", "yolov4-tiny"),
 598     dnnBackendsAndTargets()
 599 ));
 600
 601 #endif
 602
 603 TEST_P(Test_Darknet_nets, YOLOv3)
 604 {
 605     applyTestTag(
 606             CV_TEST_TAG_LONG,
 607             CV_TEST_TAG_MEMORY_2GB,
 608             CV_TEST_TAG_DEBUG_VERYLONG
 609     );
 610
 611 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
 612     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
 613         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 614     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 615         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 616 #endif
 617
 618     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 619         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);
 620
 621     // batchId, classId, confidence, left, top, right, bottom
 622     const int N0 = 3;
 623     const int N1 = 6;
 624     static const float ref_[/* (N0 + N1) * 7 */] = {
 625 0, 16, 0.998836f, 0.160024f, 0.389964f, 0.417885f, 0.943716f,
 626 0, 1, 0.987908f, 0.150913f, 0.221933f, 0.742255f, 0.746261f,
 627 0, 7, 0.952983f, 0.614621f, 0.150257f, 0.901368f, 0.289251f,
 628
 629 1, 2, 0.997412f, 0.647584f, 0.459939f, 0.821037f, 0.663947f,
 630 1, 2, 0.989633f, 0.450719f, 0.463353f, 0.496306f, 0.522258f,
 631 1, 0, 0.980053f, 0.195856f, 0.378454f, 0.258626f, 0.629257f,
 632 1, 9, 0.785341f, 0.665503f, 0.373543f, 0.688893f, 0.439244f,
 633 1, 9, 0.733275f, 0.376029f, 0.315694f, 0.401776f, 0.395165f,
 634 1, 9, 0.384815f, 0.659824f, 0.372389f, 0.673927f, 0.429412f,
 635     };
 636     Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_);
 637
 638     double scoreDiff = 8e-5, iouDiff = 3e-4;
 639     if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
 640     {
 641         scoreDiff = 0.006;
 642         iouDiff = 0.042;
 643     }
 644     else if (target == DNN_TARGET_CUDA_FP16)
 645     {
 646         scoreDiff = 0.04;
 647         iouDiff = 0.03;
 648     }
 649     std::string config_file = "yolov3.cfg";
 650     std::string weights_file = "yolov3.weights";
 651
 652 #if defined(INF_ENGINE_RELEASE)
 653     if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
 654          backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD &&
 655         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 656     {
 657         scoreDiff = 0.04;
 658         iouDiff = 0.2;
 659     }
 660 #endif
 661
 662     {
 663         SCOPED_TRACE("batch size 1");
 664         testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, 0.24, 0.4, false);
 665     }
 666
 667 #if defined(INF_ENGINE_RELEASE)
 668     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
 669     {
 670         if (target == DNN_TARGET_OPENCL)
 671             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 672         else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000))
 673             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 674         else if (target == DNN_TARGET_MYRIAD &&
 675                  getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 676             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 677     }
 678 #endif
 679
 680     {
 681         SCOPED_TRACE("batch size 2");
 682         testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, 0.4, false);
 683     }
 684 }
 685
 686 TEST_P(Test_Darknet_nets, YOLOv4)
 687 {
 688     applyTestTag(
 689             CV_TEST_TAG_LONG,
 690             CV_TEST_TAG_MEMORY_2GB,
 691             CV_TEST_TAG_DEBUG_VERYLONG
 692             );
 693
 694 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
 695     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
 696         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 697     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 698         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 699 #endif
 700 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2022010000)
 701     if (target == DNN_TARGET_MYRIAD)  // NC_OUT_OF_MEMORY
 702         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 703 #endif
 704
 705     // batchId, classId, confidence, left, top, right, bottom
 706     const int N0 = 3;
 707     const int N1 = 7;
 708     static const float ref_[/* (N0 + N1) * 7 */] = {
 709 0, 16, 0.992194f, 0.172375f, 0.402458f, 0.403918f, 0.932801f,
 710 0, 1, 0.988326f, 0.166708f, 0.228236f, 0.737208f, 0.735803f,
 711 0, 7, 0.94639f, 0.602523f, 0.130399f, 0.901623f, 0.298452f,
 712
 713 1, 2, 0.99761f, 0.646556f, 0.45985f, 0.816041f, 0.659067f,
 714 1, 0, 0.988913f, 0.201726f, 0.360282f, 0.266181f, 0.631728f,
 715 1, 2, 0.98233f, 0.452007f, 0.462217f, 0.495612f, 0.521687f,
 716 1, 9, 0.919195f, 0.374642f, 0.316524f, 0.398126f, 0.393714f,
 717 1, 9, 0.856303f, 0.666842f, 0.372215f, 0.685539f, 0.44141f,
 718 1, 9, 0.313516f, 0.656791f, 0.374734f, 0.671959f, 0.438371f,
 719 1, 9, 0.256625f, 0.940232f, 0.326931f, 0.967586f, 0.374002f,
 720     };
 721     Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_);
 722
 723     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.006 : 8e-5;
 724     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.042 : 3e-4;
 725     if (target == DNN_TARGET_CUDA_FP16)
 726     {
 727         scoreDiff = 0.008;
 728         iouDiff = 0.03;
 729     }
 730
 731     std::string config_file = "yolov4.cfg";
 732     std::string weights_file = "yolov4.weights";
 733
 734
 735 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
 736     // accuracy (batch 1): no detections
 737     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 738     {
 739         iouDiff = std::numeric_limits<double>::quiet_NaN();
 740     }
 741     // accuracy
 742     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 743     {
 744         iouDiff = std::numeric_limits<double>::quiet_NaN();
 745     }
 746 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
 747     // accuracy (batch 1)
 748     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 749     {
 750         iouDiff = std::numeric_limits<double>::quiet_NaN();
 751     }
 752 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2022010000)
 753     if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
 754          backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD &&
 755         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 756     {
 757         scoreDiff = 0.04;
 758         iouDiff = 0.2;
 759     }
 760 #endif
 761
 762     {
 763         SCOPED_TRACE("batch size 1");
 764         testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, 0.24, 0.4, false);
 765     }
 766
 767     {
 768         SCOPED_TRACE("batch size 2");
 769
 770 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
 771     // accuracy (batch 2)
 772     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 773     {
 774         iouDiff = 0.05f;
 775     }
 776     // accuracy
 777     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 778     {
 779         iouDiff = std::numeric_limits<double>::quiet_NaN();
 780     }
 781 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
 782     // accuracy (batch 2)
 783     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 784     {
 785         iouDiff = 0.45f;
 786     }
 787 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2022010000)
 788         if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
 789         {
 790             if (target == DNN_TARGET_OPENCL)
 791                 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 792             else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000))
 793                 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 794             else if (target == DNN_TARGET_MYRIAD &&
 795                      getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 796                 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 797         }
 798 #endif
 799
 800         testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, 0.4, false);
 801     }
 802
 803 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
 804     // accuracy
 805     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 806         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 807     // accuracy
 808     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 809         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 810 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
 811     // accuracy
 812     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 813         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 814 #endif
 815 }
 816
 817 TEST_P(Test_Darknet_nets, YOLOv4_tiny)
 818 {
 819     applyTestTag(
 820         target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB
 821     );
 822
 823 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000)  // nGraph compilation failure
 824     if (target == DNN_TARGET_MYRIAD)
 825         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 826 #endif
 827
 828     const double confThreshold = 0.5;
 829     // batchId, classId, confidence, left, top, right, bottom
 830     const int N0 = 2;
 831     const int N1 = 3;
 832     static const float ref_[/* (N0 + N1) * 7 */] = {
 833 0, 7, 0.85935f, 0.593484f, 0.141211f, 0.920356f, 0.291593f,
 834 0, 16, 0.795188f, 0.169207f, 0.386886f, 0.423753f, 0.933004f,
 835
 836 1, 2, 0.996832f, 0.653802f, 0.464573f, 0.815193f, 0.653292f,
 837 1, 2, 0.963325f, 0.451151f, 0.458915f, 0.496255f, 0.52241f,
 838 1, 0, 0.926244f, 0.194851f, 0.361743f, 0.260277f, 0.632364f,
 839     };
 840     Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_);
 841
 842     double scoreDiff = 0.01f;
 843     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.15 : 0.01f;
 844     if (target == DNN_TARGET_CUDA_FP16)
 845         iouDiff = 0.02;
 846
 847     std::string config_file = "yolov4-tiny.cfg";
 848     std::string weights_file = "yolov4-tiny.weights";
 849
 850 #if defined(INF_ENGINE_RELEASE)
 851     if (target == DNN_TARGET_MYRIAD)  // bad accuracy
 852         iouDiff = std::numeric_limits<double>::quiet_NaN();
 853     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL)
 854         iouDiff = std::numeric_limits<double>::quiet_NaN();
 855     if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
 856          backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16)
 857         iouDiff = std::numeric_limits<double>::quiet_NaN();
 858 #endif
 859
 860     {
 861         SCOPED_TRACE("batch size 1");
 862         testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, confThreshold);
 863     }
 864
 865     {
 866         SCOPED_TRACE("batch size 2");
 867         testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, confThreshold);
 868     }
 869
 870 #if defined(INF_ENGINE_RELEASE)
 871     if (target == DNN_TARGET_MYRIAD)  // bad accuracy
 872         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 873     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL)
 874         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 875     if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
 876          backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16)
 877         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 878 #endif
 879 }
 880
 881 TEST_P(Test_Darknet_nets, YOLOv4x_mish)
 882 {
 883     applyTestTag(
 884             CV_TEST_TAG_LONG,
 885             CV_TEST_TAG_MEMORY_2GB,
 886             CV_TEST_TAG_DEBUG_VERYLONG
 887             );
 888
 889 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
 890     // IE exception: Ngraph operation Transpose with name permute_168 has dynamic output shape on 0 port, but CPU plug-in supports only static shape
 891     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 892         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16,
 893             CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION
 894         );
 895 #endif
 896 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
 897     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
 898         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 899     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 900         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 901 #endif
 902 #if defined(INF_ENGINE_RELEASE)
 903     if (target == DNN_TARGET_MYRIAD)  // NC_OUT_OF_MEMORY
 904         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 905 #endif
 906
 907     // batchId, classId, confidence, left, top, right, bottom
 908     const int N0 = 3;
 909     const int N1 = 5;
 910     static const float ref_[/* (N0 + N1) * 7 */] = {
 911 0, 16, 0.925536f, 0.17188f,  0.386832f, 0.406138f, 0.941696f,
 912 0, 1,  0.912028f, 0.162125f, 0.208863f, 0.741316f, 0.729332f,
 913 0, 7,  0.841018f, 0.608953f, 0.128653f, 0.900692f, 0.295657f,
 914
 915 1, 2, 0.925697f, 0.650438f, 0.458118f, 0.813927f, 0.661775f,
 916 1, 0, 0.882156f, 0.203644f, 0.365763f, 0.265473f, 0.632195f,
 917 1, 2, 0.848857f, 0.451044f, 0.462997f, 0.496629f, 0.522719f,
 918 1, 9, 0.736015f, 0.374503f, 0.316029f, 0.399358f, 0.392883f,
 919 1, 9, 0.727129f, 0.662469f, 0.373687f, 0.687877f, 0.441335f,
 920     };
 921     Mat ref(N0 + N1, 7, CV_32FC1, (void*)ref_);
 922
 923     double scoreDiff = 8e-5;
 924     double iouDiff = 3e-4;
 925
 926     if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD || target == DNN_TARGET_CUDA_FP16)
 927     {
 928         scoreDiff = 0.006;
 929         iouDiff = 0.042;
 930     }
 931
 932     std::string config_file = "yolov4x-mish.cfg";
 933     std::string weights_file = "yolov4x-mish.weights";
 934
 935 #if defined(INF_ENGINE_RELEASE)
 936     if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
 937          backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_MYRIAD &&
 938         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 939     {
 940         scoreDiff = 0.04;
 941         iouDiff = 0.2;
 942     }
 943 #endif
 944
 945     {
 946         SCOPED_TRACE("batch size 1");
 947         testDarknetModel(config_file, weights_file, ref.rowRange(0, N0), scoreDiff, iouDiff, 0.24, 0.4, false);
 948     }
 949
 950     {
 951         SCOPED_TRACE("batch size 2");
 952
 953 #if defined(INF_ENGINE_RELEASE)
 954         if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
 955         {
 956             if (target == DNN_TARGET_OPENCL)
 957                 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 958             else if (target == DNN_TARGET_OPENCL_FP16 && INF_ENGINE_VER_MAJOR_LE(202010000))
 959                 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
 960             else if (target == DNN_TARGET_MYRIAD &&
 961                      getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 962                 applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 963         }
 964 #endif
 965
 966         testDarknetModel(config_file, weights_file, ref, scoreDiff, iouDiff, 0.24, 0.4, false);
 967     }
 968 }
 969
 970
 971 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_nets, dnnBackendsAndTargets());
 972
 973 TEST_P(Test_Darknet_layers, shortcut)
 974 {
 975     testDarknetLayer("shortcut");
 976 }
 977 TEST_P(Test_Darknet_layers, shortcut_leaky)
 978 {
 979     testDarknetLayer("shortcut_leaky");
 980 }
 981 TEST_P(Test_Darknet_layers, shortcut_unequal)
 982 {
 983     testDarknetLayer("shortcut_unequal");
 984 }
 985 TEST_P(Test_Darknet_layers, shortcut_unequal_2)
 986 {
 987     testDarknetLayer("shortcut_unequal_2");
 988 }
 989
 990 TEST_P(Test_Darknet_layers, upsample)
 991 {
 992 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021030000)
 993     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
 994         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);  // exception
 995 #endif
 996     testDarknetLayer("upsample");
 997 }
 998
 999 TEST_P(Test_Darknet_layers, mish)
1000 {
1001     testDarknetLayer("mish", true);
1002 }
1003
1004 TEST_P(Test_Darknet_layers, tanh)
1005 {
1006     testDarknetLayer("tanh");
1007 }
1008
1009 TEST_P(Test_Darknet_layers, avgpool_softmax)
1010 {
1011     testDarknetLayer("avgpool_softmax");
1012 }
1013
1014 TEST_P(Test_Darknet_layers, region)
1015 {
1016 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2021040000)
1017     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && INF_ENGINE_VER_MAJOR_GE(2020020000))
1018        applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
1019 #endif
1020
1021 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2022010000)
1022     // accuracy on CPU, OpenCL
1023     // Expected: (normL1) <= (l1), actual: 0.000358148 vs 1e-05
1024     //   |ref| = 1.207319974899292
1025     // Expected: (normInf) <= (lInf), actual: 0.763223 vs 0.0001
1026     //   |ref| = 1.207319974899292
1027     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_CPU)
1028         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_CPU, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
1029     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
1030         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16,
1031             CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION
1032         );
1033 #elif defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2021040000)
1034     // accuracy on CPU, OpenCL
1035     // Expected: (normInf) <= (lInf), actual: 0.763223 vs 0.0001
1036     //   |ref| = 1.207319974899292
1037     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_CPU)
1038         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_CPU, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
1039     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
1040         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16,
1041             CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION
1042         );
1043 #endif
1044
1045     testDarknetLayer("region");
1046 }
1047
1048 TEST_P(Test_Darknet_layers, reorg)
1049 {
1050     testDarknetLayer("reorg");
1051 }
1052
1053 TEST_P(Test_Darknet_layers, route)
1054 {
1055     testDarknetLayer("route");
1056     testDarknetLayer("route_multi");
1057 }
1058
1059 TEST_P(Test_Darknet_layers, maxpool)
1060 {
1061 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2020020000)
1062     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
1063         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
1064 #endif
1065     testDarknetLayer("maxpool");
1066 }
1067
1068 TEST_P(Test_Darknet_layers, convolutional)
1069 {
1070 #if defined(INF_ENGINE_RELEASE)
1071     if (target == DNN_TARGET_MYRIAD)
1072     {
1073         default_l1 = 0.01f;
1074     }
1075 #endif
1076     testDarknetLayer("convolutional", true);
1077 }
1078
1079 TEST_P(Test_Darknet_layers, scale_channels)
1080 {
1081     bool testBatches = backend == DNN_BACKEND_CUDA;
1082     testDarknetLayer("scale_channels", false, testBatches);
1083 }
1084
1085 TEST_P(Test_Darknet_layers, connected)
1086 {
1087     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
1088         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
1089     testDarknetLayer("connected", true);
1090 }
1091
1092 TEST_P(Test_Darknet_layers, relu)
1093 {
1094      if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_MYRIAD)
1095         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
1096     testDarknetLayer("relu");
1097 }
1098
1099 TEST_P(Test_Darknet_layers, sam)
1100 {
1101     testDarknetLayer("sam", true);
1102 }
1103
1104 INSTANTIATE_TEST_CASE_P(/**/, Test_Darknet_layers, dnnBackendsAndTargets());
1105
1106 }} // namespace