modules/dnn/test/test_common.impl.hpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Used in accuracy and perf tests as a content of .cpp file
   6 // Note: don't use "precomp.hpp" here
   7 #include "opencv2/ts.hpp"
   8 #include "opencv2/ts/ts_perf.hpp"
   9 #include "opencv2/core/utility.hpp"
  10 #include "opencv2/core/ocl.hpp"
  11
  12 #include "opencv2/dnn.hpp"
  13 #include "test_common.hpp"
  14
  15 #include <opencv2/core/utils/configuration.private.hpp>
  16 #include <opencv2/core/utils/logger.hpp>
  17
  18 namespace cv { namespace dnn {
  19 CV__DNN_INLINE_NS_BEGIN
  20
  21 void PrintTo(const cv::dnn::Backend& v, std::ostream* os)
  22 {
  23     switch (v) {
  24     case DNN_BACKEND_DEFAULT: *os << "DEFAULT"; return;
  25     case DNN_BACKEND_HALIDE: *os << "HALIDE"; return;
  26     case DNN_BACKEND_INFERENCE_ENGINE: *os << "DLIE*"; return;
  27     case DNN_BACKEND_VKCOM: *os << "VKCOM"; return;
  28     case DNN_BACKEND_OPENCV: *os << "OCV"; return;
  29     case DNN_BACKEND_CUDA: *os << "CUDA"; return;
  30     case DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019: *os << "DLIE"; return;
  31     case DNN_BACKEND_INFERENCE_ENGINE_NGRAPH: *os << "NGRAPH"; return;
  32     } // don't use "default:" to emit compiler warnings
  33     *os << "DNN_BACKEND_UNKNOWN(" << (int)v << ")";
  34 }
  35
  36 void PrintTo(const cv::dnn::Target& v, std::ostream* os)
  37 {
  38     switch (v) {
  39     case DNN_TARGET_CPU: *os << "CPU"; return;
  40     case DNN_TARGET_OPENCL: *os << "OCL"; return;
  41     case DNN_TARGET_OPENCL_FP16: *os << "OCL_FP16"; return;
  42     case DNN_TARGET_MYRIAD: *os << "MYRIAD"; return;
  43     case DNN_TARGET_VULKAN: *os << "VULKAN"; return;
  44     case DNN_TARGET_FPGA: *os << "FPGA"; return;
  45     case DNN_TARGET_CUDA: *os << "CUDA"; return;
  46     case DNN_TARGET_CUDA_FP16: *os << "CUDA_FP16"; return;
  47     } // don't use "default:" to emit compiler warnings
  48     *os << "DNN_TARGET_UNKNOWN(" << (int)v << ")";
  49 }
  50
  51 void PrintTo(const tuple<cv::dnn::Backend, cv::dnn::Target> v, std::ostream* os)
  52 {
  53     PrintTo(get<0>(v), os);
  54     *os << "/";
  55     PrintTo(get<1>(v), os);
  56 }
  57
  58 CV__DNN_INLINE_NS_END
  59 }} // namespace
  60
  61
  62
  63 namespace opencv_test {
  64
  65 void normAssert(
  66         cv::InputArray ref, cv::InputArray test, const char *comment /*= ""*/,
  67         double l1 /*= 0.00001*/, double lInf /*= 0.0001*/)
  68 {
  69     double normL1 = cvtest::norm(ref, test, cv::NORM_L1) / ref.getMat().total();
  70     EXPECT_LE(normL1, l1) << comment << "  |ref| = " << cvtest::norm(ref, cv::NORM_INF);
  71
  72     double normInf = cvtest::norm(ref, test, cv::NORM_INF);
  73     EXPECT_LE(normInf, lInf) << comment << "  |ref| = " << cvtest::norm(ref, cv::NORM_INF);
  74 }
  75
  76 std::vector<cv::Rect2d> matToBoxes(const cv::Mat& m)
  77 {
  78     EXPECT_EQ(m.type(), CV_32FC1);
  79     EXPECT_EQ(m.dims, 2);
  80     EXPECT_EQ(m.cols, 4);
  81
  82     std::vector<cv::Rect2d> boxes(m.rows);
  83     for (int i = 0; i < m.rows; ++i)
  84     {
  85         CV_Assert(m.row(i).isContinuous());
  86         const float* data = m.ptr<float>(i);
  87         double l = data[0], t = data[1], r = data[2], b = data[3];
  88         boxes[i] = cv::Rect2d(l, t, r - l, b - t);
  89     }
  90     return boxes;
  91 }
  92
  93 void normAssertDetections(
  94         const std::vector<int>& refClassIds,
  95         const std::vector<float>& refScores,
  96         const std::vector<cv::Rect2d>& refBoxes,
  97         const std::vector<int>& testClassIds,
  98         const std::vector<float>& testScores,
  99         const std::vector<cv::Rect2d>& testBoxes,
 100         const char *comment /*= ""*/, double confThreshold /*= 0.0*/,
 101         double scores_diff /*= 1e-5*/, double boxes_iou_diff /*= 1e-4*/)
 102 {
 103     ASSERT_FALSE(testClassIds.empty()) << "No detections";
 104     std::vector<bool> matchedRefBoxes(refBoxes.size(), false);
 105     std::vector<double> refBoxesIoUDiff(refBoxes.size(), 1.0);
 106     for (int i = 0; i < testBoxes.size(); ++i)
 107     {
 108         //cout << "Test[i=" << i << "]: score=" << testScores[i] << " id=" << testClassIds[i] << " box " << testBoxes[i] << endl;
 109         double testScore = testScores[i];
 110         if (testScore < confThreshold)
 111             continue;
 112
 113         int testClassId = testClassIds[i];
 114         const cv::Rect2d& testBox = testBoxes[i];
 115         bool matched = false;
 116         double topIoU = 0;
 117         for (int j = 0; j < refBoxes.size() && !matched; ++j)
 118         {
 119             if (!matchedRefBoxes[j] && testClassId == refClassIds[j] &&
 120                 std::abs(testScore - refScores[j]) < scores_diff)
 121             {
 122                 double interArea = (testBox & refBoxes[j]).area();
 123                 double iou = interArea / (testBox.area() + refBoxes[j].area() - interArea);
 124                 topIoU = std::max(topIoU, iou);
 125                 refBoxesIoUDiff[j] = std::min(refBoxesIoUDiff[j], 1.0f - iou);
 126                 if (1.0 - iou < boxes_iou_diff)
 127                 {
 128                     matched = true;
 129                     matchedRefBoxes[j] = true;
 130                 }
 131             }
 132         }
 133         if (!matched)
 134         {
 135             std::cout << cv::format("Unmatched prediction: class %d score %f box ",
 136                                     testClassId, testScore) << testBox << std::endl;
 137             std::cout << "Highest IoU: " << topIoU << std::endl;
 138         }
 139         EXPECT_TRUE(matched) << comment;
 140     }
 141
 142     // Check unmatched reference detections.
 143     for (int i = 0; i < refBoxes.size(); ++i)
 144     {
 145         if (!matchedRefBoxes[i] && refScores[i] > confThreshold)
 146         {
 147             std::cout << cv::format("Unmatched reference: class %d score %f box ",
 148                                     refClassIds[i], refScores[i]) << refBoxes[i]
 149                 << " IoU diff: " << refBoxesIoUDiff[i]
 150                 << std::endl;
 151             EXPECT_LE(refScores[i], confThreshold) << comment;
 152         }
 153     }
 154 }
 155
 156 // For SSD-based object detection networks which produce output of shape 1x1xNx7
 157 // where N is a number of detections and an every detection is represented by
 158 // a vector [batchId, classId, confidence, left, top, right, bottom].
 159 void normAssertDetections(
 160         cv::Mat ref, cv::Mat out, const char *comment /*= ""*/,
 161         double confThreshold /*= 0.0*/, double scores_diff /*= 1e-5*/,
 162         double boxes_iou_diff /*= 1e-4*/)
 163 {
 164     CV_Assert(ref.total() % 7 == 0);
 165     CV_Assert(out.total() % 7 == 0);
 166     ref = ref.reshape(1, ref.total() / 7);
 167     out = out.reshape(1, out.total() / 7);
 168
 169     cv::Mat refClassIds, testClassIds;
 170     ref.col(1).convertTo(refClassIds, CV_32SC1);
 171     out.col(1).convertTo(testClassIds, CV_32SC1);
 172     std::vector<float> refScores(ref.col(2)), testScores(out.col(2));
 173     std::vector<cv::Rect2d> refBoxes = matToBoxes(ref.colRange(3, 7));
 174     std::vector<cv::Rect2d> testBoxes = matToBoxes(out.colRange(3, 7));
 175     normAssertDetections(refClassIds, refScores, refBoxes, testClassIds, testScores,
 176                          testBoxes, comment, confThreshold, scores_diff, boxes_iou_diff);
 177 }
 178
 179 void readFileContent(const std::string& filename, CV_OUT std::vector<char>& content)
 180 {
 181     const std::ios::openmode mode = std::ios::in | std::ios::binary;
 182     std::ifstream ifs(filename.c_str(), mode);
 183     ASSERT_TRUE(ifs.is_open());
 184
 185     content.clear();
 186
 187     ifs.seekg(0, std::ios::end);
 188     const size_t sz = ifs.tellg();
 189     content.resize(sz);
 190     ifs.seekg(0, std::ios::beg);
 191
 192     ifs.read((char*)content.data(), sz);
 193     ASSERT_FALSE(ifs.fail());
 194 }
 195
 196
 197 testing::internal::ParamGenerator< tuple<Backend, Target> > dnnBackendsAndTargets(
 198         bool withInferenceEngine /*= true*/,
 199         bool withHalide /*= false*/,
 200         bool withCpuOCV /*= true*/,
 201         bool withVkCom /*= true*/,
 202         bool withCUDA /*= true*/,
 203         bool withNgraph /*= true*/
 204 )
 205 {
 206 #ifdef HAVE_INF_ENGINE
 207     bool withVPU = validateVPUType();
 208 #endif
 209
 210     std::vector< tuple<Backend, Target> > targets;
 211     std::vector< Target > available;
 212     if (withHalide)
 213     {
 214         available = getAvailableTargets(DNN_BACKEND_HALIDE);
 215         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 216             targets.push_back(make_tuple(DNN_BACKEND_HALIDE, *i));
 217     }
 218 #ifdef HAVE_INF_ENGINE
 219     if (withInferenceEngine)
 220     {
 221         available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019);
 222         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 223         {
 224             if (*i == DNN_TARGET_MYRIAD && !withVPU)
 225                 continue;
 226             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, *i));
 227         }
 228     }
 229     if (withNgraph)
 230     {
 231         available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH);
 232         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 233         {
 234             if (*i == DNN_TARGET_MYRIAD && !withVPU)
 235                 continue;
 236             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, *i));
 237         }
 238
 239     }
 240 #else
 241     CV_UNUSED(withInferenceEngine);
 242 #endif
 243     if (withVkCom)
 244     {
 245         available = getAvailableTargets(DNN_BACKEND_VKCOM);
 246         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 247             targets.push_back(make_tuple(DNN_BACKEND_VKCOM, *i));
 248     }
 249
 250 #ifdef HAVE_CUDA
 251     if(withCUDA)
 252     {
 253         for (auto target : getAvailableTargets(DNN_BACKEND_CUDA))
 254             targets.push_back(make_tuple(DNN_BACKEND_CUDA, target));
 255     }
 256 #endif
 257
 258     {
 259         available = getAvailableTargets(DNN_BACKEND_OPENCV);
 260         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 261         {
 262             if (!withCpuOCV && *i == DNN_TARGET_CPU)
 263                 continue;
 264             targets.push_back(make_tuple(DNN_BACKEND_OPENCV, *i));
 265         }
 266     }
 267     if (targets.empty())  // validate at least CPU mode
 268         targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU));
 269     return testing::ValuesIn(targets);
 270 }
 271
 272 testing::internal::ParamGenerator< tuple<Backend, Target> > dnnBackendsAndTargetsIE()
 273 {
 274 #ifdef HAVE_INF_ENGINE
 275     bool withVPU = validateVPUType();
 276
 277     std::vector< tuple<Backend, Target> > targets;
 278     std::vector< Target > available;
 279
 280     {
 281         available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019);
 282         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 283         {
 284             if (*i == DNN_TARGET_MYRIAD && !withVPU)
 285                 continue;
 286             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019, *i));
 287         }
 288     }
 289
 290     {
 291         available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH);
 292         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 293         {
 294             if (*i == DNN_TARGET_MYRIAD && !withVPU)
 295                 continue;
 296             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE_NGRAPH, *i));
 297         }
 298
 299     }
 300
 301     return testing::ValuesIn(targets);
 302 #else
 303     return testing::ValuesIn(std::vector< tuple<Backend, Target> >());
 304 #endif
 305 }
 306
 307 #ifdef HAVE_INF_ENGINE
 308 static std::string getTestInferenceEngineVPUType()
 309 {
 310     static std::string param_vpu_type = utils::getConfigurationParameterString("OPENCV_TEST_DNN_IE_VPU_TYPE", "");
 311     return param_vpu_type;
 312 }
 313
 314 static bool validateVPUType_()
 315 {
 316     std::string test_vpu_type = getTestInferenceEngineVPUType();
 317     if (test_vpu_type == "DISABLED" || test_vpu_type == "disabled")
 318     {
 319         return false;
 320     }
 321
 322     std::vector<Target> available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE);
 323     bool have_vpu_target = false;
 324     for (std::vector<Target>::const_iterator i = available.begin(); i != available.end(); ++i)
 325     {
 326         if (*i == DNN_TARGET_MYRIAD)
 327         {
 328             have_vpu_target = true;
 329             break;
 330         }
 331     }
 332
 333     if (test_vpu_type.empty())
 334     {
 335         if (have_vpu_target)
 336         {
 337             CV_LOG_INFO(NULL, "OpenCV-DNN-Test: VPU type for testing is not specified via 'OPENCV_TEST_DNN_IE_VPU_TYPE' parameter.")
 338         }
 339     }
 340     else
 341     {
 342         if (!have_vpu_target)
 343         {
 344             CV_LOG_FATAL(NULL, "OpenCV-DNN-Test: 'OPENCV_TEST_DNN_IE_VPU_TYPE' parameter requires VPU of type = '" << test_vpu_type << "', but VPU is not detected. STOP.");
 345             exit(1);
 346         }
 347         std::string dnn_vpu_type = getInferenceEngineVPUType();
 348         if (dnn_vpu_type != test_vpu_type)
 349         {
 350             CV_LOG_FATAL(NULL, "OpenCV-DNN-Test: 'testing' and 'detected' VPU types mismatch: '" << test_vpu_type << "' vs '" << dnn_vpu_type << "'. STOP.");
 351             exit(1);
 352         }
 353     }
 354     if (have_vpu_target)
 355     {
 356         std::string dnn_vpu_type = getInferenceEngineVPUType();
 357         if (dnn_vpu_type == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2)
 358             registerGlobalSkipTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 359         if (dnn_vpu_type == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 360             registerGlobalSkipTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 361     }
 362     return true;
 363 }
 364
 365 bool validateVPUType()
 366 {
 367     static bool result = validateVPUType_();
 368     return result;
 369 }
 370 #endif // HAVE_INF_ENGINE
 371
 372
 373 void initDNNTests()
 374 {
 375     const char* extraTestDataPath =
 376 #ifdef WINRT
 377         NULL;
 378 #else
 379         getenv("OPENCV_DNN_TEST_DATA_PATH");
 380 #endif
 381     if (extraTestDataPath)
 382         cvtest::addDataSearchPath(extraTestDataPath);
 383
 384     registerGlobalSkipTag(
 385         CV_TEST_TAG_DNN_SKIP_HALIDE,
 386         CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16
 387     );
 388 #if defined(INF_ENGINE_RELEASE)
 389     registerGlobalSkipTag(
 390         CV_TEST_TAG_DNN_SKIP_IE,
 391 #if INF_ENGINE_VER_MAJOR_EQ(2018050000)
 392         CV_TEST_TAG_DNN_SKIP_IE_2018R5,
 393 #elif INF_ENGINE_VER_MAJOR_EQ(2019010000)
 394         CV_TEST_TAG_DNN_SKIP_IE_2019R1,
 395 # if INF_ENGINE_RELEASE == 2019010100
 396         CV_TEST_TAG_DNN_SKIP_IE_2019R1_1,
 397 # endif
 398 #elif INF_ENGINE_VER_MAJOR_EQ(2019020000)
 399         CV_TEST_TAG_DNN_SKIP_IE_2019R2,
 400 #elif INF_ENGINE_VER_MAJOR_EQ(2019030000)
 401         CV_TEST_TAG_DNN_SKIP_IE_2019R3,
 402 #endif
 403 #ifdef HAVE_DNN_NGRAPH
 404         CV_TEST_TAG_DNN_SKIP_IE_NGRAPH,
 405 #endif
 406 #ifdef HAVE_DNN_IE_NN_BUILDER_2019
 407         CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER,
 408 #endif
 409         ""
 410     );
 411 #endif
 412     registerGlobalSkipTag(
 413         // see validateVPUType(): CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2, CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X
 414         CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16
 415     );
 416 #ifdef HAVE_VULKAN
 417     registerGlobalSkipTag(
 418         CV_TEST_TAG_DNN_SKIP_VULKAN
 419     );
 420 #endif
 421
 422 #ifdef HAVE_CUDA
 423     registerGlobalSkipTag(
 424         CV_TEST_TAG_DNN_SKIP_CUDA, CV_TEST_TAG_DNN_SKIP_CUDA_FP32, CV_TEST_TAG_DNN_SKIP_CUDA_FP16
 425     );
 426 #endif
 427 }
 428
 429 } // namespace