modules/dnn/test/test_common.impl.hpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Used in accuracy and perf tests as a content of .cpp file
   6 // Note: don't use "precomp.hpp" here
   7 #include "opencv2/ts.hpp"
   8 #include "opencv2/ts/ts_perf.hpp"
   9 #include "opencv2/core/utility.hpp"
  10 #include "opencv2/core/ocl.hpp"
  11
  12 #include "opencv2/dnn.hpp"
  13 #include "test_common.hpp"
  14
  15 #include <opencv2/core/utils/configuration.private.hpp>
  16 #include <opencv2/core/utils/logger.hpp>
  17
  18 namespace cv { namespace dnn {
  19 CV__DNN_INLINE_NS_BEGIN
  20
  21 void PrintTo(const cv::dnn::Backend& v, std::ostream* os)
  22 {
  23     switch (v) {
  24     case DNN_BACKEND_DEFAULT: *os << "DEFAULT"; return;
  25     case DNN_BACKEND_HALIDE: *os << "HALIDE"; return;
  26     case DNN_BACKEND_INFERENCE_ENGINE: *os << "DLIE"; return;
  27     case DNN_BACKEND_VKCOM: *os << "VKCOM"; return;
  28     case DNN_BACKEND_OPENCV: *os << "OCV"; return;
  29     } // don't use "default:" to emit compiler warnings
  30     *os << "DNN_BACKEND_UNKNOWN(" << (int)v << ")";
  31 }
  32
  33 void PrintTo(const cv::dnn::Target& v, std::ostream* os)
  34 {
  35     switch (v) {
  36     case DNN_TARGET_CPU: *os << "CPU"; return;
  37     case DNN_TARGET_OPENCL: *os << "OCL"; return;
  38     case DNN_TARGET_OPENCL_FP16: *os << "OCL_FP16"; return;
  39     case DNN_TARGET_MYRIAD: *os << "MYRIAD"; return;
  40     case DNN_TARGET_VULKAN: *os << "VULKAN"; return;
  41     case DNN_TARGET_FPGA: *os << "FPGA"; return;
  42     } // don't use "default:" to emit compiler warnings
  43     *os << "DNN_TARGET_UNKNOWN(" << (int)v << ")";
  44 }
  45
  46 void PrintTo(const tuple<cv::dnn::Backend, cv::dnn::Target> v, std::ostream* os)
  47 {
  48     PrintTo(get<0>(v), os);
  49     *os << "/";
  50     PrintTo(get<1>(v), os);
  51 }
  52
  53 CV__DNN_INLINE_NS_END
  54 }} // namespace
  55
  56
  57
  58 namespace opencv_test {
  59
  60 void normAssert(
  61         cv::InputArray ref, cv::InputArray test, const char *comment /*= ""*/,
  62         double l1 /*= 0.00001*/, double lInf /*= 0.0001*/)
  63 {
  64     double normL1 = cvtest::norm(ref, test, cv::NORM_L1) / ref.getMat().total();
  65     EXPECT_LE(normL1, l1) << comment;
  66
  67     double normInf = cvtest::norm(ref, test, cv::NORM_INF);
  68     EXPECT_LE(normInf, lInf) << comment;
  69 }
  70
  71 std::vector<cv::Rect2d> matToBoxes(const cv::Mat& m)
  72 {
  73     EXPECT_EQ(m.type(), CV_32FC1);
  74     EXPECT_EQ(m.dims, 2);
  75     EXPECT_EQ(m.cols, 4);
  76
  77     std::vector<cv::Rect2d> boxes(m.rows);
  78     for (int i = 0; i < m.rows; ++i)
  79     {
  80         CV_Assert(m.row(i).isContinuous());
  81         const float* data = m.ptr<float>(i);
  82         double l = data[0], t = data[1], r = data[2], b = data[3];
  83         boxes[i] = cv::Rect2d(l, t, r - l, b - t);
  84     }
  85     return boxes;
  86 }
  87
  88 void normAssertDetections(
  89         const std::vector<int>& refClassIds,
  90         const std::vector<float>& refScores,
  91         const std::vector<cv::Rect2d>& refBoxes,
  92         const std::vector<int>& testClassIds,
  93         const std::vector<float>& testScores,
  94         const std::vector<cv::Rect2d>& testBoxes,
  95         const char *comment /*= ""*/, double confThreshold /*= 0.0*/,
  96         double scores_diff /*= 1e-5*/, double boxes_iou_diff /*= 1e-4*/)
  97 {
  98     std::vector<bool> matchedRefBoxes(refBoxes.size(), false);
  99     for (int i = 0; i < testBoxes.size(); ++i)
 100     {
 101         double testScore = testScores[i];
 102         if (testScore < confThreshold)
 103             continue;
 104
 105         int testClassId = testClassIds[i];
 106         const cv::Rect2d& testBox = testBoxes[i];
 107         bool matched = false;
 108         for (int j = 0; j < refBoxes.size() && !matched; ++j)
 109         {
 110             if (!matchedRefBoxes[j] && testClassId == refClassIds[j] &&
 111                 std::abs(testScore - refScores[j]) < scores_diff)
 112             {
 113                 double interArea = (testBox & refBoxes[j]).area();
 114                 double iou = interArea / (testBox.area() + refBoxes[j].area() - interArea);
 115                 if (std::abs(iou - 1.0) < boxes_iou_diff)
 116                 {
 117                     matched = true;
 118                     matchedRefBoxes[j] = true;
 119                 }
 120             }
 121         }
 122         if (!matched)
 123             std::cout << cv::format("Unmatched prediction: class %d score %f box ",
 124                                     testClassId, testScore) << testBox << std::endl;
 125         EXPECT_TRUE(matched) << comment;
 126     }
 127
 128     // Check unmatched reference detections.
 129     for (int i = 0; i < refBoxes.size(); ++i)
 130     {
 131         if (!matchedRefBoxes[i] && refScores[i] > confThreshold)
 132         {
 133             std::cout << cv::format("Unmatched reference: class %d score %f box ",
 134                                     refClassIds[i], refScores[i]) << refBoxes[i] << std::endl;
 135             EXPECT_LE(refScores[i], confThreshold) << comment;
 136         }
 137     }
 138 }
 139
 140 // For SSD-based object detection networks which produce output of shape 1x1xNx7
 141 // where N is a number of detections and an every detection is represented by
 142 // a vector [batchId, classId, confidence, left, top, right, bottom].
 143 void normAssertDetections(
 144         cv::Mat ref, cv::Mat out, const char *comment /*= ""*/,
 145         double confThreshold /*= 0.0*/, double scores_diff /*= 1e-5*/,
 146         double boxes_iou_diff /*= 1e-4*/)
 147 {
 148     CV_Assert(ref.total() % 7 == 0);
 149     CV_Assert(out.total() % 7 == 0);
 150     ref = ref.reshape(1, ref.total() / 7);
 151     out = out.reshape(1, out.total() / 7);
 152
 153     cv::Mat refClassIds, testClassIds;
 154     ref.col(1).convertTo(refClassIds, CV_32SC1);
 155     out.col(1).convertTo(testClassIds, CV_32SC1);
 156     std::vector<float> refScores(ref.col(2)), testScores(out.col(2));
 157     std::vector<cv::Rect2d> refBoxes = matToBoxes(ref.colRange(3, 7));
 158     std::vector<cv::Rect2d> testBoxes = matToBoxes(out.colRange(3, 7));
 159     normAssertDetections(refClassIds, refScores, refBoxes, testClassIds, testScores,
 160                          testBoxes, comment, confThreshold, scores_diff, boxes_iou_diff);
 161 }
 162
 163 void readFileContent(const std::string& filename, CV_OUT std::vector<char>& content)
 164 {
 165     const std::ios::openmode mode = std::ios::in | std::ios::binary;
 166     std::ifstream ifs(filename.c_str(), mode);
 167     ASSERT_TRUE(ifs.is_open());
 168
 169     content.clear();
 170
 171     ifs.seekg(0, std::ios::end);
 172     const size_t sz = ifs.tellg();
 173     content.resize(sz);
 174     ifs.seekg(0, std::ios::beg);
 175
 176     ifs.read((char*)content.data(), sz);
 177     ASSERT_FALSE(ifs.fail());
 178 }
 179
 180
 181 testing::internal::ParamGenerator< tuple<Backend, Target> > dnnBackendsAndTargets(
 182         bool withInferenceEngine /*= true*/,
 183         bool withHalide /*= false*/,
 184         bool withCpuOCV /*= true*/,
 185         bool withVkCom /*= true*/
 186 )
 187 {
 188 #ifdef HAVE_INF_ENGINE
 189     bool withVPU = validateVPUType();
 190 #endif
 191
 192     std::vector< tuple<Backend, Target> > targets;
 193     std::vector< Target > available;
 194     if (withHalide)
 195     {
 196         available = getAvailableTargets(DNN_BACKEND_HALIDE);
 197         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 198             targets.push_back(make_tuple(DNN_BACKEND_HALIDE, *i));
 199     }
 200 #ifdef HAVE_INF_ENGINE
 201     if (withInferenceEngine)
 202     {
 203         available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE);
 204         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 205         {
 206             if (*i == DNN_TARGET_MYRIAD && !withVPU)
 207                 continue;
 208             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, *i));
 209         }
 210     }
 211 #else
 212     CV_UNUSED(withInferenceEngine);
 213 #endif
 214     if (withVkCom)
 215     {
 216         available = getAvailableTargets(DNN_BACKEND_VKCOM);
 217         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 218             targets.push_back(make_tuple(DNN_BACKEND_VKCOM, *i));
 219     }
 220     {
 221         available = getAvailableTargets(DNN_BACKEND_OPENCV);
 222         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 223         {
 224             if (!withCpuOCV && *i == DNN_TARGET_CPU)
 225                 continue;
 226             targets.push_back(make_tuple(DNN_BACKEND_OPENCV, *i));
 227         }
 228     }
 229     if (targets.empty())  // validate at least CPU mode
 230         targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU));
 231     return testing::ValuesIn(targets);
 232 }
 233
 234
 235 #ifdef HAVE_INF_ENGINE
 236 static std::string getTestInferenceEngineVPUType()
 237 {
 238     static std::string param_vpu_type = utils::getConfigurationParameterString("OPENCV_TEST_DNN_IE_VPU_TYPE", "");
 239     return param_vpu_type;
 240 }
 241
 242 static bool validateVPUType_()
 243 {
 244     std::string test_vpu_type = getTestInferenceEngineVPUType();
 245     if (test_vpu_type == "DISABLED" || test_vpu_type == "disabled")
 246     {
 247         return false;
 248     }
 249
 250     std::vector<Target> available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE);
 251     bool have_vpu_target = false;
 252     for (std::vector<Target>::const_iterator i = available.begin(); i != available.end(); ++i)
 253     {
 254         if (*i == DNN_TARGET_MYRIAD)
 255         {
 256             have_vpu_target = true;
 257             break;
 258         }
 259     }
 260
 261     if (test_vpu_type.empty())
 262     {
 263         if (have_vpu_target)
 264         {
 265             CV_LOG_INFO(NULL, "OpenCV-DNN-Test: VPU type for testing is not specified via 'OPENCV_TEST_DNN_IE_VPU_TYPE' parameter.")
 266         }
 267     }
 268     else
 269     {
 270         if (!have_vpu_target)
 271         {
 272             CV_LOG_FATAL(NULL, "OpenCV-DNN-Test: 'OPENCV_TEST_DNN_IE_VPU_TYPE' parameter requires VPU of type = '" << test_vpu_type << "', but VPU is not detected. STOP.");
 273             exit(1);
 274         }
 275         std::string dnn_vpu_type = getInferenceEngineVPUType();
 276         if (dnn_vpu_type != test_vpu_type)
 277         {
 278             CV_LOG_FATAL(NULL, "OpenCV-DNN-Test: 'testing' and 'detected' VPU types mismatch: '" << test_vpu_type << "' vs '" << dnn_vpu_type << "'. STOP.");
 279             exit(1);
 280         }
 281     }
 282     if (have_vpu_target)
 283     {
 284         std::string dnn_vpu_type = getInferenceEngineVPUType();
 285         if (dnn_vpu_type == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2)
 286             registerGlobalSkipTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 287         if (dnn_vpu_type == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 288             registerGlobalSkipTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 289     }
 290     return true;
 291 }
 292
 293 bool validateVPUType()
 294 {
 295     static bool result = validateVPUType_();
 296     return result;
 297 }
 298 #endif // HAVE_INF_ENGINE
 299
 300
 301 void initDNNTests()
 302 {
 303     const char* extraTestDataPath =
 304 #ifdef WINRT
 305         NULL;
 306 #else
 307         getenv("OPENCV_DNN_TEST_DATA_PATH");
 308 #endif
 309     if (extraTestDataPath)
 310         cvtest::addDataSearchPath(extraTestDataPath);
 311
 312     registerGlobalSkipTag(
 313         CV_TEST_TAG_DNN_SKIP_HALIDE,
 314         CV_TEST_TAG_DNN_SKIP_OPENCL, CV_TEST_TAG_DNN_SKIP_OPENCL_FP16
 315     );
 316 #if defined(INF_ENGINE_RELEASE)
 317     registerGlobalSkipTag(
 318 #if INF_ENGINE_VER_MAJOR_EQ(2018050000)
 319         CV_TEST_TAG_DNN_SKIP_IE_2018R5,
 320 #elif INF_ENGINE_VER_MAJOR_EQ(2019010000)
 321         CV_TEST_TAG_DNN_SKIP_IE_2019R1,
 322 # if INF_ENGINE_RELEASE == 2019010100
 323         CV_TEST_TAG_DNN_SKIP_IE_2019R1_1,
 324 # endif
 325 #elif INF_ENGINE_VER_MAJOR_EQ(2019020000)
 326         CV_TEST_TAG_DNN_SKIP_IE_2019R2,
 327 #elif INF_ENGINE_VER_MAJOR_EQ(2019030000)
 328         CV_TEST_TAG_DNN_SKIP_IE_2019R3,
 329 #endif
 330         CV_TEST_TAG_DNN_SKIP_IE
 331     );
 332 #endif
 333     registerGlobalSkipTag(
 334         // see validateVPUType(): CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2, CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X
 335         CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16
 336     );
 337 #ifdef HAVE_VULKAN
 338     registerGlobalSkipTag(
 339         CV_TEST_TAG_DNN_SKIP_VULKAN
 340     );
 341 #endif
 342 }
 343
 344 } // namespace