modules/dnn/test/test_common.impl.hpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Used in accuracy and perf tests as a content of .cpp file
   6 // Note: don't use "precomp.hpp" here
   7 #include "opencv2/ts.hpp"
   8 #include "opencv2/ts/ts_perf.hpp"
   9 #include "opencv2/core/utility.hpp"
  10 #include "opencv2/core/ocl.hpp"
  11
  12 #include "opencv2/dnn.hpp"
  13 #include "test_common.hpp"
  14
  15 #include <opencv2/core/utils/configuration.private.hpp>
  16 #include <opencv2/core/utils/logger.hpp>
  17
  18 namespace cv { namespace dnn {
  19 CV__DNN_EXPERIMENTAL_NS_BEGIN
  20
  21 void PrintTo(const cv::dnn::Backend& v, std::ostream* os)
  22 {
  23     switch (v) {
  24     case DNN_BACKEND_DEFAULT: *os << "DEFAULT"; return;
  25     case DNN_BACKEND_HALIDE: *os << "HALIDE"; return;
  26     case DNN_BACKEND_INFERENCE_ENGINE: *os << "DLIE"; return;
  27     case DNN_BACKEND_OPENCV: *os << "OCV"; return;
  28     } // don't use "default:" to emit compiler warnings
  29     *os << "DNN_BACKEND_UNKNOWN(" << (int)v << ")";
  30 }
  31
  32 void PrintTo(const cv::dnn::Target& v, std::ostream* os)
  33 {
  34     switch (v) {
  35     case DNN_TARGET_CPU: *os << "CPU"; return;
  36     case DNN_TARGET_OPENCL: *os << "OCL"; return;
  37     case DNN_TARGET_OPENCL_FP16: *os << "OCL_FP16"; return;
  38     case DNN_TARGET_MYRIAD: *os << "MYRIAD"; return;
  39     case DNN_TARGET_FPGA: *os << "FPGA"; return;
  40     } // don't use "default:" to emit compiler warnings
  41     *os << "DNN_TARGET_UNKNOWN(" << (int)v << ")";
  42 }
  43
  44 void PrintTo(const tuple<cv::dnn::Backend, cv::dnn::Target> v, std::ostream* os)
  45 {
  46     PrintTo(get<0>(v), os);
  47     *os << "/";
  48     PrintTo(get<1>(v), os);
  49 }
  50
  51 CV__DNN_EXPERIMENTAL_NS_END
  52 }} // namespace
  53
  54
  55
  56 namespace opencv_test {
  57
  58 void normAssert(
  59         cv::InputArray ref, cv::InputArray test, const char *comment /*= ""*/,
  60         double l1 /*= 0.00001*/, double lInf /*= 0.0001*/)
  61 {
  62     double normL1 = cvtest::norm(ref, test, cv::NORM_L1) / ref.getMat().total();
  63     EXPECT_LE(normL1, l1) << comment;
  64
  65     double normInf = cvtest::norm(ref, test, cv::NORM_INF);
  66     EXPECT_LE(normInf, lInf) << comment;
  67 }
  68
  69 std::vector<cv::Rect2d> matToBoxes(const cv::Mat& m)
  70 {
  71     EXPECT_EQ(m.type(), CV_32FC1);
  72     EXPECT_EQ(m.dims, 2);
  73     EXPECT_EQ(m.cols, 4);
  74
  75     std::vector<cv::Rect2d> boxes(m.rows);
  76     for (int i = 0; i < m.rows; ++i)
  77     {
  78         CV_Assert(m.row(i).isContinuous());
  79         const float* data = m.ptr<float>(i);
  80         double l = data[0], t = data[1], r = data[2], b = data[3];
  81         boxes[i] = cv::Rect2d(l, t, r - l, b - t);
  82     }
  83     return boxes;
  84 }
  85
  86 void normAssertDetections(
  87         const std::vector<int>& refClassIds,
  88         const std::vector<float>& refScores,
  89         const std::vector<cv::Rect2d>& refBoxes,
  90         const std::vector<int>& testClassIds,
  91         const std::vector<float>& testScores,
  92         const std::vector<cv::Rect2d>& testBoxes,
  93         const char *comment /*= ""*/, double confThreshold /*= 0.0*/,
  94         double scores_diff /*= 1e-5*/, double boxes_iou_diff /*= 1e-4*/)
  95 {
  96     std::vector<bool> matchedRefBoxes(refBoxes.size(), false);
  97     for (int i = 0; i < testBoxes.size(); ++i)
  98     {
  99         double testScore = testScores[i];
 100         if (testScore < confThreshold)
 101             continue;
 102
 103         int testClassId = testClassIds[i];
 104         const cv::Rect2d& testBox = testBoxes[i];
 105         bool matched = false;
 106         for (int j = 0; j < refBoxes.size() && !matched; ++j)
 107         {
 108             if (!matchedRefBoxes[j] && testClassId == refClassIds[j] &&
 109                 std::abs(testScore - refScores[j]) < scores_diff)
 110             {
 111                 double interArea = (testBox & refBoxes[j]).area();
 112                 double iou = interArea / (testBox.area() + refBoxes[j].area() - interArea);
 113                 if (std::abs(iou - 1.0) < boxes_iou_diff)
 114                 {
 115                     matched = true;
 116                     matchedRefBoxes[j] = true;
 117                 }
 118             }
 119         }
 120         if (!matched)
 121             std::cout << cv::format("Unmatched prediction: class %d score %f box ",
 122                                     testClassId, testScore) << testBox << std::endl;
 123         EXPECT_TRUE(matched) << comment;
 124     }
 125
 126     // Check unmatched reference detections.
 127     for (int i = 0; i < refBoxes.size(); ++i)
 128     {
 129         if (!matchedRefBoxes[i] && refScores[i] > confThreshold)
 130         {
 131             std::cout << cv::format("Unmatched reference: class %d score %f box ",
 132                                     refClassIds[i], refScores[i]) << refBoxes[i] << std::endl;
 133             EXPECT_LE(refScores[i], confThreshold) << comment;
 134         }
 135     }
 136 }
 137
 138 // For SSD-based object detection networks which produce output of shape 1x1xNx7
 139 // where N is a number of detections and an every detection is represented by
 140 // a vector [batchId, classId, confidence, left, top, right, bottom].
 141 void normAssertDetections(
 142         cv::Mat ref, cv::Mat out, const char *comment /*= ""*/,
 143         double confThreshold /*= 0.0*/, double scores_diff /*= 1e-5*/,
 144         double boxes_iou_diff /*= 1e-4*/)
 145 {
 146     CV_Assert(ref.total() % 7 == 0);
 147     CV_Assert(out.total() % 7 == 0);
 148     ref = ref.reshape(1, ref.total() / 7);
 149     out = out.reshape(1, out.total() / 7);
 150
 151     cv::Mat refClassIds, testClassIds;
 152     ref.col(1).convertTo(refClassIds, CV_32SC1);
 153     out.col(1).convertTo(testClassIds, CV_32SC1);
 154     std::vector<float> refScores(ref.col(2)), testScores(out.col(2));
 155     std::vector<cv::Rect2d> refBoxes = matToBoxes(ref.colRange(3, 7));
 156     std::vector<cv::Rect2d> testBoxes = matToBoxes(out.colRange(3, 7));
 157     normAssertDetections(refClassIds, refScores, refBoxes, testClassIds, testScores,
 158                          testBoxes, comment, confThreshold, scores_diff, boxes_iou_diff);
 159 }
 160
 161 void readFileContent(const std::string& filename, CV_OUT std::vector<char>& content)
 162 {
 163     const std::ios::openmode mode = std::ios::in | std::ios::binary;
 164     std::ifstream ifs(filename.c_str(), mode);
 165     ASSERT_TRUE(ifs.is_open());
 166
 167     content.clear();
 168
 169     ifs.seekg(0, std::ios::end);
 170     const size_t sz = ifs.tellg();
 171     content.resize(sz);
 172     ifs.seekg(0, std::ios::beg);
 173
 174     ifs.read((char*)content.data(), sz);
 175     ASSERT_FALSE(ifs.fail());
 176 }
 177
 178
 179 testing::internal::ParamGenerator< tuple<Backend, Target> > dnnBackendsAndTargets(
 180         bool withInferenceEngine /*= true*/,
 181         bool withHalide /*= false*/,
 182         bool withCpuOCV /*= true*/
 183 )
 184 {
 185 #ifdef HAVE_INF_ENGINE
 186     bool withVPU = validateVPUType();
 187 #endif
 188
 189     std::vector< tuple<Backend, Target> > targets;
 190     std::vector< Target > available;
 191     if (withHalide)
 192     {
 193         available = getAvailableTargets(DNN_BACKEND_HALIDE);
 194         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 195             targets.push_back(make_tuple(DNN_BACKEND_HALIDE, *i));
 196     }
 197 #ifdef HAVE_INF_ENGINE
 198     if (withInferenceEngine)
 199     {
 200         available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE);
 201         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 202         {
 203             if (*i == DNN_TARGET_MYRIAD && !withVPU)
 204                 continue;
 205             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, *i));
 206         }
 207     }
 208 #else
 209     CV_UNUSED(withInferenceEngine);
 210 #endif
 211     {
 212         available = getAvailableTargets(DNN_BACKEND_OPENCV);
 213         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 214         {
 215             if (!withCpuOCV && *i == DNN_TARGET_CPU)
 216                 continue;
 217             targets.push_back(make_tuple(DNN_BACKEND_OPENCV, *i));
 218         }
 219     }
 220     if (targets.empty())  // validate at least CPU mode
 221         targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU));
 222     return testing::ValuesIn(targets);
 223 }
 224
 225
 226 #ifdef HAVE_INF_ENGINE
 227 static std::string getTestInferenceEngineVPUType()
 228 {
 229     static std::string param_vpu_type = utils::getConfigurationParameterString("OPENCV_TEST_DNN_IE_VPU_TYPE", "");
 230     return param_vpu_type;
 231 }
 232
 233 static bool validateVPUType_()
 234 {
 235     std::string test_vpu_type = getTestInferenceEngineVPUType();
 236     if (test_vpu_type == "DISABLED" || test_vpu_type == "disabled")
 237     {
 238         return false;
 239     }
 240
 241     std::vector<Target> available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE);
 242     bool have_vpu_target = false;
 243     for (std::vector<Target>::const_iterator i = available.begin(); i != available.end(); ++i)
 244     {
 245         if (*i == DNN_TARGET_MYRIAD)
 246         {
 247             have_vpu_target = true;
 248             break;
 249         }
 250     }
 251
 252     if (test_vpu_type.empty())
 253     {
 254         if (have_vpu_target)
 255         {
 256             CV_LOG_INFO(NULL, "OpenCV-DNN-Test: VPU type for testing is not specified via 'OPENCV_TEST_DNN_IE_VPU_TYPE' parameter.")
 257         }
 258     }
 259     else
 260     {
 261         if (!have_vpu_target)
 262         {
 263             CV_LOG_FATAL(NULL, "OpenCV-DNN-Test: 'OPENCV_TEST_DNN_IE_VPU_TYPE' parameter requires VPU of type = '" << test_vpu_type << "', but VPU is not detected. STOP.");
 264             exit(1);
 265         }
 266         std::string dnn_vpu_type = getInferenceEngineVPUType();
 267         if (dnn_vpu_type != test_vpu_type)
 268         {
 269             CV_LOG_FATAL(NULL, "OpenCV-DNN-Test: 'testing' and 'detected' VPU types mismatch: '" << test_vpu_type << "' vs '" << dnn_vpu_type << "'. STOP.");
 270             exit(1);
 271         }
 272     }
 273     return true;
 274 }
 275
 276 bool validateVPUType()
 277 {
 278     static bool result = validateVPUType_();
 279     return result;
 280 }
 281 #endif // HAVE_INF_ENGINE
 282
 283 } // namespace