modules/dnn/test/test_common.impl.hpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Used in accuracy and perf tests as a content of .cpp file
   6 // Note: don't use "precomp.hpp" here
   7 #include "opencv2/ts.hpp"
   8 #include "opencv2/ts/ts_perf.hpp"
   9 #include "opencv2/core/utility.hpp"
  10 #include "opencv2/core/ocl.hpp"
  11
  12 #include "opencv2/dnn.hpp"
  13 #include "test_common.hpp"
  14
  15 #include <opencv2/core/utils/configuration.private.hpp>
  16 #include <opencv2/core/utils/logger.hpp>
  17
  18 namespace cv { namespace dnn {
  19 CV__DNN_EXPERIMENTAL_NS_BEGIN
  20
  21 void PrintTo(const cv::dnn::Backend& v, std::ostream* os)
  22 {
  23     switch (v) {
  24     case DNN_BACKEND_DEFAULT: *os << "DEFAULT"; return;
  25     case DNN_BACKEND_HALIDE: *os << "HALIDE"; return;
  26     case DNN_BACKEND_INFERENCE_ENGINE: *os << "DLIE"; return;
  27     case DNN_BACKEND_OPENCV: *os << "OCV"; return;
  28     } // don't use "default:" to emit compiler warnings
  29     *os << "DNN_BACKEND_UNKNOWN(" << (int)v << ")";
  30 }
  31
  32 void PrintTo(const cv::dnn::Target& v, std::ostream* os)
  33 {
  34     switch (v) {
  35     case DNN_TARGET_CPU: *os << "CPU"; return;
  36     case DNN_TARGET_OPENCL: *os << "OCL"; return;
  37     case DNN_TARGET_OPENCL_FP16: *os << "OCL_FP16"; return;
  38     case DNN_TARGET_MYRIAD: *os << "MYRIAD"; return;
  39     case DNN_TARGET_FPGA: *os << "FPGA"; return;
  40     } // don't use "default:" to emit compiler warnings
  41     *os << "DNN_TARGET_UNKNOWN(" << (int)v << ")";
  42 }
  43
  44 void PrintTo(const tuple<cv::dnn::Backend, cv::dnn::Target> v, std::ostream* os)
  45 {
  46     PrintTo(get<0>(v), os);
  47     *os << "/";
  48     PrintTo(get<1>(v), os);
  49 }
  50
  51 CV__DNN_EXPERIMENTAL_NS_END
  52 }} // namespace
  53
  54
  55
  56 namespace opencv_test {
  57
  58 void normAssert(
  59         cv::InputArray ref, cv::InputArray test, const char *comment /*= ""*/,
  60         double l1 /*= 0.00001*/, double lInf /*= 0.0001*/)
  61 {
  62     double normL1 = cvtest::norm(ref, test, cv::NORM_L1) / ref.getMat().total();
  63     EXPECT_LE(normL1, l1) << comment;
  64
  65     double normInf = cvtest::norm(ref, test, cv::NORM_INF);
  66     EXPECT_LE(normInf, lInf) << comment;
  67 }
  68
  69 std::vector<cv::Rect2d> matToBoxes(const cv::Mat& m)
  70 {
  71     EXPECT_EQ(m.type(), CV_32FC1);
  72     EXPECT_EQ(m.dims, 2);
  73     EXPECT_EQ(m.cols, 4);
  74
  75     std::vector<cv::Rect2d> boxes(m.rows);
  76     for (int i = 0; i < m.rows; ++i)
  77     {
  78         CV_Assert(m.row(i).isContinuous());
  79         const float* data = m.ptr<float>(i);
  80         double l = data[0], t = data[1], r = data[2], b = data[3];
  81         boxes[i] = cv::Rect2d(l, t, r - l, b - t);
  82     }
  83     return boxes;
  84 }
  85
  86 void normAssertDetections(
  87         const std::vector<int>& refClassIds,
  88         const std::vector<float>& refScores,
  89         const std::vector<cv::Rect2d>& refBoxes,
  90         const std::vector<int>& testClassIds,
  91         const std::vector<float>& testScores,
  92         const std::vector<cv::Rect2d>& testBoxes,
  93         const char *comment /*= ""*/, double confThreshold /*= 0.0*/,
  94         double scores_diff /*= 1e-5*/, double boxes_iou_diff /*= 1e-4*/)
  95 {
  96     std::vector<bool> matchedRefBoxes(refBoxes.size(), false);
  97     for (int i = 0; i < testBoxes.size(); ++i)
  98     {
  99         double testScore = testScores[i];
 100         if (testScore < confThreshold)
 101             continue;
 102
 103         int testClassId = testClassIds[i];
 104         const cv::Rect2d& testBox = testBoxes[i];
 105         bool matched = false;
 106         for (int j = 0; j < refBoxes.size() && !matched; ++j)
 107         {
 108             if (!matchedRefBoxes[j] && testClassId == refClassIds[j] &&
 109                 std::abs(testScore - refScores[j]) < scores_diff)
 110             {
 111                 double interArea = (testBox & refBoxes[j]).area();
 112                 double iou = interArea / (testBox.area() + refBoxes[j].area() - interArea);
 113                 if (std::abs(iou - 1.0) < boxes_iou_diff)
 114                 {
 115                     matched = true;
 116                     matchedRefBoxes[j] = true;
 117                 }
 118             }
 119         }
 120         if (!matched)
 121             std::cout << cv::format("Unmatched prediction: class %d score %f box ",
 122                                     testClassId, testScore) << testBox << std::endl;
 123         EXPECT_TRUE(matched) << comment;
 124     }
 125
 126     // Check unmatched reference detections.
 127     for (int i = 0; i < refBoxes.size(); ++i)
 128     {
 129         if (!matchedRefBoxes[i] && refScores[i] > confThreshold)
 130         {
 131             std::cout << cv::format("Unmatched reference: class %d score %f box ",
 132                                     refClassIds[i], refScores[i]) << refBoxes[i] << std::endl;
 133             EXPECT_LE(refScores[i], confThreshold) << comment;
 134         }
 135     }
 136 }
 137
 138 // For SSD-based object detection networks which produce output of shape 1x1xNx7
 139 // where N is a number of detections and an every detection is represented by
 140 // a vector [batchId, classId, confidence, left, top, right, bottom].
 141 void normAssertDetections(
 142         cv::Mat ref, cv::Mat out, const char *comment /*= ""*/,
 143         double confThreshold /*= 0.0*/, double scores_diff /*= 1e-5*/,
 144         double boxes_iou_diff /*= 1e-4*/)
 145 {
 146     CV_Assert(ref.total() % 7 == 0);
 147     CV_Assert(out.total() % 7 == 0);
 148     ref = ref.reshape(1, ref.total() / 7);
 149     out = out.reshape(1, out.total() / 7);
 150
 151     cv::Mat refClassIds, testClassIds;
 152     ref.col(1).convertTo(refClassIds, CV_32SC1);
 153     out.col(1).convertTo(testClassIds, CV_32SC1);
 154     std::vector<float> refScores(ref.col(2)), testScores(out.col(2));
 155     std::vector<cv::Rect2d> refBoxes = matToBoxes(ref.colRange(3, 7));
 156     std::vector<cv::Rect2d> testBoxes = matToBoxes(out.colRange(3, 7));
 157     normAssertDetections(refClassIds, refScores, refBoxes, testClassIds, testScores,
 158                          testBoxes, comment, confThreshold, scores_diff, boxes_iou_diff);
 159 }
 160
 161 bool readFileInMemory(const std::string& filename, std::string& content)
 162 {
 163     std::ios::openmode mode = std::ios::in | std::ios::binary;
 164     std::ifstream ifs(filename.c_str(), mode);
 165     if (!ifs.is_open())
 166         return false;
 167
 168     content.clear();
 169
 170     ifs.seekg(0, std::ios::end);
 171     content.reserve(ifs.tellg());
 172     ifs.seekg(0, std::ios::beg);
 173
 174     content.assign((std::istreambuf_iterator<char>(ifs)),
 175                    std::istreambuf_iterator<char>());
 176
 177     return true;
 178 }
 179
 180
 181 testing::internal::ParamGenerator< tuple<Backend, Target> > dnnBackendsAndTargets(
 182         bool withInferenceEngine /*= true*/,
 183         bool withHalide /*= false*/,
 184         bool withCpuOCV /*= true*/
 185 )
 186 {
 187 #ifdef HAVE_INF_ENGINE
 188     bool withVPU = validateVPUType();
 189 #endif
 190
 191     std::vector< tuple<Backend, Target> > targets;
 192     std::vector< Target > available;
 193     if (withHalide)
 194     {
 195         available = getAvailableTargets(DNN_BACKEND_HALIDE);
 196         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 197             targets.push_back(make_tuple(DNN_BACKEND_HALIDE, *i));
 198     }
 199 #ifdef HAVE_INF_ENGINE
 200     if (withInferenceEngine)
 201     {
 202         available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE);
 203         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 204         {
 205             if (*i == DNN_TARGET_MYRIAD && !withVPU)
 206                 continue;
 207             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, *i));
 208         }
 209     }
 210 #else
 211     CV_UNUSED(withInferenceEngine);
 212 #endif
 213     {
 214         available = getAvailableTargets(DNN_BACKEND_OPENCV);
 215         for (std::vector< Target >::const_iterator i = available.begin(); i != available.end(); ++i)
 216         {
 217             if (!withCpuOCV && *i == DNN_TARGET_CPU)
 218                 continue;
 219             targets.push_back(make_tuple(DNN_BACKEND_OPENCV, *i));
 220         }
 221     }
 222     if (targets.empty())  // validate at least CPU mode
 223         targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU));
 224     return testing::ValuesIn(targets);
 225 }
 226
 227
 228 #ifdef HAVE_INF_ENGINE
 229 static std::string getTestInferenceEngineVPUType()
 230 {
 231     static std::string param_vpu_type = utils::getConfigurationParameterString("OPENCV_TEST_DNN_IE_VPU_TYPE", "");
 232     return param_vpu_type;
 233 }
 234
 235 static bool validateVPUType_()
 236 {
 237     std::string test_vpu_type = getTestInferenceEngineVPUType();
 238     if (test_vpu_type == "DISABLED" || test_vpu_type == "disabled")
 239     {
 240         return false;
 241     }
 242
 243     std::vector<Target> available = getAvailableTargets(DNN_BACKEND_INFERENCE_ENGINE);
 244     bool have_vpu_target = false;
 245     for (std::vector<Target>::const_iterator i = available.begin(); i != available.end(); ++i)
 246     {
 247         if (*i == DNN_TARGET_MYRIAD)
 248         {
 249             have_vpu_target = true;
 250             break;
 251         }
 252     }
 253
 254     if (test_vpu_type.empty())
 255     {
 256         if (have_vpu_target)
 257         {
 258             CV_LOG_INFO(NULL, "OpenCV-DNN-Test: VPU type for testing is not specified via 'OPENCV_TEST_DNN_IE_VPU_TYPE' parameter.")
 259         }
 260     }
 261     else
 262     {
 263         if (!have_vpu_target)
 264         {
 265             CV_LOG_FATAL(NULL, "OpenCV-DNN-Test: 'OPENCV_TEST_DNN_IE_VPU_TYPE' parameter requires VPU of type = '" << test_vpu_type << "', but VPU is not detected. STOP.");
 266             exit(1);
 267         }
 268         std::string dnn_vpu_type = getInferenceEngineVPUType();
 269         if (dnn_vpu_type != test_vpu_type)
 270         {
 271             CV_LOG_FATAL(NULL, "OpenCV-DNN-Test: 'testing' and 'detected' VPU types mismatch: '" << test_vpu_type << "' vs '" << dnn_vpu_type << "'. STOP.");
 272             exit(1);
 273         }
 274     }
 275     return true;
 276 }
 277
 278 bool validateVPUType()
 279 {
 280     static bool result = validateVPUType_();
 281     return result;
 282 }
 283 #endif // HAVE_INF_ENGINE
 284
 285 } // namespace