modules/dnn/test/test_common.hpp

   1 /*M///////////////////////////////////////////////////////////////////////////////////////
   2 //
   3 //  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
   4 //
   5 //  By downloading, copying, installing or using the software you agree to this license.
   6 //  If you do not agree to this license, do not download, install,
   7 //  copy or use the software.
   8 //
   9 //
  10 //                           License Agreement
  11 //                For Open Source Computer Vision Library
  12 //
  13 // Copyright (C) 2013, OpenCV Foundation, all rights reserved.
  14 // Third party copyrights are property of their respective owners.
  15 //
  16 // Redistribution and use in source and binary forms, with or without modification,
  17 // are permitted provided that the following conditions are met:
  18 //
  19 //   * Redistribution's of source code must retain the above copyright notice,
  20 //     this list of conditions and the following disclaimer.
  21 //
  22 //   * Redistribution's in binary form must reproduce the above copyright notice,
  23 //     this list of conditions and the following disclaimer in the documentation
  24 //     and/or other materials provided with the distribution.
  25 //
  26 //   * The name of the copyright holders may not be used to endorse or promote products
  27 //     derived from this software without specific prior written permission.
  28 //
  29 // This software is provided by the copyright holders and contributors "as is" and
  30 // any express or implied warranties, including, but not limited to, the implied
  31 // warranties of merchantability and fitness for a particular purpose are disclaimed.
  32 // In no event shall the Intel Corporation or contributors be liable for any direct,
  33 // indirect, incidental, special, exemplary, or consequential damages
  34 // (including, but not limited to, procurement of substitute goods or services;
  35 // loss of use, data, or profits; or business interruption) however caused
  36 // and on any theory of liability, whether in contract, strict liability,
  37 // or tort (including negligence or otherwise) arising in any way out of
  38 // the use of this software, even if advised of the possibility of such damage.
  39 //
  40 //M*/
  41
  42 #ifndef __OPENCV_TEST_COMMON_HPP__
  43 #define __OPENCV_TEST_COMMON_HPP__
  44
  45 #ifdef HAVE_OPENCL
  46 #include "opencv2/core/ocl.hpp"
  47 #endif
  48
  49 namespace cv { namespace dnn {
  50 CV__DNN_EXPERIMENTAL_NS_BEGIN
  51 static inline void PrintTo(const cv::dnn::Backend& v, std::ostream* os)
  52 {
  53     switch (v) {
  54     case DNN_BACKEND_DEFAULT: *os << "DEFAULT"; return;
  55     case DNN_BACKEND_HALIDE: *os << "HALIDE"; return;
  56     case DNN_BACKEND_INFERENCE_ENGINE: *os << "DLIE"; return;
  57     case DNN_BACKEND_OPENCV: *os << "OCV"; return;
  58     } // don't use "default:" to emit compiler warnings
  59     *os << "DNN_BACKEND_UNKNOWN(" << (int)v << ")";
  60 }
  61
  62 static inline void PrintTo(const cv::dnn::Target& v, std::ostream* os)
  63 {
  64     switch (v) {
  65     case DNN_TARGET_CPU: *os << "CPU"; return;
  66     case DNN_TARGET_OPENCL: *os << "OCL"; return;
  67     case DNN_TARGET_OPENCL_FP16: *os << "OCL_FP16"; return;
  68     case DNN_TARGET_MYRIAD: *os << "MYRIAD"; return;
  69     } // don't use "default:" to emit compiler warnings
  70     *os << "DNN_TARGET_UNKNOWN(" << (int)v << ")";
  71 }
  72
  73 using opencv_test::tuple;
  74 using opencv_test::get;
  75 static inline void PrintTo(const tuple<cv::dnn::Backend, cv::dnn::Target> v, std::ostream* os)
  76 {
  77     PrintTo(get<0>(v), os);
  78     *os << "/";
  79     PrintTo(get<1>(v), os);
  80 }
  81
  82 CV__DNN_EXPERIMENTAL_NS_END
  83 }} // namespace
  84
  85
  86 static inline const std::string &getOpenCVExtraDir()
  87 {
  88     return cvtest::TS::ptr()->get_data_path();
  89 }
  90
  91 static inline void normAssert(cv::InputArray ref, cv::InputArray test, const char *comment = "",
  92                        double l1 = 0.00001, double lInf = 0.0001)
  93 {
  94     double normL1 = cvtest::norm(ref, test, cv::NORM_L1) / ref.getMat().total();
  95     EXPECT_LE(normL1, l1) << comment;
  96
  97     double normInf = cvtest::norm(ref, test, cv::NORM_INF);
  98     EXPECT_LE(normInf, lInf) << comment;
  99 }
 100
 101 static std::vector<cv::Rect2d> matToBoxes(const cv::Mat& m)
 102 {
 103     EXPECT_EQ(m.type(), CV_32FC1);
 104     EXPECT_EQ(m.dims, 2);
 105     EXPECT_EQ(m.cols, 4);
 106
 107     std::vector<cv::Rect2d> boxes(m.rows);
 108     for (int i = 0; i < m.rows; ++i)
 109     {
 110         CV_Assert(m.row(i).isContinuous());
 111         const float* data = m.ptr<float>(i);
 112         double l = data[0], t = data[1], r = data[2], b = data[3];
 113         boxes[i] = cv::Rect2d(l, t, r - l, b - t);
 114     }
 115     return boxes;
 116 }
 117
 118 static inline void normAssertDetections(const std::vector<int>& refClassIds,
 119                                  const std::vector<float>& refScores,
 120                                  const std::vector<cv::Rect2d>& refBoxes,
 121                                  const std::vector<int>& testClassIds,
 122                                  const std::vector<float>& testScores,
 123                                  const std::vector<cv::Rect2d>& testBoxes,
 124                                  const char *comment = "", double confThreshold = 0.0,
 125                                  double scores_diff = 1e-5, double boxes_iou_diff = 1e-4)
 126 {
 127     std::vector<bool> matchedRefBoxes(refBoxes.size(), false);
 128     for (int i = 0; i < testBoxes.size(); ++i)
 129     {
 130         double testScore = testScores[i];
 131         if (testScore < confThreshold)
 132             continue;
 133
 134         int testClassId = testClassIds[i];
 135         const cv::Rect2d& testBox = testBoxes[i];
 136         bool matched = false;
 137         for (int j = 0; j < refBoxes.size() && !matched; ++j)
 138         {
 139             if (!matchedRefBoxes[j] && testClassId == refClassIds[j] &&
 140                 std::abs(testScore - refScores[j]) < scores_diff)
 141             {
 142                 double interArea = (testBox & refBoxes[j]).area();
 143                 double iou = interArea / (testBox.area() + refBoxes[j].area() - interArea);
 144                 if (std::abs(iou - 1.0) < boxes_iou_diff)
 145                 {
 146                     matched = true;
 147                     matchedRefBoxes[j] = true;
 148                 }
 149             }
 150         }
 151         if (!matched)
 152             std::cout << cv::format("Unmatched prediction: class %d score %f box ",
 153                                     testClassId, testScore) << testBox << std::endl;
 154         EXPECT_TRUE(matched) << comment;
 155     }
 156
 157     // Check unmatched reference detections.
 158     for (int i = 0; i < refBoxes.size(); ++i)
 159     {
 160         if (!matchedRefBoxes[i] && refScores[i] > confThreshold)
 161         {
 162             std::cout << cv::format("Unmatched reference: class %d score %f box ",
 163                                     refClassIds[i], refScores[i]) << refBoxes[i] << std::endl;
 164             EXPECT_LE(refScores[i], confThreshold) << comment;
 165         }
 166     }
 167 }
 168
 169 // For SSD-based object detection networks which produce output of shape 1x1xNx7
 170 // where N is a number of detections and an every detection is represented by
 171 // a vector [batchId, classId, confidence, left, top, right, bottom].
 172 static inline void normAssertDetections(cv::Mat ref, cv::Mat out, const char *comment = "",
 173                                  double confThreshold = 0.0, double scores_diff = 1e-5,
 174                                  double boxes_iou_diff = 1e-4)
 175 {
 176     CV_Assert(ref.total() % 7 == 0);
 177     CV_Assert(out.total() % 7 == 0);
 178     ref = ref.reshape(1, ref.total() / 7);
 179     out = out.reshape(1, out.total() / 7);
 180
 181     cv::Mat refClassIds, testClassIds;
 182     ref.col(1).convertTo(refClassIds, CV_32SC1);
 183     out.col(1).convertTo(testClassIds, CV_32SC1);
 184     std::vector<float> refScores(ref.col(2)), testScores(out.col(2));
 185     std::vector<cv::Rect2d> refBoxes = matToBoxes(ref.colRange(3, 7));
 186     std::vector<cv::Rect2d> testBoxes = matToBoxes(out.colRange(3, 7));
 187     normAssertDetections(refClassIds, refScores, refBoxes, testClassIds, testScores,
 188                          testBoxes, comment, confThreshold, scores_diff, boxes_iou_diff);
 189 }
 190
 191 static inline bool checkMyriadTarget()
 192 {
 193 #ifndef HAVE_INF_ENGINE
 194     return false;
 195 #else
 196     cv::dnn::Net net;
 197     cv::dnn::LayerParams lp;
 198     net.addLayerToPrev("testLayer", "Identity", lp);
 199     net.setPreferableBackend(cv::dnn::DNN_BACKEND_INFERENCE_ENGINE);
 200     net.setPreferableTarget(cv::dnn::DNN_TARGET_MYRIAD);
 201     static int inpDims[] = {1, 2, 3, 4};
 202     net.setInput(cv::Mat(4, &inpDims[0], CV_32FC1, cv::Scalar(0)));
 203     try
 204     {
 205         net.forward();
 206     }
 207     catch(...)
 208     {
 209         return false;
 210     }
 211     return true;
 212 #endif
 213 }
 214
 215 static inline bool readFileInMemory(const std::string& filename, std::string& content)
 216 {
 217     std::ios::openmode mode = std::ios::in | std::ios::binary;
 218     std::ifstream ifs(filename.c_str(), mode);
 219     if (!ifs.is_open())
 220         return false;
 221
 222     content.clear();
 223
 224     ifs.seekg(0, std::ios::end);
 225     content.reserve(ifs.tellg());
 226     ifs.seekg(0, std::ios::beg);
 227
 228     content.assign((std::istreambuf_iterator<char>(ifs)),
 229                    std::istreambuf_iterator<char>());
 230
 231     return true;
 232 }
 233
 234 namespace opencv_test {
 235
 236 using namespace cv::dnn;
 237
 238 static inline
 239 testing::internal::ParamGenerator<tuple<Backend, Target> > dnnBackendsAndTargets(
 240         bool withInferenceEngine = true,
 241         bool withHalide = false,
 242         bool withCpuOCV = true
 243 )
 244 {
 245     std::vector<tuple<Backend, Target> > targets;
 246 #ifdef HAVE_HALIDE
 247     if (withHalide)
 248     {
 249         targets.push_back(make_tuple(DNN_BACKEND_HALIDE, DNN_TARGET_CPU));
 250 #ifdef HAVE_OPENCL
 251         if (cv::ocl::useOpenCL())
 252             targets.push_back(make_tuple(DNN_BACKEND_HALIDE, DNN_TARGET_OPENCL));
 253 #endif
 254     }
 255 #endif
 256 #ifdef HAVE_INF_ENGINE
 257     if (withInferenceEngine)
 258     {
 259         targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_CPU));
 260 #ifdef HAVE_OPENCL
 261         if (cv::ocl::useOpenCL() && ocl::Device::getDefault().isIntel())
 262         {
 263             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL));
 264             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL_FP16));
 265         }
 266 #endif
 267         if (checkMyriadTarget())
 268             targets.push_back(make_tuple(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_MYRIAD));
 269     }
 270 #endif
 271     if (withCpuOCV)
 272         targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU));
 273 #ifdef HAVE_OPENCL
 274     if (cv::ocl::useOpenCL())
 275     {
 276         targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_OPENCL));
 277         targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_OPENCL_FP16));
 278     }
 279 #endif
 280     if (targets.empty())  // validate at least CPU mode
 281         targets.push_back(make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU));
 282     return testing::ValuesIn(targets);
 283 }
 284
 285 } // namespace
 286
 287
 288 namespace opencv_test {
 289 using namespace cv::dnn;
 290
 291 static inline
 292 testing::internal::ParamGenerator<Target> availableDnnTargets()
 293 {
 294     static std::vector<Target> targets;
 295     if (targets.empty())
 296     {
 297         targets.push_back(DNN_TARGET_CPU);
 298 #ifdef HAVE_OPENCL
 299         if (cv::ocl::useOpenCL())
 300             targets.push_back(DNN_TARGET_OPENCL);
 301 #endif
 302     }
 303     return testing::ValuesIn(targets);
 304 }
 305
 306 class DNNTestLayer : public TestWithParam<tuple<Backend, Target> >
 307 {
 308 public:
 309     dnn::Backend backend;
 310     dnn::Target target;
 311     double default_l1, default_lInf;
 312
 313     DNNTestLayer()
 314     {
 315         backend = (dnn::Backend)(int)get<0>(GetParam());
 316         target = (dnn::Target)(int)get<1>(GetParam());
 317         getDefaultThresholds(backend, target, &default_l1, &default_lInf);
 318     }
 319
 320    static void getDefaultThresholds(int backend, int target, double* l1, double* lInf)
 321    {
 322        if (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD)
 323        {
 324            *l1 = 4e-3;
 325            *lInf = 2e-2;
 326        }
 327        else
 328        {
 329            *l1 = 1e-5;
 330            *lInf = 1e-4;
 331        }
 332    }
 333
 334    static void checkBackend(int backend, int target, Mat* inp = 0, Mat* ref = 0)
 335    {
 336        if (backend == DNN_BACKEND_OPENCV && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 337        {
 338 #ifdef HAVE_OPENCL
 339            if (!cv::ocl::useOpenCL())
 340 #endif
 341            {
 342                throw SkipTestException("OpenCL is not available/disabled in OpenCV");
 343            }
 344        }
 345        if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 346        {
 347            if (!checkMyriadTarget())
 348            {
 349                throw SkipTestException("Myriad is not available/disabled in OpenCV");
 350            }
 351 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE < 2018030000
 352            if (inp && ref && inp->size[0] != 1)
 353            {
 354                // Myriad plugin supports only batch size 1. Slice a single sample.
 355                if (inp->size[0] == ref->size[0])
 356                {
 357                    std::vector<cv::Range> range(inp->dims, Range::all());
 358                    range[0] = Range(0, 1);
 359                    *inp = inp->operator()(range);
 360
 361                    range = std::vector<cv::Range>(ref->dims, Range::all());
 362                    range[0] = Range(0, 1);
 363                    *ref = ref->operator()(range);
 364                }
 365                else
 366                    throw SkipTestException("Myriad plugin supports only batch size 1");
 367            }
 368 #else
 369            if (inp && ref && inp->dims == 4 && ref->dims == 4 &&
 370                inp->size[0] != 1 && inp->size[0] != ref->size[0])
 371                throw SkipTestException("Inconsistent batch size of input and output blobs for Myriad plugin");
 372
 373 #endif
 374        }
 375    }
 376
 377 protected:
 378     void checkBackend(Mat* inp = 0, Mat* ref = 0)
 379     {
 380         checkBackend(backend, target, inp, ref);
 381     }
 382 };
 383
 384 } // namespace
 385
 386 #endif