modules/dnn/perf/perf_net.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4 //
   5 // Copyright (C) 2017, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 #include "perf_precomp.hpp"
   9 #include "opencv2/core/ocl.hpp"
  10
  11 #include "opencv2/dnn/shape_utils.hpp"
  12
  13 #include "../test/test_common.hpp"
  14
  15 namespace opencv_test {
  16
  17 class DNNTestNetwork : public ::perf::TestBaseWithParam< tuple<Backend, Target> >
  18 {
  19 public:
  20     dnn::Backend backend;
  21     dnn::Target target;
  22
  23     dnn::Net net;
  24
  25     DNNTestNetwork()
  26     {
  27         backend = (dnn::Backend)(int)get<0>(GetParam());
  28         target = (dnn::Target)(int)get<1>(GetParam());
  29     }
  30
  31     void processNet(std::string weights, std::string proto, std::string halide_scheduler,
  32                     const Mat& input, const std::string& outputLayer = "")
  33     {
  34         randu(input, 0.0f, 1.0f);
  35
  36         weights = findDataFile(weights, false);
  37         if (!proto.empty())
  38             proto = findDataFile(proto);
  39         if (backend == DNN_BACKEND_HALIDE)
  40         {
  41             if (halide_scheduler == "disabled")
  42                 throw cvtest::SkipTestException("Halide test is disabled");
  43             if (!halide_scheduler.empty())
  44                 halide_scheduler = findDataFile(std::string("dnn/halide_scheduler_") + (target == DNN_TARGET_OPENCL ? "opencl_" : "") + halide_scheduler, true);
  45         }
  46         net = readNet(proto, weights);
  47         net.setInput(blobFromImage(input, 1.0, Size(), Scalar(), false));
  48         net.setPreferableBackend(backend);
  49         net.setPreferableTarget(target);
  50         if (backend == DNN_BACKEND_HALIDE)
  51         {
  52             net.setHalideScheduler(halide_scheduler);
  53         }
  54
  55         MatShape netInputShape = shape(1, 3, input.rows, input.cols);
  56         size_t weightsMemory = 0, blobsMemory = 0;
  57         net.getMemoryConsumption(netInputShape, weightsMemory, blobsMemory);
  58         int64 flops = net.getFLOPS(netInputShape);
  59         CV_Assert(flops > 0);
  60
  61         net.forward(outputLayer); // warmup
  62
  63         std::cout << "Memory consumption:" << std::endl;
  64         std::cout << "    Weights(parameters): " << divUp(weightsMemory, 1u<<20) << " Mb" << std::endl;
  65         std::cout << "    Blobs: " << divUp(blobsMemory, 1u<<20) << " Mb" << std::endl;
  66         std::cout << "Calculation complexity: " << flops * 1e-9 << " GFlops" << std::endl;
  67
  68         PERF_SAMPLE_BEGIN()
  69             net.forward();
  70         PERF_SAMPLE_END()
  71
  72         SANITY_CHECK_NOTHING();
  73     }
  74 };
  75
  76
  77 PERF_TEST_P_(DNNTestNetwork, AlexNet)
  78 {
  79     processNet("dnn/bvlc_alexnet.caffemodel", "dnn/bvlc_alexnet.prototxt",
  80             "alexnet.yml", Mat(cv::Size(227, 227), CV_32FC3));
  81 }
  82
  83 PERF_TEST_P_(DNNTestNetwork, GoogLeNet)
  84 {
  85     processNet("dnn/bvlc_googlenet.caffemodel", "dnn/bvlc_googlenet.prototxt",
  86             "", Mat(cv::Size(224, 224), CV_32FC3));
  87 }
  88
  89 PERF_TEST_P_(DNNTestNetwork, ResNet_50)
  90 {
  91     processNet("dnn/ResNet-50-model.caffemodel", "dnn/ResNet-50-deploy.prototxt",
  92             "resnet_50.yml", Mat(cv::Size(224, 224), CV_32FC3));
  93 }
  94
  95 PERF_TEST_P_(DNNTestNetwork, SqueezeNet_v1_1)
  96 {
  97     processNet("dnn/squeezenet_v1.1.caffemodel", "dnn/squeezenet_v1.1.prototxt",
  98             "squeezenet_v1_1.yml", Mat(cv::Size(227, 227), CV_32FC3));
  99 }
 100
 101 PERF_TEST_P_(DNNTestNetwork, Inception_5h)
 102 {
 103     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019) throw SkipTestException("");
 104     processNet("dnn/tensorflow_inception_graph.pb", "",
 105             "inception_5h.yml",
 106             Mat(cv::Size(224, 224), CV_32FC3), "softmax2");
 107 }
 108
 109 PERF_TEST_P_(DNNTestNetwork, ENet)
 110 {
 111     if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) ||
 112         (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16))
 113         throw SkipTestException("");
 114 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000)
 115     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
 116         throw SkipTestException("");
 117 #endif
 118     processNet("dnn/Enet-model-best.net", "", "enet.yml",
 119             Mat(cv::Size(512, 256), CV_32FC3));
 120 }
 121
 122 PERF_TEST_P_(DNNTestNetwork, SSD)
 123 {
 124     processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel", "dnn/ssd_vgg16.prototxt", "disabled",
 125             Mat(cv::Size(300, 300), CV_32FC3));
 126 }
 127
 128 PERF_TEST_P_(DNNTestNetwork, OpenFace)
 129 {
 130     if (backend == DNN_BACKEND_HALIDE)
 131         throw SkipTestException("");
 132 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 133     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_HDDL))
 134         throw SkipTestException("");
 135 #endif
 136     processNet("dnn/openface_nn4.small2.v1.t7", "", "",
 137             Mat(cv::Size(96, 96), CV_32FC3));
 138 }
 139
 140 PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_Caffe)
 141 {
 142     if (backend == DNN_BACKEND_HALIDE)
 143         throw SkipTestException("");
 144     processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt", "",
 145             Mat(cv::Size(300, 300), CV_32FC3));
 146 }
 147
 148 PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_v1_TensorFlow)
 149 {
 150     if (backend == DNN_BACKEND_HALIDE)
 151         throw SkipTestException("");
 152     processNet("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", "ssd_mobilenet_v1_coco_2017_11_17.pbtxt", "",
 153             Mat(cv::Size(300, 300), CV_32FC3));
 154 }
 155
 156 PERF_TEST_P_(DNNTestNetwork, MobileNet_SSD_v2_TensorFlow)
 157 {
 158     if (backend == DNN_BACKEND_HALIDE)
 159         throw SkipTestException("");
 160     processNet("dnn/ssd_mobilenet_v2_coco_2018_03_29.pb", "ssd_mobilenet_v2_coco_2018_03_29.pbtxt", "",
 161             Mat(cv::Size(300, 300), CV_32FC3));
 162 }
 163
 164 PERF_TEST_P_(DNNTestNetwork, DenseNet_121)
 165 {
 166     if (backend == DNN_BACKEND_HALIDE)
 167         throw SkipTestException("");
 168     processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", "",
 169                Mat(cv::Size(224, 224), CV_32FC3));
 170 }
 171
 172 PERF_TEST_P_(DNNTestNetwork, OpenPose_pose_mpi_faster_4_stages)
 173 {
 174     if (backend == DNN_BACKEND_HALIDE ||
 175         (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && (target == DNN_TARGET_MYRIAD || target == DNN_TARGET_HDDL)))
 176         throw SkipTestException("");
 177     // The same .caffemodel but modified .prototxt
 178     // See https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/pose/poseParameters.cpp
 179     processNet("dnn/openpose_pose_mpi.caffemodel", "dnn/openpose_pose_mpi_faster_4_stages.prototxt", "",
 180                Mat(cv::Size(368, 368), CV_32FC3));
 181 }
 182
 183 PERF_TEST_P_(DNNTestNetwork, opencv_face_detector)
 184 {
 185     if (backend == DNN_BACKEND_HALIDE)
 186         throw SkipTestException("");
 187     processNet("dnn/opencv_face_detector.caffemodel", "dnn/opencv_face_detector.prototxt", "",
 188                Mat(cv::Size(300, 300), CV_32FC3));
 189 }
 190
 191 PERF_TEST_P_(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
 192 {
 193     if (backend == DNN_BACKEND_HALIDE)
 194         throw SkipTestException("");
 195     processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "ssd_inception_v2_coco_2017_11_17.pbtxt", "",
 196             Mat(cv::Size(300, 300), CV_32FC3));
 197 }
 198
 199 PERF_TEST_P_(DNNTestNetwork, YOLOv3)
 200 {
 201     applyTestTag(CV_TEST_TAG_MEMORY_2GB);
 202     if (backend == DNN_BACKEND_HALIDE)
 203         throw SkipTestException("");
 204 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
 205     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
 206         throw SkipTestException("Test is disabled in OpenVINO 2020.4");
 207     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 208         throw SkipTestException("Test is disabled in OpenVINO 2020.4");
 209 #endif
 210 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000)  // nGraph compilation failure
 211     if (target == DNN_TARGET_MYRIAD)
 212         throw SkipTestException("");
 213 #endif
 214
 215     Mat sample = imread(findDataFile("dnn/dog416.png"));
 216     cvtColor(sample, sample, COLOR_BGR2RGB);
 217     Mat inp;
 218     sample.convertTo(inp, CV_32FC3, 1.0f / 255, 0);
 219     processNet("dnn/yolov3.weights", "dnn/yolov3.cfg", "", inp);
 220 }
 221
 222 PERF_TEST_P_(DNNTestNetwork, YOLOv4)
 223 {
 224     applyTestTag(CV_TEST_TAG_MEMORY_2GB);
 225     if (backend == DNN_BACKEND_HALIDE)
 226         throw SkipTestException("");
 227     if (target == DNN_TARGET_MYRIAD)  // not enough resources
 228         throw SkipTestException("");
 229 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)  // nGraph compilation failure
 230     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
 231         throw SkipTestException("Test is disabled in OpenVINO 2020.4");
 232     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
 233         throw SkipTestException("Test is disabled in OpenVINO 2020.4");
 234 #endif
 235     Mat sample = imread(findDataFile("dnn/dog416.png"));
 236     cvtColor(sample, sample, COLOR_BGR2RGB);
 237     Mat inp;
 238     sample.convertTo(inp, CV_32FC3, 1.0f / 255, 0);
 239     processNet("dnn/yolov4.weights", "dnn/yolov4.cfg", "", inp);
 240 }
 241
 242 PERF_TEST_P_(DNNTestNetwork, YOLOv4_tiny)
 243 {
 244     if (backend == DNN_BACKEND_HALIDE)
 245         throw SkipTestException("");
 246 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000)  // nGraph compilation failure
 247     if (target == DNN_TARGET_MYRIAD)
 248         throw SkipTestException("");
 249 #endif
 250     Mat sample = imread(findDataFile("dnn/dog416.png"));
 251     cvtColor(sample, sample, COLOR_BGR2RGB);
 252     Mat inp;
 253     sample.convertTo(inp, CV_32FC3, 1.0f / 255, 0);
 254     processNet("dnn/yolov4-tiny.weights", "dnn/yolov4-tiny.cfg", "", inp);
 255 }
 256
 257 PERF_TEST_P_(DNNTestNetwork, EAST_text_detection)
 258 {
 259     if (backend == DNN_BACKEND_HALIDE)
 260         throw SkipTestException("");
 261     processNet("dnn/frozen_east_text_detection.pb", "", "", Mat(cv::Size(320, 320), CV_32FC3));
 262 }
 263
 264 PERF_TEST_P_(DNNTestNetwork, FastNeuralStyle_eccv16)
 265 {
 266     if (backend == DNN_BACKEND_HALIDE)
 267         throw SkipTestException("");
 268     processNet("dnn/fast_neural_style_eccv16_starry_night.t7", "", "", Mat(cv::Size(320, 240), CV_32FC3));
 269 }
 270
 271 PERF_TEST_P_(DNNTestNetwork, Inception_v2_Faster_RCNN)
 272 {
 273 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019010000)
 274     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
 275         throw SkipTestException("Test is disabled in OpenVINO 2019R1");
 276 #endif
 277 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000)
 278     if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
 279         throw SkipTestException("Test is disabled in OpenVINO 2019R2");
 280 #endif
 281 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2021010000)
 282     if (target == DNN_TARGET_MYRIAD)
 283         throw SkipTestException("Test is disabled in OpenVINO 2021.1+ / MYRIAD");
 284 #endif
 285     if (backend == DNN_BACKEND_HALIDE ||
 286         (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target != DNN_TARGET_CPU) ||
 287         (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16))
 288         throw SkipTestException("");
 289     processNet("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb",
 290                "dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt", "",
 291                Mat(cv::Size(800, 600), CV_32FC3));
 292 }
 293
 294 PERF_TEST_P_(DNNTestNetwork, EfficientDet)
 295 {
 296     if (backend == DNN_BACKEND_HALIDE || target != DNN_TARGET_CPU)
 297         throw SkipTestException("");
 298     Mat sample = imread(findDataFile("dnn/dog416.png"));
 299     resize(sample, sample, Size(512, 512));
 300     Mat inp;
 301     sample.convertTo(inp, CV_32FC3, 1.0/255);
 302     processNet("dnn/efficientdet-d0.pb", "dnn/efficientdet-d0.pbtxt", "", inp);
 303 }
 304
 305 INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, dnnBackendsAndTargets());
 306
 307 } // namespace