modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (backend != DNN_BACKEND_INFERENCE_ENGINE || target != DNN_TARGET_CPU)
 140             throw SkipTestException("Only DLIE backend on CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("keras_pad_concat");
 149 }
 150
 151 TEST_P(Test_TensorFlow_layers, padding_same)
 152 {
 153 #if defined(INF_ENGINE_RELEASE)
 154     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 155             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 156     )
 157         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 158 #endif
 159     // Reference output values are in range [0.0006, 2.798]
 160     runTensorFlowNet("padding_same");
 161 }
 162
 163 TEST_P(Test_TensorFlow_layers, eltwise)
 164 {
 165     runTensorFlowNet("eltwise_add_mul");
 166     runTensorFlowNet("eltwise_sub");
 167 }
 168
 169 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 170 {
 171     runTensorFlowNet("pad_and_concat");
 172 }
 173
 174 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 175 {
 176     runTensorFlowNet("concat_axis_1");
 177 }
 178
 179 TEST_P(Test_TensorFlow_layers, batch_norm)
 180 {
 181     runTensorFlowNet("batch_norm");
 182     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 183     runTensorFlowNet("fused_batch_norm");
 184     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 185     runTensorFlowNet("batch_norm_text", true);
 186     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 187     runTensorFlowNet("unfused_batch_norm");
 188     runTensorFlowNet("fused_batch_norm_no_gamma");
 189     runTensorFlowNet("unfused_batch_norm_no_gamma");
 190     runTensorFlowNet("mvn_batch_norm");
 191     runTensorFlowNet("mvn_batch_norm_1x1");
 192     runTensorFlowNet("switch_identity");
 193     runTensorFlowNet("keras_batch_norm_training");
 194 }
 195
 196 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 197 {
 198     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 199     {
 200         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 201         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 202         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 203         throw SkipTestException("");
 204     }
 205     runTensorFlowNet("batch_norm3d");
 206 }
 207
 208 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 209 {
 210     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 211         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 212     // Output values range: [-40.0597, 207.827]
 213     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 214     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 215     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 216 }
 217
 218 TEST_P(Test_TensorFlow_layers, pooling)
 219 {
 220     runTensorFlowNet("max_pool_even");
 221     runTensorFlowNet("max_pool_odd_valid");
 222     runTensorFlowNet("max_pool_odd_same");
 223     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 224 }
 225
 226 // TODO: fix tests and replace to pooling
 227 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 228 {
 229     // Reference output values are in range [-0.519531, 0.112976]
 230 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 231     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 232             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 233     )
 234         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 235 #endif
 236     runTensorFlowNet("ave_pool_same");
 237 }
 238
 239 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 240 {
 241 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 242     throw SkipTestException("Test is enabled starts from 2019R1");
 243 #endif
 244     if (backend != DNN_BACKEND_INFERENCE_ENGINE || target != DNN_TARGET_CPU)
 245         throw SkipTestException("Only DLIE backend on CPU is supported");
 246     runTensorFlowNet("max_pool3d");
 247 }
 248
 249 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 250 {
 251 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 252     throw SkipTestException("Test is enabled starts from 2019R1");
 253 #endif
 254     if (backend != DNN_BACKEND_INFERENCE_ENGINE || target != DNN_TARGET_CPU)
 255         throw SkipTestException("Only DLIE backend on CPU is supported");
 256     runTensorFlowNet("ave_pool3d");
 257 }
 258
 259 TEST_P(Test_TensorFlow_layers, deconvolution)
 260 {
 261     runTensorFlowNet("deconvolution");
 262     runTensorFlowNet("deconvolution_same");
 263     runTensorFlowNet("deconvolution_stride_2_same");
 264     runTensorFlowNet("deconvolution_adj_pad_valid");
 265     runTensorFlowNet("deconvolution_adj_pad_same");
 266     runTensorFlowNet("keras_deconv_valid");
 267     runTensorFlowNet("keras_deconv_same");
 268     runTensorFlowNet("keras_deconv_same_v2");
 269 }
 270
 271 TEST_P(Test_TensorFlow_layers, matmul)
 272 {
 273     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 274         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 275     runTensorFlowNet("matmul");
 276     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 277     // Reference output values are in range [-5.688, 4.484]
 278     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 279     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 280
 281 }
 282
 283 TEST_P(Test_TensorFlow_layers, reshape)
 284 {
 285     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 286         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 287     runTensorFlowNet("shift_reshape_no_reorder");
 288     runTensorFlowNet("reshape_no_reorder");
 289     runTensorFlowNet("reshape_reduce");
 290     runTensorFlowNet("reshape_as_shape");
 291 }
 292
 293 TEST_P(Test_TensorFlow_layers, flatten)
 294 {
 295 #if defined(INF_ENGINE_RELEASE)
 296     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 297             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 298     )
 299         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 300 #endif
 301
 302     runTensorFlowNet("flatten", true);
 303 }
 304
 305 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 306 {
 307     runTensorFlowNet("unfused_flatten");
 308     runTensorFlowNet("unfused_flatten_unknown_batch");
 309 }
 310
 311 TEST_P(Test_TensorFlow_layers, leaky_relu)
 312 {
 313 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 314     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 315         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 316 #endif
 317     runTensorFlowNet("leaky_relu_order1");
 318     runTensorFlowNet("leaky_relu_order2");
 319     runTensorFlowNet("leaky_relu_order3");
 320 }
 321
 322 TEST_P(Test_TensorFlow_layers, l2_normalize)
 323 {
 324 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 325     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 326             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 327     )
 328         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 329 #endif
 330
 331     runTensorFlowNet("l2_normalize");
 332 }
 333
 334 // TODO: fix it and add to l2_normalize
 335 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 336 {
 337 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 338     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 339             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 340     )
 341         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 342 #endif
 343 #if defined(INF_ENGINE_RELEASE)
 344     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 345         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 346 #endif
 347
 348     runTensorFlowNet("l2_normalize_3d");
 349 }
 350
 351 class Test_TensorFlow_nets : public DNNTestLayer {};
 352
 353 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 354 {
 355 #if defined(INF_ENGINE_RELEASE)
 356     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 357     {
 358 #if INF_ENGINE_VER_MAJOR_GE(2019010000)
 359         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 360             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 361 #else
 362             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 363 #endif
 364     }
 365 #endif
 366
 367     checkBackend();
 368     std::string imgPath = findDataFile("dnn/street.png");
 369     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 370     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 371
 372     Mat inp;
 373     resize(imread(imgPath), inp, Size(300, 300));
 374     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 375
 376     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 377
 378     Net net = readNetFromTensorflow(netPath, netConfig);
 379     net.setPreferableBackend(backend);
 380     net.setPreferableTarget(target);
 381
 382     net.setInput(inp);
 383     Mat out = net.forward();
 384
 385     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 386     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 387     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 388 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 389     expectNoFallbacksFromIE(net);
 390 #endif
 391 }
 392
 393 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 394 {
 395     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 396
 397 #if defined(INF_ENGINE_RELEASE)
 398     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 399             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 400     )
 401         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 402 #endif
 403
 404     checkBackend();
 405     Mat img = imread(findDataFile("dnn/street.png"));
 406     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 407     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 408
 409     Net net = readNetFromTensorflow(model, proto);
 410     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 411
 412     net.setPreferableBackend(backend);
 413     net.setPreferableTarget(target);
 414
 415     net.setInput(blob);
 416     // Output has shape 1x1xNx7 where N - number of detections.
 417     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 418     Mat out = net.forward();
 419     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 420                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 421                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 422                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 423                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 424
 425     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 426     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 427     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 428     expectNoFallbacksFromIE(net);
 429 }
 430
 431 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 432 {
 433     checkBackend();
 434
 435 #if defined(INF_ENGINE_RELEASE)
 436     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 437             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 438     )
 439         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 440 #endif
 441
 442     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 443     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 444
 445     Net net = readNetFromTensorflow(model, proto);
 446     Mat img = imread(findDataFile("dnn/dog416.png"));
 447     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 448
 449     net.setPreferableBackend(backend);
 450     net.setPreferableTarget(target);
 451
 452     net.setInput(blob);
 453     Mat out = net.forward();
 454
 455     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 456     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 7e-3 : 1.5e-5;
 457     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 458     normAssertDetections(ref, out, "", 0.3, scoreDiff, iouDiff);
 459     expectNoFallbacksFromIE(net);
 460 }
 461
 462 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 463 {
 464     // FIXIT split test
 465     applyTestTag(
 466         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 467         CV_TEST_TAG_LONG,
 468         CV_TEST_TAG_DEBUG_VERYLONG
 469     );
 470     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 471                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 472
 473     checkBackend();
 474     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 475         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 476     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 477         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 478
 479     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 480     for (int i = 0; i < 2; ++i)
 481     {
 482         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 483         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 484
 485         Net net = readNetFromTensorflow(model, proto);
 486         net.setPreferableBackend(backend);
 487         net.setPreferableTarget(target);
 488         Mat img = imread(findDataFile("dnn/dog416.png"));
 489         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 490
 491         net.setInput(blob);
 492         Mat out = net.forward();
 493
 494         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 495         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 496     }
 497 }
 498
 499 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 500 {
 501 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 502     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 503         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 504 #endif
 505
 506     checkBackend();
 507     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 508     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 509
 510     Net net = readNetFromTensorflow(model, proto);
 511     Mat img = imread(findDataFile("dnn/dog416.png"));
 512     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 513     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 514
 515     net.setPreferableBackend(backend);
 516     net.setPreferableTarget(target);
 517
 518     net.setInput(blob);
 519     Mat out = net.forward();
 520
 521     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 522     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 523     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 524     expectNoFallbacksFromIE(net);
 525 }
 526
 527 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 528 {
 529     checkBackend();
 530     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 531     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 532
 533     Net net = readNetFromTensorflow(model, proto);
 534     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 535     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 536
 537     net.setPreferableBackend(backend);
 538     net.setPreferableTarget(target);
 539     net.setInput(blob);
 540     // Output has shape 1x1xNx7 where N - number of detections.
 541     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 542     Mat out = net.forward();
 543
 544     // References are from test for Caffe model.
 545     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 546                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 547                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 548                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 549                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 550                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 551     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 552     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 553     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 554     expectNoFallbacksFromIE(net);
 555 }
 556
 557 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 558 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 559 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 560 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 561 //                 feed_dict={'input_images:0': inp})
 562 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 563 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 564 // np.save('east_text_detection.scores.npy', scores)
 565 // np.save('east_text_detection.geometry.npy', geometry)
 566 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 567 {
 568     applyTestTag(
 569         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 570         CV_TEST_TAG_DEBUG_LONG
 571     );
 572
 573 #if defined(INF_ENGINE_RELEASE)
 574     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 575         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 576 #endif
 577
 578     checkBackend();
 579
 580     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 581     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 582     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 583     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 584
 585     Net net = readNet(netPath);
 586
 587     net.setPreferableBackend(backend);
 588     net.setPreferableTarget(target);
 589
 590     Mat img = imread(imgPath);
 591     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 592     net.setInput(inp);
 593
 594     std::vector<Mat> outs;
 595     std::vector<String> outNames(2);
 596     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 597     outNames[1] = "feature_fusion/concat_3";
 598     net.forward(outs, outNames);
 599
 600     Mat scores = outs[0];
 601     Mat geometry = outs[1];
 602
 603     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 604     double l1_scores = default_l1, lInf_scores = default_lInf;
 605     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 606     if (target == DNN_TARGET_OPENCL_FP16)
 607     {
 608         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 609         l1_geometry = 0.28; lInf_geometry = 5.94;
 610     }
 611     else if (target == DNN_TARGET_MYRIAD)
 612     {
 613         lInf_scores = 0.41;
 614         l1_geometry = 0.28; lInf_geometry = 5.94;
 615     }
 616     else
 617     {
 618         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 619     }
 620     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 621     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 622     expectNoFallbacksFromIE(net);
 623 }
 624
 625 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 626
 627 TEST_P(Test_TensorFlow_layers, fp16_weights)
 628 {
 629     float l1 = 0.00078;
 630     float lInf = 0.012;
 631     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 632     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 633     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 634     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 635     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 636     // Reference output values are in range [0.0889, 1.651]
 637     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 638     if (target == DNN_TARGET_MYRIAD) {
 639         l1 = 0.0041;
 640         lInf = 0.024;
 641     }
 642     // Reference output values are in range [0, 10.75]
 643     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 644     // Reference output values are in range [0.418, 2.297]
 645     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 646 }
 647
 648 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 649 {
 650 #if defined(INF_ENGINE_RELEASE)
 651     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 652             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 653     )
 654         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 655 #endif
 656
 657     // Reference output values are in range [-3.504, -0.002]
 658     runTensorFlowNet("fp16_padding_same", false, 6e-4, 4e-3);
 659 }
 660
 661 TEST_P(Test_TensorFlow_layers, defun)
 662 {
 663     runTensorFlowNet("defun_dropout");
 664 }
 665
 666 TEST_P(Test_TensorFlow_layers, quantized)
 667 {
 668     runTensorFlowNet("uint8_single_conv");
 669 }
 670
 671 TEST_P(Test_TensorFlow_layers, lstm)
 672 {
 673     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 674         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 675     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 676         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 677     runTensorFlowNet("lstm", true);
 678     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 679 }
 680
 681 TEST_P(Test_TensorFlow_layers, split)
 682 {
 683     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 684         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 685     runTensorFlowNet("split_equals");
 686 }
 687
 688 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 689 {
 690     runTensorFlowNet("resize_nearest_neighbor");
 691     runTensorFlowNet("keras_upsampling2d");
 692 }
 693
 694 TEST_P(Test_TensorFlow_layers, slice)
 695 {
 696     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 697         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 698         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 699     runTensorFlowNet("slice_4d");
 700     runTensorFlowNet("strided_slice");
 701 }
 702
 703 TEST_P(Test_TensorFlow_layers, softmax)
 704 {
 705     runTensorFlowNet("keras_softmax");
 706     runTensorFlowNet("slim_softmax");
 707 }
 708
 709 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 710 {
 711 #if defined(INF_ENGINE_RELEASE)
 712     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 713         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 714     )
 715         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 716 #endif
 717     runTensorFlowNet("slim_softmax_v2");
 718 }
 719
 720 TEST_P(Test_TensorFlow_layers, relu6)
 721 {
 722     runTensorFlowNet("keras_relu6");
 723     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 724 }
 725
 726 TEST_P(Test_TensorFlow_layers, subpixel)
 727 {
 728     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 729         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 730     runTensorFlowNet("subpixel");
 731 }
 732
 733 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 734 {
 735     runTensorFlowNet("keras_mobilenet_head");
 736 }
 737
 738 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 739 {
 740     runTensorFlowNet("resize_bilinear");
 741     runTensorFlowNet("resize_bilinear_factor");
 742 }
 743
 744 TEST_P(Test_TensorFlow_layers, squeeze)
 745 {
 746 #if defined(INF_ENGINE_RELEASE)
 747     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 748             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 749     )
 750         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 751 #endif
 752     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 753     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 754     int squeeze_dims[] = {0, 2, 3, -1};
 755     for (int i = 0; i < 4; ++i)
 756     {
 757         SCOPED_TRACE(format("i=%d", i));
 758         std::string pbtxt =
 759             "node { name: \"input\" op: \"Placeholder\""
 760             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 761             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 762               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 763         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 764         net.setPreferableBackend(backend);
 765         net.setPreferableTarget(target);
 766         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 767         randu(tfInp, -1, 1);
 768
 769         // NHWC to NCHW
 770         CV_Assert(inpShapes[i][0] == 1);
 771         std::swap(inpShapes[i][2], inpShapes[i][3]);
 772         std::swap(inpShapes[i][1], inpShapes[i][2]);
 773         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 774         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 775
 776         net.setInput(cvInp);
 777         Mat out = net.forward();
 778         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 779     }
 780 }
 781
 782 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 783
 784 TEST(Test_TensorFlow, two_inputs)
 785 {
 786     Net net = readNet(path("two_inputs_net.pbtxt"));
 787     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 788
 789     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 790     randu(firstInput, -1, 1);
 791     randu(secondInput, -1, 1);
 792
 793     net.setInput(firstInput, "first_input");
 794     net.setInput(secondInput, "second_input");
 795     Mat out = net.forward();
 796
 797     normAssert(out, firstInput + secondInput);
 798 }
 799
 800 TEST(Test_TensorFlow, Mask_RCNN)
 801 {
 802     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 803     Mat img = imread(findDataFile("dnn/street.png"));
 804     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 805     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 806
 807     Net net = readNetFromTensorflow(model, proto);
 808     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 809     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 810     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 811
 812     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 813
 814     net.setInput(blob);
 815
 816     // Mask-RCNN predicts bounding boxes and segmentation masks.
 817     std::vector<String> outNames(2);
 818     outNames[0] = "detection_out_final";
 819     outNames[1] = "detection_masks";
 820
 821     std::vector<Mat> outs;
 822     net.forward(outs, outNames);
 823
 824     Mat outDetections = outs[0];
 825     Mat outMasks = outs[1];
 826     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 827
 828     // Output size of masks is NxCxHxW where
 829     // N - number of detected boxes
 830     // C - number of classes (excluding background)
 831     // HxW - segmentation shape
 832     const int numDetections = outDetections.size[2];
 833
 834     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 835     Mat masks(4, &masksSize[0], CV_32F);
 836
 837     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 838     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 839
 840     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 841     for (int i = 0; i < numDetections; ++i)
 842     {
 843         // Get a class id for this bounding box and copy mask only for that class.
 844         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 845         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 846         srcRanges[1] = cv::Range(classId, classId + 1);
 847         outMasks(srcRanges).copyTo(masks(dstRanges));
 848     }
 849     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 850     normAssert(masks, refMasks(&topRefMasks[0]));
 851 }
 852
 853 }