modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU && backend != DNN_BACKEND_CUDA)
 140         throw SkipTestException("Only CPU and CUDA is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("mirror_pad");
 149 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000)
 150     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 151     {
 152         if (target == DNN_TARGET_MYRIAD)
 153             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_2019R3, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
 154         if (target == DNN_TARGET_OPENCL_FP16)
 155             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R3, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
 156     }
 157 #endif
 158     runTensorFlowNet("keras_pad_concat");
 159 }
 160
 161 TEST_P(Test_TensorFlow_layers, padding_same)
 162 {
 163     // Reference output values are in range [0.0006, 2.798]
 164     runTensorFlowNet("padding_same");
 165 }
 166
 167 TEST_P(Test_TensorFlow_layers, eltwise)
 168 {
 169     runTensorFlowNet("eltwise_add_mul");
 170     runTensorFlowNet("eltwise_sub");
 171 }
 172
 173 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 174 {
 175     runTensorFlowNet("pad_and_concat");
 176 }
 177
 178 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 179 {
 180     runTensorFlowNet("concat_axis_1");
 181 }
 182
 183 TEST_P(Test_TensorFlow_layers, batch_norm)
 184 {
 185     runTensorFlowNet("batch_norm");
 186     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 187     runTensorFlowNet("fused_batch_norm");
 188     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 189     runTensorFlowNet("batch_norm_text", true);
 190     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 191     runTensorFlowNet("unfused_batch_norm");
 192     runTensorFlowNet("fused_batch_norm_no_gamma");
 193     runTensorFlowNet("unfused_batch_norm_no_gamma");
 194     runTensorFlowNet("mvn_batch_norm");
 195     runTensorFlowNet("mvn_batch_norm_1x1");
 196     runTensorFlowNet("switch_identity");
 197     runTensorFlowNet("keras_batch_norm_training");
 198 }
 199
 200 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 201 {
 202     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 203     {
 204         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 205         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 206         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 207         throw SkipTestException("");
 208     }
 209     runTensorFlowNet("batch_norm3d");
 210 }
 211
 212 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 213 {
 214     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 215         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 216     // Output values range: [-40.0597, 207.827]
 217     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 218     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 219     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 220 }
 221
 222 TEST_P(Test_TensorFlow_layers, pooling)
 223 {
 224     runTensorFlowNet("max_pool_even");
 225     runTensorFlowNet("max_pool_odd_valid");
 226     runTensorFlowNet("max_pool_odd_same");
 227     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 228 }
 229
 230 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 231 {
 232     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 233         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 234     runTensorFlowNet("max_pool_grad");
 235 }
 236
 237 // TODO: fix tests and replace to pooling
 238 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 239 {
 240     // Reference output values are in range [-0.519531, 0.112976]
 241 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 242     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 243             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 244     )
 245         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 246 #endif
 247     runTensorFlowNet("ave_pool_same");
 248 }
 249
 250 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 251 {
 252 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 253     throw SkipTestException("Test is enabled starts from 2019R1");
 254 #endif
 255     if (target != DNN_TARGET_CPU && backend != DNN_BACKEND_CUDA)
 256         throw SkipTestException("Only CPU and CUDA is supported");
 257     runTensorFlowNet("max_pool3d");
 258 }
 259
 260 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 261 {
 262 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 263     throw SkipTestException("Test is enabled starts from 2019R1");
 264 #endif
 265     if (target != DNN_TARGET_CPU && backend != DNN_BACKEND_CUDA)
 266         throw SkipTestException("Only CPU and CUDA is supported");
 267     runTensorFlowNet("ave_pool3d");
 268 }
 269
 270 TEST_P(Test_TensorFlow_layers, deconvolution)
 271 {
 272     if(backend == DNN_BACKEND_CUDA)
 273         applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); /* bugged */
 274
 275     runTensorFlowNet("deconvolution");
 276     runTensorFlowNet("deconvolution_same");
 277     runTensorFlowNet("deconvolution_stride_2_same");
 278     runTensorFlowNet("deconvolution_adj_pad_valid");
 279     runTensorFlowNet("deconvolution_adj_pad_same");
 280     runTensorFlowNet("keras_deconv_valid");
 281     runTensorFlowNet("keras_deconv_same");
 282     runTensorFlowNet("keras_deconv_same_v2");
 283 }
 284
 285 TEST_P(Test_TensorFlow_layers, matmul)
 286 {
 287     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 288         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 289     runTensorFlowNet("matmul");
 290     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 291     // Reference output values are in range [-5.688, 4.484]
 292     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 293     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 294     runTensorFlowNet("matmul_layout");
 295 }
 296
 297 TEST_P(Test_TensorFlow_layers, reshape)
 298 {
 299     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 300         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 301     runTensorFlowNet("shift_reshape_no_reorder");
 302     runTensorFlowNet("reshape_no_reorder");
 303     runTensorFlowNet("reshape_reduce");
 304     runTensorFlowNet("reshape_as_shape");
 305 }
 306
 307 TEST_P(Test_TensorFlow_layers, flatten)
 308 {
 309 #if defined(INF_ENGINE_RELEASE)
 310     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 311             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 312     )
 313         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 314 #endif
 315
 316     runTensorFlowNet("flatten", true);
 317 }
 318
 319 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 320 {
 321     runTensorFlowNet("unfused_flatten");
 322     runTensorFlowNet("unfused_flatten_unknown_batch");
 323 }
 324
 325 TEST_P(Test_TensorFlow_layers, leaky_relu)
 326 {
 327 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 328     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 329         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 330 #endif
 331     runTensorFlowNet("leaky_relu_order1");
 332     runTensorFlowNet("leaky_relu_order2");
 333     runTensorFlowNet("leaky_relu_order3");
 334 }
 335
 336 TEST_P(Test_TensorFlow_layers, l2_normalize)
 337 {
 338 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 339     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 340             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 341     )
 342         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 343 #endif
 344
 345     runTensorFlowNet("l2_normalize");
 346 }
 347
 348 // TODO: fix it and add to l2_normalize
 349 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 350 {
 351 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 352     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 353             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 354     )
 355         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 356 #endif
 357 #if defined(INF_ENGINE_RELEASE)
 358     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 359         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 360 #endif
 361
 362     runTensorFlowNet("l2_normalize_3d");
 363 }
 364
 365 class Test_TensorFlow_nets : public DNNTestLayer {};
 366
 367 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 368 {
 369 #if defined(INF_ENGINE_RELEASE)
 370     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 371     {
 372 #if INF_ENGINE_VER_MAJOR_GE(2019020000)
 373         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 374             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 375 #endif
 376     }
 377 #endif
 378
 379     checkBackend();
 380     std::string imgPath = findDataFile("dnn/street.png");
 381     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 382     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 383
 384     Mat inp;
 385     resize(imread(imgPath), inp, Size(300, 300));
 386     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 387
 388     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 389
 390     Net net = readNetFromTensorflow(netPath, netConfig);
 391     net.setPreferableBackend(backend);
 392     net.setPreferableTarget(target);
 393
 394     net.setInput(inp);
 395     Mat out = net.forward();
 396
 397     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 398     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 399     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 400 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 401     expectNoFallbacksFromIE(net);
 402 #endif
 403 }
 404
 405 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 406 {
 407     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 408 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
 409     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 410         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 411         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 412 #endif
 413
 414     checkBackend();
 415     Mat img = imread(findDataFile("dnn/street.png"));
 416     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 417     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 418
 419     Net net = readNetFromTensorflow(model, proto);
 420     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 421
 422     net.setPreferableBackend(backend);
 423     net.setPreferableTarget(target);
 424
 425     net.setInput(blob);
 426     // Output has shape 1x1xNx7 where N - number of detections.
 427     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 428     Mat out = net.forward();
 429     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 430                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 431                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 432                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 433                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 434
 435     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 436     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 437     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 438     expectNoFallbacksFromIE(net);
 439 }
 440
 441 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 442 {
 443     checkBackend();
 444     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 445     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 446
 447     Net net = readNetFromTensorflow(model, proto);
 448     Mat img = imread(findDataFile("dnn/dog416.png"));
 449     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 450
 451     net.setPreferableBackend(backend);
 452     net.setPreferableTarget(target);
 453
 454     net.setInput(blob);
 455     Mat out = net.forward();
 456
 457     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 458     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 1.5e-5;
 459     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 460     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 461
 462 #if defined(INF_ENGINE_RELEASE)
 463     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 464         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 465     {
 466         scoreDiff = 0.061;
 467         iouDiff = 0.12;
 468         detectionConfThresh = 0.36;
 469     }
 470 #endif
 471     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 472     expectNoFallbacksFromIE(net);
 473 }
 474
 475 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 476 {
 477     // FIXIT split test
 478     applyTestTag(
 479         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 480         CV_TEST_TAG_LONG,
 481         CV_TEST_TAG_DEBUG_VERYLONG
 482     );
 483     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 484                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 485
 486     checkBackend();
 487 #ifdef INF_ENGINE_RELEASE
 488     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 489         (INF_ENGINE_VER_MAJOR_LT(2019020000) || target != DNN_TARGET_CPU))
 490         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 491 #endif
 492     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 493         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 494
 495     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 496     for (int i = 0; i < 2; ++i)
 497     {
 498         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 499         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 500
 501         Net net = readNetFromTensorflow(model, proto);
 502         net.setPreferableBackend(backend);
 503         net.setPreferableTarget(target);
 504         Mat img = imread(findDataFile("dnn/dog416.png"));
 505         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 506
 507         net.setInput(blob);
 508         Mat out = net.forward();
 509
 510         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 511         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 512     }
 513 }
 514
 515 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 516 {
 517 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 518     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 519         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 520 #endif
 521     checkBackend();
 522     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 523     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 524
 525     Net net = readNetFromTensorflow(model, proto);
 526     Mat img = imread(findDataFile("dnn/dog416.png"));
 527     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 528     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 529
 530     net.setPreferableBackend(backend);
 531     net.setPreferableTarget(target);
 532
 533     net.setInput(blob);
 534     Mat out = net.forward();
 535
 536     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 537     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 538     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 539     expectNoFallbacksFromIE(net);
 540 }
 541
 542 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 543 {
 544     checkBackend();
 545     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 546     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 547
 548     Net net = readNetFromTensorflow(model, proto);
 549     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 550     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 551
 552     net.setPreferableBackend(backend);
 553     net.setPreferableTarget(target);
 554     net.setInput(blob);
 555     // Output has shape 1x1xNx7 where N - number of detections.
 556     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 557     Mat out = net.forward();
 558
 559     // References are from test for Caffe model.
 560     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 561                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 562                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 563                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 564                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 565                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 566     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 567     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 568     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 569     expectNoFallbacksFromIE(net);
 570 }
 571
 572 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 573 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 574 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 575 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 576 //                 feed_dict={'input_images:0': inp})
 577 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 578 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 579 // np.save('east_text_detection.scores.npy', scores)
 580 // np.save('east_text_detection.geometry.npy', geometry)
 581 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 582 {
 583     applyTestTag(
 584         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 585         CV_TEST_TAG_DEBUG_LONG
 586     );
 587
 588 #if defined(INF_ENGINE_RELEASE)
 589     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 590         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 591
 592     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16 &&
 593         INF_ENGINE_VER_MAJOR_EQ(2019020000))
 594         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
 595 #endif
 596
 597     checkBackend();
 598
 599     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 600     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 601     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 602     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 603
 604     Net net = readNet(netPath);
 605
 606     net.setPreferableBackend(backend);
 607     net.setPreferableTarget(target);
 608
 609     Mat img = imread(imgPath);
 610     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 611     net.setInput(inp);
 612
 613     std::vector<Mat> outs;
 614     std::vector<String> outNames(2);
 615     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 616     outNames[1] = "feature_fusion/concat_3";
 617     net.forward(outs, outNames);
 618
 619     Mat scores = outs[0];
 620     Mat geometry = outs[1];
 621
 622     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 623     double l1_scores = default_l1, lInf_scores = default_lInf;
 624     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 625     if (target == DNN_TARGET_OPENCL_FP16)
 626     {
 627         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 628         l1_geometry = 0.28; lInf_geometry = 5.94;
 629     }
 630     else if (target == DNN_TARGET_MYRIAD)
 631     {
 632         lInf_scores = 0.41;
 633         l1_geometry = 0.28; lInf_geometry = 5.94;
 634     }
 635     else
 636     {
 637         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 638     }
 639     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 640     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 641     expectNoFallbacksFromIE(net);
 642 }
 643
 644 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 645
 646 TEST_P(Test_TensorFlow_layers, fp16_weights)
 647 {
 648     float l1 = 0.00078;
 649     float lInf = 0.012;
 650     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 651     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 652     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 653     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 654     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 655     // Reference output values are in range [0.0889, 1.651]
 656     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 657     if (target == DNN_TARGET_MYRIAD) {
 658         l1 = 0.0041;
 659         lInf = 0.024;
 660     }
 661     // Reference output values are in range [0, 10.75]
 662     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 663     // Reference output values are in range [0.418, 2.297]
 664     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 665 }
 666
 667 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 668 {
 669     // Reference output values are in range [-3.504, -0.002]
 670     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 671 }
 672
 673 TEST_P(Test_TensorFlow_layers, defun)
 674 {
 675     runTensorFlowNet("defun_dropout");
 676 }
 677
 678 TEST_P(Test_TensorFlow_layers, quantized)
 679 {
 680     runTensorFlowNet("uint8_single_conv");
 681 }
 682
 683 TEST_P(Test_TensorFlow_layers, lstm)
 684 {
 685     if(backend == DNN_BACKEND_CUDA)
 686         applyTestTag(CV_TEST_TAG_DNN_SKIP_CUDA); /* not supported */
 687     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 688         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 689     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 690         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 691     runTensorFlowNet("lstm", true);
 692     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 693 }
 694
 695 TEST_P(Test_TensorFlow_layers, split)
 696 {
 697     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 698         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2)
 699         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 700     runTensorFlowNet("split");
 701     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 702         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 703     runTensorFlowNet("split_equals");
 704 }
 705
 706 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 707 {
 708     runTensorFlowNet("resize_nearest_neighbor");
 709     runTensorFlowNet("keras_upsampling2d");
 710 }
 711
 712 TEST_P(Test_TensorFlow_layers, slice)
 713 {
 714     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 715         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 716         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 717     runTensorFlowNet("slice_4d");
 718     runTensorFlowNet("strided_slice");
 719 }
 720
 721 TEST_P(Test_TensorFlow_layers, softmax)
 722 {
 723     runTensorFlowNet("keras_softmax");
 724     runTensorFlowNet("slim_softmax");
 725 }
 726
 727 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 728 {
 729 #if defined(INF_ENGINE_RELEASE)
 730     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 731         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 732     )
 733         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 734 #endif
 735     runTensorFlowNet("slim_softmax_v2");
 736 }
 737
 738 TEST_P(Test_TensorFlow_layers, relu6)
 739 {
 740     runTensorFlowNet("keras_relu6");
 741     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 742 }
 743
 744 TEST_P(Test_TensorFlow_layers, subpixel)
 745 {
 746     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 747         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 748     runTensorFlowNet("subpixel");
 749 }
 750
 751 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 752 {
 753     runTensorFlowNet("keras_mobilenet_head");
 754     runTensorFlowNet("keras_learning_phase");
 755 }
 756
 757 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 758 {
 759     runTensorFlowNet("resize_bilinear");
 760     runTensorFlowNet("resize_bilinear_factor");
 761 }
 762
 763 TEST_P(Test_TensorFlow_layers, squeeze)
 764 {
 765 #if defined(INF_ENGINE_RELEASE)
 766     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 767             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 768     )
 769         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 770 #endif
 771     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 772     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 773     int squeeze_dims[] = {0, 2, 3, -1};
 774     for (int i = 0; i < 4; ++i)
 775     {
 776         SCOPED_TRACE(format("i=%d", i));
 777         std::string pbtxt =
 778             "node { name: \"input\" op: \"Placeholder\""
 779             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 780             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 781               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 782         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 783         net.setPreferableBackend(backend);
 784         net.setPreferableTarget(target);
 785         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 786         randu(tfInp, -1, 1);
 787
 788         // NHWC to NCHW
 789         CV_Assert(inpShapes[i][0] == 1);
 790         std::swap(inpShapes[i][2], inpShapes[i][3]);
 791         std::swap(inpShapes[i][1], inpShapes[i][2]);
 792         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 793         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 794
 795         net.setInput(cvInp);
 796         Mat out = net.forward();
 797         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 798     }
 799 }
 800
 801 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 802
 803 TEST(Test_TensorFlow, two_inputs)
 804 {
 805     Net net = readNet(path("two_inputs_net.pbtxt"));
 806     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 807
 808     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 809     randu(firstInput, -1, 1);
 810     randu(secondInput, -1, 1);
 811
 812     net.setInput(firstInput, "first_input");
 813     net.setInput(secondInput, "second_input");
 814     Mat out = net.forward();
 815
 816     normAssert(out, firstInput + secondInput);
 817 }
 818
 819 TEST(Test_TensorFlow, Mask_RCNN)
 820 {
 821     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 822     Mat img = imread(findDataFile("dnn/street.png"));
 823     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 824     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 825
 826     Net net = readNetFromTensorflow(model, proto);
 827     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 828     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 829     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 830
 831     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 832
 833     net.setInput(blob);
 834
 835     // Mask-RCNN predicts bounding boxes and segmentation masks.
 836     std::vector<String> outNames(2);
 837     outNames[0] = "detection_out_final";
 838     outNames[1] = "detection_masks";
 839
 840     std::vector<Mat> outs;
 841     net.forward(outs, outNames);
 842
 843     Mat outDetections = outs[0];
 844     Mat outMasks = outs[1];
 845     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 846
 847     // Output size of masks is NxCxHxW where
 848     // N - number of detected boxes
 849     // C - number of classes (excluding background)
 850     // HxW - segmentation shape
 851     const int numDetections = outDetections.size[2];
 852
 853     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 854     Mat masks(4, &masksSize[0], CV_32F);
 855
 856     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 857     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 858
 859     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 860     for (int i = 0; i < numDetections; ++i)
 861     {
 862         // Get a class id for this bounding box and copy mask only for that class.
 863         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 864         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 865         srcRanges[1] = cv::Range(classId, classId + 1);
 866         outMasks(srcRanges).copyTo(masks(dstRanges));
 867     }
 868     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 869     normAssert(masks, refMasks(&topRefMasks[0]));
 870 }
 871
 872 }