modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU)
 140         throw SkipTestException("Only CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("mirror_pad");
 149 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000)
 150     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 151     {
 152         if (target == DNN_TARGET_MYRIAD)
 153             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
 154         if (target == DNN_TARGET_OPENCL_FP16)
 155             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
 156     }
 157 #endif
 158     runTensorFlowNet("keras_pad_concat");
 159 }
 160
 161 TEST_P(Test_TensorFlow_layers, padding_same)
 162 {
 163     // Reference output values are in range [0.0006, 2.798]
 164     runTensorFlowNet("padding_same");
 165 }
 166
 167 TEST_P(Test_TensorFlow_layers, eltwise)
 168 {
 169     runTensorFlowNet("eltwise_add_mul");
 170     runTensorFlowNet("eltwise_sub");
 171 }
 172
 173 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 174 {
 175     runTensorFlowNet("pad_and_concat");
 176 }
 177
 178 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 179 {
 180     runTensorFlowNet("concat_axis_1");
 181 }
 182
 183 TEST_P(Test_TensorFlow_layers, batch_norm)
 184 {
 185     runTensorFlowNet("batch_norm");
 186     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 187     runTensorFlowNet("fused_batch_norm");
 188     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 189     runTensorFlowNet("batch_norm_text", true);
 190     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 191     runTensorFlowNet("unfused_batch_norm");
 192     runTensorFlowNet("fused_batch_norm_no_gamma");
 193     runTensorFlowNet("unfused_batch_norm_no_gamma");
 194     runTensorFlowNet("mvn_batch_norm");
 195     runTensorFlowNet("mvn_batch_norm_1x1");
 196     runTensorFlowNet("switch_identity");
 197     runTensorFlowNet("keras_batch_norm_training");
 198 }
 199
 200 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 201 {
 202     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 203     {
 204         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 205         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 206         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 207         throw SkipTestException("");
 208     }
 209     runTensorFlowNet("batch_norm3d");
 210 }
 211
 212 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 213 {
 214     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 215         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 216     // Output values range: [-40.0597, 207.827]
 217     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 218     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 219     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 220 }
 221
 222 TEST_P(Test_TensorFlow_layers, pooling)
 223 {
 224     runTensorFlowNet("max_pool_even");
 225     runTensorFlowNet("max_pool_odd_valid");
 226     runTensorFlowNet("max_pool_odd_same");
 227     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 228 }
 229
 230 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 231 {
 232     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 233         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 234     runTensorFlowNet("max_pool_grad");
 235 }
 236
 237 // TODO: fix tests and replace to pooling
 238 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 239 {
 240     // Reference output values are in range [-0.519531, 0.112976]
 241 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 242     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 243             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 244     )
 245         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 246 #endif
 247     runTensorFlowNet("ave_pool_same");
 248 }
 249
 250 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 251 {
 252 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 253     throw SkipTestException("Test is enabled starts from 2019R1");
 254 #endif
 255     if (target != DNN_TARGET_CPU)
 256         throw SkipTestException("Only CPU is supported");
 257     runTensorFlowNet("max_pool3d");
 258 }
 259
 260 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 261 {
 262 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 263     throw SkipTestException("Test is enabled starts from 2019R1");
 264 #endif
 265     if (target != DNN_TARGET_CPU)
 266         throw SkipTestException("Only CPU is supported");
 267     runTensorFlowNet("ave_pool3d");
 268 }
 269
 270 TEST_P(Test_TensorFlow_layers, deconvolution)
 271 {
 272     runTensorFlowNet("deconvolution");
 273     runTensorFlowNet("deconvolution_same");
 274     runTensorFlowNet("deconvolution_stride_2_same");
 275     runTensorFlowNet("deconvolution_adj_pad_valid");
 276     runTensorFlowNet("deconvolution_adj_pad_same");
 277     runTensorFlowNet("keras_deconv_valid");
 278     runTensorFlowNet("keras_deconv_same");
 279     runTensorFlowNet("keras_deconv_same_v2");
 280 }
 281
 282 TEST_P(Test_TensorFlow_layers, matmul)
 283 {
 284     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 285         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 286     runTensorFlowNet("matmul");
 287     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 288     // Reference output values are in range [-5.688, 4.484]
 289     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 290     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 291     runTensorFlowNet("matmul_layout");
 292 }
 293
 294 TEST_P(Test_TensorFlow_layers, reshape)
 295 {
 296     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 297         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 298     runTensorFlowNet("shift_reshape_no_reorder");
 299     runTensorFlowNet("reshape_no_reorder");
 300     runTensorFlowNet("reshape_reduce");
 301     runTensorFlowNet("reshape_as_shape");
 302 }
 303
 304 TEST_P(Test_TensorFlow_layers, flatten)
 305 {
 306 #if defined(INF_ENGINE_RELEASE)
 307     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 308             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 309     )
 310         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 311 #endif
 312
 313     runTensorFlowNet("flatten", true);
 314 }
 315
 316 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 317 {
 318     runTensorFlowNet("unfused_flatten");
 319     runTensorFlowNet("unfused_flatten_unknown_batch");
 320 }
 321
 322 TEST_P(Test_TensorFlow_layers, leaky_relu)
 323 {
 324 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 325     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 326         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 327 #endif
 328     runTensorFlowNet("leaky_relu_order1");
 329     runTensorFlowNet("leaky_relu_order2");
 330     runTensorFlowNet("leaky_relu_order3");
 331 }
 332
 333 TEST_P(Test_TensorFlow_layers, l2_normalize)
 334 {
 335 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 336     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 337             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 338     )
 339         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 340 #endif
 341
 342     runTensorFlowNet("l2_normalize");
 343 }
 344
 345 // TODO: fix it and add to l2_normalize
 346 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 347 {
 348 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 349     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 350             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 351     )
 352         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 353 #endif
 354 #if defined(INF_ENGINE_RELEASE)
 355     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 356         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 357 #endif
 358
 359     runTensorFlowNet("l2_normalize_3d");
 360 }
 361
 362 class Test_TensorFlow_nets : public DNNTestLayer {};
 363
 364 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 365 {
 366 #if defined(INF_ENGINE_RELEASE)
 367     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 368     {
 369 #if INF_ENGINE_VER_MAJOR_GE(2019020000)
 370         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 371             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 372 #endif
 373     }
 374 #endif
 375
 376     checkBackend();
 377     std::string imgPath = findDataFile("dnn/street.png");
 378     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 379     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 380
 381     Mat inp;
 382     resize(imread(imgPath), inp, Size(300, 300));
 383     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 384
 385     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 386
 387     Net net = readNetFromTensorflow(netPath, netConfig);
 388     net.setPreferableBackend(backend);
 389     net.setPreferableTarget(target);
 390
 391     net.setInput(inp);
 392     Mat out = net.forward();
 393
 394     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 395     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 396     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 397 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 398     expectNoFallbacksFromIE(net);
 399 #endif
 400 }
 401
 402 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 403 {
 404     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 405 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
 406     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 407         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 408         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 409 #endif
 410
 411     checkBackend();
 412     Mat img = imread(findDataFile("dnn/street.png"));
 413     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 414     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 415
 416     Net net = readNetFromTensorflow(model, proto);
 417     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 418
 419     net.setPreferableBackend(backend);
 420     net.setPreferableTarget(target);
 421
 422     net.setInput(blob);
 423     // Output has shape 1x1xNx7 where N - number of detections.
 424     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 425     Mat out = net.forward();
 426     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 427                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 428                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 429                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 430                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 431
 432     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 433     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 434     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 435     expectNoFallbacksFromIE(net);
 436 }
 437
 438 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 439 {
 440     checkBackend();
 441     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 442     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 443
 444     Net net = readNetFromTensorflow(model, proto);
 445     Mat img = imread(findDataFile("dnn/dog416.png"));
 446     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 447
 448     net.setPreferableBackend(backend);
 449     net.setPreferableTarget(target);
 450
 451     net.setInput(blob);
 452     Mat out = net.forward();
 453
 454     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 455     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 1.5e-5;
 456     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 457     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 458
 459 #if defined(INF_ENGINE_RELEASE)
 460     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 461         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 462     {
 463         scoreDiff = 0.061;
 464         iouDiff = 0.12;
 465         detectionConfThresh = 0.36;
 466     }
 467 #endif
 468     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 469     expectNoFallbacksFromIE(net);
 470 }
 471
 472 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 473 {
 474     // FIXIT split test
 475     applyTestTag(
 476         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 477         CV_TEST_TAG_LONG,
 478         CV_TEST_TAG_DEBUG_VERYLONG
 479     );
 480     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 481                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 482
 483     checkBackend();
 484     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 485         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 486     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 487         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 488
 489     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 490     for (int i = 0; i < 2; ++i)
 491     {
 492         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 493         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 494
 495         Net net = readNetFromTensorflow(model, proto);
 496         net.setPreferableBackend(backend);
 497         net.setPreferableTarget(target);
 498         Mat img = imread(findDataFile("dnn/dog416.png"));
 499         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 500
 501         net.setInput(blob);
 502         Mat out = net.forward();
 503
 504         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 505         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 506     }
 507 }
 508
 509 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 510 {
 511 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 512     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 513         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 514 #endif
 515     checkBackend();
 516     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 517     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 518
 519     Net net = readNetFromTensorflow(model, proto);
 520     Mat img = imread(findDataFile("dnn/dog416.png"));
 521     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 522     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 523
 524     net.setPreferableBackend(backend);
 525     net.setPreferableTarget(target);
 526
 527     net.setInput(blob);
 528     Mat out = net.forward();
 529
 530     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 531     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 532     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 533     expectNoFallbacksFromIE(net);
 534 }
 535
 536 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 537 {
 538     checkBackend();
 539     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 540     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 541
 542     Net net = readNetFromTensorflow(model, proto);
 543     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 544     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 545
 546     net.setPreferableBackend(backend);
 547     net.setPreferableTarget(target);
 548     net.setInput(blob);
 549     // Output has shape 1x1xNx7 where N - number of detections.
 550     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 551     Mat out = net.forward();
 552
 553     // References are from test for Caffe model.
 554     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 555                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 556                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 557                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 558                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 559                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 560     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 561     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 562     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 563     expectNoFallbacksFromIE(net);
 564 }
 565
 566 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 567 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 568 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 569 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 570 //                 feed_dict={'input_images:0': inp})
 571 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 572 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 573 // np.save('east_text_detection.scores.npy', scores)
 574 // np.save('east_text_detection.geometry.npy', geometry)
 575 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 576 {
 577     applyTestTag(
 578         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 579         CV_TEST_TAG_DEBUG_LONG
 580     );
 581
 582 #if defined(INF_ENGINE_RELEASE)
 583     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 584         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 585
 586     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16 &&
 587         INF_ENGINE_VER_MAJOR_EQ(2019020000))
 588         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
 589 #endif
 590
 591     checkBackend();
 592
 593     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 594     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 595     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 596     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 597
 598     Net net = readNet(netPath);
 599
 600     net.setPreferableBackend(backend);
 601     net.setPreferableTarget(target);
 602
 603     Mat img = imread(imgPath);
 604     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 605     net.setInput(inp);
 606
 607     std::vector<Mat> outs;
 608     std::vector<String> outNames(2);
 609     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 610     outNames[1] = "feature_fusion/concat_3";
 611     net.forward(outs, outNames);
 612
 613     Mat scores = outs[0];
 614     Mat geometry = outs[1];
 615
 616     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 617     double l1_scores = default_l1, lInf_scores = default_lInf;
 618     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 619     if (target == DNN_TARGET_OPENCL_FP16)
 620     {
 621         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 622         l1_geometry = 0.28; lInf_geometry = 5.94;
 623     }
 624     else if (target == DNN_TARGET_MYRIAD)
 625     {
 626         lInf_scores = 0.41;
 627         l1_geometry = 0.28; lInf_geometry = 5.94;
 628     }
 629     else
 630     {
 631         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 632     }
 633     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 634     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 635     expectNoFallbacksFromIE(net);
 636 }
 637
 638 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 639
 640 TEST_P(Test_TensorFlow_layers, fp16_weights)
 641 {
 642     float l1 = 0.00078;
 643     float lInf = 0.012;
 644     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 645     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 646     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 647     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 648     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 649     // Reference output values are in range [0.0889, 1.651]
 650     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 651     if (target == DNN_TARGET_MYRIAD) {
 652         l1 = 0.0041;
 653         lInf = 0.024;
 654     }
 655     // Reference output values are in range [0, 10.75]
 656     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 657     // Reference output values are in range [0.418, 2.297]
 658     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 659 }
 660
 661 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 662 {
 663     // Reference output values are in range [-3.504, -0.002]
 664     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 665 }
 666
 667 TEST_P(Test_TensorFlow_layers, defun)
 668 {
 669     runTensorFlowNet("defun_dropout");
 670 }
 671
 672 TEST_P(Test_TensorFlow_layers, quantized)
 673 {
 674     runTensorFlowNet("uint8_single_conv");
 675 }
 676
 677 TEST_P(Test_TensorFlow_layers, lstm)
 678 {
 679     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 680         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 681     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 682         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 683     runTensorFlowNet("lstm", true);
 684     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 685 }
 686
 687 TEST_P(Test_TensorFlow_layers, split)
 688 {
 689     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 690         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2)
 691         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 692     runTensorFlowNet("split");
 693     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 694         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 695     runTensorFlowNet("split_equals");
 696 }
 697
 698 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 699 {
 700     runTensorFlowNet("resize_nearest_neighbor");
 701     runTensorFlowNet("keras_upsampling2d");
 702 }
 703
 704 TEST_P(Test_TensorFlow_layers, slice)
 705 {
 706     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 707         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 708         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 709     runTensorFlowNet("slice_4d");
 710     runTensorFlowNet("strided_slice");
 711 }
 712
 713 TEST_P(Test_TensorFlow_layers, softmax)
 714 {
 715     runTensorFlowNet("keras_softmax");
 716     runTensorFlowNet("slim_softmax");
 717 }
 718
 719 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 720 {
 721 #if defined(INF_ENGINE_RELEASE)
 722     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 723         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 724     )
 725         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 726 #endif
 727     runTensorFlowNet("slim_softmax_v2");
 728 }
 729
 730 TEST_P(Test_TensorFlow_layers, relu6)
 731 {
 732     runTensorFlowNet("keras_relu6");
 733     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 734 }
 735
 736 TEST_P(Test_TensorFlow_layers, subpixel)
 737 {
 738     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 739         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 740     runTensorFlowNet("subpixel");
 741 }
 742
 743 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 744 {
 745     runTensorFlowNet("keras_mobilenet_head");
 746     runTensorFlowNet("keras_learning_phase");
 747 }
 748
 749 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 750 {
 751     runTensorFlowNet("resize_bilinear");
 752     runTensorFlowNet("resize_bilinear_factor");
 753 }
 754
 755 TEST_P(Test_TensorFlow_layers, squeeze)
 756 {
 757 #if defined(INF_ENGINE_RELEASE)
 758     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 759             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 760     )
 761         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 762 #endif
 763     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 764     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 765     int squeeze_dims[] = {0, 2, 3, -1};
 766     for (int i = 0; i < 4; ++i)
 767     {
 768         SCOPED_TRACE(format("i=%d", i));
 769         std::string pbtxt =
 770             "node { name: \"input\" op: \"Placeholder\""
 771             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 772             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 773               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 774         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 775         net.setPreferableBackend(backend);
 776         net.setPreferableTarget(target);
 777         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 778         randu(tfInp, -1, 1);
 779
 780         // NHWC to NCHW
 781         CV_Assert(inpShapes[i][0] == 1);
 782         std::swap(inpShapes[i][2], inpShapes[i][3]);
 783         std::swap(inpShapes[i][1], inpShapes[i][2]);
 784         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 785         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 786
 787         net.setInput(cvInp);
 788         Mat out = net.forward();
 789         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 790     }
 791 }
 792
 793 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 794
 795 TEST(Test_TensorFlow, two_inputs)
 796 {
 797     Net net = readNet(path("two_inputs_net.pbtxt"));
 798     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 799
 800     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 801     randu(firstInput, -1, 1);
 802     randu(secondInput, -1, 1);
 803
 804     net.setInput(firstInput, "first_input");
 805     net.setInput(secondInput, "second_input");
 806     Mat out = net.forward();
 807
 808     normAssert(out, firstInput + secondInput);
 809 }
 810
 811 TEST(Test_TensorFlow, Mask_RCNN)
 812 {
 813     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 814     Mat img = imread(findDataFile("dnn/street.png"));
 815     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 816     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 817
 818     Net net = readNetFromTensorflow(model, proto);
 819     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 820     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 821     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 822
 823     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 824
 825     net.setInput(blob);
 826
 827     // Mask-RCNN predicts bounding boxes and segmentation masks.
 828     std::vector<String> outNames(2);
 829     outNames[0] = "detection_out_final";
 830     outNames[1] = "detection_masks";
 831
 832     std::vector<Mat> outs;
 833     net.forward(outs, outNames);
 834
 835     Mat outDetections = outs[0];
 836     Mat outMasks = outs[1];
 837     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 838
 839     // Output size of masks is NxCxHxW where
 840     // N - number of detected boxes
 841     // C - number of classes (excluding background)
 842     // HxW - segmentation shape
 843     const int numDetections = outDetections.size[2];
 844
 845     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 846     Mat masks(4, &masksSize[0], CV_32F);
 847
 848     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 849     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 850
 851     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 852     for (int i = 0; i < numDetections; ++i)
 853     {
 854         // Get a class id for this bounding box and copy mask only for that class.
 855         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 856         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 857         srcRanges[1] = cv::Range(classId, classId + 1);
 858         outMasks(srcRanges).copyTo(masks(dstRanges));
 859     }
 860     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 861     normAssert(masks, refMasks(&topRefMasks[0]));
 862 }
 863
 864 }