modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU)
 140         throw SkipTestException("Only CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("keras_pad_concat");
 149     runTensorFlowNet("mirror_pad");
 150 }
 151
 152 TEST_P(Test_TensorFlow_layers, padding_same)
 153 {
 154     // Reference output values are in range [0.0006, 2.798]
 155     runTensorFlowNet("padding_same");
 156 }
 157
 158 TEST_P(Test_TensorFlow_layers, eltwise)
 159 {
 160     runTensorFlowNet("eltwise_add_mul");
 161     runTensorFlowNet("eltwise_sub");
 162 }
 163
 164 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 165 {
 166     runTensorFlowNet("pad_and_concat");
 167 }
 168
 169 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 170 {
 171     runTensorFlowNet("concat_axis_1");
 172 }
 173
 174 TEST_P(Test_TensorFlow_layers, batch_norm)
 175 {
 176     runTensorFlowNet("batch_norm");
 177     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 178     runTensorFlowNet("fused_batch_norm");
 179     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 180     runTensorFlowNet("batch_norm_text", true);
 181     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 182     runTensorFlowNet("unfused_batch_norm");
 183     runTensorFlowNet("fused_batch_norm_no_gamma");
 184     runTensorFlowNet("unfused_batch_norm_no_gamma");
 185     runTensorFlowNet("mvn_batch_norm");
 186     runTensorFlowNet("mvn_batch_norm_1x1");
 187     runTensorFlowNet("switch_identity");
 188     runTensorFlowNet("keras_batch_norm_training");
 189 }
 190
 191 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 192 {
 193     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 194     {
 195         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 196         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 197         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 198         throw SkipTestException("");
 199     }
 200     runTensorFlowNet("batch_norm3d");
 201 }
 202
 203 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 204 {
 205     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 206         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 207     // Output values range: [-40.0597, 207.827]
 208     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 209     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 210     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 211 }
 212
 213 TEST_P(Test_TensorFlow_layers, pooling)
 214 {
 215     runTensorFlowNet("max_pool_even");
 216     runTensorFlowNet("max_pool_odd_valid");
 217     runTensorFlowNet("max_pool_odd_same");
 218     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 219 }
 220
 221 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 222 {
 223     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 224         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 225     runTensorFlowNet("max_pool_grad");
 226 }
 227
 228 // TODO: fix tests and replace to pooling
 229 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 230 {
 231     // Reference output values are in range [-0.519531, 0.112976]
 232 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 233     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 234             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 235     )
 236         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 237 #endif
 238     runTensorFlowNet("ave_pool_same");
 239 }
 240
 241 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 242 {
 243 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 244     throw SkipTestException("Test is enabled starts from 2019R1");
 245 #endif
 246     if (target != DNN_TARGET_CPU)
 247         throw SkipTestException("Only CPU is supported");
 248     runTensorFlowNet("max_pool3d");
 249 }
 250
 251 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 252 {
 253 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 254     throw SkipTestException("Test is enabled starts from 2019R1");
 255 #endif
 256     if (target != DNN_TARGET_CPU)
 257         throw SkipTestException("Only CPU is supported");
 258     runTensorFlowNet("ave_pool3d");
 259 }
 260
 261 TEST_P(Test_TensorFlow_layers, deconvolution)
 262 {
 263     runTensorFlowNet("deconvolution");
 264     runTensorFlowNet("deconvolution_same");
 265     runTensorFlowNet("deconvolution_stride_2_same");
 266     runTensorFlowNet("deconvolution_adj_pad_valid");
 267     runTensorFlowNet("deconvolution_adj_pad_same");
 268     runTensorFlowNet("keras_deconv_valid");
 269     runTensorFlowNet("keras_deconv_same");
 270     runTensorFlowNet("keras_deconv_same_v2");
 271 }
 272
 273 TEST_P(Test_TensorFlow_layers, matmul)
 274 {
 275     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 276         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 277     runTensorFlowNet("matmul");
 278     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 279     // Reference output values are in range [-5.688, 4.484]
 280     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 281     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 282
 283 }
 284
 285 TEST_P(Test_TensorFlow_layers, reshape)
 286 {
 287     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 288         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 289     runTensorFlowNet("shift_reshape_no_reorder");
 290     runTensorFlowNet("reshape_no_reorder");
 291     runTensorFlowNet("reshape_reduce");
 292     runTensorFlowNet("reshape_as_shape");
 293 }
 294
 295 TEST_P(Test_TensorFlow_layers, flatten)
 296 {
 297 #if defined(INF_ENGINE_RELEASE)
 298     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 299             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 300     )
 301         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 302 #endif
 303
 304     runTensorFlowNet("flatten", true);
 305 }
 306
 307 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 308 {
 309     runTensorFlowNet("unfused_flatten");
 310     runTensorFlowNet("unfused_flatten_unknown_batch");
 311 }
 312
 313 TEST_P(Test_TensorFlow_layers, leaky_relu)
 314 {
 315 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 316     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 317         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 318 #endif
 319     runTensorFlowNet("leaky_relu_order1");
 320     runTensorFlowNet("leaky_relu_order2");
 321     runTensorFlowNet("leaky_relu_order3");
 322 }
 323
 324 TEST_P(Test_TensorFlow_layers, l2_normalize)
 325 {
 326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 327     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 328             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 329     )
 330         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 331 #endif
 332
 333     runTensorFlowNet("l2_normalize");
 334 }
 335
 336 // TODO: fix it and add to l2_normalize
 337 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 338 {
 339 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 340     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 341             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 342     )
 343         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 344 #endif
 345 #if defined(INF_ENGINE_RELEASE)
 346     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 347         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 348 #endif
 349
 350     runTensorFlowNet("l2_normalize_3d");
 351 }
 352
 353 class Test_TensorFlow_nets : public DNNTestLayer {};
 354
 355 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 356 {
 357 #if defined(INF_ENGINE_RELEASE)
 358     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 359     {
 360 #if INF_ENGINE_VER_MAJOR_EQ(2019010000)
 361         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 362             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 363 #else
 364         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 365 #endif
 366     }
 367 #endif
 368
 369     checkBackend();
 370     std::string imgPath = findDataFile("dnn/street.png");
 371     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 372     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 373
 374     Mat inp;
 375     resize(imread(imgPath), inp, Size(300, 300));
 376     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 377
 378     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 379
 380     Net net = readNetFromTensorflow(netPath, netConfig);
 381     net.setPreferableBackend(backend);
 382     net.setPreferableTarget(target);
 383
 384     net.setInput(inp);
 385     Mat out = net.forward();
 386
 387     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 388     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 389     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 390 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 391     expectNoFallbacksFromIE(net);
 392 #endif
 393 }
 394
 395 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 396 {
 397     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 398 #if defined(INF_ENGINE_RELEASE)
 399     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 400     {
 401 #if INF_ENGINE_VER_MAJOR_LE(2019010000)
 402         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 403             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 404 #else
 405         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 406 #endif
 407     }
 408 #endif
 409
 410     checkBackend();
 411     Mat img = imread(findDataFile("dnn/street.png"));
 412     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 413     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 414
 415     Net net = readNetFromTensorflow(model, proto);
 416     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 417
 418     net.setPreferableBackend(backend);
 419     net.setPreferableTarget(target);
 420
 421     net.setInput(blob);
 422     // Output has shape 1x1xNx7 where N - number of detections.
 423     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 424     Mat out = net.forward();
 425     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 426                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 427                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 428                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 429                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 430
 431     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 432     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 433     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 434     expectNoFallbacksFromIE(net);
 435 }
 436
 437 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 438 {
 439 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000)
 440     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 441         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
 442 #endif
 443
 444     checkBackend();
 445     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 446     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 447
 448     Net net = readNetFromTensorflow(model, proto);
 449     Mat img = imread(findDataFile("dnn/dog416.png"));
 450     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 451
 452     net.setPreferableBackend(backend);
 453     net.setPreferableTarget(target);
 454
 455     net.setInput(blob);
 456     Mat out = net.forward();
 457
 458     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 459     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 7e-3 : 1.5e-5;
 460     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 461     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 462
 463 #if defined(INF_ENGINE_RELEASE)
 464     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 465             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 466     )
 467         scoreDiff = 0.061;
 468         iouDiff = 0.12;
 469         detectionConfThresh = 0.36;
 470 #endif
 471     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 472     expectNoFallbacksFromIE(net);
 473 }
 474
 475 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 476 {
 477     // FIXIT split test
 478     applyTestTag(
 479         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 480         CV_TEST_TAG_LONG,
 481         CV_TEST_TAG_DEBUG_VERYLONG
 482     );
 483     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 484                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 485
 486     checkBackend();
 487     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 488         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 489     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 490         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 491
 492     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 493     for (int i = 0; i < 2; ++i)
 494     {
 495         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 496         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 497
 498         Net net = readNetFromTensorflow(model, proto);
 499         net.setPreferableBackend(backend);
 500         net.setPreferableTarget(target);
 501         Mat img = imread(findDataFile("dnn/dog416.png"));
 502         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 503
 504         net.setInput(blob);
 505         Mat out = net.forward();
 506
 507         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 508         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 509     }
 510 }
 511
 512 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 513 {
 514 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 515     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 516         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 517 #endif
 518 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000)
 519     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 520         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE, CV_TEST_TAG_DNN_SKIP_IE_2019R2);
 521 #endif
 522
 523     checkBackend();
 524     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 525     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 526
 527     Net net = readNetFromTensorflow(model, proto);
 528     Mat img = imread(findDataFile("dnn/dog416.png"));
 529     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 530     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 531
 532     net.setPreferableBackend(backend);
 533     net.setPreferableTarget(target);
 534
 535     net.setInput(blob);
 536     Mat out = net.forward();
 537
 538     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 539     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 540     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 541     expectNoFallbacksFromIE(net);
 542 }
 543
 544 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 545 {
 546     checkBackend();
 547     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 548     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 549
 550     Net net = readNetFromTensorflow(model, proto);
 551     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 552     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 553
 554     net.setPreferableBackend(backend);
 555     net.setPreferableTarget(target);
 556     net.setInput(blob);
 557     // Output has shape 1x1xNx7 where N - number of detections.
 558     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 559     Mat out = net.forward();
 560
 561     // References are from test for Caffe model.
 562     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 563                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 564                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 565                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 566                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 567                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 568     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 569     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 570     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 571     expectNoFallbacksFromIE(net);
 572 }
 573
 574 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 575 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 576 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 577 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 578 //                 feed_dict={'input_images:0': inp})
 579 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 580 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 581 // np.save('east_text_detection.scores.npy', scores)
 582 // np.save('east_text_detection.geometry.npy', geometry)
 583 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 584 {
 585     applyTestTag(
 586         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 587         CV_TEST_TAG_DEBUG_LONG
 588     );
 589
 590 #if defined(INF_ENGINE_RELEASE)
 591     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 592         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 593 #endif
 594
 595     checkBackend();
 596
 597     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 598     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 599     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 600     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 601
 602     Net net = readNet(netPath);
 603
 604     net.setPreferableBackend(backend);
 605     net.setPreferableTarget(target);
 606
 607     Mat img = imread(imgPath);
 608     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 609     net.setInput(inp);
 610
 611     std::vector<Mat> outs;
 612     std::vector<String> outNames(2);
 613     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 614     outNames[1] = "feature_fusion/concat_3";
 615     net.forward(outs, outNames);
 616
 617     Mat scores = outs[0];
 618     Mat geometry = outs[1];
 619
 620     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 621     double l1_scores = default_l1, lInf_scores = default_lInf;
 622     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 623     if (target == DNN_TARGET_OPENCL_FP16)
 624     {
 625         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 626         l1_geometry = 0.28; lInf_geometry = 5.94;
 627     }
 628     else if (target == DNN_TARGET_MYRIAD)
 629     {
 630         lInf_scores = 0.41;
 631         l1_geometry = 0.28; lInf_geometry = 5.94;
 632     }
 633     else
 634     {
 635         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 636     }
 637     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 638     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 639     expectNoFallbacksFromIE(net);
 640 }
 641
 642 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 643
 644 TEST_P(Test_TensorFlow_layers, fp16_weights)
 645 {
 646     float l1 = 0.00078;
 647     float lInf = 0.012;
 648     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 649     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 650     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 651     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 652     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 653     // Reference output values are in range [0.0889, 1.651]
 654     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 655     if (target == DNN_TARGET_MYRIAD) {
 656         l1 = 0.0041;
 657         lInf = 0.024;
 658     }
 659     // Reference output values are in range [0, 10.75]
 660     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 661     // Reference output values are in range [0.418, 2.297]
 662     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 663 }
 664
 665 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 666 {
 667     // Reference output values are in range [-3.504, -0.002]
 668     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 669 }
 670
 671 TEST_P(Test_TensorFlow_layers, defun)
 672 {
 673     runTensorFlowNet("defun_dropout");
 674 }
 675
 676 TEST_P(Test_TensorFlow_layers, quantized)
 677 {
 678     runTensorFlowNet("uint8_single_conv");
 679 }
 680
 681 TEST_P(Test_TensorFlow_layers, lstm)
 682 {
 683     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 684         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 685     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 686         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 687     runTensorFlowNet("lstm", true);
 688     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 689 }
 690
 691 TEST_P(Test_TensorFlow_layers, split)
 692 {
 693     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 694         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 695     runTensorFlowNet("split");
 696     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 697         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 698     runTensorFlowNet("split_equals");
 699 }
 700
 701 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 702 {
 703     runTensorFlowNet("resize_nearest_neighbor");
 704     runTensorFlowNet("keras_upsampling2d");
 705 }
 706
 707 TEST_P(Test_TensorFlow_layers, slice)
 708 {
 709     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 710         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 711         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 712     runTensorFlowNet("slice_4d");
 713     runTensorFlowNet("strided_slice");
 714 }
 715
 716 TEST_P(Test_TensorFlow_layers, softmax)
 717 {
 718     runTensorFlowNet("keras_softmax");
 719     runTensorFlowNet("slim_softmax");
 720 }
 721
 722 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 723 {
 724 #if defined(INF_ENGINE_RELEASE)
 725     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 726         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 727     )
 728         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 729 #endif
 730     runTensorFlowNet("slim_softmax_v2");
 731 }
 732
 733 TEST_P(Test_TensorFlow_layers, relu6)
 734 {
 735     runTensorFlowNet("keras_relu6");
 736     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 737 }
 738
 739 TEST_P(Test_TensorFlow_layers, subpixel)
 740 {
 741     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 742         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 743     runTensorFlowNet("subpixel");
 744 }
 745
 746 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 747 {
 748     runTensorFlowNet("keras_mobilenet_head");
 749 }
 750
 751 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 752 {
 753     runTensorFlowNet("resize_bilinear");
 754     runTensorFlowNet("resize_bilinear_factor");
 755 }
 756
 757 TEST_P(Test_TensorFlow_layers, squeeze)
 758 {
 759 #if defined(INF_ENGINE_RELEASE)
 760     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 761             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 762     )
 763         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 764 #endif
 765     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 766     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 767     int squeeze_dims[] = {0, 2, 3, -1};
 768     for (int i = 0; i < 4; ++i)
 769     {
 770         SCOPED_TRACE(format("i=%d", i));
 771         std::string pbtxt =
 772             "node { name: \"input\" op: \"Placeholder\""
 773             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 774             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 775               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 776         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 777         net.setPreferableBackend(backend);
 778         net.setPreferableTarget(target);
 779         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 780         randu(tfInp, -1, 1);
 781
 782         // NHWC to NCHW
 783         CV_Assert(inpShapes[i][0] == 1);
 784         std::swap(inpShapes[i][2], inpShapes[i][3]);
 785         std::swap(inpShapes[i][1], inpShapes[i][2]);
 786         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 787         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 788
 789         net.setInput(cvInp);
 790         Mat out = net.forward();
 791         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 792     }
 793 }
 794
 795 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 796
 797 TEST(Test_TensorFlow, two_inputs)
 798 {
 799     Net net = readNet(path("two_inputs_net.pbtxt"));
 800     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 801
 802     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 803     randu(firstInput, -1, 1);
 804     randu(secondInput, -1, 1);
 805
 806     net.setInput(firstInput, "first_input");
 807     net.setInput(secondInput, "second_input");
 808     Mat out = net.forward();
 809
 810     normAssert(out, firstInput + secondInput);
 811 }
 812
 813 TEST(Test_TensorFlow, Mask_RCNN)
 814 {
 815     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 816     Mat img = imread(findDataFile("dnn/street.png"));
 817     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 818     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 819
 820     Net net = readNetFromTensorflow(model, proto);
 821     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 822     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 823     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 824
 825     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 826
 827     net.setInput(blob);
 828
 829     // Mask-RCNN predicts bounding boxes and segmentation masks.
 830     std::vector<String> outNames(2);
 831     outNames[0] = "detection_out_final";
 832     outNames[1] = "detection_masks";
 833
 834     std::vector<Mat> outs;
 835     net.forward(outs, outNames);
 836
 837     Mat outDetections = outs[0];
 838     Mat outMasks = outs[1];
 839     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 840
 841     // Output size of masks is NxCxHxW where
 842     // N - number of detected boxes
 843     // C - number of classes (excluding background)
 844     // HxW - segmentation shape
 845     const int numDetections = outDetections.size[2];
 846
 847     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 848     Mat masks(4, &masksSize[0], CV_32F);
 849
 850     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 851     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 852
 853     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 854     for (int i = 0; i < numDetections; ++i)
 855     {
 856         // Get a class id for this bounding box and copy mask only for that class.
 857         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 858         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 859         srcRanges[1] = cv::Range(classId, classId + 1);
 860         outMasks(srcRanges).copyTo(masks(dstRanges));
 861     }
 862     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 863     normAssert(masks, refMasks(&topRefMasks[0]));
 864 }
 865
 866 }