modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU)
 140         throw SkipTestException("Only CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("keras_pad_concat");
 149     runTensorFlowNet("mirror_pad");
 150 }
 151
 152 TEST_P(Test_TensorFlow_layers, padding_same)
 153 {
 154     // Reference output values are in range [0.0006, 2.798]
 155     runTensorFlowNet("padding_same");
 156 }
 157
 158 TEST_P(Test_TensorFlow_layers, eltwise)
 159 {
 160     runTensorFlowNet("eltwise_add_mul");
 161     runTensorFlowNet("eltwise_sub");
 162 }
 163
 164 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 165 {
 166     runTensorFlowNet("pad_and_concat");
 167 }
 168
 169 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 170 {
 171     runTensorFlowNet("concat_axis_1");
 172 }
 173
 174 TEST_P(Test_TensorFlow_layers, batch_norm)
 175 {
 176     runTensorFlowNet("batch_norm");
 177     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 178     runTensorFlowNet("fused_batch_norm");
 179     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 180     runTensorFlowNet("batch_norm_text", true);
 181     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 182     runTensorFlowNet("unfused_batch_norm");
 183     runTensorFlowNet("fused_batch_norm_no_gamma");
 184     runTensorFlowNet("unfused_batch_norm_no_gamma");
 185     runTensorFlowNet("mvn_batch_norm");
 186     runTensorFlowNet("mvn_batch_norm_1x1");
 187     runTensorFlowNet("switch_identity");
 188     runTensorFlowNet("keras_batch_norm_training");
 189 }
 190
 191 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 192 {
 193     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 194     {
 195         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 196         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 197         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 198         throw SkipTestException("");
 199     }
 200     runTensorFlowNet("batch_norm3d");
 201 }
 202
 203 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 204 {
 205     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 206         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 207     // Output values range: [-40.0597, 207.827]
 208     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 209     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 210     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 211 }
 212
 213 TEST_P(Test_TensorFlow_layers, pooling)
 214 {
 215     runTensorFlowNet("max_pool_even");
 216     runTensorFlowNet("max_pool_odd_valid");
 217     runTensorFlowNet("max_pool_odd_same");
 218     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 219 }
 220
 221 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 222 {
 223     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 224         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 225     runTensorFlowNet("max_pool_grad");
 226 }
 227
 228 // TODO: fix tests and replace to pooling
 229 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 230 {
 231     // Reference output values are in range [-0.519531, 0.112976]
 232 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 233     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 234             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 235     )
 236         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 237 #endif
 238     runTensorFlowNet("ave_pool_same");
 239 }
 240
 241 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 242 {
 243 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 244     throw SkipTestException("Test is enabled starts from 2019R1");
 245 #endif
 246     if (target != DNN_TARGET_CPU)
 247         throw SkipTestException("Only CPU is supported");
 248     runTensorFlowNet("max_pool3d");
 249 }
 250
 251 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 252 {
 253 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 254     throw SkipTestException("Test is enabled starts from 2019R1");
 255 #endif
 256     if (target != DNN_TARGET_CPU)
 257         throw SkipTestException("Only CPU is supported");
 258     runTensorFlowNet("ave_pool3d");
 259 }
 260
 261 TEST_P(Test_TensorFlow_layers, deconvolution)
 262 {
 263     runTensorFlowNet("deconvolution");
 264     runTensorFlowNet("deconvolution_same");
 265     runTensorFlowNet("deconvolution_stride_2_same");
 266     runTensorFlowNet("deconvolution_adj_pad_valid");
 267     runTensorFlowNet("deconvolution_adj_pad_same");
 268     runTensorFlowNet("keras_deconv_valid");
 269     runTensorFlowNet("keras_deconv_same");
 270     runTensorFlowNet("keras_deconv_same_v2");
 271 }
 272
 273 TEST_P(Test_TensorFlow_layers, matmul)
 274 {
 275     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 276         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 277     runTensorFlowNet("matmul");
 278     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 279     // Reference output values are in range [-5.688, 4.484]
 280     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 281     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 282
 283 }
 284
 285 TEST_P(Test_TensorFlow_layers, reshape)
 286 {
 287     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 288         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 289     runTensorFlowNet("shift_reshape_no_reorder");
 290     runTensorFlowNet("reshape_no_reorder");
 291     runTensorFlowNet("reshape_reduce");
 292     runTensorFlowNet("reshape_as_shape");
 293 }
 294
 295 TEST_P(Test_TensorFlow_layers, flatten)
 296 {
 297 #if defined(INF_ENGINE_RELEASE)
 298     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 299             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 300     )
 301         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 302 #endif
 303
 304     runTensorFlowNet("flatten", true);
 305 }
 306
 307 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 308 {
 309     runTensorFlowNet("unfused_flatten");
 310     runTensorFlowNet("unfused_flatten_unknown_batch");
 311 }
 312
 313 TEST_P(Test_TensorFlow_layers, leaky_relu)
 314 {
 315 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 316     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 317         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 318 #endif
 319     runTensorFlowNet("leaky_relu_order1");
 320     runTensorFlowNet("leaky_relu_order2");
 321     runTensorFlowNet("leaky_relu_order3");
 322 }
 323
 324 TEST_P(Test_TensorFlow_layers, l2_normalize)
 325 {
 326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 327     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 328             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 329     )
 330         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 331 #endif
 332
 333     runTensorFlowNet("l2_normalize");
 334 }
 335
 336 // TODO: fix it and add to l2_normalize
 337 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 338 {
 339 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 340     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 341             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 342     )
 343         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 344 #endif
 345 #if defined(INF_ENGINE_RELEASE)
 346     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 347         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 348 #endif
 349
 350     runTensorFlowNet("l2_normalize_3d");
 351 }
 352
 353 class Test_TensorFlow_nets : public DNNTestLayer {};
 354
 355 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 356 {
 357 #if defined(INF_ENGINE_RELEASE)
 358     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 359     {
 360 #if INF_ENGINE_VER_MAJOR_EQ(2019010000)
 361         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 362             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 363 #else
 364         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 365 #endif
 366     }
 367 #endif
 368
 369     checkBackend();
 370     std::string imgPath = findDataFile("dnn/street.png");
 371     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 372     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 373
 374     Mat inp;
 375     resize(imread(imgPath), inp, Size(300, 300));
 376     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 377
 378     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 379
 380     Net net = readNetFromTensorflow(netPath, netConfig);
 381     net.setPreferableBackend(backend);
 382     net.setPreferableTarget(target);
 383
 384     net.setInput(inp);
 385     Mat out = net.forward();
 386
 387     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 388     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 389     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 390 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 391     expectNoFallbacksFromIE(net);
 392 #endif
 393 }
 394
 395 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 396 {
 397     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 398 #if defined(INF_ENGINE_RELEASE)
 399     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 400     {
 401 #if INF_ENGINE_VER_MAJOR_LE(2019010000)
 402         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 403             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 404 #else
 405         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 406 #endif
 407     }
 408 #endif
 409
 410     checkBackend();
 411     Mat img = imread(findDataFile("dnn/street.png"));
 412     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 413     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 414
 415     Net net = readNetFromTensorflow(model, proto);
 416     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 417
 418     net.setPreferableBackend(backend);
 419     net.setPreferableTarget(target);
 420
 421     net.setInput(blob);
 422     // Output has shape 1x1xNx7 where N - number of detections.
 423     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 424     Mat out = net.forward();
 425     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 426                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 427                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 428                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 429                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 430
 431     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 432     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 433     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 434     expectNoFallbacksFromIE(net);
 435 }
 436
 437 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 438 {
 439     checkBackend();
 440     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 441     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 442
 443     Net net = readNetFromTensorflow(model, proto);
 444     Mat img = imread(findDataFile("dnn/dog416.png"));
 445     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 446
 447     net.setPreferableBackend(backend);
 448     net.setPreferableTarget(target);
 449
 450     net.setInput(blob);
 451     Mat out = net.forward();
 452
 453     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 454     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 1.5e-5;
 455     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 456     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 457
 458 #if defined(INF_ENGINE_RELEASE)
 459     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 460             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 461     )
 462         scoreDiff = 0.061;
 463         iouDiff = 0.12;
 464         detectionConfThresh = 0.36;
 465 #endif
 466     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 467     expectNoFallbacksFromIE(net);
 468 }
 469
 470 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 471 {
 472     // FIXIT split test
 473     applyTestTag(
 474         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 475         CV_TEST_TAG_LONG,
 476         CV_TEST_TAG_DEBUG_VERYLONG
 477     );
 478     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 479                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 480
 481     checkBackend();
 482     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 483         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 484     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 485         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 486
 487     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 488     for (int i = 0; i < 2; ++i)
 489     {
 490         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 491         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 492
 493         Net net = readNetFromTensorflow(model, proto);
 494         net.setPreferableBackend(backend);
 495         net.setPreferableTarget(target);
 496         Mat img = imread(findDataFile("dnn/dog416.png"));
 497         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 498
 499         net.setInput(blob);
 500         Mat out = net.forward();
 501
 502         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 503         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 504     }
 505 }
 506
 507 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 508 {
 509 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 510     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 511         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 512 #endif
 513     checkBackend();
 514     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 515     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 516
 517     Net net = readNetFromTensorflow(model, proto);
 518     Mat img = imread(findDataFile("dnn/dog416.png"));
 519     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 520     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 521
 522     net.setPreferableBackend(backend);
 523     net.setPreferableTarget(target);
 524
 525     net.setInput(blob);
 526     Mat out = net.forward();
 527
 528     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 529     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 530     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 531     expectNoFallbacksFromIE(net);
 532 }
 533
 534 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 535 {
 536     checkBackend();
 537     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 538     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 539
 540     Net net = readNetFromTensorflow(model, proto);
 541     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 542     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 543
 544     net.setPreferableBackend(backend);
 545     net.setPreferableTarget(target);
 546     net.setInput(blob);
 547     // Output has shape 1x1xNx7 where N - number of detections.
 548     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 549     Mat out = net.forward();
 550
 551     // References are from test for Caffe model.
 552     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 553                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 554                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 555                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 556                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 557                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 558     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 559     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 560     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 561     expectNoFallbacksFromIE(net);
 562 }
 563
 564 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 565 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 566 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 567 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 568 //                 feed_dict={'input_images:0': inp})
 569 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 570 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 571 // np.save('east_text_detection.scores.npy', scores)
 572 // np.save('east_text_detection.geometry.npy', geometry)
 573 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 574 {
 575     applyTestTag(
 576         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 577         CV_TEST_TAG_DEBUG_LONG
 578     );
 579
 580 #if defined(INF_ENGINE_RELEASE)
 581     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 582         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 583 #endif
 584
 585     checkBackend();
 586
 587     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 588     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 589     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 590     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 591
 592     Net net = readNet(netPath);
 593
 594     net.setPreferableBackend(backend);
 595     net.setPreferableTarget(target);
 596
 597     Mat img = imread(imgPath);
 598     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 599     net.setInput(inp);
 600
 601     std::vector<Mat> outs;
 602     std::vector<String> outNames(2);
 603     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 604     outNames[1] = "feature_fusion/concat_3";
 605     net.forward(outs, outNames);
 606
 607     Mat scores = outs[0];
 608     Mat geometry = outs[1];
 609
 610     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 611     double l1_scores = default_l1, lInf_scores = default_lInf;
 612     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 613     if (target == DNN_TARGET_OPENCL_FP16)
 614     {
 615         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 616         l1_geometry = 0.28; lInf_geometry = 5.94;
 617     }
 618     else if (target == DNN_TARGET_MYRIAD)
 619     {
 620         lInf_scores = 0.41;
 621         l1_geometry = 0.28; lInf_geometry = 5.94;
 622     }
 623     else
 624     {
 625         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 626     }
 627     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 628     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 629     expectNoFallbacksFromIE(net);
 630 }
 631
 632 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 633
 634 TEST_P(Test_TensorFlow_layers, fp16_weights)
 635 {
 636     float l1 = 0.00078;
 637     float lInf = 0.012;
 638     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 639     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 640     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 641     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 642     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 643     // Reference output values are in range [0.0889, 1.651]
 644     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 645     if (target == DNN_TARGET_MYRIAD) {
 646         l1 = 0.0041;
 647         lInf = 0.024;
 648     }
 649     // Reference output values are in range [0, 10.75]
 650     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 651     // Reference output values are in range [0.418, 2.297]
 652     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 653 }
 654
 655 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 656 {
 657     // Reference output values are in range [-3.504, -0.002]
 658     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 659 }
 660
 661 TEST_P(Test_TensorFlow_layers, defun)
 662 {
 663     runTensorFlowNet("defun_dropout");
 664 }
 665
 666 TEST_P(Test_TensorFlow_layers, quantized)
 667 {
 668     runTensorFlowNet("uint8_single_conv");
 669 }
 670
 671 TEST_P(Test_TensorFlow_layers, lstm)
 672 {
 673     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 674         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 675     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 676         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 677     runTensorFlowNet("lstm", true);
 678     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 679 }
 680
 681 TEST_P(Test_TensorFlow_layers, split)
 682 {
 683     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 684         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 685     runTensorFlowNet("split");
 686     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 687         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 688     runTensorFlowNet("split_equals");
 689 }
 690
 691 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 692 {
 693     runTensorFlowNet("resize_nearest_neighbor");
 694     runTensorFlowNet("keras_upsampling2d");
 695 }
 696
 697 TEST_P(Test_TensorFlow_layers, slice)
 698 {
 699     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 700         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 701         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 702     runTensorFlowNet("slice_4d");
 703     runTensorFlowNet("strided_slice");
 704 }
 705
 706 TEST_P(Test_TensorFlow_layers, softmax)
 707 {
 708     runTensorFlowNet("keras_softmax");
 709     runTensorFlowNet("slim_softmax");
 710 }
 711
 712 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 713 {
 714 #if defined(INF_ENGINE_RELEASE)
 715     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 716         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 717     )
 718         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 719 #endif
 720     runTensorFlowNet("slim_softmax_v2");
 721 }
 722
 723 TEST_P(Test_TensorFlow_layers, relu6)
 724 {
 725     runTensorFlowNet("keras_relu6");
 726     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 727 }
 728
 729 TEST_P(Test_TensorFlow_layers, subpixel)
 730 {
 731     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 732         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 733     runTensorFlowNet("subpixel");
 734 }
 735
 736 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 737 {
 738     runTensorFlowNet("keras_mobilenet_head");
 739 }
 740
 741 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 742 {
 743     runTensorFlowNet("resize_bilinear");
 744     runTensorFlowNet("resize_bilinear_factor");
 745 }
 746
 747 TEST_P(Test_TensorFlow_layers, squeeze)
 748 {
 749 #if defined(INF_ENGINE_RELEASE)
 750     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 751             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 752     )
 753         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 754 #endif
 755     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 756     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 757     int squeeze_dims[] = {0, 2, 3, -1};
 758     for (int i = 0; i < 4; ++i)
 759     {
 760         SCOPED_TRACE(format("i=%d", i));
 761         std::string pbtxt =
 762             "node { name: \"input\" op: \"Placeholder\""
 763             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 764             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 765               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 766         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 767         net.setPreferableBackend(backend);
 768         net.setPreferableTarget(target);
 769         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 770         randu(tfInp, -1, 1);
 771
 772         // NHWC to NCHW
 773         CV_Assert(inpShapes[i][0] == 1);
 774         std::swap(inpShapes[i][2], inpShapes[i][3]);
 775         std::swap(inpShapes[i][1], inpShapes[i][2]);
 776         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 777         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 778
 779         net.setInput(cvInp);
 780         Mat out = net.forward();
 781         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 782     }
 783 }
 784
 785 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 786
 787 TEST(Test_TensorFlow, two_inputs)
 788 {
 789     Net net = readNet(path("two_inputs_net.pbtxt"));
 790     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 791
 792     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 793     randu(firstInput, -1, 1);
 794     randu(secondInput, -1, 1);
 795
 796     net.setInput(firstInput, "first_input");
 797     net.setInput(secondInput, "second_input");
 798     Mat out = net.forward();
 799
 800     normAssert(out, firstInput + secondInput);
 801 }
 802
 803 TEST(Test_TensorFlow, Mask_RCNN)
 804 {
 805     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 806     Mat img = imread(findDataFile("dnn/street.png"));
 807     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 808     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 809
 810     Net net = readNetFromTensorflow(model, proto);
 811     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 812     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 813     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 814
 815     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 816
 817     net.setInput(blob);
 818
 819     // Mask-RCNN predicts bounding boxes and segmentation masks.
 820     std::vector<String> outNames(2);
 821     outNames[0] = "detection_out_final";
 822     outNames[1] = "detection_masks";
 823
 824     std::vector<Mat> outs;
 825     net.forward(outs, outNames);
 826
 827     Mat outDetections = outs[0];
 828     Mat outMasks = outs[1];
 829     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 830
 831     // Output size of masks is NxCxHxW where
 832     // N - number of detected boxes
 833     // C - number of classes (excluding background)
 834     // HxW - segmentation shape
 835     const int numDetections = outDetections.size[2];
 836
 837     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 838     Mat masks(4, &masksSize[0], CV_32F);
 839
 840     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 841     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 842
 843     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 844     for (int i = 0; i < numDetections; ++i)
 845     {
 846         // Get a class id for this bounding box and copy mask only for that class.
 847         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 848         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 849         srcRanges[1] = cv::Range(classId, classId + 1);
 850         outMasks(srcRanges).copyTo(masks(dstRanges));
 851     }
 852     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 853     normAssert(masks, refMasks(&topRefMasks[0]));
 854 }
 855
 856 }