modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU)
 140         throw SkipTestException("Only CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("keras_pad_concat");
 149     runTensorFlowNet("mirror_pad");
 150 }
 151
 152 TEST_P(Test_TensorFlow_layers, padding_same)
 153 {
 154     // Reference output values are in range [0.0006, 2.798]
 155     runTensorFlowNet("padding_same");
 156 }
 157
 158 TEST_P(Test_TensorFlow_layers, eltwise)
 159 {
 160     runTensorFlowNet("eltwise_add_mul");
 161     runTensorFlowNet("eltwise_sub");
 162 }
 163
 164 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 165 {
 166     runTensorFlowNet("pad_and_concat");
 167 }
 168
 169 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 170 {
 171     runTensorFlowNet("concat_axis_1");
 172 }
 173
 174 TEST_P(Test_TensorFlow_layers, batch_norm)
 175 {
 176     runTensorFlowNet("batch_norm");
 177     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 178     runTensorFlowNet("fused_batch_norm");
 179     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 180     runTensorFlowNet("batch_norm_text", true);
 181     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 182     runTensorFlowNet("unfused_batch_norm");
 183     runTensorFlowNet("fused_batch_norm_no_gamma");
 184     runTensorFlowNet("unfused_batch_norm_no_gamma");
 185     runTensorFlowNet("mvn_batch_norm");
 186     runTensorFlowNet("mvn_batch_norm_1x1");
 187     runTensorFlowNet("switch_identity");
 188     runTensorFlowNet("keras_batch_norm_training");
 189 }
 190
 191 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 192 {
 193     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 194     {
 195         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 196         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 197         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 198         throw SkipTestException("");
 199     }
 200     runTensorFlowNet("batch_norm3d");
 201 }
 202
 203 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 204 {
 205     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 206         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 207     // Output values range: [-40.0597, 207.827]
 208     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 209     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 210     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 211 }
 212
 213 TEST_P(Test_TensorFlow_layers, pooling)
 214 {
 215     runTensorFlowNet("max_pool_even");
 216     runTensorFlowNet("max_pool_odd_valid");
 217     runTensorFlowNet("max_pool_odd_same");
 218     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 219 }
 220
 221 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 222 {
 223     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 224         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 225     runTensorFlowNet("max_pool_grad");
 226 }
 227
 228 // TODO: fix tests and replace to pooling
 229 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 230 {
 231     // Reference output values are in range [-0.519531, 0.112976]
 232 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 233     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 234             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 235     )
 236         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 237 #endif
 238     runTensorFlowNet("ave_pool_same");
 239 }
 240
 241 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 242 {
 243 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 244     throw SkipTestException("Test is enabled starts from 2019R1");
 245 #endif
 246     if (target != DNN_TARGET_CPU)
 247         throw SkipTestException("Only CPU is supported");
 248     runTensorFlowNet("max_pool3d");
 249 }
 250
 251 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 252 {
 253 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 254     throw SkipTestException("Test is enabled starts from 2019R1");
 255 #endif
 256     if (target != DNN_TARGET_CPU)
 257         throw SkipTestException("Only CPU is supported");
 258     runTensorFlowNet("ave_pool3d");
 259 }
 260
 261 TEST_P(Test_TensorFlow_layers, deconvolution)
 262 {
 263     runTensorFlowNet("deconvolution");
 264     runTensorFlowNet("deconvolution_same");
 265     runTensorFlowNet("deconvolution_stride_2_same");
 266     runTensorFlowNet("deconvolution_adj_pad_valid");
 267     runTensorFlowNet("deconvolution_adj_pad_same");
 268     runTensorFlowNet("keras_deconv_valid");
 269     runTensorFlowNet("keras_deconv_same");
 270     runTensorFlowNet("keras_deconv_same_v2");
 271 }
 272
 273 TEST_P(Test_TensorFlow_layers, matmul)
 274 {
 275     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 276         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 277     runTensorFlowNet("matmul");
 278     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 279     // Reference output values are in range [-5.688, 4.484]
 280     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 281     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 282
 283 }
 284
 285 TEST_P(Test_TensorFlow_layers, reshape)
 286 {
 287     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 288         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 289     runTensorFlowNet("shift_reshape_no_reorder");
 290     runTensorFlowNet("reshape_no_reorder");
 291     runTensorFlowNet("reshape_reduce");
 292     runTensorFlowNet("reshape_as_shape");
 293 }
 294
 295 TEST_P(Test_TensorFlow_layers, flatten)
 296 {
 297 #if defined(INF_ENGINE_RELEASE)
 298     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 299             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 300     )
 301         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 302 #endif
 303
 304     runTensorFlowNet("flatten", true);
 305 }
 306
 307 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 308 {
 309     runTensorFlowNet("unfused_flatten");
 310     runTensorFlowNet("unfused_flatten_unknown_batch");
 311 }
 312
 313 TEST_P(Test_TensorFlow_layers, leaky_relu)
 314 {
 315 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 316     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 317         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 318 #endif
 319     runTensorFlowNet("leaky_relu_order1");
 320     runTensorFlowNet("leaky_relu_order2");
 321     runTensorFlowNet("leaky_relu_order3");
 322 }
 323
 324 TEST_P(Test_TensorFlow_layers, l2_normalize)
 325 {
 326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 327     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 328             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 329     )
 330         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 331 #endif
 332
 333     runTensorFlowNet("l2_normalize");
 334 }
 335
 336 // TODO: fix it and add to l2_normalize
 337 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 338 {
 339 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 340     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 341             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 342     )
 343         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 344 #endif
 345 #if defined(INF_ENGINE_RELEASE)
 346     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 347         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 348 #endif
 349
 350     runTensorFlowNet("l2_normalize_3d");
 351 }
 352
 353 class Test_TensorFlow_nets : public DNNTestLayer {};
 354
 355 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 356 {
 357 #if defined(INF_ENGINE_RELEASE)
 358     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 359     {
 360 #if INF_ENGINE_VER_MAJOR_GE(2019020000)
 361         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 362             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 363 #endif
 364     }
 365 #endif
 366
 367     checkBackend();
 368     std::string imgPath = findDataFile("dnn/street.png");
 369     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 370     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 371
 372     Mat inp;
 373     resize(imread(imgPath), inp, Size(300, 300));
 374     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 375
 376     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 377
 378     Net net = readNetFromTensorflow(netPath, netConfig);
 379     net.setPreferableBackend(backend);
 380     net.setPreferableTarget(target);
 381
 382     net.setInput(inp);
 383     Mat out = net.forward();
 384
 385     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 386     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 387     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 388 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 389     expectNoFallbacksFromIE(net);
 390 #endif
 391 }
 392
 393 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 394 {
 395     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 396 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
 397     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 398         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 399         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 400 #endif
 401
 402     checkBackend();
 403     Mat img = imread(findDataFile("dnn/street.png"));
 404     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 405     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 406
 407     Net net = readNetFromTensorflow(model, proto);
 408     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 409
 410     net.setPreferableBackend(backend);
 411     net.setPreferableTarget(target);
 412
 413     net.setInput(blob);
 414     // Output has shape 1x1xNx7 where N - number of detections.
 415     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 416     Mat out = net.forward();
 417     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 418                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 419                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 420                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 421                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 422
 423     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 424     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 425     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 426     expectNoFallbacksFromIE(net);
 427 }
 428
 429 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 430 {
 431     checkBackend();
 432     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 433     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 434
 435     Net net = readNetFromTensorflow(model, proto);
 436     Mat img = imread(findDataFile("dnn/dog416.png"));
 437     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 438
 439     net.setPreferableBackend(backend);
 440     net.setPreferableTarget(target);
 441
 442     net.setInput(blob);
 443     Mat out = net.forward();
 444
 445     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 446     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 1.5e-5;
 447     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 448     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 449
 450 #if defined(INF_ENGINE_RELEASE)
 451     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 452         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 453     {
 454         scoreDiff = 0.061;
 455         iouDiff = 0.12;
 456         detectionConfThresh = 0.36;
 457     }
 458 #endif
 459     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 460     expectNoFallbacksFromIE(net);
 461 }
 462
 463 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 464 {
 465     // FIXIT split test
 466     applyTestTag(
 467         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 468         CV_TEST_TAG_LONG,
 469         CV_TEST_TAG_DEBUG_VERYLONG
 470     );
 471     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 472                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 473
 474     checkBackend();
 475     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 476         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 477     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 478         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 479
 480     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 481     for (int i = 0; i < 2; ++i)
 482     {
 483         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 484         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 485
 486         Net net = readNetFromTensorflow(model, proto);
 487         net.setPreferableBackend(backend);
 488         net.setPreferableTarget(target);
 489         Mat img = imread(findDataFile("dnn/dog416.png"));
 490         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 491
 492         net.setInput(blob);
 493         Mat out = net.forward();
 494
 495         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 496         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 497     }
 498 }
 499
 500 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 501 {
 502 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 503     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 504         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 505 #endif
 506     checkBackend();
 507     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 508     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 509
 510     Net net = readNetFromTensorflow(model, proto);
 511     Mat img = imread(findDataFile("dnn/dog416.png"));
 512     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 513     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 514
 515     net.setPreferableBackend(backend);
 516     net.setPreferableTarget(target);
 517
 518     net.setInput(blob);
 519     Mat out = net.forward();
 520
 521     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 522     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 523     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 524     expectNoFallbacksFromIE(net);
 525 }
 526
 527 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 528 {
 529     checkBackend();
 530     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 531     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 532
 533     Net net = readNetFromTensorflow(model, proto);
 534     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 535     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 536
 537     net.setPreferableBackend(backend);
 538     net.setPreferableTarget(target);
 539     net.setInput(blob);
 540     // Output has shape 1x1xNx7 where N - number of detections.
 541     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 542     Mat out = net.forward();
 543
 544     // References are from test for Caffe model.
 545     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 546                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 547                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 548                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 549                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 550                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 551     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 552     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 553     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 554     expectNoFallbacksFromIE(net);
 555 }
 556
 557 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 558 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 559 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 560 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 561 //                 feed_dict={'input_images:0': inp})
 562 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 563 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 564 // np.save('east_text_detection.scores.npy', scores)
 565 // np.save('east_text_detection.geometry.npy', geometry)
 566 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 567 {
 568     applyTestTag(
 569         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 570         CV_TEST_TAG_DEBUG_LONG
 571     );
 572
 573 #if defined(INF_ENGINE_RELEASE)
 574     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 575         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 576 #endif
 577
 578     checkBackend();
 579
 580     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 581     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 582     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 583     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 584
 585     Net net = readNet(netPath);
 586
 587     net.setPreferableBackend(backend);
 588     net.setPreferableTarget(target);
 589
 590     Mat img = imread(imgPath);
 591     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 592     net.setInput(inp);
 593
 594     std::vector<Mat> outs;
 595     std::vector<String> outNames(2);
 596     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 597     outNames[1] = "feature_fusion/concat_3";
 598     net.forward(outs, outNames);
 599
 600     Mat scores = outs[0];
 601     Mat geometry = outs[1];
 602
 603     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 604     double l1_scores = default_l1, lInf_scores = default_lInf;
 605     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 606     if (target == DNN_TARGET_OPENCL_FP16)
 607     {
 608         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 609         l1_geometry = 0.28; lInf_geometry = 5.94;
 610     }
 611     else if (target == DNN_TARGET_MYRIAD)
 612     {
 613         lInf_scores = 0.41;
 614         l1_geometry = 0.28; lInf_geometry = 5.94;
 615     }
 616     else
 617     {
 618         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 619     }
 620     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 621     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 622     expectNoFallbacksFromIE(net);
 623 }
 624
 625 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 626
 627 TEST_P(Test_TensorFlow_layers, fp16_weights)
 628 {
 629     float l1 = 0.00078;
 630     float lInf = 0.012;
 631     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 632     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 633     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 634     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 635     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 636     // Reference output values are in range [0.0889, 1.651]
 637     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 638     if (target == DNN_TARGET_MYRIAD) {
 639         l1 = 0.0041;
 640         lInf = 0.024;
 641     }
 642     // Reference output values are in range [0, 10.75]
 643     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 644     // Reference output values are in range [0.418, 2.297]
 645     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 646 }
 647
 648 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 649 {
 650     // Reference output values are in range [-3.504, -0.002]
 651     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 652 }
 653
 654 TEST_P(Test_TensorFlow_layers, defun)
 655 {
 656     runTensorFlowNet("defun_dropout");
 657 }
 658
 659 TEST_P(Test_TensorFlow_layers, quantized)
 660 {
 661     runTensorFlowNet("uint8_single_conv");
 662 }
 663
 664 TEST_P(Test_TensorFlow_layers, lstm)
 665 {
 666     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 667         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 668     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 669         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 670     runTensorFlowNet("lstm", true);
 671     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 672 }
 673
 674 TEST_P(Test_TensorFlow_layers, split)
 675 {
 676     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 677         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 678     runTensorFlowNet("split");
 679     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 680         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 681     runTensorFlowNet("split_equals");
 682 }
 683
 684 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 685 {
 686     runTensorFlowNet("resize_nearest_neighbor");
 687     runTensorFlowNet("keras_upsampling2d");
 688 }
 689
 690 TEST_P(Test_TensorFlow_layers, slice)
 691 {
 692     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 693         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 694         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 695     runTensorFlowNet("slice_4d");
 696     runTensorFlowNet("strided_slice");
 697 }
 698
 699 TEST_P(Test_TensorFlow_layers, softmax)
 700 {
 701     runTensorFlowNet("keras_softmax");
 702     runTensorFlowNet("slim_softmax");
 703 }
 704
 705 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 706 {
 707 #if defined(INF_ENGINE_RELEASE)
 708     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 709         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 710     )
 711         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 712 #endif
 713     runTensorFlowNet("slim_softmax_v2");
 714 }
 715
 716 TEST_P(Test_TensorFlow_layers, relu6)
 717 {
 718     runTensorFlowNet("keras_relu6");
 719     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 720 }
 721
 722 TEST_P(Test_TensorFlow_layers, subpixel)
 723 {
 724     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 725         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 726     runTensorFlowNet("subpixel");
 727 }
 728
 729 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 730 {
 731     runTensorFlowNet("keras_mobilenet_head");
 732 }
 733
 734 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 735 {
 736     runTensorFlowNet("resize_bilinear");
 737     runTensorFlowNet("resize_bilinear_factor");
 738 }
 739
 740 TEST_P(Test_TensorFlow_layers, squeeze)
 741 {
 742 #if defined(INF_ENGINE_RELEASE)
 743     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 744             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 745     )
 746         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 747 #endif
 748     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 749     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 750     int squeeze_dims[] = {0, 2, 3, -1};
 751     for (int i = 0; i < 4; ++i)
 752     {
 753         SCOPED_TRACE(format("i=%d", i));
 754         std::string pbtxt =
 755             "node { name: \"input\" op: \"Placeholder\""
 756             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 757             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 758               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 759         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 760         net.setPreferableBackend(backend);
 761         net.setPreferableTarget(target);
 762         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 763         randu(tfInp, -1, 1);
 764
 765         // NHWC to NCHW
 766         CV_Assert(inpShapes[i][0] == 1);
 767         std::swap(inpShapes[i][2], inpShapes[i][3]);
 768         std::swap(inpShapes[i][1], inpShapes[i][2]);
 769         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 770         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 771
 772         net.setInput(cvInp);
 773         Mat out = net.forward();
 774         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 775     }
 776 }
 777
 778 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 779
 780 TEST(Test_TensorFlow, two_inputs)
 781 {
 782     Net net = readNet(path("two_inputs_net.pbtxt"));
 783     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 784
 785     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 786     randu(firstInput, -1, 1);
 787     randu(secondInput, -1, 1);
 788
 789     net.setInput(firstInput, "first_input");
 790     net.setInput(secondInput, "second_input");
 791     Mat out = net.forward();
 792
 793     normAssert(out, firstInput + secondInput);
 794 }
 795
 796 TEST(Test_TensorFlow, Mask_RCNN)
 797 {
 798     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 799     Mat img = imread(findDataFile("dnn/street.png"));
 800     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 801     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 802
 803     Net net = readNetFromTensorflow(model, proto);
 804     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 805     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 806     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 807
 808     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 809
 810     net.setInput(blob);
 811
 812     // Mask-RCNN predicts bounding boxes and segmentation masks.
 813     std::vector<String> outNames(2);
 814     outNames[0] = "detection_out_final";
 815     outNames[1] = "detection_masks";
 816
 817     std::vector<Mat> outs;
 818     net.forward(outs, outNames);
 819
 820     Mat outDetections = outs[0];
 821     Mat outMasks = outs[1];
 822     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 823
 824     // Output size of masks is NxCxHxW where
 825     // N - number of detected boxes
 826     // C - number of classes (excluding background)
 827     // HxW - segmentation shape
 828     const int numDetections = outDetections.size[2];
 829
 830     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 831     Mat masks(4, &masksSize[0], CV_32F);
 832
 833     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 834     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 835
 836     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 837     for (int i = 0; i < numDetections; ++i)
 838     {
 839         // Get a class id for this bounding box and copy mask only for that class.
 840         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 841         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 842         srcRanges[1] = cv::Range(classId, classId + 1);
 843         outMasks(srcRanges).copyTo(masks(dstRanges));
 844     }
 845     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 846     normAssert(masks, refMasks(&topRefMasks[0]));
 847 }
 848
 849 }