modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU)
 140         throw SkipTestException("Only CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("keras_pad_concat");
 149     runTensorFlowNet("mirror_pad");
 150 }
 151
 152 TEST_P(Test_TensorFlow_layers, padding_same)
 153 {
 154     // Reference output values are in range [0.0006, 2.798]
 155     runTensorFlowNet("padding_same");
 156 }
 157
 158 TEST_P(Test_TensorFlow_layers, eltwise)
 159 {
 160     runTensorFlowNet("eltwise_add_mul");
 161     runTensorFlowNet("eltwise_sub");
 162 }
 163
 164 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 165 {
 166     runTensorFlowNet("pad_and_concat");
 167 }
 168
 169 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 170 {
 171     runTensorFlowNet("concat_axis_1");
 172 }
 173
 174 TEST_P(Test_TensorFlow_layers, batch_norm)
 175 {
 176     runTensorFlowNet("batch_norm");
 177     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 178     runTensorFlowNet("fused_batch_norm");
 179     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 180     runTensorFlowNet("batch_norm_text", true);
 181     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 182     runTensorFlowNet("unfused_batch_norm");
 183     runTensorFlowNet("fused_batch_norm_no_gamma");
 184     runTensorFlowNet("unfused_batch_norm_no_gamma");
 185     runTensorFlowNet("mvn_batch_norm");
 186     runTensorFlowNet("mvn_batch_norm_1x1");
 187     runTensorFlowNet("switch_identity");
 188     runTensorFlowNet("keras_batch_norm_training");
 189 }
 190
 191 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 192 {
 193     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 194     {
 195         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 196         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 197         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 198         throw SkipTestException("");
 199     }
 200     runTensorFlowNet("batch_norm3d");
 201 }
 202
 203 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 204 {
 205     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 206         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 207     // Output values range: [-40.0597, 207.827]
 208     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 209     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 210     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 211 }
 212
 213 TEST_P(Test_TensorFlow_layers, pooling)
 214 {
 215     runTensorFlowNet("max_pool_even");
 216     runTensorFlowNet("max_pool_odd_valid");
 217     runTensorFlowNet("max_pool_odd_same");
 218     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 219 }
 220
 221 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 222 {
 223     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 224         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 225     runTensorFlowNet("max_pool_grad");
 226 }
 227
 228 // TODO: fix tests and replace to pooling
 229 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 230 {
 231     // Reference output values are in range [-0.519531, 0.112976]
 232 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 233     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 234             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 235     )
 236         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 237 #endif
 238     runTensorFlowNet("ave_pool_same");
 239 }
 240
 241 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 242 {
 243 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 244     throw SkipTestException("Test is enabled starts from 2019R1");
 245 #endif
 246     if (target != DNN_TARGET_CPU)
 247         throw SkipTestException("Only CPU is supported");
 248     runTensorFlowNet("max_pool3d");
 249 }
 250
 251 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 252 {
 253 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 254     throw SkipTestException("Test is enabled starts from 2019R1");
 255 #endif
 256     if (target != DNN_TARGET_CPU)
 257         throw SkipTestException("Only CPU is supported");
 258     runTensorFlowNet("ave_pool3d");
 259 }
 260
 261 TEST_P(Test_TensorFlow_layers, deconvolution)
 262 {
 263     runTensorFlowNet("deconvolution");
 264     runTensorFlowNet("deconvolution_same");
 265     runTensorFlowNet("deconvolution_stride_2_same");
 266     runTensorFlowNet("deconvolution_adj_pad_valid");
 267     runTensorFlowNet("deconvolution_adj_pad_same");
 268     runTensorFlowNet("keras_deconv_valid");
 269     runTensorFlowNet("keras_deconv_same");
 270     runTensorFlowNet("keras_deconv_same_v2");
 271 }
 272
 273 TEST_P(Test_TensorFlow_layers, matmul)
 274 {
 275     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 276         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 277     runTensorFlowNet("matmul");
 278     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 279     // Reference output values are in range [-5.688, 4.484]
 280     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 281     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 282     runTensorFlowNet("matmul_layout");
 283 }
 284
 285 TEST_P(Test_TensorFlow_layers, reshape)
 286 {
 287     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 288         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 289     runTensorFlowNet("shift_reshape_no_reorder");
 290     runTensorFlowNet("reshape_no_reorder");
 291     runTensorFlowNet("reshape_reduce");
 292     runTensorFlowNet("reshape_as_shape");
 293 }
 294
 295 TEST_P(Test_TensorFlow_layers, flatten)
 296 {
 297 #if defined(INF_ENGINE_RELEASE)
 298     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 299             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 300     )
 301         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 302 #endif
 303
 304     runTensorFlowNet("flatten", true);
 305 }
 306
 307 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 308 {
 309     runTensorFlowNet("unfused_flatten");
 310     runTensorFlowNet("unfused_flatten_unknown_batch");
 311 }
 312
 313 TEST_P(Test_TensorFlow_layers, leaky_relu)
 314 {
 315 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 316     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 317         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 318 #endif
 319     runTensorFlowNet("leaky_relu_order1");
 320     runTensorFlowNet("leaky_relu_order2");
 321     runTensorFlowNet("leaky_relu_order3");
 322 }
 323
 324 TEST_P(Test_TensorFlow_layers, l2_normalize)
 325 {
 326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 327     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 328             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 329     )
 330         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 331 #endif
 332
 333     runTensorFlowNet("l2_normalize");
 334 }
 335
 336 // TODO: fix it and add to l2_normalize
 337 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 338 {
 339 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 340     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 341             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 342     )
 343         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 344 #endif
 345 #if defined(INF_ENGINE_RELEASE)
 346     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 347         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 348 #endif
 349
 350     runTensorFlowNet("l2_normalize_3d");
 351 }
 352
 353 class Test_TensorFlow_nets : public DNNTestLayer {};
 354
 355 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 356 {
 357 #if defined(INF_ENGINE_RELEASE)
 358     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 359     {
 360 #if INF_ENGINE_VER_MAJOR_GE(2019020000)
 361         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 362             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 363 #endif
 364     }
 365 #endif
 366
 367     checkBackend();
 368     std::string imgPath = findDataFile("dnn/street.png");
 369     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 370     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 371
 372     Mat inp;
 373     resize(imread(imgPath), inp, Size(300, 300));
 374     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 375
 376     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 377
 378     Net net = readNetFromTensorflow(netPath, netConfig);
 379     net.setPreferableBackend(backend);
 380     net.setPreferableTarget(target);
 381
 382     net.setInput(inp);
 383     Mat out = net.forward();
 384
 385     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 386     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 387     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 388 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 389     expectNoFallbacksFromIE(net);
 390 #endif
 391 }
 392
 393 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 394 {
 395     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 396 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LE(2019010000)
 397     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 398         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 399         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 400 #endif
 401
 402     checkBackend();
 403     Mat img = imread(findDataFile("dnn/street.png"));
 404     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 405     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 406
 407     Net net = readNetFromTensorflow(model, proto);
 408     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 409
 410     net.setPreferableBackend(backend);
 411     net.setPreferableTarget(target);
 412
 413     net.setInput(blob);
 414     // Output has shape 1x1xNx7 where N - number of detections.
 415     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 416     Mat out = net.forward();
 417     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 418                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 419                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 420                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 421                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 422
 423     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 424     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 425     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 426     expectNoFallbacksFromIE(net);
 427 }
 428
 429 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 430 {
 431     checkBackend();
 432     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 433     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 434
 435     Net net = readNetFromTensorflow(model, proto);
 436     Mat img = imread(findDataFile("dnn/dog416.png"));
 437     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 438
 439     net.setPreferableBackend(backend);
 440     net.setPreferableTarget(target);
 441
 442     net.setInput(blob);
 443     Mat out = net.forward();
 444
 445     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 446     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.011 : 1.5e-5;
 447     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 448     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 449
 450 #if defined(INF_ENGINE_RELEASE)
 451     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 452         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 453     {
 454         scoreDiff = 0.061;
 455         iouDiff = 0.12;
 456         detectionConfThresh = 0.36;
 457     }
 458 #endif
 459     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 460     expectNoFallbacksFromIE(net);
 461 }
 462
 463 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 464 {
 465     // FIXIT split test
 466     applyTestTag(
 467         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 468         CV_TEST_TAG_LONG,
 469         CV_TEST_TAG_DEBUG_VERYLONG
 470     );
 471     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 472                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 473
 474     checkBackend();
 475     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 476         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 477     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 478         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 479
 480     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 481     for (int i = 0; i < 2; ++i)
 482     {
 483         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 484         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 485
 486         Net net = readNetFromTensorflow(model, proto);
 487         net.setPreferableBackend(backend);
 488         net.setPreferableTarget(target);
 489         Mat img = imread(findDataFile("dnn/dog416.png"));
 490         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 491
 492         net.setInput(blob);
 493         Mat out = net.forward();
 494
 495         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 496         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 497     }
 498 }
 499
 500 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 501 {
 502 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 503     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 504         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 505 #endif
 506     checkBackend();
 507     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 508     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 509
 510     Net net = readNetFromTensorflow(model, proto);
 511     Mat img = imread(findDataFile("dnn/dog416.png"));
 512     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 513     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 514
 515     net.setPreferableBackend(backend);
 516     net.setPreferableTarget(target);
 517
 518     net.setInput(blob);
 519     Mat out = net.forward();
 520
 521     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 522     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 523     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 524     expectNoFallbacksFromIE(net);
 525 }
 526
 527 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 528 {
 529     checkBackend();
 530     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 531     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 532
 533     Net net = readNetFromTensorflow(model, proto);
 534     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 535     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 536
 537     net.setPreferableBackend(backend);
 538     net.setPreferableTarget(target);
 539     net.setInput(blob);
 540     // Output has shape 1x1xNx7 where N - number of detections.
 541     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 542     Mat out = net.forward();
 543
 544     // References are from test for Caffe model.
 545     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 546                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 547                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 548                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 549                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 550                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 551     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 552     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 553     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 554     expectNoFallbacksFromIE(net);
 555 }
 556
 557 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 558 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 559 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 560 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 561 //                 feed_dict={'input_images:0': inp})
 562 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 563 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 564 // np.save('east_text_detection.scores.npy', scores)
 565 // np.save('east_text_detection.geometry.npy', geometry)
 566 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 567 {
 568     applyTestTag(
 569         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 570         CV_TEST_TAG_DEBUG_LONG
 571     );
 572
 573 #if defined(INF_ENGINE_RELEASE)
 574     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 575         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 576 #endif
 577
 578     checkBackend();
 579
 580     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 581     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 582     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 583     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 584
 585     Net net = readNet(netPath);
 586
 587     net.setPreferableBackend(backend);
 588     net.setPreferableTarget(target);
 589
 590     Mat img = imread(imgPath);
 591     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 592     net.setInput(inp);
 593
 594     std::vector<Mat> outs;
 595     std::vector<String> outNames(2);
 596     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 597     outNames[1] = "feature_fusion/concat_3";
 598     net.forward(outs, outNames);
 599
 600     Mat scores = outs[0];
 601     Mat geometry = outs[1];
 602
 603     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 604     double l1_scores = default_l1, lInf_scores = default_lInf;
 605     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 606     if (target == DNN_TARGET_OPENCL_FP16)
 607     {
 608         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 609         l1_geometry = 0.28; lInf_geometry = 5.94;
 610     }
 611     else if (target == DNN_TARGET_MYRIAD)
 612     {
 613         lInf_scores = 0.41;
 614         l1_geometry = 0.28; lInf_geometry = 5.94;
 615     }
 616     else
 617     {
 618         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 619     }
 620     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 621     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 622     expectNoFallbacksFromIE(net);
 623 }
 624
 625 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 626
 627 TEST_P(Test_TensorFlow_layers, fp16_weights)
 628 {
 629     float l1 = 0.00078;
 630     float lInf = 0.012;
 631     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 632     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 633     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 634     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 635     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 636     // Reference output values are in range [0.0889, 1.651]
 637     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 638     if (target == DNN_TARGET_MYRIAD) {
 639         l1 = 0.0041;
 640         lInf = 0.024;
 641     }
 642     // Reference output values are in range [0, 10.75]
 643     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 644     // Reference output values are in range [0.418, 2.297]
 645     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 646 }
 647
 648 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 649 {
 650     // Reference output values are in range [-3.504, -0.002]
 651     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 652 }
 653
 654 TEST_P(Test_TensorFlow_layers, defun)
 655 {
 656     runTensorFlowNet("defun_dropout");
 657 }
 658
 659 TEST_P(Test_TensorFlow_layers, quantized)
 660 {
 661     runTensorFlowNet("uint8_single_conv");
 662 }
 663
 664 TEST_P(Test_TensorFlow_layers, lstm)
 665 {
 666     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 667         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 668     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 669         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 670     runTensorFlowNet("lstm", true);
 671     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 672 }
 673
 674 TEST_P(Test_TensorFlow_layers, split)
 675 {
 676     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 677         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 678     runTensorFlowNet("split");
 679     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 680         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 681     runTensorFlowNet("split_equals");
 682 }
 683
 684 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 685 {
 686     runTensorFlowNet("resize_nearest_neighbor");
 687     runTensorFlowNet("keras_upsampling2d");
 688 }
 689
 690 TEST_P(Test_TensorFlow_layers, slice)
 691 {
 692     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 693         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 694         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 695     runTensorFlowNet("slice_4d");
 696     runTensorFlowNet("strided_slice");
 697 }
 698
 699 TEST_P(Test_TensorFlow_layers, softmax)
 700 {
 701     runTensorFlowNet("keras_softmax");
 702     runTensorFlowNet("slim_softmax");
 703 }
 704
 705 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 706 {
 707 #if defined(INF_ENGINE_RELEASE)
 708     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 709         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 710     )
 711         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 712 #endif
 713     runTensorFlowNet("slim_softmax_v2");
 714 }
 715
 716 TEST_P(Test_TensorFlow_layers, relu6)
 717 {
 718     runTensorFlowNet("keras_relu6");
 719     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 720 }
 721
 722 TEST_P(Test_TensorFlow_layers, subpixel)
 723 {
 724     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 725         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 726     runTensorFlowNet("subpixel");
 727 }
 728
 729 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 730 {
 731     runTensorFlowNet("keras_mobilenet_head");
 732     runTensorFlowNet("keras_learning_phase");
 733 }
 734
 735 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 736 {
 737     runTensorFlowNet("resize_bilinear");
 738     runTensorFlowNet("resize_bilinear_factor");
 739 }
 740
 741 TEST_P(Test_TensorFlow_layers, squeeze)
 742 {
 743 #if defined(INF_ENGINE_RELEASE)
 744     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 745             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 746     )
 747         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 748 #endif
 749     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 750     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 751     int squeeze_dims[] = {0, 2, 3, -1};
 752     for (int i = 0; i < 4; ++i)
 753     {
 754         SCOPED_TRACE(format("i=%d", i));
 755         std::string pbtxt =
 756             "node { name: \"input\" op: \"Placeholder\""
 757             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 758             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 759               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 760         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 761         net.setPreferableBackend(backend);
 762         net.setPreferableTarget(target);
 763         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 764         randu(tfInp, -1, 1);
 765
 766         // NHWC to NCHW
 767         CV_Assert(inpShapes[i][0] == 1);
 768         std::swap(inpShapes[i][2], inpShapes[i][3]);
 769         std::swap(inpShapes[i][1], inpShapes[i][2]);
 770         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 771         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 772
 773         net.setInput(cvInp);
 774         Mat out = net.forward();
 775         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 776     }
 777 }
 778
 779 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 780
 781 TEST(Test_TensorFlow, two_inputs)
 782 {
 783     Net net = readNet(path("two_inputs_net.pbtxt"));
 784     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 785
 786     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 787     randu(firstInput, -1, 1);
 788     randu(secondInput, -1, 1);
 789
 790     net.setInput(firstInput, "first_input");
 791     net.setInput(secondInput, "second_input");
 792     Mat out = net.forward();
 793
 794     normAssert(out, firstInput + secondInput);
 795 }
 796
 797 TEST(Test_TensorFlow, Mask_RCNN)
 798 {
 799     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 800     Mat img = imread(findDataFile("dnn/street.png"));
 801     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 802     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 803
 804     Net net = readNetFromTensorflow(model, proto);
 805     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 806     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 807     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 808
 809     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 810
 811     net.setInput(blob);
 812
 813     // Mask-RCNN predicts bounding boxes and segmentation masks.
 814     std::vector<String> outNames(2);
 815     outNames[0] = "detection_out_final";
 816     outNames[1] = "detection_masks";
 817
 818     std::vector<Mat> outs;
 819     net.forward(outs, outNames);
 820
 821     Mat outDetections = outs[0];
 822     Mat outMasks = outs[1];
 823     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 824
 825     // Output size of masks is NxCxHxW where
 826     // N - number of detected boxes
 827     // C - number of classes (excluding background)
 828     // HxW - segmentation shape
 829     const int numDetections = outDetections.size[2];
 830
 831     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 832     Mat masks(4, &masksSize[0], CV_32F);
 833
 834     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 835     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 836
 837     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 838     for (int i = 0; i < numDetections; ++i)
 839     {
 840         // Get a class id for this bounding box and copy mask only for that class.
 841         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 842         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 843         srcRanges[1] = cv::Range(classId, classId + 1);
 844         outMasks(srcRanges).copyTo(masks(dstRanges));
 845     }
 846     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 847     normAssert(masks, refMasks(&topRefMasks[0]));
 848 }
 849
 850 }