modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU)
 140         throw SkipTestException("Only CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("keras_pad_concat");
 149     runTensorFlowNet("mirror_pad");
 150 }
 151
 152 TEST_P(Test_TensorFlow_layers, padding_same)
 153 {
 154     // Reference output values are in range [0.0006, 2.798]
 155     runTensorFlowNet("padding_same");
 156 }
 157
 158 TEST_P(Test_TensorFlow_layers, eltwise)
 159 {
 160     runTensorFlowNet("eltwise_add_mul");
 161     runTensorFlowNet("eltwise_sub");
 162 }
 163
 164 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 165 {
 166     runTensorFlowNet("pad_and_concat");
 167 }
 168
 169 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 170 {
 171     runTensorFlowNet("concat_axis_1");
 172 }
 173
 174 TEST_P(Test_TensorFlow_layers, batch_norm)
 175 {
 176     runTensorFlowNet("batch_norm");
 177     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 178     runTensorFlowNet("fused_batch_norm");
 179     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 180     runTensorFlowNet("batch_norm_text", true);
 181     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 182     runTensorFlowNet("unfused_batch_norm");
 183     runTensorFlowNet("fused_batch_norm_no_gamma");
 184     runTensorFlowNet("unfused_batch_norm_no_gamma");
 185     runTensorFlowNet("mvn_batch_norm");
 186     runTensorFlowNet("mvn_batch_norm_1x1");
 187     runTensorFlowNet("switch_identity");
 188     runTensorFlowNet("keras_batch_norm_training");
 189 }
 190
 191 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 192 {
 193     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 194     {
 195         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 196         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 197         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 198         throw SkipTestException("");
 199     }
 200     runTensorFlowNet("batch_norm3d");
 201 }
 202
 203 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 204 {
 205     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 206         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 207     // Output values range: [-40.0597, 207.827]
 208     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 209     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 210     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 211 }
 212
 213 TEST_P(Test_TensorFlow_layers, pooling)
 214 {
 215     runTensorFlowNet("max_pool_even");
 216     runTensorFlowNet("max_pool_odd_valid");
 217     runTensorFlowNet("max_pool_odd_same");
 218     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 219 }
 220
 221 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 222 {
 223     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 224         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 225     runTensorFlowNet("max_pool_grad");
 226 }
 227
 228 // TODO: fix tests and replace to pooling
 229 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 230 {
 231     // Reference output values are in range [-0.519531, 0.112976]
 232 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 233     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 234             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 235     )
 236         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 237 #endif
 238     runTensorFlowNet("ave_pool_same");
 239 }
 240
 241 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 242 {
 243 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 244     throw SkipTestException("Test is enabled starts from 2019R1");
 245 #endif
 246     if (target != DNN_TARGET_CPU)
 247         throw SkipTestException("Only CPU is supported");
 248     runTensorFlowNet("max_pool3d");
 249 }
 250
 251 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 252 {
 253 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 254     throw SkipTestException("Test is enabled starts from 2019R1");
 255 #endif
 256     if (target != DNN_TARGET_CPU)
 257         throw SkipTestException("Only CPU is supported");
 258     runTensorFlowNet("ave_pool3d");
 259 }
 260
 261 TEST_P(Test_TensorFlow_layers, deconvolution)
 262 {
 263     runTensorFlowNet("deconvolution");
 264     runTensorFlowNet("deconvolution_same");
 265     runTensorFlowNet("deconvolution_stride_2_same");
 266     runTensorFlowNet("deconvolution_adj_pad_valid");
 267     runTensorFlowNet("deconvolution_adj_pad_same");
 268     runTensorFlowNet("keras_deconv_valid");
 269     runTensorFlowNet("keras_deconv_same");
 270     runTensorFlowNet("keras_deconv_same_v2");
 271 }
 272
 273 TEST_P(Test_TensorFlow_layers, matmul)
 274 {
 275     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 276         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 277     runTensorFlowNet("matmul");
 278     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 279     // Reference output values are in range [-5.688, 4.484]
 280     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 281     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 282
 283 }
 284
 285 TEST_P(Test_TensorFlow_layers, reshape)
 286 {
 287     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 288         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 289     runTensorFlowNet("shift_reshape_no_reorder");
 290     runTensorFlowNet("reshape_no_reorder");
 291     runTensorFlowNet("reshape_reduce");
 292     runTensorFlowNet("reshape_as_shape");
 293 }
 294
 295 TEST_P(Test_TensorFlow_layers, flatten)
 296 {
 297 #if defined(INF_ENGINE_RELEASE)
 298     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 299             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 300     )
 301         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 302 #endif
 303
 304     runTensorFlowNet("flatten", true);
 305 }
 306
 307 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 308 {
 309     runTensorFlowNet("unfused_flatten");
 310     runTensorFlowNet("unfused_flatten_unknown_batch");
 311 }
 312
 313 TEST_P(Test_TensorFlow_layers, leaky_relu)
 314 {
 315 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 316     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 317         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 318 #endif
 319     runTensorFlowNet("leaky_relu_order1");
 320     runTensorFlowNet("leaky_relu_order2");
 321     runTensorFlowNet("leaky_relu_order3");
 322 }
 323
 324 TEST_P(Test_TensorFlow_layers, l2_normalize)
 325 {
 326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 327     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 328             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 329     )
 330         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 331 #endif
 332
 333     runTensorFlowNet("l2_normalize");
 334 }
 335
 336 // TODO: fix it and add to l2_normalize
 337 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 338 {
 339 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 340     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 341             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 342     )
 343         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 344 #endif
 345 #if defined(INF_ENGINE_RELEASE)
 346     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 347         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 348 #endif
 349
 350     runTensorFlowNet("l2_normalize_3d");
 351 }
 352
 353 TEST_P(Test_TensorFlow_layers, Split)
 354 {
 355     runTensorFlowNet("split");
 356 }
 357
 358 class Test_TensorFlow_nets : public DNNTestLayer {};
 359
 360 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 361 {
 362 #if defined(INF_ENGINE_RELEASE)
 363     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 364     {
 365 #if INF_ENGINE_VER_MAJOR_GE(2019010000)
 366         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 367             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 368 #else
 369             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 370 #endif
 371     }
 372 #endif
 373
 374     checkBackend();
 375     std::string imgPath = findDataFile("dnn/street.png");
 376     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 377     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 378
 379     Mat inp;
 380     resize(imread(imgPath), inp, Size(300, 300));
 381     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 382
 383     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 384
 385     Net net = readNetFromTensorflow(netPath, netConfig);
 386     net.setPreferableBackend(backend);
 387     net.setPreferableTarget(target);
 388
 389     net.setInput(inp);
 390     Mat out = net.forward();
 391
 392     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 393     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 394     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 395 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 396     expectNoFallbacksFromIE(net);
 397 #endif
 398 }
 399
 400 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 401 {
 402     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 403
 404 #if defined(INF_ENGINE_RELEASE)
 405     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 406             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 407     )
 408         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 409 #endif
 410
 411     checkBackend();
 412     Mat img = imread(findDataFile("dnn/street.png"));
 413     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 414     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 415
 416     Net net = readNetFromTensorflow(model, proto);
 417     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 418
 419     net.setPreferableBackend(backend);
 420     net.setPreferableTarget(target);
 421
 422     net.setInput(blob);
 423     // Output has shape 1x1xNx7 where N - number of detections.
 424     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 425     Mat out = net.forward();
 426     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 427                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 428                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 429                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 430                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 431
 432     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 433     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 434     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 435     expectNoFallbacksFromIE(net);
 436 }
 437
 438 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 439 {
 440     checkBackend();
 441     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 442     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 443
 444     Net net = readNetFromTensorflow(model, proto);
 445     Mat img = imread(findDataFile("dnn/dog416.png"));
 446     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 447
 448     net.setPreferableBackend(backend);
 449     net.setPreferableTarget(target);
 450
 451     net.setInput(blob);
 452     Mat out = net.forward();
 453
 454     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 455     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 7e-3 : 1.5e-5;
 456     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 457     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 458
 459 #if defined(INF_ENGINE_RELEASE)
 460     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 461             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 462     )
 463         scoreDiff = 0.061;
 464         iouDiff = 0.12;
 465         detectionConfThresh = 0.36;
 466 #endif
 467     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 468     expectNoFallbacksFromIE(net);
 469 }
 470
 471 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 472 {
 473     // FIXIT split test
 474     applyTestTag(
 475         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 476         CV_TEST_TAG_LONG,
 477         CV_TEST_TAG_DEBUG_VERYLONG
 478     );
 479     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 480                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 481
 482     checkBackend();
 483     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 484         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 485     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 486         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 487
 488     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 489     for (int i = 0; i < 2; ++i)
 490     {
 491         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 492         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 493
 494         Net net = readNetFromTensorflow(model, proto);
 495         net.setPreferableBackend(backend);
 496         net.setPreferableTarget(target);
 497         Mat img = imread(findDataFile("dnn/dog416.png"));
 498         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 499
 500         net.setInput(blob);
 501         Mat out = net.forward();
 502
 503         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 504         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 505     }
 506 }
 507
 508 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 509 {
 510 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 511     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 512         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 513 #endif
 514
 515     checkBackend();
 516     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 517     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 518
 519     Net net = readNetFromTensorflow(model, proto);
 520     Mat img = imread(findDataFile("dnn/dog416.png"));
 521     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 522     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 523
 524     net.setPreferableBackend(backend);
 525     net.setPreferableTarget(target);
 526
 527     net.setInput(blob);
 528     Mat out = net.forward();
 529
 530     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 531     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 532     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 533     expectNoFallbacksFromIE(net);
 534 }
 535
 536 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 537 {
 538     checkBackend();
 539     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 540     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 541
 542     Net net = readNetFromTensorflow(model, proto);
 543     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 544     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 545
 546     net.setPreferableBackend(backend);
 547     net.setPreferableTarget(target);
 548     net.setInput(blob);
 549     // Output has shape 1x1xNx7 where N - number of detections.
 550     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 551     Mat out = net.forward();
 552
 553     // References are from test for Caffe model.
 554     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 555                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 556                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 557                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 558                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 559                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 560     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 561     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 562     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 563     expectNoFallbacksFromIE(net);
 564 }
 565
 566 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 567 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 568 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 569 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 570 //                 feed_dict={'input_images:0': inp})
 571 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 572 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 573 // np.save('east_text_detection.scores.npy', scores)
 574 // np.save('east_text_detection.geometry.npy', geometry)
 575 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 576 {
 577     applyTestTag(
 578         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 579         CV_TEST_TAG_DEBUG_LONG
 580     );
 581
 582 #if defined(INF_ENGINE_RELEASE)
 583     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 584         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 585 #endif
 586
 587     checkBackend();
 588
 589     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 590     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 591     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 592     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 593
 594     Net net = readNet(netPath);
 595
 596     net.setPreferableBackend(backend);
 597     net.setPreferableTarget(target);
 598
 599     Mat img = imread(imgPath);
 600     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 601     net.setInput(inp);
 602
 603     std::vector<Mat> outs;
 604     std::vector<String> outNames(2);
 605     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 606     outNames[1] = "feature_fusion/concat_3";
 607     net.forward(outs, outNames);
 608
 609     Mat scores = outs[0];
 610     Mat geometry = outs[1];
 611
 612     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 613     double l1_scores = default_l1, lInf_scores = default_lInf;
 614     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 615     if (target == DNN_TARGET_OPENCL_FP16)
 616     {
 617         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 618         l1_geometry = 0.28; lInf_geometry = 5.94;
 619     }
 620     else if (target == DNN_TARGET_MYRIAD)
 621     {
 622         lInf_scores = 0.41;
 623         l1_geometry = 0.28; lInf_geometry = 5.94;
 624     }
 625     else
 626     {
 627         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 628     }
 629     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 630     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 631     expectNoFallbacksFromIE(net);
 632 }
 633
 634 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 635
 636 TEST_P(Test_TensorFlow_layers, fp16_weights)
 637 {
 638     float l1 = 0.00078;
 639     float lInf = 0.012;
 640     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 641     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 642     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 643     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 644     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 645     // Reference output values are in range [0.0889, 1.651]
 646     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 647     if (target == DNN_TARGET_MYRIAD) {
 648         l1 = 0.0041;
 649         lInf = 0.024;
 650     }
 651     // Reference output values are in range [0, 10.75]
 652     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 653     // Reference output values are in range [0.418, 2.297]
 654     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 655 }
 656
 657 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 658 {
 659     // Reference output values are in range [-3.504, -0.002]
 660     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 661 }
 662
 663 TEST_P(Test_TensorFlow_layers, defun)
 664 {
 665     runTensorFlowNet("defun_dropout");
 666 }
 667
 668 TEST_P(Test_TensorFlow_layers, quantized)
 669 {
 670     runTensorFlowNet("uint8_single_conv");
 671 }
 672
 673 TEST_P(Test_TensorFlow_layers, lstm)
 674 {
 675     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 676         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 677     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 678         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 679     runTensorFlowNet("lstm", true);
 680     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 681 }
 682
 683 TEST_P(Test_TensorFlow_layers, split)
 684 {
 685     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 686         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 687     runTensorFlowNet("split_equals");
 688 }
 689
 690 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 691 {
 692     runTensorFlowNet("resize_nearest_neighbor");
 693     runTensorFlowNet("keras_upsampling2d");
 694 }
 695
 696 TEST_P(Test_TensorFlow_layers, slice)
 697 {
 698     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 699         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 700         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 701     runTensorFlowNet("slice_4d");
 702     runTensorFlowNet("strided_slice");
 703 }
 704
 705 TEST_P(Test_TensorFlow_layers, softmax)
 706 {
 707     runTensorFlowNet("keras_softmax");
 708     runTensorFlowNet("slim_softmax");
 709 }
 710
 711 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 712 {
 713 #if defined(INF_ENGINE_RELEASE)
 714     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 715         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 716     )
 717         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 718 #endif
 719     runTensorFlowNet("slim_softmax_v2");
 720 }
 721
 722 TEST_P(Test_TensorFlow_layers, relu6)
 723 {
 724     runTensorFlowNet("keras_relu6");
 725     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 726 }
 727
 728 TEST_P(Test_TensorFlow_layers, subpixel)
 729 {
 730     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 731         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 732     runTensorFlowNet("subpixel");
 733 }
 734
 735 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 736 {
 737     runTensorFlowNet("keras_mobilenet_head");
 738 }
 739
 740 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 741 {
 742     runTensorFlowNet("resize_bilinear");
 743     runTensorFlowNet("resize_bilinear_factor");
 744 }
 745
 746 TEST_P(Test_TensorFlow_layers, squeeze)
 747 {
 748 #if defined(INF_ENGINE_RELEASE)
 749     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 750             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 751     )
 752         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 753 #endif
 754     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 755     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 756     int squeeze_dims[] = {0, 2, 3, -1};
 757     for (int i = 0; i < 4; ++i)
 758     {
 759         SCOPED_TRACE(format("i=%d", i));
 760         std::string pbtxt =
 761             "node { name: \"input\" op: \"Placeholder\""
 762             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 763             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 764               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 765         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 766         net.setPreferableBackend(backend);
 767         net.setPreferableTarget(target);
 768         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 769         randu(tfInp, -1, 1);
 770
 771         // NHWC to NCHW
 772         CV_Assert(inpShapes[i][0] == 1);
 773         std::swap(inpShapes[i][2], inpShapes[i][3]);
 774         std::swap(inpShapes[i][1], inpShapes[i][2]);
 775         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 776         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 777
 778         net.setInput(cvInp);
 779         Mat out = net.forward();
 780         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 781     }
 782 }
 783
 784 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 785
 786 TEST(Test_TensorFlow, two_inputs)
 787 {
 788     Net net = readNet(path("two_inputs_net.pbtxt"));
 789     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 790
 791     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 792     randu(firstInput, -1, 1);
 793     randu(secondInput, -1, 1);
 794
 795     net.setInput(firstInput, "first_input");
 796     net.setInput(secondInput, "second_input");
 797     Mat out = net.forward();
 798
 799     normAssert(out, firstInput + secondInput);
 800 }
 801
 802 TEST(Test_TensorFlow, Mask_RCNN)
 803 {
 804     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 805     Mat img = imread(findDataFile("dnn/street.png"));
 806     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 807     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 808
 809     Net net = readNetFromTensorflow(model, proto);
 810     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 811     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 812     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 813
 814     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 815
 816     net.setInput(blob);
 817
 818     // Mask-RCNN predicts bounding boxes and segmentation masks.
 819     std::vector<String> outNames(2);
 820     outNames[0] = "detection_out_final";
 821     outNames[1] = "detection_masks";
 822
 823     std::vector<Mat> outs;
 824     net.forward(outs, outNames);
 825
 826     Mat outDetections = outs[0];
 827     Mat outMasks = outs[1];
 828     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 829
 830     // Output size of masks is NxCxHxW where
 831     // N - number of detected boxes
 832     // C - number of classes (excluding background)
 833     // HxW - segmentation shape
 834     const int numDetections = outDetections.size[2];
 835
 836     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 837     Mat masks(4, &masksSize[0], CV_32F);
 838
 839     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 840     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 841
 842     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 843     for (int i = 0; i < numDetections; ++i)
 844     {
 845         // Get a class id for this bounding box and copy mask only for that class.
 846         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 847         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 848         srcRanges[1] = cv::Range(classId, classId + 1);
 849         outMasks(srcRanges).copyTo(masks(dstRanges));
 850     }
 851     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 852     normAssert(masks, refMasks(&topRefMasks[0]));
 853 }
 854
 855 }