modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU)
 140         throw SkipTestException("Only CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("keras_pad_concat");
 149     runTensorFlowNet("mirror_pad");
 150 }
 151
 152 TEST_P(Test_TensorFlow_layers, padding_same)
 153 {
 154     // Reference output values are in range [0.0006, 2.798]
 155     runTensorFlowNet("padding_same");
 156 }
 157
 158 TEST_P(Test_TensorFlow_layers, eltwise)
 159 {
 160     runTensorFlowNet("eltwise_add_mul");
 161     runTensorFlowNet("eltwise_sub");
 162 }
 163
 164 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 165 {
 166     runTensorFlowNet("pad_and_concat");
 167 }
 168
 169 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 170 {
 171     runTensorFlowNet("concat_axis_1");
 172 }
 173
 174 TEST_P(Test_TensorFlow_layers, batch_norm)
 175 {
 176     runTensorFlowNet("batch_norm");
 177     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 178     runTensorFlowNet("fused_batch_norm");
 179     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 180     runTensorFlowNet("batch_norm_text", true);
 181     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 182     runTensorFlowNet("unfused_batch_norm");
 183     runTensorFlowNet("fused_batch_norm_no_gamma");
 184     runTensorFlowNet("unfused_batch_norm_no_gamma");
 185     runTensorFlowNet("mvn_batch_norm");
 186     runTensorFlowNet("mvn_batch_norm_1x1");
 187     runTensorFlowNet("switch_identity");
 188     runTensorFlowNet("keras_batch_norm_training");
 189 }
 190
 191 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 192 {
 193     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 194     {
 195         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 196         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 197         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 198         throw SkipTestException("");
 199     }
 200     runTensorFlowNet("batch_norm3d");
 201 }
 202
 203 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 204 {
 205     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 206         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 207     // Output values range: [-40.0597, 207.827]
 208     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 209     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 210     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 211 }
 212
 213 TEST_P(Test_TensorFlow_layers, pooling)
 214 {
 215     runTensorFlowNet("max_pool_even");
 216     runTensorFlowNet("max_pool_odd_valid");
 217     runTensorFlowNet("max_pool_odd_same");
 218     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 219 }
 220
 221 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 222 {
 223     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 224         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 225     runTensorFlowNet("max_pool_grad");
 226 }
 227
 228 // TODO: fix tests and replace to pooling
 229 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 230 {
 231     // Reference output values are in range [-0.519531, 0.112976]
 232 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 233     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 234             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 235     )
 236         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 237 #endif
 238     runTensorFlowNet("ave_pool_same");
 239 }
 240
 241 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 242 {
 243 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 244     throw SkipTestException("Test is enabled starts from 2019R1");
 245 #endif
 246     if (target != DNN_TARGET_CPU)
 247         throw SkipTestException("Only CPU is supported");
 248     runTensorFlowNet("max_pool3d");
 249 }
 250
 251 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 252 {
 253 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 254     throw SkipTestException("Test is enabled starts from 2019R1");
 255 #endif
 256     if (target != DNN_TARGET_CPU)
 257         throw SkipTestException("Only CPU is supported");
 258     runTensorFlowNet("ave_pool3d");
 259 }
 260
 261 TEST_P(Test_TensorFlow_layers, deconvolution)
 262 {
 263     runTensorFlowNet("deconvolution");
 264     runTensorFlowNet("deconvolution_same");
 265     runTensorFlowNet("deconvolution_stride_2_same");
 266     runTensorFlowNet("deconvolution_adj_pad_valid");
 267     runTensorFlowNet("deconvolution_adj_pad_same");
 268     runTensorFlowNet("keras_deconv_valid");
 269     runTensorFlowNet("keras_deconv_same");
 270     runTensorFlowNet("keras_deconv_same_v2");
 271 }
 272
 273 TEST_P(Test_TensorFlow_layers, matmul)
 274 {
 275     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 276         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 277     runTensorFlowNet("matmul");
 278     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 279     // Reference output values are in range [-5.688, 4.484]
 280     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 281     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 282
 283 }
 284
 285 TEST_P(Test_TensorFlow_layers, reshape)
 286 {
 287     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 288         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 289     runTensorFlowNet("shift_reshape_no_reorder");
 290     runTensorFlowNet("reshape_no_reorder");
 291     runTensorFlowNet("reshape_reduce");
 292     runTensorFlowNet("reshape_as_shape");
 293 }
 294
 295 TEST_P(Test_TensorFlow_layers, flatten)
 296 {
 297 #if defined(INF_ENGINE_RELEASE)
 298     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 299             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 300     )
 301         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 302 #endif
 303
 304     runTensorFlowNet("flatten", true);
 305 }
 306
 307 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 308 {
 309     runTensorFlowNet("unfused_flatten");
 310     runTensorFlowNet("unfused_flatten_unknown_batch");
 311 }
 312
 313 TEST_P(Test_TensorFlow_layers, leaky_relu)
 314 {
 315 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 316     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 317         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 318 #endif
 319     runTensorFlowNet("leaky_relu_order1");
 320     runTensorFlowNet("leaky_relu_order2");
 321     runTensorFlowNet("leaky_relu_order3");
 322 }
 323
 324 TEST_P(Test_TensorFlow_layers, l2_normalize)
 325 {
 326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 327     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 328             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 329     )
 330         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 331 #endif
 332
 333     runTensorFlowNet("l2_normalize");
 334 }
 335
 336 // TODO: fix it and add to l2_normalize
 337 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 338 {
 339 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 340     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 341             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 342     )
 343         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 344 #endif
 345 #if defined(INF_ENGINE_RELEASE)
 346     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 347         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 348 #endif
 349
 350     runTensorFlowNet("l2_normalize_3d");
 351 }
 352
 353 class Test_TensorFlow_nets : public DNNTestLayer {};
 354
 355 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 356 {
 357 #if defined(INF_ENGINE_RELEASE)
 358     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 359     {
 360 #if INF_ENGINE_VER_MAJOR_GE(2019010000)
 361         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 362             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 363 #else
 364             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 365 #endif
 366     }
 367 #endif
 368
 369     checkBackend();
 370     std::string imgPath = findDataFile("dnn/street.png");
 371     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 372     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 373
 374     Mat inp;
 375     resize(imread(imgPath), inp, Size(300, 300));
 376     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 377
 378     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 379
 380     Net net = readNetFromTensorflow(netPath, netConfig);
 381     net.setPreferableBackend(backend);
 382     net.setPreferableTarget(target);
 383
 384     net.setInput(inp);
 385     Mat out = net.forward();
 386
 387     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 388     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 389     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 390 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 391     expectNoFallbacksFromIE(net);
 392 #endif
 393 }
 394
 395 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 396 {
 397     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 398
 399 #if defined(INF_ENGINE_RELEASE)
 400     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 401             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 402     )
 403         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 404 #endif
 405
 406     checkBackend();
 407     Mat img = imread(findDataFile("dnn/street.png"));
 408     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 409     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 410
 411     Net net = readNetFromTensorflow(model, proto);
 412     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 413
 414     net.setPreferableBackend(backend);
 415     net.setPreferableTarget(target);
 416
 417     net.setInput(blob);
 418     // Output has shape 1x1xNx7 where N - number of detections.
 419     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 420     Mat out = net.forward();
 421     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 422                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 423                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 424                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 425                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 426
 427     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 428     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 429     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 430     expectNoFallbacksFromIE(net);
 431 }
 432
 433 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 434 {
 435     checkBackend();
 436     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 437     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 438
 439     Net net = readNetFromTensorflow(model, proto);
 440     Mat img = imread(findDataFile("dnn/dog416.png"));
 441     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 442
 443     net.setPreferableBackend(backend);
 444     net.setPreferableTarget(target);
 445
 446     net.setInput(blob);
 447     Mat out = net.forward();
 448
 449     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 450     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 7e-3 : 1.5e-5;
 451     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 452     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 453
 454 #if defined(INF_ENGINE_RELEASE)
 455     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 456             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 457     )
 458         scoreDiff = 0.061;
 459         iouDiff = 0.12;
 460         detectionConfThresh = 0.36;
 461 #endif
 462     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 463     expectNoFallbacksFromIE(net);
 464 }
 465
 466 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 467 {
 468     // FIXIT split test
 469     applyTestTag(
 470         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 471         CV_TEST_TAG_LONG,
 472         CV_TEST_TAG_DEBUG_VERYLONG
 473     );
 474     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 475                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 476
 477     checkBackend();
 478     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 479         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 480     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 481         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 482
 483     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 484     for (int i = 0; i < 2; ++i)
 485     {
 486         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 487         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 488
 489         Net net = readNetFromTensorflow(model, proto);
 490         net.setPreferableBackend(backend);
 491         net.setPreferableTarget(target);
 492         Mat img = imread(findDataFile("dnn/dog416.png"));
 493         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 494
 495         net.setInput(blob);
 496         Mat out = net.forward();
 497
 498         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 499         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 500     }
 501 }
 502
 503 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 504 {
 505 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 506     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 507         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 508 #endif
 509
 510     checkBackend();
 511     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 512     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 513
 514     Net net = readNetFromTensorflow(model, proto);
 515     Mat img = imread(findDataFile("dnn/dog416.png"));
 516     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 517     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 518
 519     net.setPreferableBackend(backend);
 520     net.setPreferableTarget(target);
 521
 522     net.setInput(blob);
 523     Mat out = net.forward();
 524
 525     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 526     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 527     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 528     expectNoFallbacksFromIE(net);
 529 }
 530
 531 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 532 {
 533     checkBackend();
 534     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 535     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 536
 537     Net net = readNetFromTensorflow(model, proto);
 538     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 539     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 540
 541     net.setPreferableBackend(backend);
 542     net.setPreferableTarget(target);
 543     net.setInput(blob);
 544     // Output has shape 1x1xNx7 where N - number of detections.
 545     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 546     Mat out = net.forward();
 547
 548     // References are from test for Caffe model.
 549     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 550                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 551                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 552                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 553                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 554                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 555     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 556     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 557     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 558     expectNoFallbacksFromIE(net);
 559 }
 560
 561 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 562 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 563 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 564 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 565 //                 feed_dict={'input_images:0': inp})
 566 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 567 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 568 // np.save('east_text_detection.scores.npy', scores)
 569 // np.save('east_text_detection.geometry.npy', geometry)
 570 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 571 {
 572     applyTestTag(
 573         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 574         CV_TEST_TAG_DEBUG_LONG
 575     );
 576
 577 #if defined(INF_ENGINE_RELEASE)
 578     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 579         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 580 #endif
 581
 582     checkBackend();
 583
 584     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 585     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 586     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 587     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 588
 589     Net net = readNet(netPath);
 590
 591     net.setPreferableBackend(backend);
 592     net.setPreferableTarget(target);
 593
 594     Mat img = imread(imgPath);
 595     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 596     net.setInput(inp);
 597
 598     std::vector<Mat> outs;
 599     std::vector<String> outNames(2);
 600     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 601     outNames[1] = "feature_fusion/concat_3";
 602     net.forward(outs, outNames);
 603
 604     Mat scores = outs[0];
 605     Mat geometry = outs[1];
 606
 607     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 608     double l1_scores = default_l1, lInf_scores = default_lInf;
 609     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 610     if (target == DNN_TARGET_OPENCL_FP16)
 611     {
 612         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 613         l1_geometry = 0.28; lInf_geometry = 5.94;
 614     }
 615     else if (target == DNN_TARGET_MYRIAD)
 616     {
 617         lInf_scores = 0.41;
 618         l1_geometry = 0.28; lInf_geometry = 5.94;
 619     }
 620     else
 621     {
 622         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 623     }
 624     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 625     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 626     expectNoFallbacksFromIE(net);
 627 }
 628
 629 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 630
 631 TEST_P(Test_TensorFlow_layers, fp16_weights)
 632 {
 633     float l1 = 0.00078;
 634     float lInf = 0.012;
 635     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 636     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 637     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 638     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 639     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 640     // Reference output values are in range [0.0889, 1.651]
 641     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 642     if (target == DNN_TARGET_MYRIAD) {
 643         l1 = 0.0041;
 644         lInf = 0.024;
 645     }
 646     // Reference output values are in range [0, 10.75]
 647     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 648     // Reference output values are in range [0.418, 2.297]
 649     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 650 }
 651
 652 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 653 {
 654     // Reference output values are in range [-3.504, -0.002]
 655     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 656 }
 657
 658 TEST_P(Test_TensorFlow_layers, defun)
 659 {
 660     runTensorFlowNet("defun_dropout");
 661 }
 662
 663 TEST_P(Test_TensorFlow_layers, quantized)
 664 {
 665     runTensorFlowNet("uint8_single_conv");
 666 }
 667
 668 TEST_P(Test_TensorFlow_layers, lstm)
 669 {
 670     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 671         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 672     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 673         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 674     runTensorFlowNet("lstm", true);
 675     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 676 }
 677
 678 TEST_P(Test_TensorFlow_layers, split)
 679 {
 680     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 681         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 682     runTensorFlowNet("split");
 683     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 684         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 685     runTensorFlowNet("split_equals");
 686 }
 687
 688 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 689 {
 690     runTensorFlowNet("resize_nearest_neighbor");
 691     runTensorFlowNet("keras_upsampling2d");
 692 }
 693
 694 TEST_P(Test_TensorFlow_layers, slice)
 695 {
 696     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 697         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 698         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 699     runTensorFlowNet("slice_4d");
 700     runTensorFlowNet("strided_slice");
 701 }
 702
 703 TEST_P(Test_TensorFlow_layers, softmax)
 704 {
 705     runTensorFlowNet("keras_softmax");
 706     runTensorFlowNet("slim_softmax");
 707 }
 708
 709 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 710 {
 711 #if defined(INF_ENGINE_RELEASE)
 712     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 713         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 714     )
 715         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 716 #endif
 717     runTensorFlowNet("slim_softmax_v2");
 718 }
 719
 720 TEST_P(Test_TensorFlow_layers, relu6)
 721 {
 722     runTensorFlowNet("keras_relu6");
 723     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 724 }
 725
 726 TEST_P(Test_TensorFlow_layers, subpixel)
 727 {
 728     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 729         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 730     runTensorFlowNet("subpixel");
 731 }
 732
 733 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 734 {
 735     runTensorFlowNet("keras_mobilenet_head");
 736 }
 737
 738 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 739 {
 740     runTensorFlowNet("resize_bilinear");
 741     runTensorFlowNet("resize_bilinear_factor");
 742 }
 743
 744 TEST_P(Test_TensorFlow_layers, squeeze)
 745 {
 746 #if defined(INF_ENGINE_RELEASE)
 747     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 748             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 749     )
 750         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 751 #endif
 752     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 753     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 754     int squeeze_dims[] = {0, 2, 3, -1};
 755     for (int i = 0; i < 4; ++i)
 756     {
 757         SCOPED_TRACE(format("i=%d", i));
 758         std::string pbtxt =
 759             "node { name: \"input\" op: \"Placeholder\""
 760             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 761             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 762               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 763         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 764         net.setPreferableBackend(backend);
 765         net.setPreferableTarget(target);
 766         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 767         randu(tfInp, -1, 1);
 768
 769         // NHWC to NCHW
 770         CV_Assert(inpShapes[i][0] == 1);
 771         std::swap(inpShapes[i][2], inpShapes[i][3]);
 772         std::swap(inpShapes[i][1], inpShapes[i][2]);
 773         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 774         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 775
 776         net.setInput(cvInp);
 777         Mat out = net.forward();
 778         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 779     }
 780 }
 781
 782 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 783
 784 TEST(Test_TensorFlow, two_inputs)
 785 {
 786     Net net = readNet(path("two_inputs_net.pbtxt"));
 787     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 788
 789     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 790     randu(firstInput, -1, 1);
 791     randu(secondInput, -1, 1);
 792
 793     net.setInput(firstInput, "first_input");
 794     net.setInput(secondInput, "second_input");
 795     Mat out = net.forward();
 796
 797     normAssert(out, firstInput + secondInput);
 798 }
 799
 800 TEST(Test_TensorFlow, Mask_RCNN)
 801 {
 802     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 803     Mat img = imread(findDataFile("dnn/street.png"));
 804     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 805     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 806
 807     Net net = readNetFromTensorflow(model, proto);
 808     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 809     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 810     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 811
 812     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 813
 814     net.setInput(blob);
 815
 816     // Mask-RCNN predicts bounding boxes and segmentation masks.
 817     std::vector<String> outNames(2);
 818     outNames[0] = "detection_out_final";
 819     outNames[1] = "detection_masks";
 820
 821     std::vector<Mat> outs;
 822     net.forward(outs, outNames);
 823
 824     Mat outDetections = outs[0];
 825     Mat outMasks = outs[1];
 826     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 827
 828     // Output size of masks is NxCxHxW where
 829     // N - number of detected boxes
 830     // C - number of classes (excluding background)
 831     // HxW - segmentation shape
 832     const int numDetections = outDetections.size[2];
 833
 834     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 835     Mat masks(4, &masksSize[0], CV_32F);
 836
 837     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 838     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 839
 840     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 841     for (int i = 0; i < numDetections; ++i)
 842     {
 843         // Get a class id for this bounding box and copy mask only for that class.
 844         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 845         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 846         srcRanges[1] = cv::Range(classId, classId + 1);
 847         outMasks(srcRanges).copyTo(masks(dstRanges));
 848     }
 849     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 850     normAssert(masks, refMasks(&topRefMasks[0]));
 851 }
 852
 853 }