modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017-2019, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= Scalar::all(117); // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     Mat inputBlob = blobFromImage(sample, 1.0, Size(224, 224), Scalar(), /*swapRB*/true);
  66
  67     net.setInput(inputBlob, "input");
  68     Mat out = net.forward("softmax2");
  69
  70     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  71
  72     normAssert(ref, out);
  73 }
  74
  75 static std::string path(const std::string& file)
  76 {
  77     return findDataFile("dnn/tensorflow/" + file);
  78 }
  79
  80 class Test_TensorFlow_layers : public DNNTestLayer
  81 {
  82 public:
  83     void runTensorFlowNet(const std::string& prefix, bool hasText = false,
  84                           double l1 = 0.0, double lInf = 0.0, bool memoryLoad = false)
  85     {
  86         std::string netPath = path(prefix + "_net.pb");
  87         std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  88         std::string inpPath = path(prefix + "_in.npy");
  89         std::string outPath = path(prefix + "_out.npy");
  90
  91         cv::Mat input = blobFromNPY(inpPath);
  92         cv::Mat ref = blobFromNPY(outPath);
  93         checkBackend(&input, &ref);
  94
  95         Net net;
  96         if (memoryLoad)
  97         {
  98             // Load files into a memory buffers
  99             std::vector<char> dataModel;
 100             readFileContent(netPath, dataModel);
 101
 102             std::vector<char> dataConfig;
 103             if (hasText)
 104             {
 105                 readFileContent(netConfig, dataConfig);
 106             }
 107
 108             net = readNetFromTensorflow(dataModel.data(), dataModel.size(),
 109                                         dataConfig.data(), dataConfig.size());
 110         }
 111         else
 112             net = readNetFromTensorflow(netPath, netConfig);
 113
 114         ASSERT_FALSE(net.empty());
 115
 116         net.setPreferableBackend(backend);
 117         net.setPreferableTarget(target);
 118         net.setInput(input);
 119         cv::Mat output = net.forward();
 120         normAssert(ref, output, "", l1 ? l1 : default_l1, lInf ? lInf : default_lInf);
 121     }
 122 };
 123
 124 TEST_P(Test_TensorFlow_layers, conv)
 125 {
 126     runTensorFlowNet("single_conv");
 127     runTensorFlowNet("atrous_conv2d_valid");
 128     runTensorFlowNet("atrous_conv2d_same");
 129     runTensorFlowNet("depthwise_conv2d");
 130     runTensorFlowNet("keras_atrous_conv2d_same");
 131     runTensorFlowNet("conv_pool_nchw");
 132 }
 133
 134 TEST_P(Test_TensorFlow_layers, Convolution3D)
 135 {
 136 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 137     throw SkipTestException("Test is enabled starts from 2019R1");
 138 #endif
 139     if (target != DNN_TARGET_CPU)
 140         throw SkipTestException("Only CPU is supported");
 141     runTensorFlowNet("conv3d");
 142 }
 143
 144 TEST_P(Test_TensorFlow_layers, padding)
 145 {
 146     runTensorFlowNet("padding_valid");
 147     runTensorFlowNet("spatial_padding");
 148     runTensorFlowNet("keras_pad_concat");
 149     runTensorFlowNet("mirror_pad");
 150 }
 151
 152 TEST_P(Test_TensorFlow_layers, padding_same)
 153 {
 154     // Reference output values are in range [0.0006, 2.798]
 155     runTensorFlowNet("padding_same");
 156 }
 157
 158 TEST_P(Test_TensorFlow_layers, eltwise)
 159 {
 160     runTensorFlowNet("eltwise_add_mul");
 161     runTensorFlowNet("eltwise_sub");
 162 }
 163
 164 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 165 {
 166     runTensorFlowNet("pad_and_concat");
 167 }
 168
 169 TEST_P(Test_TensorFlow_layers, concat_axis_1)
 170 {
 171     runTensorFlowNet("concat_axis_1");
 172 }
 173
 174 TEST_P(Test_TensorFlow_layers, batch_norm)
 175 {
 176     runTensorFlowNet("batch_norm");
 177     runTensorFlowNet("batch_norm", false, 0.0, 0.0, true);
 178     runTensorFlowNet("fused_batch_norm");
 179     runTensorFlowNet("fused_batch_norm", false, 0.0, 0.0, true);
 180     runTensorFlowNet("batch_norm_text", true);
 181     runTensorFlowNet("batch_norm_text", true, 0.0, 0.0, true);
 182     runTensorFlowNet("unfused_batch_norm");
 183     runTensorFlowNet("fused_batch_norm_no_gamma");
 184     runTensorFlowNet("unfused_batch_norm_no_gamma");
 185     runTensorFlowNet("mvn_batch_norm");
 186     runTensorFlowNet("mvn_batch_norm_1x1");
 187     runTensorFlowNet("switch_identity");
 188     runTensorFlowNet("keras_batch_norm_training");
 189 }
 190
 191 TEST_P(Test_TensorFlow_layers, batch_norm3D)
 192 {
 193     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
 194     {
 195         if (target == DNN_TARGET_OPENCL_FP16) applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 196         if (target == DNN_TARGET_OPENCL)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL);
 197         if (target == DNN_TARGET_MYRIAD)      applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 198         throw SkipTestException("");
 199     }
 200     runTensorFlowNet("batch_norm3d");
 201 }
 202
 203 TEST_P(Test_TensorFlow_layers, slim_batch_norm)
 204 {
 205     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 206         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 207     // Output values range: [-40.0597, 207.827]
 208     double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.041 : default_l1;
 209     double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.33 : default_lInf;
 210     runTensorFlowNet("slim_batch_norm", false, l1, lInf);
 211 }
 212
 213 TEST_P(Test_TensorFlow_layers, pooling)
 214 {
 215     runTensorFlowNet("max_pool_even");
 216     runTensorFlowNet("max_pool_odd_valid");
 217     runTensorFlowNet("max_pool_odd_same");
 218     runTensorFlowNet("reduce_mean");  // an average pooling over all spatial dimensions.
 219 }
 220
 221 TEST_P(Test_TensorFlow_layers, max_pool_grad)
 222 {
 223     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 224         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 225     runTensorFlowNet("max_pool_grad");
 226 }
 227
 228 // TODO: fix tests and replace to pooling
 229 TEST_P(Test_TensorFlow_layers, ave_pool_same)
 230 {
 231     // Reference output values are in range [-0.519531, 0.112976]
 232 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 233     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 234             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 235     )
 236         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 237 #endif
 238     runTensorFlowNet("ave_pool_same");
 239 }
 240
 241 TEST_P(Test_TensorFlow_layers, MaxPooling3D)
 242 {
 243 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 244     throw SkipTestException("Test is enabled starts from 2019R1");
 245 #endif
 246     if (target != DNN_TARGET_CPU)
 247         throw SkipTestException("Only CPU is supported");
 248     runTensorFlowNet("max_pool3d");
 249 }
 250
 251 TEST_P(Test_TensorFlow_layers, AvePooling3D)
 252 {
 253 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_LT(2019010000)
 254     throw SkipTestException("Test is enabled starts from 2019R1");
 255 #endif
 256     if (target != DNN_TARGET_CPU)
 257         throw SkipTestException("Only CPU is supported");
 258     runTensorFlowNet("ave_pool3d");
 259 }
 260
 261 TEST_P(Test_TensorFlow_layers, deconvolution)
 262 {
 263     runTensorFlowNet("deconvolution");
 264     runTensorFlowNet("deconvolution_same");
 265     runTensorFlowNet("deconvolution_stride_2_same");
 266     runTensorFlowNet("deconvolution_adj_pad_valid");
 267     runTensorFlowNet("deconvolution_adj_pad_same");
 268     runTensorFlowNet("keras_deconv_valid");
 269     runTensorFlowNet("keras_deconv_same");
 270     runTensorFlowNet("keras_deconv_same_v2");
 271 }
 272
 273 TEST_P(Test_TensorFlow_layers, matmul)
 274 {
 275     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 276         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 277     runTensorFlowNet("matmul");
 278     runTensorFlowNet("nhwc_transpose_reshape_matmul");
 279     // Reference output values are in range [-5.688, 4.484]
 280     double l1 = target == DNN_TARGET_MYRIAD ? 6.1e-3 : default_l1;
 281     runTensorFlowNet("nhwc_reshape_matmul", false, l1);
 282
 283 }
 284
 285 TEST_P(Test_TensorFlow_layers, reshape)
 286 {
 287     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 288         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 289     runTensorFlowNet("shift_reshape_no_reorder");
 290     runTensorFlowNet("reshape_no_reorder");
 291     runTensorFlowNet("reshape_reduce");
 292     runTensorFlowNet("reshape_as_shape");
 293 }
 294
 295 TEST_P(Test_TensorFlow_layers, flatten)
 296 {
 297 #if defined(INF_ENGINE_RELEASE)
 298     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 299             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 300     )
 301         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 302 #endif
 303
 304     runTensorFlowNet("flatten", true);
 305 }
 306
 307 TEST_P(Test_TensorFlow_layers, unfused_flatten)
 308 {
 309     runTensorFlowNet("unfused_flatten");
 310     runTensorFlowNet("unfused_flatten_unknown_batch");
 311 }
 312
 313 TEST_P(Test_TensorFlow_layers, leaky_relu)
 314 {
 315 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 316     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL)
 317         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_2018R5);
 318 #endif
 319     runTensorFlowNet("leaky_relu_order1");
 320     runTensorFlowNet("leaky_relu_order2");
 321     runTensorFlowNet("leaky_relu_order3");
 322 }
 323
 324 TEST_P(Test_TensorFlow_layers, l2_normalize)
 325 {
 326 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
 327     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 328             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 329     )
 330         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 331 #endif
 332
 333     runTensorFlowNet("l2_normalize");
 334 }
 335
 336 // TODO: fix it and add to l2_normalize
 337 TEST_P(Test_TensorFlow_layers, l2_normalize_3d)
 338 {
 339 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 340     if (backend == DNN_BACKEND_INFERENCE_ENGINE
 341             && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16)
 342     )
 343         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 344 #endif
 345 #if defined(INF_ENGINE_RELEASE)
 346     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 347         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 348 #endif
 349
 350     runTensorFlowNet("l2_normalize_3d");
 351 }
 352
 353 class Test_TensorFlow_nets : public DNNTestLayer {};
 354
 355 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 356 {
 357 #if defined(INF_ENGINE_RELEASE)
 358     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 359     {
 360 #if INF_ENGINE_VER_MAJOR_GE(2019010000)
 361         if (getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
 362             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 363 #else
 364             applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 365 #endif
 366     }
 367 #endif
 368
 369     checkBackend();
 370     std::string imgPath = findDataFile("dnn/street.png");
 371     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt");
 372     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 373
 374     Mat inp;
 375     resize(imread(imgPath), inp, Size(300, 300));
 376     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 377
 378     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco.detection_out.npy"));
 379
 380     Net net = readNetFromTensorflow(netPath, netConfig);
 381     net.setPreferableBackend(backend);
 382     net.setPreferableTarget(target);
 383
 384     net.setInput(inp);
 385     Mat out = net.forward();
 386
 387     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0043 : default_l1;
 388     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.037 : default_lInf;
 389     normAssertDetections(ref, out, "", 0.2, scoreDiff, iouDiff);
 390 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_RELEASE >= 2019010000
 391     expectNoFallbacksFromIE(net);
 392 #endif
 393 }
 394
 395 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 396 {
 397     applyTestTag(target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB);
 398
 399 #if defined(INF_ENGINE_RELEASE)
 400     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 401             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 402     )
 403         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_X);
 404 #endif
 405
 406     checkBackend();
 407     Mat img = imread(findDataFile("dnn/street.png"));
 408     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt");
 409     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 410
 411     Net net = readNetFromTensorflow(model, proto);
 412     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 413
 414     net.setPreferableBackend(backend);
 415     net.setPreferableTarget(target);
 416
 417     net.setInput(blob);
 418     // Output has shape 1x1xNx7 where N - number of detections.
 419     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 420     Mat out = net.forward();
 421     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 422                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 423                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 424                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 425                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 426
 427     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.0097 : default_l1;
 428     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.09 : default_lInf;
 429     normAssertDetections(ref, out, "", 0.5, scoreDiff, iouDiff);
 430     expectNoFallbacksFromIE(net);
 431 }
 432
 433 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD)
 434 {
 435     checkBackend();
 436     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pbtxt");
 437     std::string model = findDataFile("dnn/ssd_mobilenet_v1_coco_2017_11_17.pb", false);
 438
 439     Net net = readNetFromTensorflow(model, proto);
 440     Mat img = imread(findDataFile("dnn/dog416.png"));
 441     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 442
 443     net.setPreferableBackend(backend);
 444     net.setPreferableTarget(target);
 445
 446     net.setInput(blob);
 447     Mat out = net.forward();
 448
 449     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco_2017_11_17.detection_out.npy"));
 450     float scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 7e-3 : 1.5e-5;
 451     float iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.012 : 1e-3;
 452     float detectionConfThresh = (target == DNN_TARGET_MYRIAD) ? 0.35 : 0.3;
 453
 454 #if defined(INF_ENGINE_RELEASE)
 455     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 456             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X
 457     )
 458         scoreDiff = 0.061;
 459         iouDiff = 0.12;
 460         detectionConfThresh = 0.36;
 461 #endif
 462     normAssertDetections(ref, out, "", detectionConfThresh, scoreDiff, iouDiff);
 463     expectNoFallbacksFromIE(net);
 464 }
 465
 466 TEST_P(Test_TensorFlow_nets, Faster_RCNN)
 467 {
 468     // FIXIT split test
 469     applyTestTag(
 470         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB),
 471         CV_TEST_TAG_LONG,
 472         CV_TEST_TAG_DEBUG_VERYLONG
 473     );
 474     static std::string names[] = {"faster_rcnn_inception_v2_coco_2018_01_28",
 475                                   "faster_rcnn_resnet50_coco_2018_01_28"};
 476
 477     checkBackend();
 478     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 479         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 480     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 481         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 482
 483     double scoresDiff = backend == DNN_BACKEND_INFERENCE_ENGINE ? 2.9e-5 : 1e-5;
 484     for (int i = 0; i < 2; ++i)
 485     {
 486         std::string proto = findDataFile("dnn/" + names[i] + ".pbtxt");
 487         std::string model = findDataFile("dnn/" + names[i] + ".pb", false);
 488
 489         Net net = readNetFromTensorflow(model, proto);
 490         net.setPreferableBackend(backend);
 491         net.setPreferableTarget(target);
 492         Mat img = imread(findDataFile("dnn/dog416.png"));
 493         Mat blob = blobFromImage(img, 1.0f, Size(800, 600), Scalar(), true, false);
 494
 495         net.setInput(blob);
 496         Mat out = net.forward();
 497
 498         Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/" + names[i] + ".detection_out.npy"));
 499         normAssertDetections(ref, out, names[i].c_str(), 0.3, scoresDiff);
 500     }
 501 }
 502
 503 TEST_P(Test_TensorFlow_nets, MobileNet_v1_SSD_PPN)
 504 {
 505 #if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2018050000)
 506     if (backend == DNN_BACKEND_INFERENCE_ENGINE && (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 507         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 508 #endif
 509
 510     checkBackend();
 511     std::string proto = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pbtxt");
 512     std::string model = findDataFile("dnn/ssd_mobilenet_v1_ppn_coco.pb", false);
 513
 514     Net net = readNetFromTensorflow(model, proto);
 515     Mat img = imread(findDataFile("dnn/dog416.png"));
 516     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/ssd_mobilenet_v1_ppn_coco.detection_out.npy"));
 517     Mat blob = blobFromImage(img, 1.0f, Size(300, 300), Scalar(), true, false);
 518
 519     net.setPreferableBackend(backend);
 520     net.setPreferableTarget(target);
 521
 522     net.setInput(blob);
 523     Mat out = net.forward();
 524
 525     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.048 : 1.1e-5;
 526     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.058 : default_lInf;
 527     normAssertDetections(ref, out, "", 0.45, scoreDiff, iouDiff);
 528     expectNoFallbacksFromIE(net);
 529 }
 530
 531 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 532 {
 533     checkBackend();
 534     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt");
 535     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 536
 537     Net net = readNetFromTensorflow(model, proto);
 538     Mat img = imread(findDataFile("gpu/lbpcascade/er.png"));
 539     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 540
 541     net.setPreferableBackend(backend);
 542     net.setPreferableTarget(target);
 543     net.setInput(blob);
 544     // Output has shape 1x1xNx7 where N - number of detections.
 545     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 546     Mat out = net.forward();
 547
 548     // References are from test for Caffe model.
 549     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 550                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 551                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 552                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 553                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 554                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 555     double scoreDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 4e-3 : 3.4e-3;
 556     double iouDiff = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.024 : 1e-2;
 557     normAssertDetections(ref, out, "", 0.9, scoreDiff, iouDiff);
 558     expectNoFallbacksFromIE(net);
 559 }
 560
 561 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 562 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 563 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 564 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 565 //                 feed_dict={'input_images:0': inp})
 566 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 567 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 568 // np.save('east_text_detection.scores.npy', scores)
 569 // np.save('east_text_detection.geometry.npy', geometry)
 570 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 571 {
 572     applyTestTag(
 573         (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_512MB : CV_TEST_TAG_MEMORY_1GB),
 574         CV_TEST_TAG_DEBUG_LONG
 575     );
 576
 577 #if defined(INF_ENGINE_RELEASE)
 578     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD)
 579         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD);
 580 #endif
 581
 582     checkBackend();
 583
 584     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 585     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png");
 586     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy");
 587     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy");
 588
 589     Net net = readNet(netPath);
 590
 591     net.setPreferableBackend(backend);
 592     net.setPreferableTarget(target);
 593
 594     Mat img = imread(imgPath);
 595     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 596     net.setInput(inp);
 597
 598     std::vector<Mat> outs;
 599     std::vector<String> outNames(2);
 600     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 601     outNames[1] = "feature_fusion/concat_3";
 602     net.forward(outs, outNames);
 603
 604     Mat scores = outs[0];
 605     Mat geometry = outs[1];
 606
 607     // Scores are in range [0, 1]. Geometry values are in range [-0.23, 290]
 608     double l1_scores = default_l1, lInf_scores = default_lInf;
 609     double l1_geometry = default_l1, lInf_geometry = default_lInf;
 610     if (target == DNN_TARGET_OPENCL_FP16)
 611     {
 612         lInf_scores = backend == DNN_BACKEND_INFERENCE_ENGINE ? 0.16 : 0.11;
 613         l1_geometry = 0.28; lInf_geometry = 5.94;
 614     }
 615     else if (target == DNN_TARGET_MYRIAD)
 616     {
 617         lInf_scores = 0.41;
 618         l1_geometry = 0.28; lInf_geometry = 5.94;
 619     }
 620     else
 621     {
 622         l1_geometry = 1e-4, lInf_geometry = 3e-3;
 623     }
 624     normAssert(scores, blobFromNPY(refScoresPath), "scores", l1_scores, lInf_scores);
 625     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", l1_geometry, lInf_geometry);
 626     expectNoFallbacksFromIE(net);
 627 }
 628
 629 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, dnnBackendsAndTargets());
 630
 631 TEST_P(Test_TensorFlow_layers, fp16_weights)
 632 {
 633     float l1 = 0.00078;
 634     float lInf = 0.012;
 635     runTensorFlowNet("fp16_single_conv", false, l1, lInf);
 636     runTensorFlowNet("fp16_max_pool_odd_same", false, l1, lInf);
 637     runTensorFlowNet("fp16_eltwise_add_mul", false, l1, lInf);
 638     runTensorFlowNet("fp16_pad_and_concat", false, l1, lInf);
 639     runTensorFlowNet("fp16_padding_valid", false, l1, lInf);
 640     // Reference output values are in range [0.0889, 1.651]
 641     runTensorFlowNet("fp16_max_pool_even", false, (target == DNN_TARGET_MYRIAD) ? 0.003 : l1, lInf);
 642     if (target == DNN_TARGET_MYRIAD) {
 643         l1 = 0.0041;
 644         lInf = 0.024;
 645     }
 646     // Reference output values are in range [0, 10.75]
 647     runTensorFlowNet("fp16_deconvolution", false, l1, lInf);
 648     // Reference output values are in range [0.418, 2.297]
 649     runTensorFlowNet("fp16_max_pool_odd_valid", false, l1, lInf);
 650 }
 651
 652 TEST_P(Test_TensorFlow_layers, fp16_padding_same)
 653 {
 654     // Reference output values are in range [-3.504, -0.002]
 655     runTensorFlowNet("fp16_padding_same", false, 7e-4, 4e-3);
 656 }
 657
 658 TEST_P(Test_TensorFlow_layers, defun)
 659 {
 660     runTensorFlowNet("defun_dropout");
 661 }
 662
 663 TEST_P(Test_TensorFlow_layers, quantized)
 664 {
 665     runTensorFlowNet("uint8_single_conv");
 666 }
 667
 668 TEST_P(Test_TensorFlow_layers, lstm)
 669 {
 670     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 671         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 672     if (backend == DNN_BACKEND_OPENCV && target == DNN_TARGET_OPENCL_FP16)
 673         applyTestTag(CV_TEST_TAG_DNN_SKIP_OPENCL_FP16);
 674     runTensorFlowNet("lstm", true);
 675     runTensorFlowNet("lstm", true, 0.0, 0.0, true);
 676 }
 677
 678 TEST_P(Test_TensorFlow_layers, split)
 679 {
 680     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 681         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 682     runTensorFlowNet("split_equals");
 683 }
 684
 685 TEST_P(Test_TensorFlow_layers, resize_nearest_neighbor)
 686 {
 687     runTensorFlowNet("resize_nearest_neighbor");
 688     runTensorFlowNet("keras_upsampling2d");
 689 }
 690
 691 TEST_P(Test_TensorFlow_layers, slice)
 692 {
 693     if (backend == DNN_BACKEND_INFERENCE_ENGINE &&
 694         (target == DNN_TARGET_OPENCL || target == DNN_TARGET_OPENCL_FP16))
 695         applyTestTag(target == DNN_TARGET_OPENCL ? CV_TEST_TAG_DNN_SKIP_IE_OPENCL : CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
 696     runTensorFlowNet("slice_4d");
 697     runTensorFlowNet("strided_slice");
 698 }
 699
 700 TEST_P(Test_TensorFlow_layers, softmax)
 701 {
 702     runTensorFlowNet("keras_softmax");
 703     runTensorFlowNet("slim_softmax");
 704 }
 705
 706 TEST_P(Test_TensorFlow_layers, slim_softmax_v2)
 707 {
 708 #if defined(INF_ENGINE_RELEASE)
 709     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD &&
 710         getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 711     )
 712         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 713 #endif
 714     runTensorFlowNet("slim_softmax_v2");
 715 }
 716
 717 TEST_P(Test_TensorFlow_layers, relu6)
 718 {
 719     runTensorFlowNet("keras_relu6");
 720     runTensorFlowNet("keras_relu6", /*hasText*/ true);
 721 }
 722
 723 TEST_P(Test_TensorFlow_layers, subpixel)
 724 {
 725     if (backend == DNN_BACKEND_INFERENCE_ENGINE)
 726         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE);
 727     runTensorFlowNet("subpixel");
 728 }
 729
 730 TEST_P(Test_TensorFlow_layers, keras_mobilenet_head)
 731 {
 732     runTensorFlowNet("keras_mobilenet_head");
 733 }
 734
 735 TEST_P(Test_TensorFlow_layers, resize_bilinear)
 736 {
 737     runTensorFlowNet("resize_bilinear");
 738     runTensorFlowNet("resize_bilinear_factor");
 739 }
 740
 741 TEST_P(Test_TensorFlow_layers, squeeze)
 742 {
 743 #if defined(INF_ENGINE_RELEASE)
 744     if (backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_MYRIAD
 745             && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_2
 746     )
 747         applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD_2);
 748 #endif
 749     int inpShapes[][4] = {{1, 3, 4, 2}, {1, 3, 1, 2}, {1, 3, 4, 1}, {1, 3, 4, 1}};  // TensorFlow's shape (NHWC)
 750     int outShapes[][3] = {{3, 4, 2}, {1, 3, 2}, {1, 3, 4}, {1, 3, 4}};
 751     int squeeze_dims[] = {0, 2, 3, -1};
 752     for (int i = 0; i < 4; ++i)
 753     {
 754         SCOPED_TRACE(format("i=%d", i));
 755         std::string pbtxt =
 756             "node { name: \"input\" op: \"Placeholder\""
 757             "attr { key: \"data_format\" value { s: \"NHWC\" } } }"
 758             "node { name: \"squeeze\" op: \"Squeeze\" input: \"input\""
 759               "attr { key: \"squeeze_dims\" value { list { i:" + format("%d", squeeze_dims[i]) + "}}}}";
 760         Net net = readNetFromTensorflow(0, 0, pbtxt.c_str(), pbtxt.size());
 761         net.setPreferableBackend(backend);
 762         net.setPreferableTarget(target);
 763         Mat tfInp(4, &inpShapes[i][0], CV_32F);
 764         randu(tfInp, -1, 1);
 765
 766         // NHWC to NCHW
 767         CV_Assert(inpShapes[i][0] == 1);
 768         std::swap(inpShapes[i][2], inpShapes[i][3]);
 769         std::swap(inpShapes[i][1], inpShapes[i][2]);
 770         Mat cvInp = tfInp.reshape(1, tfInp.total() / inpShapes[i][1]).t();
 771         cvInp = cvInp.reshape(1, 4, &inpShapes[i][0]);
 772
 773         net.setInput(cvInp);
 774         Mat out = net.forward();
 775         normAssert(tfInp.reshape(1, 3, &outShapes[i][0]), out, "", default_l1, default_lInf);
 776     }
 777 }
 778
 779 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, dnnBackendsAndTargets());
 780
 781 TEST(Test_TensorFlow, two_inputs)
 782 {
 783     Net net = readNet(path("two_inputs_net.pbtxt"));
 784     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 785
 786     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 787     randu(firstInput, -1, 1);
 788     randu(secondInput, -1, 1);
 789
 790     net.setInput(firstInput, "first_input");
 791     net.setInput(secondInput, "second_input");
 792     Mat out = net.forward();
 793
 794     normAssert(out, firstInput + secondInput);
 795 }
 796
 797 TEST(Test_TensorFlow, Mask_RCNN)
 798 {
 799     applyTestTag(CV_TEST_TAG_MEMORY_1GB, CV_TEST_TAG_DEBUG_VERYLONG);
 800     Mat img = imread(findDataFile("dnn/street.png"));
 801     std::string proto = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pbtxt");
 802     std::string model = findDataFile("dnn/mask_rcnn_inception_v2_coco_2018_01_28.pb", false);
 803
 804     Net net = readNetFromTensorflow(model, proto);
 805     Mat refDetections = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 806     Mat refMasks = blobFromNPY(path("mask_rcnn_inception_v2_coco_2018_01_28.detection_masks.npy"));
 807     Mat blob = blobFromImage(img, 1.0f, Size(800, 800), Scalar(), true, false);
 808
 809     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 810
 811     net.setInput(blob);
 812
 813     // Mask-RCNN predicts bounding boxes and segmentation masks.
 814     std::vector<String> outNames(2);
 815     outNames[0] = "detection_out_final";
 816     outNames[1] = "detection_masks";
 817
 818     std::vector<Mat> outs;
 819     net.forward(outs, outNames);
 820
 821     Mat outDetections = outs[0];
 822     Mat outMasks = outs[1];
 823     normAssertDetections(refDetections, outDetections, "", /*threshold for zero confidence*/1e-5);
 824
 825     // Output size of masks is NxCxHxW where
 826     // N - number of detected boxes
 827     // C - number of classes (excluding background)
 828     // HxW - segmentation shape
 829     const int numDetections = outDetections.size[2];
 830
 831     int masksSize[] = {1, numDetections, outMasks.size[2], outMasks.size[3]};
 832     Mat masks(4, &masksSize[0], CV_32F);
 833
 834     std::vector<cv::Range> srcRanges(4, cv::Range::all());
 835     std::vector<cv::Range> dstRanges(4, cv::Range::all());
 836
 837     outDetections = outDetections.reshape(1, outDetections.total() / 7);
 838     for (int i = 0; i < numDetections; ++i)
 839     {
 840         // Get a class id for this bounding box and copy mask only for that class.
 841         int classId = static_cast<int>(outDetections.at<float>(i, 1));
 842         srcRanges[0] = dstRanges[1] = cv::Range(i, i + 1);
 843         srcRanges[1] = cv::Range(classId, classId + 1);
 844         outMasks(srcRanges).copyTo(masks(dstRanges));
 845     }
 846     cv::Range topRefMasks[] = {Range::all(), Range(0, numDetections), Range::all(), Range::all()};
 847     normAssert(masks, refMasks(&topRefMasks[0]));
 848 }
 849
 850 }