modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= 128; // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     resize(sample, sample, Size(224, 224));
  66     Mat inputBlob = blobFromImage(sample);
  67
  68     net.setInput(inputBlob, "input");
  69     Mat out = net.forward("softmax2");
  70
  71     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  72
  73     normAssert(ref, out);
  74 }
  75
  76 static std::string path(const std::string& file)
  77 {
  78     return findDataFile("dnn/tensorflow/" + file, false);
  79 }
  80
  81 static void runTensorFlowNet(const std::string& prefix, int targetId = DNN_TARGET_CPU, bool hasText = false,
  82                              double l1 = 1e-5, double lInf = 1e-4,
  83                              bool memoryLoad = false)
  84 {
  85     std::string netPath = path(prefix + "_net.pb");
  86     std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  87     std::string inpPath = path(prefix + "_in.npy");
  88     std::string outPath = path(prefix + "_out.npy");
  89
  90     Net net;
  91     if (memoryLoad)
  92     {
  93         // Load files into a memory buffers
  94         string dataModel;
  95         ASSERT_TRUE(readFileInMemory(netPath, dataModel));
  96
  97         string dataConfig;
  98         if (hasText)
  99             ASSERT_TRUE(readFileInMemory(netConfig, dataConfig));
 100
 101         net = readNetFromTensorflow(dataModel.c_str(), dataModel.size(),
 102                                     dataConfig.c_str(), dataConfig.size());
 103     }
 104     else
 105         net = readNetFromTensorflow(netPath, netConfig);
 106
 107     ASSERT_FALSE(net.empty());
 108
 109     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 110     net.setPreferableTarget(targetId);
 111
 112     cv::Mat input = blobFromNPY(inpPath);
 113     cv::Mat target = blobFromNPY(outPath);
 114
 115     net.setInput(input);
 116     cv::Mat output = net.forward();
 117     normAssert(target, output, "", l1, lInf);
 118 }
 119
 120 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_layers;
 121
 122 TEST_P(Test_TensorFlow_layers, conv)
 123 {
 124     int targetId = GetParam();
 125     runTensorFlowNet("single_conv", targetId);
 126     runTensorFlowNet("atrous_conv2d_valid", targetId);
 127     runTensorFlowNet("atrous_conv2d_same", targetId);
 128     runTensorFlowNet("depthwise_conv2d", targetId);
 129     runTensorFlowNet("keras_atrous_conv2d_same", targetId);
 130     runTensorFlowNet("conv_pool_nchw", targetId);
 131 }
 132
 133 TEST_P(Test_TensorFlow_layers, padding)
 134 {
 135     int targetId = GetParam();
 136     runTensorFlowNet("padding_same", targetId);
 137     runTensorFlowNet("padding_valid", targetId);
 138     runTensorFlowNet("spatial_padding", targetId);
 139 }
 140
 141 TEST_P(Test_TensorFlow_layers, eltwise_add_mul)
 142 {
 143     runTensorFlowNet("eltwise_add_mul", GetParam());
 144 }
 145
 146 TEST_P(Test_TensorFlow_layers, concat)
 147 {
 148     runTensorFlowNet("pad_and_concat", GetParam());
 149     runTensorFlowNet("concat_axis_1", GetParam());
 150 }
 151
 152 TEST_P(Test_TensorFlow_layers, batch_norm)
 153 {
 154     int targetId = GetParam();
 155     runTensorFlowNet("batch_norm", targetId);
 156     runTensorFlowNet("fused_batch_norm", targetId);
 157     runTensorFlowNet("batch_norm_text", targetId, true);
 158     runTensorFlowNet("mvn_batch_norm", targetId);
 159     runTensorFlowNet("mvn_batch_norm_1x1", targetId);
 160     runTensorFlowNet("unfused_batch_norm", targetId);
 161     runTensorFlowNet("fused_batch_norm_no_gamma", targetId);
 162     runTensorFlowNet("unfused_batch_norm_no_gamma", targetId);
 163 }
 164
 165 TEST_P(Test_TensorFlow_layers, pooling)
 166 {
 167     int targetId = GetParam();
 168     cv::ocl::Device d = cv::ocl::Device::getDefault();
 169     bool loosenFlag = targetId == DNN_TARGET_OPENCL && d.isIntel() && d.type() == cv::ocl::Device::TYPE_CPU;
 170     runTensorFlowNet("max_pool_even", targetId);
 171     runTensorFlowNet("max_pool_odd_valid", targetId);
 172     runTensorFlowNet("ave_pool_same", targetId);
 173     runTensorFlowNet("max_pool_odd_same", targetId, false, loosenFlag ? 3e-5 : 1e-5, loosenFlag ? 3e-4 : 1e-4);
 174     runTensorFlowNet("reduce_mean", targetId);  // an average pooling over all spatial dimensions.
 175 }
 176
 177 TEST_P(Test_TensorFlow_layers, deconvolution)
 178 {
 179     int targetId = GetParam();
 180     runTensorFlowNet("deconvolution", targetId);
 181     runTensorFlowNet("deconvolution_same", targetId);
 182     runTensorFlowNet("deconvolution_stride_2_same", targetId);
 183     runTensorFlowNet("deconvolution_adj_pad_valid", targetId);
 184     runTensorFlowNet("deconvolution_adj_pad_same", targetId);
 185     runTensorFlowNet("keras_deconv_valid", targetId);
 186     runTensorFlowNet("keras_deconv_same", targetId);
 187 }
 188
 189 TEST_P(Test_TensorFlow_layers, matmul)
 190 {
 191     int targetId = GetParam();
 192     runTensorFlowNet("matmul", targetId);
 193     runTensorFlowNet("nhwc_reshape_matmul", targetId);
 194     runTensorFlowNet("nhwc_transpose_reshape_matmul", targetId);
 195 }
 196
 197 TEST_P(Test_TensorFlow_layers, reshape)
 198 {
 199     int targetId = GetParam();
 200     runTensorFlowNet("shift_reshape_no_reorder", targetId);
 201     runTensorFlowNet("reshape_reduce", targetId);
 202     runTensorFlowNet("flatten", targetId, true);
 203     runTensorFlowNet("unfused_flatten", targetId);
 204     runTensorFlowNet("unfused_flatten_unknown_batch", targetId);
 205 }
 206
 207 TEST_P(Test_TensorFlow_layers, l2_normalize)
 208 {
 209     int targetId = GetParam();
 210     runTensorFlowNet("l2_normalize", targetId);
 211     runTensorFlowNet("l2_normalize_3d", targetId);
 212 }
 213
 214 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, availableDnnTargets());
 215
 216 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_nets;
 217
 218 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 219 {
 220     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 221     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt", false);
 222     std::string imgPath = findDataFile("dnn/street.png", false);
 223
 224     Mat inp;
 225     resize(imread(imgPath), inp, Size(300, 300));
 226     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 227
 228     std::vector<String> outNames(3);
 229     outNames[0] = "concat";
 230     outNames[1] = "concat_1";
 231     outNames[2] = "detection_out";
 232
 233     std::vector<Mat> target(outNames.size());
 234     for (int i = 0; i < outNames.size(); ++i)
 235     {
 236         std::string path = findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco." + outNames[i] + ".npy", false);
 237         target[i] = blobFromNPY(path);
 238     }
 239
 240     Net net = readNetFromTensorflow(netPath, netConfig);
 241     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 242     net.setPreferableTarget(GetParam());
 243
 244     net.setInput(inp);
 245
 246     std::vector<Mat> output;
 247     net.forward(output, outNames);
 248
 249     normAssert(target[0].reshape(1, 1), output[0].reshape(1, 1), "", 1e-5, 1.5e-4);
 250     normAssert(target[1].reshape(1, 1), output[1].reshape(1, 1), "", 1e-5, 3e-4);
 251     normAssertDetections(target[2], output[2], "", 0.2);
 252 }
 253
 254 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 255 {
 256     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt", false);
 257     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 258
 259     Net net = readNetFromTensorflow(model, proto);
 260     Mat img = imread(findDataFile("dnn/street.png", false));
 261     Mat blob = blobFromImage(img, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), true, false);
 262
 263     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 264     net.setPreferableTarget(GetParam());
 265
 266     net.setInput(blob);
 267     // Output has shape 1x1xNx7 where N - number of detections.
 268     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 269     Mat out = net.forward();
 270     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 271                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 272                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 273                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 274                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 275     normAssertDetections(ref, out, "", 0.5);
 276 }
 277
 278 TEST_P(Test_TensorFlow_nets, Inception_v2_Faster_RCNN)
 279 {
 280     std::string proto = findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt", false);
 281     std::string model = findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false);
 282
 283     Net net = readNetFromTensorflow(model, proto);
 284     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 285     Mat img = imread(findDataFile("dnn/dog416.png", false));
 286     Mat blob = blobFromImage(img, 1.0f / 127.5, Size(800, 600), Scalar(127.5, 127.5, 127.5), true, false);
 287
 288     net.setInput(blob);
 289     Mat out = net.forward();
 290
 291     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/faster_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 292     normAssertDetections(ref, out, "", 0.3);
 293 }
 294
 295 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 296 {
 297     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt", false);
 298     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 299
 300     Net net = readNetFromTensorflow(model, proto);
 301     Mat img = imread(findDataFile("gpu/lbpcascade/er.png", false));
 302     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 303
 304     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 305     net.setPreferableTarget(GetParam());
 306
 307     net.setInput(blob);
 308     // Output has shape 1x1xNx7 where N - number of detections.
 309     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 310     Mat out = net.forward();
 311
 312     // References are from test for Caffe model.
 313     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 314                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 315                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 316                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 317                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 318                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 319     normAssertDetections(ref, out, "", 0.9, 3.4e-3, 1e-2);
 320 }
 321
 322 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 323 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 324 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 325 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 326 //                 feed_dict={'input_images:0': inp})
 327 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 328 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 329 // np.save('east_text_detection.scores.npy', scores)
 330 // np.save('east_text_detection.geometry.npy', geometry)
 331 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 332 {
 333     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 334     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png", false);
 335     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy", false);
 336     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy", false);
 337
 338     Net net = readNet(findDataFile("dnn/frozen_east_text_detection.pb", false));
 339
 340     net.setPreferableTarget(GetParam());
 341
 342     Mat img = imread(imgPath);
 343     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 344     net.setInput(inp);
 345
 346     std::vector<Mat> outs;
 347     std::vector<String> outNames(2);
 348     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 349     outNames[1] = "feature_fusion/concat_3";
 350     net.forward(outs, outNames);
 351
 352     Mat scores = outs[0];
 353     Mat geometry = outs[1];
 354
 355     normAssert(scores, blobFromNPY(refScoresPath), "scores");
 356     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", 1e-4, 3e-3);
 357 }
 358
 359 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, availableDnnTargets());
 360
 361 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_fp16;
 362
 363 TEST_P(Test_TensorFlow_fp16, tests)
 364 {
 365     int targetId = GetParam();
 366     const float l1 = 7e-4;
 367     const float lInf = 1e-2;
 368     runTensorFlowNet("fp16_single_conv", targetId, false, l1, lInf);
 369     runTensorFlowNet("fp16_deconvolution", targetId, false, l1, lInf);
 370     runTensorFlowNet("fp16_max_pool_odd_same", targetId, false, l1, lInf);
 371     runTensorFlowNet("fp16_padding_valid", targetId, false, l1, lInf);
 372     runTensorFlowNet("fp16_eltwise_add_mul", targetId, false, l1, lInf);
 373     runTensorFlowNet("fp16_max_pool_odd_valid", targetId, false, l1, lInf);
 374     runTensorFlowNet("fp16_pad_and_concat", targetId, false, l1, lInf);
 375     runTensorFlowNet("fp16_max_pool_even", targetId, false, l1, lInf);
 376     runTensorFlowNet("fp16_padding_same", targetId, false, l1, lInf);
 377 }
 378
 379 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_fp16,
 380                         Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16));
 381
 382 TEST(Test_TensorFlow, defun)
 383 {
 384     runTensorFlowNet("defun_dropout");
 385 }
 386
 387 TEST(Test_TensorFlow, quantized)
 388 {
 389     runTensorFlowNet("uint8_single_conv");
 390 }
 391
 392 TEST(Test_TensorFlow, lstm)
 393 {
 394     runTensorFlowNet("lstm", DNN_TARGET_CPU, true);
 395 }
 396
 397 TEST(Test_TensorFlow, split)
 398 {
 399     runTensorFlowNet("split_equals");
 400 }
 401
 402 TEST(Test_TensorFlow, resize_nearest_neighbor)
 403 {
 404     runTensorFlowNet("resize_nearest_neighbor");
 405 }
 406
 407 TEST(Test_TensorFlow, slice)
 408 {
 409     runTensorFlowNet("slice_4d");
 410 }
 411
 412 TEST(Test_TensorFlow, softmax)
 413 {
 414     runTensorFlowNet("keras_softmax");
 415 }
 416
 417 TEST(Test_TensorFlow, relu6)
 418 {
 419     runTensorFlowNet("keras_relu6");
 420     runTensorFlowNet("keras_relu6", DNN_TARGET_CPU, /*hasText*/ true);
 421 }
 422
 423 TEST(Test_TensorFlow, keras_mobilenet_head)
 424 {
 425     runTensorFlowNet("keras_mobilenet_head");
 426 }
 427
 428 TEST(Test_TensorFlow, memory_read)
 429 {
 430     double l1 = 1e-5;
 431     double lInf = 1e-4;
 432     runTensorFlowNet("lstm", DNN_TARGET_CPU, true, l1, lInf, true);
 433
 434     runTensorFlowNet("batch_norm", DNN_TARGET_CPU, false, l1, lInf, true);
 435     runTensorFlowNet("fused_batch_norm", DNN_TARGET_CPU, false, l1, lInf, true);
 436     runTensorFlowNet("batch_norm_text", DNN_TARGET_CPU, true, l1, lInf, true);
 437 }
 438
 439 TEST(Test_TensorFlow, resize_bilinear)
 440 {
 441     runTensorFlowNet("resize_bilinear");
 442     runTensorFlowNet("resize_bilinear_factor");
 443 }
 444
 445 TEST(Test_TensorFlow, two_inputs)
 446 {
 447     Net net = readNet(path("two_inputs_net.pbtxt"));
 448     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 449
 450     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 451     randu(firstInput, -1, 1);
 452     randu(secondInput, -1, 1);
 453
 454     net.setInput(firstInput, "first_input");
 455     net.setInput(secondInput, "second_input");
 456     Mat out = net.forward();
 457
 458     normAssert(out, firstInput + secondInput);
 459 }
 460
 461 }