modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= 128; // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     resize(sample, sample, Size(224, 224));
  66     Mat inputBlob = blobFromImage(sample);
  67
  68     net.setInput(inputBlob, "input");
  69     Mat out = net.forward("softmax2");
  70
  71     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  72
  73     normAssert(ref, out);
  74 }
  75
  76 static std::string path(const std::string& file)
  77 {
  78     return findDataFile("dnn/tensorflow/" + file, false);
  79 }
  80
  81 static void runTensorFlowNet(const std::string& prefix, int targetId = DNN_TARGET_CPU, bool hasText = false,
  82                              double l1 = 1e-5, double lInf = 1e-4,
  83                              bool memoryLoad = false)
  84 {
  85     std::string netPath = path(prefix + "_net.pb");
  86     std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  87     std::string inpPath = path(prefix + "_in.npy");
  88     std::string outPath = path(prefix + "_out.npy");
  89
  90     Net net;
  91     if (memoryLoad)
  92     {
  93         // Load files into a memory buffers
  94         string dataModel;
  95         ASSERT_TRUE(readFileInMemory(netPath, dataModel));
  96
  97         string dataConfig;
  98         if (hasText)
  99             ASSERT_TRUE(readFileInMemory(netConfig, dataConfig));
 100
 101         net = readNetFromTensorflow(dataModel.c_str(), dataModel.size(),
 102                                     dataConfig.c_str(), dataConfig.size());
 103     }
 104     else
 105         net = readNetFromTensorflow(netPath, netConfig);
 106
 107     ASSERT_FALSE(net.empty());
 108
 109     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 110     net.setPreferableTarget(targetId);
 111
 112     cv::Mat input = blobFromNPY(inpPath);
 113     cv::Mat target = blobFromNPY(outPath);
 114
 115     net.setInput(input);
 116     cv::Mat output = net.forward();
 117     normAssert(target, output, "", l1, lInf);
 118 }
 119
 120 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_layers;
 121
 122 TEST_P(Test_TensorFlow_layers, conv)
 123 {
 124     int targetId = GetParam();
 125     runTensorFlowNet("single_conv", targetId);
 126     runTensorFlowNet("atrous_conv2d_valid", targetId);
 127     runTensorFlowNet("atrous_conv2d_same", targetId);
 128     runTensorFlowNet("depthwise_conv2d", targetId);
 129     runTensorFlowNet("keras_atrous_conv2d_same", targetId);
 130 }
 131
 132 TEST_P(Test_TensorFlow_layers, padding)
 133 {
 134     int targetId = GetParam();
 135     runTensorFlowNet("padding_same", targetId);
 136     runTensorFlowNet("padding_valid", targetId);
 137     runTensorFlowNet("spatial_padding", targetId);
 138 }
 139
 140 TEST_P(Test_TensorFlow_layers, eltwise_add_mul)
 141 {
 142     runTensorFlowNet("eltwise_add_mul", GetParam());
 143 }
 144
 145 TEST_P(Test_TensorFlow_layers, pad_and_concat)
 146 {
 147     runTensorFlowNet("pad_and_concat", GetParam());
 148 }
 149
 150 TEST_P(Test_TensorFlow_layers, batch_norm)
 151 {
 152     int targetId = GetParam();
 153     runTensorFlowNet("batch_norm", targetId);
 154     runTensorFlowNet("fused_batch_norm", targetId);
 155     runTensorFlowNet("batch_norm_text", targetId, true);
 156     runTensorFlowNet("mvn_batch_norm", targetId);
 157     runTensorFlowNet("mvn_batch_norm_1x1", targetId);
 158     runTensorFlowNet("unfused_batch_norm", targetId);
 159     runTensorFlowNet("fused_batch_norm_no_gamma", targetId);
 160     runTensorFlowNet("unfused_batch_norm_no_gamma", targetId);
 161 }
 162
 163 TEST_P(Test_TensorFlow_layers, pooling)
 164 {
 165     int targetId = GetParam();
 166     cv::ocl::Device d = cv::ocl::Device::getDefault();
 167     bool loosenFlag = targetId == DNN_TARGET_OPENCL && d.isIntel() && d.type() == cv::ocl::Device::TYPE_CPU;
 168     runTensorFlowNet("max_pool_even", targetId);
 169     runTensorFlowNet("max_pool_odd_valid", targetId);
 170     runTensorFlowNet("ave_pool_same", targetId);
 171     runTensorFlowNet("max_pool_odd_same", targetId, false, loosenFlag ? 3e-5 : 1e-5, loosenFlag ? 3e-4 : 1e-4);
 172     runTensorFlowNet("reduce_mean", targetId);  // an average pooling over all spatial dimensions.
 173 }
 174
 175 TEST_P(Test_TensorFlow_layers, deconvolution)
 176 {
 177     int targetId = GetParam();
 178     runTensorFlowNet("deconvolution", targetId);
 179     runTensorFlowNet("deconvolution_same", targetId);
 180     runTensorFlowNet("deconvolution_stride_2_same", targetId);
 181     runTensorFlowNet("deconvolution_adj_pad_valid", targetId);
 182     runTensorFlowNet("deconvolution_adj_pad_same", targetId);
 183     runTensorFlowNet("keras_deconv_valid", targetId);
 184     runTensorFlowNet("keras_deconv_same", targetId);
 185 }
 186
 187 TEST_P(Test_TensorFlow_layers, matmul)
 188 {
 189     int targetId = GetParam();
 190     runTensorFlowNet("matmul", targetId);
 191     runTensorFlowNet("nhwc_reshape_matmul", targetId);
 192     runTensorFlowNet("nhwc_transpose_reshape_matmul", targetId);
 193 }
 194
 195 TEST_P(Test_TensorFlow_layers, reshape)
 196 {
 197     int targetId = GetParam();
 198     runTensorFlowNet("shift_reshape_no_reorder", targetId);
 199     runTensorFlowNet("reshape_reduce", targetId);
 200     runTensorFlowNet("flatten", targetId, true);
 201     runTensorFlowNet("unfused_flatten", targetId);
 202     runTensorFlowNet("unfused_flatten_unknown_batch", targetId);
 203 }
 204
 205 TEST_P(Test_TensorFlow_layers, l2_normalize)
 206 {
 207     int targetId = GetParam();
 208     runTensorFlowNet("l2_normalize", targetId);
 209     runTensorFlowNet("l2_normalize_3d", targetId);
 210 }
 211
 212 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, availableDnnTargets());
 213
 214 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_nets;
 215
 216 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 217 {
 218     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 219     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt", false);
 220     std::string imgPath = findDataFile("dnn/street.png", false);
 221
 222     Mat inp;
 223     resize(imread(imgPath), inp, Size(300, 300));
 224     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 225
 226     std::vector<String> outNames(3);
 227     outNames[0] = "concat";
 228     outNames[1] = "concat_1";
 229     outNames[2] = "detection_out";
 230
 231     std::vector<Mat> target(outNames.size());
 232     for (int i = 0; i < outNames.size(); ++i)
 233     {
 234         std::string path = findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco." + outNames[i] + ".npy", false);
 235         target[i] = blobFromNPY(path);
 236     }
 237
 238     Net net = readNetFromTensorflow(netPath, netConfig);
 239     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 240     net.setPreferableTarget(GetParam());
 241
 242     net.setInput(inp);
 243
 244     std::vector<Mat> output;
 245     net.forward(output, outNames);
 246
 247     normAssert(target[0].reshape(1, 1), output[0].reshape(1, 1), "", 1e-5, 1.5e-4);
 248     normAssert(target[1].reshape(1, 1), output[1].reshape(1, 1), "", 1e-5, 3e-4);
 249     normAssertDetections(target[2], output[2], "", 0.2);
 250 }
 251
 252 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 253 {
 254     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt", false);
 255     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 256
 257     Net net = readNetFromTensorflow(model, proto);
 258     Mat img = imread(findDataFile("dnn/street.png", false));
 259     Mat blob = blobFromImage(img, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), true, false);
 260
 261     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 262     net.setPreferableTarget(GetParam());
 263
 264     net.setInput(blob);
 265     // Output has shape 1x1xNx7 where N - number of detections.
 266     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 267     Mat out = net.forward();
 268     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 269                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 270                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 271                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 272                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 273     normAssertDetections(ref, out, "", 0.5);
 274 }
 275
 276 TEST_P(Test_TensorFlow_nets, Inception_v2_Faster_RCNN)
 277 {
 278     std::string proto = findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt", false);
 279     std::string model = findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false);
 280
 281     Net net = readNetFromTensorflow(model, proto);
 282     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 283     Mat img = imread(findDataFile("dnn/dog416.png", false));
 284     Mat blob = blobFromImage(img, 1.0f / 127.5, Size(800, 600), Scalar(127.5, 127.5, 127.5), true, false);
 285
 286     net.setInput(blob);
 287     Mat out = net.forward();
 288
 289     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/faster_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 290     normAssertDetections(ref, out, "", 0.3);
 291 }
 292
 293 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 294 {
 295     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt", false);
 296     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 297
 298     Net net = readNetFromTensorflow(model, proto);
 299     Mat img = imread(findDataFile("gpu/lbpcascade/er.png", false));
 300     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 301
 302     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 303     net.setPreferableTarget(GetParam());
 304
 305     net.setInput(blob);
 306     // Output has shape 1x1xNx7 where N - number of detections.
 307     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 308     Mat out = net.forward();
 309
 310     // References are from test for Caffe model.
 311     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 312                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 313                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 314                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 315                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 316                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 317     normAssertDetections(ref, out, "", 0.9, 3.4e-3, 1e-2);
 318 }
 319
 320 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 321 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 322 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 323 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 324 //                 feed_dict={'input_images:0': inp})
 325 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 326 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 327 // np.save('east_text_detection.scores.npy', scores)
 328 // np.save('east_text_detection.geometry.npy', geometry)
 329 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 330 {
 331     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 332     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png", false);
 333     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy", false);
 334     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy", false);
 335
 336     Net net = readNet(findDataFile("dnn/frozen_east_text_detection.pb", false));
 337
 338     net.setPreferableTarget(GetParam());
 339
 340     Mat img = imread(imgPath);
 341     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 342     net.setInput(inp);
 343
 344     std::vector<Mat> outs;
 345     std::vector<String> outNames(2);
 346     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 347     outNames[1] = "feature_fusion/concat_3";
 348     net.forward(outs, outNames);
 349
 350     Mat scores = outs[0];
 351     Mat geometry = outs[1];
 352
 353     normAssert(scores, blobFromNPY(refScoresPath), "scores");
 354     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", 1e-4, 3e-3);
 355 }
 356
 357 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, availableDnnTargets());
 358
 359 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_fp16;
 360
 361 TEST_P(Test_TensorFlow_fp16, tests)
 362 {
 363     int targetId = GetParam();
 364     const float l1 = 7e-4;
 365     const float lInf = 1e-2;
 366     runTensorFlowNet("fp16_single_conv", targetId, false, l1, lInf);
 367     runTensorFlowNet("fp16_deconvolution", targetId, false, l1, lInf);
 368     runTensorFlowNet("fp16_max_pool_odd_same", targetId, false, l1, lInf);
 369     runTensorFlowNet("fp16_padding_valid", targetId, false, l1, lInf);
 370     runTensorFlowNet("fp16_eltwise_add_mul", targetId, false, l1, lInf);
 371     runTensorFlowNet("fp16_max_pool_odd_valid", targetId, false, l1, lInf);
 372     runTensorFlowNet("fp16_pad_and_concat", targetId, false, l1, lInf);
 373     runTensorFlowNet("fp16_max_pool_even", targetId, false, l1, lInf);
 374     runTensorFlowNet("fp16_padding_same", targetId, false, l1, lInf);
 375 }
 376
 377 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_fp16,
 378                         Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16));
 379
 380 TEST(Test_TensorFlow, defun)
 381 {
 382     runTensorFlowNet("defun_dropout");
 383 }
 384
 385 TEST(Test_TensorFlow, quantized)
 386 {
 387     runTensorFlowNet("uint8_single_conv");
 388 }
 389
 390 TEST(Test_TensorFlow, lstm)
 391 {
 392     runTensorFlowNet("lstm", DNN_TARGET_CPU, true);
 393 }
 394
 395 TEST(Test_TensorFlow, split)
 396 {
 397     runTensorFlowNet("split_equals");
 398 }
 399
 400 TEST(Test_TensorFlow, resize_nearest_neighbor)
 401 {
 402     runTensorFlowNet("resize_nearest_neighbor");
 403 }
 404
 405 TEST(Test_TensorFlow, slice)
 406 {
 407     runTensorFlowNet("slice_4d");
 408 }
 409
 410 TEST(Test_TensorFlow, softmax)
 411 {
 412     runTensorFlowNet("keras_softmax");
 413 }
 414
 415 TEST(Test_TensorFlow, relu6)
 416 {
 417     runTensorFlowNet("keras_relu6");
 418     runTensorFlowNet("keras_relu6", DNN_TARGET_CPU, /*hasText*/ true);
 419 }
 420
 421 TEST(Test_TensorFlow, keras_mobilenet_head)
 422 {
 423     runTensorFlowNet("keras_mobilenet_head");
 424 }
 425
 426 TEST(Test_TensorFlow, memory_read)
 427 {
 428     double l1 = 1e-5;
 429     double lInf = 1e-4;
 430     runTensorFlowNet("lstm", DNN_TARGET_CPU, true, l1, lInf, true);
 431
 432     runTensorFlowNet("batch_norm", DNN_TARGET_CPU, false, l1, lInf, true);
 433     runTensorFlowNet("fused_batch_norm", DNN_TARGET_CPU, false, l1, lInf, true);
 434     runTensorFlowNet("batch_norm_text", DNN_TARGET_CPU, true, l1, lInf, true);
 435 }
 436
 437 TEST(Test_TensorFlow, resize_bilinear)
 438 {
 439     runTensorFlowNet("resize_bilinear");
 440     runTensorFlowNet("resize_bilinear_factor");
 441 }
 442
 443 }