modules/dnn/test/test_tf_importer.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4
   5 // Copyright (C) 2017, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 /*
   9 Test for Tensorflow models loading
  10 */
  11
  12 #include "test_precomp.hpp"
  13 #include "npy_blob.hpp"
  14
  15 #include <opencv2/dnn/layer.details.hpp>  // CV_DNN_REGISTER_LAYER_CLASS
  16
  17 namespace opencv_test
  18 {
  19
  20 using namespace cv;
  21 using namespace cv::dnn;
  22
  23 template<typename TString>
  24 static std::string _tf(TString filename)
  25 {
  26     return (getOpenCVExtraDir() + "/dnn/") + filename;
  27 }
  28
  29 TEST(Test_TensorFlow, read_inception)
  30 {
  31     Net net;
  32     {
  33         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  34         net = readNetFromTensorflow(model);
  35         ASSERT_FALSE(net.empty());
  36     }
  37     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  38
  39     Mat sample = imread(_tf("grace_hopper_227.png"));
  40     ASSERT_TRUE(!sample.empty());
  41     Mat input;
  42     resize(sample, input, Size(224, 224));
  43     input -= 128; // mean sub
  44
  45     Mat inputBlob = blobFromImage(input);
  46
  47     net.setInput(inputBlob, "input");
  48     Mat out = net.forward("softmax2");
  49
  50     std::cout << out.dims << std::endl;
  51 }
  52
  53 TEST(Test_TensorFlow, inception_accuracy)
  54 {
  55     Net net;
  56     {
  57         const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
  58         net = readNetFromTensorflow(model);
  59         ASSERT_FALSE(net.empty());
  60     }
  61     net.setPreferableBackend(DNN_BACKEND_OPENCV);
  62
  63     Mat sample = imread(_tf("grace_hopper_227.png"));
  64     ASSERT_TRUE(!sample.empty());
  65     resize(sample, sample, Size(224, 224));
  66     Mat inputBlob = blobFromImage(sample);
  67
  68     net.setInput(inputBlob, "input");
  69     Mat out = net.forward("softmax2");
  70
  71     Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
  72
  73     normAssert(ref, out);
  74 }
  75
  76 static std::string path(const std::string& file)
  77 {
  78     return findDataFile("dnn/tensorflow/" + file, false);
  79 }
  80
  81 static void runTensorFlowNet(const std::string& prefix, int targetId = DNN_TARGET_CPU, bool hasText = false,
  82                              double l1 = 1e-5, double lInf = 1e-4,
  83                              bool memoryLoad = false)
  84 {
  85     std::string netPath = path(prefix + "_net.pb");
  86     std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
  87     std::string inpPath = path(prefix + "_in.npy");
  88     std::string outPath = path(prefix + "_out.npy");
  89
  90     Net net;
  91     if (memoryLoad)
  92     {
  93         // Load files into a memory buffers
  94         string dataModel;
  95         ASSERT_TRUE(readFileInMemory(netPath, dataModel));
  96
  97         string dataConfig;
  98         if (hasText)
  99             ASSERT_TRUE(readFileInMemory(netConfig, dataConfig));
 100
 101         net = readNetFromTensorflow(dataModel.c_str(), dataModel.size(),
 102                                     dataConfig.c_str(), dataConfig.size());
 103     }
 104     else
 105         net = readNetFromTensorflow(netPath, netConfig);
 106
 107     ASSERT_FALSE(net.empty());
 108
 109     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 110     net.setPreferableTarget(targetId);
 111
 112     cv::Mat input = blobFromNPY(inpPath);
 113     cv::Mat target = blobFromNPY(outPath);
 114
 115     net.setInput(input);
 116     cv::Mat output = net.forward();
 117     normAssert(target, output, "", l1, lInf);
 118 }
 119
 120 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_layers;
 121
 122 TEST_P(Test_TensorFlow_layers, conv)
 123 {
 124     int targetId = GetParam();
 125     runTensorFlowNet("single_conv", targetId);
 126     runTensorFlowNet("atrous_conv2d_valid", targetId);
 127     runTensorFlowNet("atrous_conv2d_same", targetId);
 128     runTensorFlowNet("depthwise_conv2d", targetId);
 129     runTensorFlowNet("keras_atrous_conv2d_same", targetId);
 130 }
 131
 132 TEST_P(Test_TensorFlow_layers, padding)
 133 {
 134     int targetId = GetParam();
 135     runTensorFlowNet("padding_same", targetId);
 136     runTensorFlowNet("padding_valid", targetId);
 137     runTensorFlowNet("spatial_padding", targetId);
 138 }
 139
 140 TEST_P(Test_TensorFlow_layers, eltwise_add_mul)
 141 {
 142     runTensorFlowNet("eltwise_add_mul", GetParam());
 143 }
 144
 145 TEST_P(Test_TensorFlow_layers, concat)
 146 {
 147     runTensorFlowNet("pad_and_concat", GetParam());
 148     runTensorFlowNet("concat_axis_1", GetParam());
 149 }
 150
 151 TEST_P(Test_TensorFlow_layers, batch_norm)
 152 {
 153     int targetId = GetParam();
 154     runTensorFlowNet("batch_norm", targetId);
 155     runTensorFlowNet("fused_batch_norm", targetId);
 156     runTensorFlowNet("batch_norm_text", targetId, true);
 157     runTensorFlowNet("mvn_batch_norm", targetId);
 158     runTensorFlowNet("mvn_batch_norm_1x1", targetId);
 159     runTensorFlowNet("unfused_batch_norm", targetId);
 160     runTensorFlowNet("fused_batch_norm_no_gamma", targetId);
 161     runTensorFlowNet("unfused_batch_norm_no_gamma", targetId);
 162 }
 163
 164 TEST_P(Test_TensorFlow_layers, pooling)
 165 {
 166     int targetId = GetParam();
 167     cv::ocl::Device d = cv::ocl::Device::getDefault();
 168     bool loosenFlag = targetId == DNN_TARGET_OPENCL && d.isIntel() && d.type() == cv::ocl::Device::TYPE_CPU;
 169     runTensorFlowNet("max_pool_even", targetId);
 170     runTensorFlowNet("max_pool_odd_valid", targetId);
 171     runTensorFlowNet("ave_pool_same", targetId);
 172     runTensorFlowNet("max_pool_odd_same", targetId, false, loosenFlag ? 3e-5 : 1e-5, loosenFlag ? 3e-4 : 1e-4);
 173     runTensorFlowNet("reduce_mean", targetId);  // an average pooling over all spatial dimensions.
 174 }
 175
 176 TEST_P(Test_TensorFlow_layers, deconvolution)
 177 {
 178     int targetId = GetParam();
 179     runTensorFlowNet("deconvolution", targetId);
 180     runTensorFlowNet("deconvolution_same", targetId);
 181     runTensorFlowNet("deconvolution_stride_2_same", targetId);
 182     runTensorFlowNet("deconvolution_adj_pad_valid", targetId);
 183     runTensorFlowNet("deconvolution_adj_pad_same", targetId);
 184     runTensorFlowNet("keras_deconv_valid", targetId);
 185     runTensorFlowNet("keras_deconv_same", targetId);
 186 }
 187
 188 TEST_P(Test_TensorFlow_layers, matmul)
 189 {
 190     int targetId = GetParam();
 191     runTensorFlowNet("matmul", targetId);
 192     runTensorFlowNet("nhwc_reshape_matmul", targetId);
 193     runTensorFlowNet("nhwc_transpose_reshape_matmul", targetId);
 194 }
 195
 196 TEST_P(Test_TensorFlow_layers, reshape)
 197 {
 198     int targetId = GetParam();
 199     runTensorFlowNet("shift_reshape_no_reorder", targetId);
 200     runTensorFlowNet("reshape_reduce", targetId);
 201     runTensorFlowNet("flatten", targetId, true);
 202     runTensorFlowNet("unfused_flatten", targetId);
 203     runTensorFlowNet("unfused_flatten_unknown_batch", targetId);
 204 }
 205
 206 TEST_P(Test_TensorFlow_layers, l2_normalize)
 207 {
 208     int targetId = GetParam();
 209     runTensorFlowNet("l2_normalize", targetId);
 210     runTensorFlowNet("l2_normalize_3d", targetId);
 211 }
 212
 213 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, availableDnnTargets());
 214
 215 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_nets;
 216
 217 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
 218 {
 219     std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
 220     std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt", false);
 221     std::string imgPath = findDataFile("dnn/street.png", false);
 222
 223     Mat inp;
 224     resize(imread(imgPath), inp, Size(300, 300));
 225     inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
 226
 227     std::vector<String> outNames(3);
 228     outNames[0] = "concat";
 229     outNames[1] = "concat_1";
 230     outNames[2] = "detection_out";
 231
 232     std::vector<Mat> target(outNames.size());
 233     for (int i = 0; i < outNames.size(); ++i)
 234     {
 235         std::string path = findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco." + outNames[i] + ".npy", false);
 236         target[i] = blobFromNPY(path);
 237     }
 238
 239     Net net = readNetFromTensorflow(netPath, netConfig);
 240     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 241     net.setPreferableTarget(GetParam());
 242
 243     net.setInput(inp);
 244
 245     std::vector<Mat> output;
 246     net.forward(output, outNames);
 247
 248     normAssert(target[0].reshape(1, 1), output[0].reshape(1, 1), "", 1e-5, 1.5e-4);
 249     normAssert(target[1].reshape(1, 1), output[1].reshape(1, 1), "", 1e-5, 3e-4);
 250     normAssertDetections(target[2], output[2], "", 0.2);
 251 }
 252
 253 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
 254 {
 255     std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt", false);
 256     std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
 257
 258     Net net = readNetFromTensorflow(model, proto);
 259     Mat img = imread(findDataFile("dnn/street.png", false));
 260     Mat blob = blobFromImage(img, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), true, false);
 261
 262     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 263     net.setPreferableTarget(GetParam());
 264
 265     net.setInput(blob);
 266     // Output has shape 1x1xNx7 where N - number of detections.
 267     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 268     Mat out = net.forward();
 269     Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
 270                                     0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
 271                                     0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
 272                                     0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
 273                                     0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
 274     normAssertDetections(ref, out, "", 0.5);
 275 }
 276
 277 TEST_P(Test_TensorFlow_nets, Inception_v2_Faster_RCNN)
 278 {
 279     std::string proto = findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt", false);
 280     std::string model = findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false);
 281
 282     Net net = readNetFromTensorflow(model, proto);
 283     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 284     Mat img = imread(findDataFile("dnn/dog416.png", false));
 285     Mat blob = blobFromImage(img, 1.0f / 127.5, Size(800, 600), Scalar(127.5, 127.5, 127.5), true, false);
 286
 287     net.setInput(blob);
 288     Mat out = net.forward();
 289
 290     Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/faster_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
 291     normAssertDetections(ref, out, "", 0.3);
 292 }
 293
 294 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
 295 {
 296     std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt", false);
 297     std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
 298
 299     Net net = readNetFromTensorflow(model, proto);
 300     Mat img = imread(findDataFile("gpu/lbpcascade/er.png", false));
 301     Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
 302
 303     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 304     net.setPreferableTarget(GetParam());
 305
 306     net.setInput(blob);
 307     // Output has shape 1x1xNx7 where N - number of detections.
 308     // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
 309     Mat out = net.forward();
 310
 311     // References are from test for Caffe model.
 312     Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
 313                                     0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
 314                                     0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
 315                                     0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
 316                                     0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
 317                                     0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
 318     normAssertDetections(ref, out, "", 0.9, 3.4e-3, 1e-2);
 319 }
 320
 321 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
 322 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
 323 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
 324 //                  sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
 325 //                 feed_dict={'input_images:0': inp})
 326 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
 327 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
 328 // np.save('east_text_detection.scores.npy', scores)
 329 // np.save('east_text_detection.geometry.npy', geometry)
 330 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
 331 {
 332     std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
 333     std::string imgPath = findDataFile("cv/ximgproc/sources/08.png", false);
 334     std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy", false);
 335     std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy", false);
 336
 337     Net net = readNet(findDataFile("dnn/frozen_east_text_detection.pb", false));
 338
 339     net.setPreferableTarget(GetParam());
 340
 341     Mat img = imread(imgPath);
 342     Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
 343     net.setInput(inp);
 344
 345     std::vector<Mat> outs;
 346     std::vector<String> outNames(2);
 347     outNames[0] = "feature_fusion/Conv_7/Sigmoid";
 348     outNames[1] = "feature_fusion/concat_3";
 349     net.forward(outs, outNames);
 350
 351     Mat scores = outs[0];
 352     Mat geometry = outs[1];
 353
 354     normAssert(scores, blobFromNPY(refScoresPath), "scores");
 355     normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", 1e-4, 3e-3);
 356 }
 357
 358 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, availableDnnTargets());
 359
 360 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_fp16;
 361
 362 TEST_P(Test_TensorFlow_fp16, tests)
 363 {
 364     int targetId = GetParam();
 365     const float l1 = 7e-4;
 366     const float lInf = 1e-2;
 367     runTensorFlowNet("fp16_single_conv", targetId, false, l1, lInf);
 368     runTensorFlowNet("fp16_deconvolution", targetId, false, l1, lInf);
 369     runTensorFlowNet("fp16_max_pool_odd_same", targetId, false, l1, lInf);
 370     runTensorFlowNet("fp16_padding_valid", targetId, false, l1, lInf);
 371     runTensorFlowNet("fp16_eltwise_add_mul", targetId, false, l1, lInf);
 372     runTensorFlowNet("fp16_max_pool_odd_valid", targetId, false, l1, lInf);
 373     runTensorFlowNet("fp16_pad_and_concat", targetId, false, l1, lInf);
 374     runTensorFlowNet("fp16_max_pool_even", targetId, false, l1, lInf);
 375     runTensorFlowNet("fp16_padding_same", targetId, false, l1, lInf);
 376 }
 377
 378 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_fp16,
 379                         Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16));
 380
 381 TEST(Test_TensorFlow, defun)
 382 {
 383     runTensorFlowNet("defun_dropout");
 384 }
 385
 386 TEST(Test_TensorFlow, quantized)
 387 {
 388     runTensorFlowNet("uint8_single_conv");
 389 }
 390
 391 TEST(Test_TensorFlow, lstm)
 392 {
 393     runTensorFlowNet("lstm", DNN_TARGET_CPU, true);
 394 }
 395
 396 TEST(Test_TensorFlow, split)
 397 {
 398     runTensorFlowNet("split_equals");
 399 }
 400
 401 TEST(Test_TensorFlow, resize_nearest_neighbor)
 402 {
 403     runTensorFlowNet("resize_nearest_neighbor");
 404 }
 405
 406 TEST(Test_TensorFlow, slice)
 407 {
 408     runTensorFlowNet("slice_4d");
 409 }
 410
 411 TEST(Test_TensorFlow, softmax)
 412 {
 413     runTensorFlowNet("keras_softmax");
 414 }
 415
 416 TEST(Test_TensorFlow, relu6)
 417 {
 418     runTensorFlowNet("keras_relu6");
 419     runTensorFlowNet("keras_relu6", DNN_TARGET_CPU, /*hasText*/ true);
 420 }
 421
 422 TEST(Test_TensorFlow, keras_mobilenet_head)
 423 {
 424     runTensorFlowNet("keras_mobilenet_head");
 425 }
 426
 427 TEST(Test_TensorFlow, memory_read)
 428 {
 429     double l1 = 1e-5;
 430     double lInf = 1e-4;
 431     runTensorFlowNet("lstm", DNN_TARGET_CPU, true, l1, lInf, true);
 432
 433     runTensorFlowNet("batch_norm", DNN_TARGET_CPU, false, l1, lInf, true);
 434     runTensorFlowNet("fused_batch_norm", DNN_TARGET_CPU, false, l1, lInf, true);
 435     runTensorFlowNet("batch_norm_text", DNN_TARGET_CPU, true, l1, lInf, true);
 436 }
 437
 438 TEST(Test_TensorFlow, resize_bilinear)
 439 {
 440     runTensorFlowNet("resize_bilinear");
 441     runTensorFlowNet("resize_bilinear_factor");
 442 }
 443
 444 TEST(Test_TensorFlow, two_inputs)
 445 {
 446     Net net = readNet(path("two_inputs_net.pbtxt"));
 447     net.setPreferableBackend(DNN_BACKEND_OPENCV);
 448
 449     Mat firstInput(2, 3, CV_32FC1), secondInput(2, 3, CV_32FC1);
 450     randu(firstInput, -1, 1);
 451     randu(secondInput, -1, 1);
 452
 453     net.setInput(firstInput, "first_input");
 454     net.setInput(secondInput, "second_input");
 455     Mat out = net.forward();
 456
 457     normAssert(out, firstInput + secondInput);
 458 }
 459
 460 }