1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2017, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
9 Test for Tensorflow models loading
12 #include "test_precomp.hpp"
13 #include "npy_blob.hpp"
15 #include <opencv2/dnn/layer.details.hpp> // CV_DNN_REGISTER_LAYER_CLASS
21 using namespace cv::dnn;
23 template<typename TString>
24 static std::string _tf(TString filename)
26 return (getOpenCVExtraDir() + "/dnn/") + filename;
29 TEST(Test_TensorFlow, read_inception)
33 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
34 net = readNetFromTensorflow(model);
35 ASSERT_FALSE(net.empty());
37 net.setPreferableBackend(DNN_BACKEND_OPENCV);
39 Mat sample = imread(_tf("grace_hopper_227.png"));
40 ASSERT_TRUE(!sample.empty());
42 resize(sample, input, Size(224, 224));
43 input -= 128; // mean sub
45 Mat inputBlob = blobFromImage(input);
47 net.setInput(inputBlob, "input");
48 Mat out = net.forward("softmax2");
50 std::cout << out.dims << std::endl;
53 TEST(Test_TensorFlow, inception_accuracy)
57 const string model = findDataFile("dnn/tensorflow_inception_graph.pb", false);
58 net = readNetFromTensorflow(model);
59 ASSERT_FALSE(net.empty());
61 net.setPreferableBackend(DNN_BACKEND_OPENCV);
63 Mat sample = imread(_tf("grace_hopper_227.png"));
64 ASSERT_TRUE(!sample.empty());
65 resize(sample, sample, Size(224, 224));
66 Mat inputBlob = blobFromImage(sample);
68 net.setInput(inputBlob, "input");
69 Mat out = net.forward("softmax2");
71 Mat ref = blobFromNPY(_tf("tf_inception_prob.npy"));
76 static std::string path(const std::string& file)
78 return findDataFile("dnn/tensorflow/" + file, false);
81 static void runTensorFlowNet(const std::string& prefix, int targetId = DNN_TARGET_CPU, bool hasText = false,
82 double l1 = 1e-5, double lInf = 1e-4,
83 bool memoryLoad = false)
85 std::string netPath = path(prefix + "_net.pb");
86 std::string netConfig = (hasText ? path(prefix + "_net.pbtxt") : "");
87 std::string inpPath = path(prefix + "_in.npy");
88 std::string outPath = path(prefix + "_out.npy");
93 // Load files into a memory buffers
95 ASSERT_TRUE(readFileInMemory(netPath, dataModel));
99 ASSERT_TRUE(readFileInMemory(netConfig, dataConfig));
101 net = readNetFromTensorflow(dataModel.c_str(), dataModel.size(),
102 dataConfig.c_str(), dataConfig.size());
105 net = readNetFromTensorflow(netPath, netConfig);
107 ASSERT_FALSE(net.empty());
109 net.setPreferableBackend(DNN_BACKEND_OPENCV);
110 net.setPreferableTarget(targetId);
112 cv::Mat input = blobFromNPY(inpPath);
113 cv::Mat target = blobFromNPY(outPath);
116 cv::Mat output = net.forward();
117 normAssert(target, output, "", l1, lInf);
120 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_layers;
122 TEST_P(Test_TensorFlow_layers, conv)
124 int targetId = GetParam();
125 runTensorFlowNet("single_conv", targetId);
126 runTensorFlowNet("atrous_conv2d_valid", targetId);
127 runTensorFlowNet("atrous_conv2d_same", targetId);
128 runTensorFlowNet("depthwise_conv2d", targetId);
129 runTensorFlowNet("keras_atrous_conv2d_same", targetId);
132 TEST_P(Test_TensorFlow_layers, padding)
134 int targetId = GetParam();
135 runTensorFlowNet("padding_same", targetId);
136 runTensorFlowNet("padding_valid", targetId);
137 runTensorFlowNet("spatial_padding", targetId);
140 TEST_P(Test_TensorFlow_layers, eltwise_add_mul)
142 runTensorFlowNet("eltwise_add_mul", GetParam());
145 TEST_P(Test_TensorFlow_layers, pad_and_concat)
147 runTensorFlowNet("pad_and_concat", GetParam());
150 TEST_P(Test_TensorFlow_layers, batch_norm)
152 int targetId = GetParam();
153 runTensorFlowNet("batch_norm", targetId);
154 runTensorFlowNet("fused_batch_norm", targetId);
155 runTensorFlowNet("batch_norm_text", targetId, true);
156 runTensorFlowNet("mvn_batch_norm", targetId);
157 runTensorFlowNet("mvn_batch_norm_1x1", targetId);
158 runTensorFlowNet("unfused_batch_norm", targetId);
159 runTensorFlowNet("fused_batch_norm_no_gamma", targetId);
160 runTensorFlowNet("unfused_batch_norm_no_gamma", targetId);
163 TEST_P(Test_TensorFlow_layers, pooling)
165 int targetId = GetParam();
166 cv::ocl::Device d = cv::ocl::Device::getDefault();
167 bool loosenFlag = targetId == DNN_TARGET_OPENCL && d.isIntel() && d.type() == cv::ocl::Device::TYPE_CPU;
168 runTensorFlowNet("max_pool_even", targetId);
169 runTensorFlowNet("max_pool_odd_valid", targetId);
170 runTensorFlowNet("ave_pool_same", targetId);
171 runTensorFlowNet("max_pool_odd_same", targetId, false, loosenFlag ? 3e-5 : 1e-5, loosenFlag ? 3e-4 : 1e-4);
172 runTensorFlowNet("reduce_mean", targetId); // an average pooling over all spatial dimensions.
175 TEST_P(Test_TensorFlow_layers, deconvolution)
177 int targetId = GetParam();
178 runTensorFlowNet("deconvolution", targetId);
179 runTensorFlowNet("deconvolution_same", targetId);
180 runTensorFlowNet("deconvolution_stride_2_same", targetId);
181 runTensorFlowNet("deconvolution_adj_pad_valid", targetId);
182 runTensorFlowNet("deconvolution_adj_pad_same", targetId);
183 runTensorFlowNet("keras_deconv_valid", targetId);
184 runTensorFlowNet("keras_deconv_same", targetId);
187 TEST_P(Test_TensorFlow_layers, matmul)
189 int targetId = GetParam();
190 runTensorFlowNet("matmul", targetId);
191 runTensorFlowNet("nhwc_reshape_matmul", targetId);
192 runTensorFlowNet("nhwc_transpose_reshape_matmul", targetId);
195 TEST_P(Test_TensorFlow_layers, reshape)
197 int targetId = GetParam();
198 runTensorFlowNet("shift_reshape_no_reorder", targetId);
199 runTensorFlowNet("reshape_reduce", targetId);
200 runTensorFlowNet("flatten", targetId, true);
201 runTensorFlowNet("unfused_flatten", targetId);
202 runTensorFlowNet("unfused_flatten_unknown_batch", targetId);
205 TEST_P(Test_TensorFlow_layers, l2_normalize)
207 int targetId = GetParam();
208 runTensorFlowNet("l2_normalize", targetId);
209 runTensorFlowNet("l2_normalize_3d", targetId);
212 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_layers, availableDnnTargets());
214 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_nets;
216 TEST_P(Test_TensorFlow_nets, MobileNet_SSD)
218 std::string netPath = findDataFile("dnn/ssd_mobilenet_v1_coco.pb", false);
219 std::string netConfig = findDataFile("dnn/ssd_mobilenet_v1_coco.pbtxt", false);
220 std::string imgPath = findDataFile("dnn/street.png", false);
223 resize(imread(imgPath), inp, Size(300, 300));
224 inp = blobFromImage(inp, 1.0f / 127.5, Size(), Scalar(127.5, 127.5, 127.5), true);
226 std::vector<String> outNames(3);
227 outNames[0] = "concat";
228 outNames[1] = "concat_1";
229 outNames[2] = "detection_out";
231 std::vector<Mat> target(outNames.size());
232 for (int i = 0; i < outNames.size(); ++i)
234 std::string path = findDataFile("dnn/tensorflow/ssd_mobilenet_v1_coco." + outNames[i] + ".npy", false);
235 target[i] = blobFromNPY(path);
238 Net net = readNetFromTensorflow(netPath, netConfig);
239 net.setPreferableBackend(DNN_BACKEND_OPENCV);
240 net.setPreferableTarget(GetParam());
244 std::vector<Mat> output;
245 net.forward(output, outNames);
247 normAssert(target[0].reshape(1, 1), output[0].reshape(1, 1), "", 1e-5, 1.5e-4);
248 normAssert(target[1].reshape(1, 1), output[1].reshape(1, 1), "", 1e-5, 3e-4);
249 normAssertDetections(target[2], output[2], "", 0.2);
252 TEST_P(Test_TensorFlow_nets, Inception_v2_SSD)
254 std::string proto = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pbtxt", false);
255 std::string model = findDataFile("dnn/ssd_inception_v2_coco_2017_11_17.pb", false);
257 Net net = readNetFromTensorflow(model, proto);
258 Mat img = imread(findDataFile("dnn/street.png", false));
259 Mat blob = blobFromImage(img, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), true, false);
261 net.setPreferableBackend(DNN_BACKEND_OPENCV);
262 net.setPreferableTarget(GetParam());
265 // Output has shape 1x1xNx7 where N - number of detections.
266 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
267 Mat out = net.forward();
268 Mat ref = (Mat_<float>(5, 7) << 0, 1, 0.90176028, 0.19872092, 0.36311883, 0.26461923, 0.63498729,
269 0, 3, 0.93569964, 0.64865261, 0.45906419, 0.80675775, 0.65708131,
270 0, 3, 0.75838411, 0.44668293, 0.45907149, 0.49459291, 0.52197015,
271 0, 10, 0.95932811, 0.38349164, 0.32528657, 0.40387636, 0.39165527,
272 0, 10, 0.93973452, 0.66561931, 0.37841269, 0.68074018, 0.42907384);
273 normAssertDetections(ref, out, "", 0.5);
276 TEST_P(Test_TensorFlow_nets, Inception_v2_Faster_RCNN)
278 std::string proto = findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pbtxt", false);
279 std::string model = findDataFile("dnn/faster_rcnn_inception_v2_coco_2018_01_28.pb", false);
281 Net net = readNetFromTensorflow(model, proto);
282 net.setPreferableBackend(DNN_BACKEND_OPENCV);
283 Mat img = imread(findDataFile("dnn/dog416.png", false));
284 Mat blob = blobFromImage(img, 1.0f / 127.5, Size(800, 600), Scalar(127.5, 127.5, 127.5), true, false);
287 Mat out = net.forward();
289 Mat ref = blobFromNPY(findDataFile("dnn/tensorflow/faster_rcnn_inception_v2_coco_2018_01_28.detection_out.npy"));
290 normAssertDetections(ref, out, "", 0.3);
293 TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
295 std::string proto = findDataFile("dnn/opencv_face_detector.pbtxt", false);
296 std::string model = findDataFile("dnn/opencv_face_detector_uint8.pb", false);
298 Net net = readNetFromTensorflow(model, proto);
299 Mat img = imread(findDataFile("gpu/lbpcascade/er.png", false));
300 Mat blob = blobFromImage(img, 1.0, Size(), Scalar(104.0, 177.0, 123.0), false, false);
302 net.setPreferableBackend(DNN_BACKEND_OPENCV);
303 net.setPreferableTarget(GetParam());
306 // Output has shape 1x1xNx7 where N - number of detections.
307 // An every detection is a vector of values [id, classId, confidence, left, top, right, bottom]
308 Mat out = net.forward();
310 // References are from test for Caffe model.
311 Mat ref = (Mat_<float>(6, 7) << 0, 1, 0.99520785, 0.80997437, 0.16379407, 0.87996572, 0.26685631,
312 0, 1, 0.9934696, 0.2831718, 0.50738752, 0.345781, 0.5985168,
313 0, 1, 0.99096733, 0.13629119, 0.24892329, 0.19756334, 0.3310290,
314 0, 1, 0.98977017, 0.23901358, 0.09084064, 0.29902688, 0.1769477,
315 0, 1, 0.97203469, 0.67965847, 0.06876482, 0.73999709, 0.1513494,
316 0, 1, 0.95097077, 0.51901293, 0.45863652, 0.5777427, 0.5347801);
317 normAssertDetections(ref, out, "", 0.9, 3.4e-3, 1e-2);
320 // inp = cv.imread('opencv_extra/testdata/cv/ximgproc/sources/08.png')
321 // inp = inp[:,:,[2, 1, 0]].astype(np.float32).reshape(1, 512, 512, 3)
322 // outs = sess.run([sess.graph.get_tensor_by_name('feature_fusion/Conv_7/Sigmoid:0'),
323 // sess.graph.get_tensor_by_name('feature_fusion/concat_3:0')],
324 // feed_dict={'input_images:0': inp})
325 // scores = np.ascontiguousarray(outs[0].transpose(0, 3, 1, 2))
326 // geometry = np.ascontiguousarray(outs[1].transpose(0, 3, 1, 2))
327 // np.save('east_text_detection.scores.npy', scores)
328 // np.save('east_text_detection.geometry.npy', geometry)
329 TEST_P(Test_TensorFlow_nets, EAST_text_detection)
331 std::string netPath = findDataFile("dnn/frozen_east_text_detection.pb", false);
332 std::string imgPath = findDataFile("cv/ximgproc/sources/08.png", false);
333 std::string refScoresPath = findDataFile("dnn/east_text_detection.scores.npy", false);
334 std::string refGeometryPath = findDataFile("dnn/east_text_detection.geometry.npy", false);
336 Net net = readNet(findDataFile("dnn/frozen_east_text_detection.pb", false));
338 net.setPreferableTarget(GetParam());
340 Mat img = imread(imgPath);
341 Mat inp = blobFromImage(img, 1.0, Size(), Scalar(123.68, 116.78, 103.94), true, false);
344 std::vector<Mat> outs;
345 std::vector<String> outNames(2);
346 outNames[0] = "feature_fusion/Conv_7/Sigmoid";
347 outNames[1] = "feature_fusion/concat_3";
348 net.forward(outs, outNames);
350 Mat scores = outs[0];
351 Mat geometry = outs[1];
353 normAssert(scores, blobFromNPY(refScoresPath), "scores");
354 normAssert(geometry, blobFromNPY(refGeometryPath), "geometry", 1e-4, 3e-3);
357 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, availableDnnTargets());
359 typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_fp16;
361 TEST_P(Test_TensorFlow_fp16, tests)
363 int targetId = GetParam();
364 const float l1 = 7e-4;
365 const float lInf = 1e-2;
366 runTensorFlowNet("fp16_single_conv", targetId, false, l1, lInf);
367 runTensorFlowNet("fp16_deconvolution", targetId, false, l1, lInf);
368 runTensorFlowNet("fp16_max_pool_odd_same", targetId, false, l1, lInf);
369 runTensorFlowNet("fp16_padding_valid", targetId, false, l1, lInf);
370 runTensorFlowNet("fp16_eltwise_add_mul", targetId, false, l1, lInf);
371 runTensorFlowNet("fp16_max_pool_odd_valid", targetId, false, l1, lInf);
372 runTensorFlowNet("fp16_pad_and_concat", targetId, false, l1, lInf);
373 runTensorFlowNet("fp16_max_pool_even", targetId, false, l1, lInf);
374 runTensorFlowNet("fp16_padding_same", targetId, false, l1, lInf);
377 INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_fp16,
378 Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16));
380 TEST(Test_TensorFlow, defun)
382 runTensorFlowNet("defun_dropout");
385 TEST(Test_TensorFlow, quantized)
387 runTensorFlowNet("uint8_single_conv");
390 TEST(Test_TensorFlow, lstm)
392 runTensorFlowNet("lstm", DNN_TARGET_CPU, true);
395 TEST(Test_TensorFlow, split)
397 runTensorFlowNet("split_equals");
400 TEST(Test_TensorFlow, resize_nearest_neighbor)
402 runTensorFlowNet("resize_nearest_neighbor");
405 TEST(Test_TensorFlow, slice)
407 runTensorFlowNet("slice_4d");
410 TEST(Test_TensorFlow, softmax)
412 runTensorFlowNet("keras_softmax");
415 TEST(Test_TensorFlow, relu6)
417 runTensorFlowNet("keras_relu6");
418 runTensorFlowNet("keras_relu6", DNN_TARGET_CPU, /*hasText*/ true);
421 TEST(Test_TensorFlow, keras_mobilenet_head)
423 runTensorFlowNet("keras_mobilenet_head");
426 TEST(Test_TensorFlow, memory_read)
430 runTensorFlowNet("lstm", DNN_TARGET_CPU, true, l1, lInf, true);
432 runTensorFlowNet("batch_norm", DNN_TARGET_CPU, false, l1, lInf, true);
433 runTensorFlowNet("fused_batch_norm", DNN_TARGET_CPU, false, l1, lInf, true);
434 runTensorFlowNet("batch_norm_text", DNN_TARGET_CPU, true, l1, lInf, true);
437 TEST(Test_TensorFlow, resize_bilinear)
439 runTensorFlowNet("resize_bilinear");
440 runTensorFlowNet("resize_bilinear_factor");