1 // Copyright (C) 2018 Intel Corporation
3 // SPDX-License-Identifier: Apache-2.0
6 #include <gtest/gtest.h>
7 #include <gmock/gmock-spec-builders.h>
8 #include "mkldnn_plugin/mkldnn_graph.h"
9 #include "mock_mkldnn_primitive.hpp"
11 #include "test_graph.hpp"
13 #include "single_layer_common.hpp"
14 #include <mkldnn_plugin/mkldnn_extension_utils.h>
15 #include "tests_common.hpp"
17 using namespace ::testing;
19 using namespace mkldnn;
22 struct simplernms_test_params {
55 MKLDNNPlugin::impl_desc_type selectedType;
57 std::vector<std::function<void(MKLDNNPlugin::PrimitiveDescInfo)>> comp;
60 struct anchor { float start_x; float start_y; float end_x; float end_y; };
62 template <typename data_t>
63 struct simpler_nms_roi_t
65 data_t x0, y0, x1, y1;
67 constexpr static inline const data_t clamp_v(const data_t v, const data_t v_min, const data_t v_max)
69 return (std::max)(v_min, (std::min)(v, v_max));
72 data_t area() const { return std::max<data_t>(0, y1 - y0 + 1) * std::max<data_t>(0, x1 - x0 + 1); }
74 simpler_nms_roi_t intersect (simpler_nms_roi_t other) const
78 (std::max)(x0, other.x0),
79 (std::max)(y0, other.y0),
80 (std::min)(x1, other.x1),
81 (std::min)(y1, other.y1)
84 simpler_nms_roi_t clamp (simpler_nms_roi_t other) const
88 clamp_v(x0, other.x0, other.x1),
89 clamp_v(y0, other.y0, other.y1),
90 clamp_v(x1, other.x0, other.x1),
91 clamp_v(y1, other.y0, other.y1)
96 template <typename data_t>
97 struct simpler_nms_proposal_t { simpler_nms_roi_t<data_t> roi; data_t confidence; size_t ord; };
98 template <typename data_t>
99 struct simpler_nms_delta_t { data_t shift_x, shift_y, log_w, log_h; };
101 template <typename data_t>
102 inline simpler_nms_roi_t<data_t> simpler_nms_gen_bbox(
104 const simpler_nms_delta_t<data_t>& delta,
108 auto anchor_w = box.end_x - box.start_x + 1;
109 auto anchor_h = box.end_y - box.start_y + 1;
110 auto center_x = box.start_x + anchor_w * .5f;
111 auto center_y = box.start_y + anchor_h *.5f;
113 data_t pred_center_x = delta.shift_x * anchor_w + center_x + anchor_shift_x;
114 data_t pred_center_y = delta.shift_y * anchor_h + center_y + anchor_shift_y;
115 data_t half_pred_w = exp(delta.log_w) * anchor_w * .5f;
116 data_t half_pred_h = exp(delta.log_h) * anchor_h * .5f;
118 return { pred_center_x - half_pred_w,
119 pred_center_y - half_pred_h,
120 pred_center_x + half_pred_w,
121 pred_center_y + half_pred_h };
123 template <typename data_t>
124 inline void sort_and_keep_at_most_top_n(std::vector<simpler_nms_proposal_t<data_t>>& proposals, size_t top_n)
126 const auto cmp_fn = [](const simpler_nms_proposal_t<data_t>& a,
127 const simpler_nms_proposal_t<data_t>& b)
129 return a.confidence > b.confidence || (a.confidence == b.confidence && a.ord > b.ord);
132 if (proposals.size() > top_n) {
133 std::partial_sort(proposals.begin(), proposals.begin() + top_n, proposals.end(), cmp_fn);
134 proposals.resize(top_n);
137 std::sort(proposals.begin(), proposals.end(), cmp_fn);
141 template <typename data_t>
142 std::vector<simpler_nms_roi_t<data_t>> simpler_nms_perform_nms(const std::vector<simpler_nms_proposal_t<data_t>>& proposals,
143 float iou_threshold, size_t top_n) {
144 //TODO(ruv): can I mark the 1st arg, proposals as const? ifndef DONT_PRECALC_AREA, i can
145 //TODO(ruv): is it better to do the precalc or not? since we need to fetch the floats from memory anyway for -
146 // intersect calc, it's only a question of whether it's faster to do (f-f)*(f-f) or fetch another val
147 #define DONT_PRECALC_AREA
149 #ifndef DONT_PRECALC_AREA
150 std::vector<Dtype> areas;
151 areas.reserve(proposals.size());
152 std::transform(proposals.begin(), proposals.end(), areas.begin(), [](const simpler_nms_proposals_t>& v)
158 std::vector<simpler_nms_roi_t<data_t>> res;
160 #ifdef DONT_PRECALC_AREA
161 for (const auto & prop : proposals) {
162 const auto bbox = prop.roi;
163 const data_t area = bbox.area();
165 size_t proposal_count = proposals.size();
166 for (size_t proposalIndex = 0; proposalIndex < proposal_count; ++proposalIndex) {
167 const auto & bbox = proposals[proposalIndex].roi;
170 // For any realistic WL, this condition is true for all top_n values anyway
171 if (prop.confidence > 0) {
172 bool overlaps = std::any_of(res.begin(), res.end(), [&](const simpler_nms_roi_t<data_t>& res_bbox)
174 data_t interArea = bbox.intersect(res_bbox).area();
175 #ifdef DONT_PRECALC_AREA
176 data_t unionArea = res_bbox.area() + area - interArea;
178 data_t unionArea = res_bbox.area() + areas[proposalIndex] - interArea;
180 return interArea > iou_threshold * unionArea;
185 if (res.size() == top_n) break;
193 template <typename data_t>
194 void ref_simplernms(const InferenceEngine::TBlob<data_t> &src_cls, const InferenceEngine::TBlob<data_t> &src_delta, const InferenceEngine::TBlob<data_t> &src_info, InferenceEngine::TBlob<data_t> &dst_blob, simplernms_test_params prm) {
195 int anchors_num = 3 * 3;
196 data_t *anchors_ = new data_t[anchors_num * sizeof(anchor) / sizeof(float)];
197 const anchor* anchors = (anchor*)anchors_;
199 int H = src_cls.dims()[2];
200 int W = src_cls.dims()[3];
204 data_t* dst = dst_blob.data();
206 const data_t* cls_scores = src_cls.readOnly();
207 const data_t* delta_pred = src_delta.readOnly();
208 const data_t* im_info = src_info.readOnly();
214 int scaled_min_bbox_size = prm.minBoxSize * IS;
216 std::vector<simpler_nms_proposal_t<data_t>> sorted_proposals_confidence;
218 for (auto y = 0; y < H; ++y)
220 int anchor_shift_y = y * prm.featStride;
222 for (auto x = 0; x < W; ++x) {
223 int anchor_shift_x = x * prm.featStride;
224 int location_index = y * W + x;
226 // we assume proposals are grouped by window location
227 for (int anchor_index = 0; anchor_index < anchors_num ; anchor_index++) {
228 data_t dx0 = delta_pred[location_index + SZ * (anchor_index * 4 + 0)];
229 data_t dy0 = delta_pred[location_index + SZ * (anchor_index * 4 + 1)];
230 data_t dx1 = delta_pred[location_index + SZ * (anchor_index * 4 + 2)];
231 data_t dy1 = delta_pred[location_index + SZ * (anchor_index * 4 + 3)];
233 simpler_nms_delta_t<data_t> bbox_delta { dx0, dy0, dx1, dy1 };
235 data_t proposal_confidence = cls_scores[location_index + SZ * (anchor_index + anchors_num * 1)];
237 simpler_nms_roi_t<data_t> tmp_roi = simpler_nms_gen_bbox(anchors[anchor_index], bbox_delta, anchor_shift_x, anchor_shift_y);
238 simpler_nms_roi_t<data_t> roi = tmp_roi.clamp({ 0, 0, data_t(IW - 1), data_t(IH - 1) });
240 int bbox_w = roi.x1 - roi.x0 + 1;
241 int bbox_h = roi.y1 - roi.y0 + 1;
243 if (bbox_w >= scaled_min_bbox_size && bbox_h >= scaled_min_bbox_size) {
244 simpler_nms_proposal_t<data_t> proposal { roi, proposal_confidence, sorted_proposals_confidence.size() };
245 sorted_proposals_confidence.push_back(proposal);
251 sort_and_keep_at_most_top_n(sorted_proposals_confidence, prm.preNmsTopn);
252 auto res = simpler_nms_perform_nms(sorted_proposals_confidence, prm.iouThreshold, prm.postNmsTopn);
254 size_t res_num_rois = res.size();
256 for (size_t i = 0; i < res_num_rois; ++i) {
257 dst[5 * i + 0] = 0; // roi_batch_ind, always zero on test time
258 dst[5 * i + 1] = res[i].x0;
259 dst[5 * i + 2] = res[i].y0;
260 dst[5 * i + 3] = res[i].x1;
261 dst[5 * i + 4] = res[i].y1;
267 class MKLDNNGraphSimplerNMSTests: public TestsCommon,
268 public WithParamInterface<simplernms_test_params> {
269 std::string model_t = R"V0G0N(
270 <Net Name="Lrn_Only" version="2" precision="FP32" batch="1">
272 <layer name="in1" type="Input" precision="FP32" id="0">
282 <layer name="in2" type="Input" precision="FP32" id="1">
292 <layer name="in3" type="Input" precision="FP32" id="2">
300 <layer name="proposal" id="3" type="SimplerNMS" precision="FP32">
301 <data cls_threshold="0.500000" max_num_proposals="300" iou_threshold="_IOU_THRESHOLD_"
302 min_bbox_size="_MIN_BOX_SIZE_" feat_stride="_FSRD_" pre_nms_topn="_PRENT_" post_nms_topn="_POSTNT_"
303 scale="8.000000,16.000000,32.000000"/>
332 <edge from-layer="0" from-port="0" to-layer="3" to-port="3"/>
333 <edge from-layer="1" from-port="1" to-layer="3" to-port="4"/>
334 <edge from-layer="2" from-port="2" to-layer="3" to-port="5"/>
339 std::string getModel(simplernms_test_params p) {
340 std::string model = model_t;
342 REPLACE_WITH_NUM(model, "_IWC_", p.in_cls.w);
343 REPLACE_WITH_NUM(model, "_IHC_", p.in_cls.h);
344 REPLACE_WITH_NUM(model, "_ICC_", p.in_cls.c);
345 REPLACE_WITH_NUM(model, "_INC_", p.in_cls.n);
347 REPLACE_WITH_NUM(model, "_IWD_", p.in_delta.w);
348 REPLACE_WITH_NUM(model, "_IHD_", p.in_delta.h);
349 REPLACE_WITH_NUM(model, "_ICD_", p.in_delta.c);
350 REPLACE_WITH_NUM(model, "_IND_", p.in_delta.n);
352 REPLACE_WITH_NUM(model, "_ICI_", p.in_info.c);
353 REPLACE_WITH_NUM(model, "_INI_", p.in_info.n);
355 REPLACE_WITH_NUM(model, "_OC_", p.out.c);
356 REPLACE_WITH_NUM(model, "_ON_", p.out.n);
358 REPLACE_WITH_NUM(model, "_MIN_BOX_SIZE_", p.minBoxSize);
359 REPLACE_WITH_NUM(model, "_FSRD_", p.featStride);
360 REPLACE_WITH_NUM(model, "_PRENT_", p.preNmsTopn);
361 REPLACE_WITH_NUM(model, "_POSTNT_", p.postNmsTopn);
362 REPLACE_WITH_NUM(model, "_IOU_THRESHOLD_", p.iouThreshold);
368 virtual void TearDown() {
371 virtual void SetUp() {
373 TestsCommon::SetUp();
374 simplernms_test_params p = ::testing::WithParamInterface<simplernms_test_params>::GetParam();
375 std::string model = getModel(p);
377 InferenceEngine::CNNNetReader net_reader;
378 ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
380 MKLDNNGraphTestClass graph;
381 graph.CreateGraph(net_reader.getNetwork());
382 auto& nodes = graph.getNodes();
383 for (int i = 0; i < nodes.size(); i++) {
384 if (nodes[i]->getType() == MKLDNNPlugin::SimplerNMS) {
385 ASSERT_EQ(p.num_prim_desc, nodes[i]->getSupportedPrimitiveDescriptors().size());
386 for (size_t j = 0; j < p.num_prim_desc && j < p.comp.size(); j++) {
387 p.comp.at(j)(nodes[i]->getSupportedPrimitiveDescriptors().at(j));
389 ASSERT_NE(nullptr, nodes[i]->getSelectedPrimitiveDescriptor());
390 ASSERT_EQ(p.selectedType, nodes[i]->getSelectedPrimitiveDescriptor()->getImplementationType());
393 InferenceEngine::SizeVector dims_src_cls = {p.in_cls.n, p.in_cls.c, p.in_cls.h, p.in_cls.w};
395 InferenceEngine::Blob::Ptr src_cls = InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src_cls);
397 fill_data(src_cls->buffer(), src_cls->size());
399 InferenceEngine::TBlob<float>* srcClsPtr = dynamic_cast<InferenceEngine::TBlob<float>*>(src_cls.get());
401 if (srcClsPtr == nullptr)
402 FAIL() << "Cannot cast blob to TBlob<float>.";
404 InferenceEngine::SizeVector dims_delta = {p.in_delta.n, p.in_delta.c, p.in_delta.h, p.in_delta.w};
406 InferenceEngine::Blob::Ptr src_delta = InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_delta);
407 src_delta->allocate();
408 fill_data(src_delta->buffer(), src_delta->size());
410 InferenceEngine::TBlob<float>* srcDeltaPtr = dynamic_cast<InferenceEngine::TBlob<float>*>(src_delta.get());
412 if (srcDeltaPtr == nullptr)
413 FAIL() << "Cannot cast blob to TBlob<float>.";
415 InferenceEngine::SizeVector dims_info = {p.in_info.n, p.in_info.c};
417 InferenceEngine::Blob::Ptr src_info = InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NC, dims_info);
418 src_info->allocate();
419 fill_data(src_info->buffer(), src_info->size());
420 float * data_info = src_info->buffer();
425 InferenceEngine::TBlob<float>* srcInfoPtr = dynamic_cast<InferenceEngine::TBlob<float>*>(src_info.get());
427 if (srcInfoPtr == nullptr)
428 FAIL() << "Cannot cast blob to TBlob<float>.";
430 InferenceEngine::BlobMap srcs;
431 srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in1", src_cls));
432 srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in2", src_delta));
433 srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in3", src_info));
435 InferenceEngine::OutputsDataMap out;
436 out = net_reader.getNetwork().getOutputsInfo();
437 InferenceEngine::BlobMap outputBlobs;
439 std::pair<std::string, InferenceEngine::DataPtr> item = *out.begin();
441 InferenceEngine::TBlob<float>::Ptr output;
442 output = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc());
444 outputBlobs[item.first] = output;
446 graph.Infer(srcs, outputBlobs);
448 InferenceEngine::TBlob<float> dst_ref(item.second->getTensorDesc());
451 ref_simplernms(*srcClsPtr, *srcDeltaPtr, *srcInfoPtr, dst_ref, p);
453 compare(*output, dst_ref);
454 } catch (const InferenceEngine::details::InferenceEngineException &e) {
460 TEST_P(MKLDNNGraphSimplerNMSTests, TestsSimplerNMS) {}
463 INSTANTIATE_TEST_CASE_P(
464 DISABLED_TestsSimplerNMS, MKLDNNGraphSimplerNMSTests,
466 simplernms_test_params{{1, 18, 39, 64}, {1, 36, 39, 64}, {1, 3}, {150, 5}, 16, 16, 6000, 150, 0.7f, 1,
467 MKLDNNPlugin::impl_desc_type::ref, {
468 [](MKLDNNPlugin::PrimitiveDescInfo impl) {
469 ASSERT_EQ(MKLDNNPlugin::impl_desc_type::ref, impl.getImplementationType());
470 ASSERT_EQ(3, impl.getConfig().inConfs.size());
471 ASSERT_EQ(1, impl.getConfig().outConfs.size());
472 ASSERT_EQ(InferenceEngine::Layout::ANY, impl.getConfig().inConfs.at(0).desc.getLayout());
473 ASSERT_EQ(InferenceEngine::Layout::ANY, impl.getConfig().inConfs.at(1).desc.getLayout());
474 ASSERT_EQ(InferenceEngine::Layout::ANY, impl.getConfig().inConfs.at(2).desc.getLayout());
475 ASSERT_EQ(InferenceEngine::Layout::ANY, impl.getConfig().outConfs.at(0).desc.getLayout());