From 8badf7f35441c02897104be40ba60ee7f4002b9d Mon Sep 17 00:00:00 2001 From: Liubov Batanina Date: Tue, 21 Apr 2020 12:26:58 +0300 Subject: [PATCH] Merge pull request #17112 from l-bat:ie_region * Support nGraph Region * Support region since OpenVINO 2020.2 * Skip myriad --- modules/dnn/src/layers/region_layer.cpp | 210 +++++++++++++++++++++++++++++ modules/dnn/test/test_darknet_importer.cpp | 4 + 2 files changed, 214 insertions(+) diff --git a/modules/dnn/src/layers/region_layer.cpp b/modules/dnn/src/layers/region_layer.cpp index c33c1cb..e2a0733 100644 --- a/modules/dnn/src/layers/region_layer.cpp +++ b/modules/dnn/src/layers/region_layer.cpp @@ -49,6 +49,11 @@ #include "opencl_kernels_dnn.hpp" #endif +#ifdef HAVE_DNN_NGRAPH +#include "../ie_ngraph.hpp" +#endif + + namespace cv { namespace dnn @@ -103,6 +108,15 @@ public: return false; } + virtual bool supportBackend(int backendId) CV_OVERRIDE + { +#ifdef HAVE_DNN_NGRAPH + if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) + return INF_ENGINE_VER_MAJOR_GE(INF_ENGINE_RELEASE_2020_2) && preferableTarget != DNN_TARGET_MYRIAD; +#endif + return backendId == DNN_BACKEND_OPENCV; + } + float logistic_activate(float x) { return 1.F / (1.F + exp(-x)); } void softmax_activate(const float* input, const int n, const float temp, float* output) @@ -344,6 +358,202 @@ public: } return flops; } + +#ifdef HAVE_DNN_NGRAPH + virtual Ptr initNgraph(const std::vector > &inputs, + const std::vector >& nodes) CV_OVERRIDE + { + auto& input = nodes[0].dynamicCast()->node; + auto parent_shape = input->get_shape(); + int64_t b = parent_shape[0]; + int64_t h = parent_shape[1]; + int64_t w = parent_shape[2]; + int64_t c = parent_shape[3]; + + int64_t cols = b * h * w * anchors; + int64_t rows = c / anchors; + auto shape_node = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{cols, rows}); + auto tr_axes = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{1, 0}); + + std::shared_ptr input2d; + { + input2d = std::make_shared(input, shape_node, true); + input2d = std::make_shared(input2d, tr_axes); + } + + std::shared_ptr region; + { + auto new_axes = std::make_shared(ngraph::element::i64, ngraph::Shape{4}, std::vector{0, 3, 1, 2}); + auto tr_input = std::make_shared(input, new_axes); + + std::vector anchors_vec(blobs[0].ptr(), blobs[0].ptr() + blobs[0].total()); + std::vector mask(anchors, 1); + region = std::make_shared(tr_input, coords, classes, anchors, useSoftmax, mask, 1, 3, anchors_vec); + + auto shape_as_inp = std::make_shared(ngraph::element::i64, + ngraph::Shape{tr_input->get_shape().size()}, tr_input->get_shape().data()); + + region = std::make_shared(region, shape_as_inp, true); + new_axes = std::make_shared(ngraph::element::i64, ngraph::Shape{4}, std::vector{0, 2, 3, 1}); + region = std::make_shared(region, new_axes); + + region = std::make_shared(region, shape_node, true); + region = std::make_shared(region, tr_axes); + } + + auto strides = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{1, 1}); + std::vector boxes_shape{b, anchors, h, w}; + auto shape_3d = std::make_shared(ngraph::element::i64, ngraph::Shape{boxes_shape.size()}, boxes_shape.data()); + + ngraph::Shape box_broad_shape{1, (size_t)anchors, (size_t)h, (size_t)w}; + + std::shared_ptr box_x; + { + auto lower_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{0, 0}); + auto upper_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{1, cols}); + box_x = std::make_shared(input2d, lower_bounds, upper_bounds, strides, std::vector{}, std::vector{}); + box_x = std::make_shared(box_x); + box_x = std::make_shared(box_x, shape_3d, true); + + std::vector x_indices(w * h * anchors); + auto begin = x_indices.begin(); + for (int i = 0; i < h; i++) + { + std::fill(begin + i * anchors, begin + (i + 1) * anchors, i); + } + + for (int j = 1; j < w; j++) + { + std::copy(begin, begin + h * anchors, begin + j * h * anchors); + } + auto horiz = std::make_shared(ngraph::element::f32, box_broad_shape, x_indices.data()); + box_x = std::make_shared(box_x, horiz, ngraph::op::AutoBroadcastType::NUMPY); + + auto cols_node = std::make_shared(ngraph::element::f32, ngraph::Shape{1}, std::vector{float(w)}); + box_x = std::make_shared(box_x, cols_node, ngraph::op::AutoBroadcastType::NUMPY); + } + + std::shared_ptr box_y; + { + auto lower_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{1, 0}); + auto upper_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{2, cols}); + box_y = std::make_shared(input2d, lower_bounds, upper_bounds, strides, std::vector{}, std::vector{}); + box_y = std::make_shared(box_y); + box_y = std::make_shared(box_y, shape_3d, true); + + std::vector y_indices(h * anchors); + for (int i = 0; i < h; i++) + { + std::fill(y_indices.begin() + i * anchors, y_indices.begin() + (i + 1) * anchors, i); + } + + auto vert = std::make_shared(ngraph::element::f32, ngraph::Shape{1, (size_t)anchors, (size_t)h, 1}, y_indices.data()); + box_y = std::make_shared(box_y, vert, ngraph::op::AutoBroadcastType::NUMPY); + auto rows_node = std::make_shared(ngraph::element::f32, ngraph::Shape{1}, std::vector{float(h)}); + box_y = std::make_shared(box_y, rows_node, ngraph::op::AutoBroadcastType::NUMPY); + } + + std::shared_ptr box_w, box_h; + { + int hNorm, wNorm; + if (nodes.size() > 1) + { + auto node_1_shape = nodes[1].dynamicCast()->node->get_shape(); + hNorm = node_1_shape[2]; + wNorm = node_1_shape[3]; + } + else + { + hNorm = h; + wNorm = w; + } + + std::vector anchors_w(anchors), anchors_h(anchors); + for (size_t a = 0; a < anchors; ++a) + { + anchors_w[a] = blobs[0].at(0, 2 * a) / wNorm; + anchors_h[a] = blobs[0].at(0, 2 * a + 1) / hNorm; + } + + std::vector bias_w(w * h * anchors), bias_h(w * h * anchors); + for (int j = 0; j < h; j++) + { + std::copy(anchors_w.begin(), anchors_w.end(), bias_w.begin() + j * anchors); + std::copy(anchors_h.begin(), anchors_h.end(), bias_h.begin() + j * anchors); + } + + for (int i = 1; i < w; i++) + { + std::copy(bias_w.begin(), bias_w.begin() + h * anchors, bias_w.begin() + i * h * anchors); + std::copy(bias_h.begin(), bias_h.begin() + h * anchors, bias_h.begin() + i * h * anchors); + } + + auto lower_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{2, 0}); + auto upper_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{3, cols}); + box_w = std::make_shared(input2d, lower_bounds, upper_bounds, strides, std::vector{}, std::vector{}); + box_w = std::make_shared(box_w); + box_w = std::make_shared(box_w, shape_3d, true); + auto anchor_w_node = std::make_shared(ngraph::element::f32, box_broad_shape, bias_w.data()); + box_w = std::make_shared(box_w, anchor_w_node, ngraph::op::AutoBroadcastType::NUMPY); + + lower_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{3, 0}); + upper_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{4, cols}); + box_h = std::make_shared(input2d, lower_bounds, upper_bounds, strides, std::vector{}, std::vector{}); + box_h = std::make_shared(box_h); + box_h = std::make_shared(box_h, shape_3d, true); + auto anchor_h_node = std::make_shared(ngraph::element::f32, box_broad_shape, bias_h.data()); + box_h = std::make_shared(box_h, anchor_h_node, ngraph::op::AutoBroadcastType::NUMPY); + } + + std::shared_ptr scale; + { + auto lower_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{4, 0}); + auto upper_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{5, cols}); + scale = std::make_shared(region, lower_bounds, upper_bounds, strides, std::vector{}, std::vector{}); + + if (classfix == -1) + { + auto thresh_node = std::make_shared(ngraph::element::f32, ngraph::Shape{1}, std::vector{0.5}); + auto mask = std::make_shared(scale, thresh_node); + auto zero_node = std::make_shared(ngraph::element::f32, mask->get_shape(), std::vector(b * cols, 0)); + scale = std::make_shared(mask, scale, zero_node); + } + } + + std::shared_ptr probs; + { + auto lower_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{5, 0}); + auto upper_bounds = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{rows, cols}); + auto classes = std::make_shared(region, lower_bounds, upper_bounds, strides, std::vector{}, std::vector{}); + probs = std::make_shared(classes, scale, ngraph::op::AutoBroadcastType::NUMPY); + + auto thresh_node = std::make_shared(ngraph::element::f32, ngraph::Shape{1}, &thresh); + auto mask = std::make_shared(probs, thresh_node); + auto zero_node = std::make_shared(ngraph::element::f32, mask->get_shape(), std::vector((rows - 5) * cols, 0)); + probs = std::make_shared(mask, probs, zero_node); + } + + + auto concat_shape = std::make_shared(ngraph::element::i64, ngraph::Shape{2}, std::vector{1, cols}); + box_x = std::make_shared(box_x, concat_shape, true); + box_y = std::make_shared(box_y, concat_shape, true); + box_w = std::make_shared(box_w, concat_shape, true); + box_h = std::make_shared(box_h, concat_shape, true); + + ngraph::NodeVector inp_nodes{box_x, box_y, box_w, box_h, scale, probs}; + std::shared_ptr result = std::make_shared(inp_nodes, 0); + result = std::make_shared(result, tr_axes); + if (b > 1) + { + std::vector sizes = {(size_t)b, result->get_shape()[0] / b, result->get_shape()[1]}; + auto shape_node = std::make_shared(ngraph::element::i64, ngraph::Shape{sizes.size()}, sizes.data()); + result = std::make_shared(result, shape_node, true); + } + + return Ptr(new InfEngineNgraphNode(result)); + } +#endif // HAVE_DNN_NGRAPH + }; Ptr RegionLayer::create(const LayerParams& params) diff --git a/modules/dnn/test/test_darknet_importer.cpp b/modules/dnn/test/test_darknet_importer.cpp index 58faaa1..6d88e3c 100644 --- a/modules/dnn/test/test_darknet_importer.cpp +++ b/modules/dnn/test/test_darknet_importer.cpp @@ -530,6 +530,10 @@ TEST_P(Test_Darknet_layers, avgpool_softmax) TEST_P(Test_Darknet_layers, region) { +#if defined(INF_ENGINE_RELEASE) + if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && INF_ENGINE_VER_MAJOR_GE(2020020000)) + applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION); +#endif testDarknetLayer("region"); } -- 2.7.4