[IE][VPU]: Implement VPU Ngraph EIP->ReorgYolo transformation (#3066)

* We need to convert ExtractImagePatches op to ReorgYolo to restore the working capacity of myriad plugin while compiling Yolo-v2 models.
* It was previously removed in #2687

diff --git a/inference-engine/src/vpu/common/include/vpu/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.hpp b/inference-engine/src/vpu/common/include/vpu/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.hpp
new file mode 100644 (file)
index 0000000..aeef206
--- /dev/null
+++ b/inference-engine/src/vpu/common/include/vpu/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.hpp
@@ -0,0 +1,16 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include <ngraph/pass/graph_rewrite.hpp>
+
+namespace vpu {
+
+class ConvertExtractImagePatchesToReorgYolo : public ngraph::pass::MatcherPass {
+public:
+    ConvertExtractImagePatchesToReorgYolo();
+};
+
+}  // namespace vpu

diff --git a/inference-engine/src/vpu/common/src/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.cpp b/inference-engine/src/vpu/common/src/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.cpp
new file mode 100644 (file)
index 0000000..4a28a14
--- /dev/null
+++ b/inference-engine/src/vpu/common/src/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.cpp
@@ -0,0 +1,80 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "vpu/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.hpp"
+
+#include <ngraph/opsets/opset5.hpp>
+#include <ngraph/rt_info.hpp>
+
+#include <memory>
+#include <vector>
+
+namespace vpu {
+
+ConvertExtractImagePatchesToReorgYolo::ConvertExtractImagePatchesToReorgYolo() {
+    const auto image = std::make_shared<ngraph::pattern::op::Label>(ngraph::element::f32, ngraph::Shape{1, 1, 1, 1});
+    const auto eip = std::make_shared<ngraph::opset5::ExtractImagePatches>(
+            image, ngraph::Shape{1, 1}, ngraph::Strides{1, 1}, ngraph::Shape{1, 1}, ngraph::op::PadType::VALID);
+
+    ngraph::matcher_pass_callback callback = [=](ngraph::pattern::Matcher &m) {
+        const auto extractImagePatches =  std::dynamic_pointer_cast<ngraph::opset5::ExtractImagePatches>(m.get_match_root());
+
+        /*
+         * In this transformation we replace ExtractImagePatches operation to ReorgYolo operation
+         * if ExtractImagePatches operation attributes obey the following conditions:
+         *
+         * EIP.sizes = EIP.strides
+         * EIP.rates = {1, 1}
+         * Spatial dimensions of input tensor must be divisible by EIP.strides
+         */
+
+        if (!extractImagePatches || m_transformation_callback(extractImagePatches)) {
+            return false;
+        }
+
+
+        if (extractImagePatches->get_strides() != extractImagePatches->get_sizes()) {
+            return false;
+        }
+
+        const auto& inputPartialShape = extractImagePatches->get_input_partial_shape(0);
+        const auto& sizes = extractImagePatches->get_sizes();
+        const auto& strides = extractImagePatches->get_strides();
+        const auto& rates = extractImagePatches->get_rates();
+
+        if (!inputPartialShape.rank().is_static() || inputPartialShape.rank().get_length() != 4) {
+            return false;
+        }
+
+        if (inputPartialShape[2].is_dynamic() || inputPartialShape[3].is_dynamic()) {
+            return false;
+        }
+
+        if (inputPartialShape[2].get_length() % strides[0] != 0 || inputPartialShape[3].get_length() % strides[1] != 0) {
+            return false;
+        }
+
+        if (sizes[0] != strides[0] || sizes[1] != strides[1]) {
+            return false;
+        }
+
+        if (rates[0] != 1 || rates[1] != 1) {
+            return false;
+        }
+
+        const auto reorgYolo = std::make_shared<ngraph::opset5::ReorgYolo>(
+                extractImagePatches->input(0).get_source_output(),
+                ngraph::Strides{extractImagePatches->get_strides()});
+
+        reorgYolo->set_friendly_name(extractImagePatches->get_friendly_name());
+        ngraph::copy_runtime_info(extractImagePatches, reorgYolo);
+        ngraph::replace_node(extractImagePatches, reorgYolo);
+        return true;
+    };
+
+    const auto matcher = std::make_shared<ngraph::pattern::Matcher>(eip, "ConvertExtractImagePatchesToReorgYolo");
+    register_matcher(matcher, callback);
+}
+
+}  // namespace vpu

diff --git a/inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp b/inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp
index dd99362..f4fa3b6 100644 (file)
--- a/inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp
+++ b/inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp
@@ -31,6 +31,7 @@
 #include <legacy/transformations/convert_opset1_to_legacy/convert_prior_to_ie_prior.hpp>
 #include <transformations/common_optimizations/common_optimizations.hpp>
 #include <transformations/init_node_info.hpp>
+#include <vpu/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.hpp>
 #include <vpu/ngraph/transformations/merge_subsequent_dsr_operations.hpp>
 #include "vpu/ngraph/transformations/dynamic_to_static_shape.hpp"
 #include "vpu/ngraph/transformations/eliminate_shapeof_after_dsr.hpp"
@@ -179,6 +180,7 @@ ie::ICNNNetwork::Ptr FrontEnd::convertNetwork(ie::ICNNNetwork& network) {
     manager.register_pass<ngraph::pass::CommonOptimizations>();
     manager.register_pass<vpu::DynamicToStaticShape>();
     manager.register_pass<vpu::EliminateShapeOfAfterDSR>();
+    manager.register_pass<vpu::ConvertExtractImagePatchesToReorgYolo>();
     manager.register_pass<ngraph::pass::ConvertOpSet3ToOpSet2>();
     manager.register_pass<ngraph::pass::ConvertOpSet2ToOpSet1>();
     manager.register_pass<ngraph::pass::ConvertOpSet1ToLegacy>();

diff --git a/inference-engine/tests/functional/plugin/myriad/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo_vpu.cpp b/inference-engine/tests/functional/plugin/myriad/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo_vpu.cpp
new file mode 100644 (file)
index 0000000..6812f64
--- /dev/null
+++ b/inference-engine/tests/functional/plugin/myriad/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo_vpu.cpp
@@ -0,0 +1,180 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <vpu/ngraph/transformations/convert_extract_image_patches_to_reorg_yolo.hpp>
+#include "common_test_utils/ngraph_test_utils.hpp"
+
+#include <ngraph/opsets/opset5.hpp>
+#include <ngraph/pass/manager.hpp>
+#include <transformations/init_node_info.hpp>
+
+#include <string>
+#include <memory>
+
+namespace {
+
+using EIPParams = std::tuple<
+        ngraph::PartialShape, ngraph::Shape, ngraph::Strides, ngraph::Shape, ngraph::op::PadType>;
+
+class ConvertEIPToReorgYoloTest : public CommonTestUtils::TestsCommon,
+                                  public testing::WithParamInterface<EIPParams> {
+public:
+    std::pair<bool, std::string> compare() {
+        const auto& parameters = GetParam();
+        const auto& dataShape  = std::get<0>(parameters);
+        const auto& sizes = std::get<1>(parameters);
+        const auto& strides = std::get<2>(parameters);
+        const auto& rates = std::get<3>(parameters);
+        const auto& padMode = std::get<4>(parameters);
+
+        return compare_functions(
+                transform(dataShape, sizes, strides, rates, padMode),
+                reference(dataShape, strides));
+    }
+protected:
+    static std::shared_ptr<ngraph::Function> transform(
+            const ngraph::PartialShape& dataShape,
+            const ngraph::Shape& sizes,
+            const ngraph::Strides& strides,
+            const ngraph::Shape& rates,
+            const ngraph::op::PadType& padMode) {
+        const auto param = std::make_shared<ngraph::opset5::Parameter>(
+                ngraph::element::f32,
+                dataShape);
+
+        const auto eip = std::make_shared<ngraph::opset5::ExtractImagePatches>(
+                param, sizes, strides, rates, padMode);
+
+        auto function = std::make_shared<ngraph::Function>(
+                ngraph::NodeVector{eip},
+                ngraph::ParameterVector{param},
+                "Actual");
+
+        ngraph::pass::Manager manager;
+        manager.register_pass<ngraph::pass::InitNodeInfo>();
+        manager.register_pass<vpu::ConvertExtractImagePatchesToReorgYolo>();
+        manager.run_passes(function);
+
+        return function;
+    }
+
+    static std::shared_ptr<ngraph::Function> reference(
+            const ngraph::PartialShape& dataShape,
+            const ngraph::Strides& strides) {
+        const auto param = std::make_shared<ngraph::opset5::Parameter>(
+                ngraph::element::f32,
+                dataShape);
+
+        auto reorgYolo = std::make_shared<ngraph::opset5::ReorgYolo>(param, strides);
+
+        return std::make_shared<ngraph::Function>(
+                ngraph::NodeVector{reorgYolo},
+                ngraph::ParameterVector{param},
+                "Expected");
+    }
+};
+
+//
+// Positive tests
+//
+
+class ConvertEIPToReorgYoloPositiveTest : public ConvertEIPToReorgYoloTest {};
+TEST_P(ConvertEIPToReorgYoloPositiveTest, CompareFunctions) {
+    const auto res = compare();
+    ASSERT_TRUE(res.first) << res.second;
+}
+
+INSTANTIATE_TEST_CASE_P(smoke_NGraph, ConvertEIPToReorgYoloPositiveTest, testing::Combine(
+        testing::Values(ngraph::Shape{1, 64, 500, 500}),
+        testing::Values(ngraph::Shape{5, 5}),
+        testing::Values(ngraph::Strides{5, 5}),
+        testing::Values(ngraph::Shape{1, 1}),
+        testing::Values(
+                ngraph::op::PadType::VALID,
+                ngraph::op::PadType::SAME_LOWER,
+                ngraph::op::PadType::SAME_UPPER)
+));
+
+//
+// Negative tests
+//
+
+class DoNotConvertEIPToReorgYoloOnDiffSizeAndStride : public ConvertEIPToReorgYoloTest {};
+TEST_P(DoNotConvertEIPToReorgYoloOnDiffSizeAndStride, CompareFunctions) {
+    const auto res = compare();
+    ASSERT_FALSE(res.first) << res.second;
+}
+
+INSTANTIATE_TEST_CASE_P(smoke_NGraph, DoNotConvertEIPToReorgYoloOnDiffSizeAndStride, testing::Combine(
+        testing::Values(ngraph::PartialShape{1, 64, 500, 500}),
+        testing::Values(ngraph::Shape{5, 5}),
+        testing::Values(ngraph::Strides{4, 4}),
+        testing::Values(ngraph::Shape{1, 1}),
+        testing::Values(ngraph::op::PadType::VALID)
+));
+
+class DoNotConvertEIPToReorgYoloOnNot4DInput : public ConvertEIPToReorgYoloTest {};
+TEST_P(DoNotConvertEIPToReorgYoloOnNot4DInput, CompareFunctions) {
+    const auto& parameters = GetParam();
+    const auto& dataShape  = std::get<0>(parameters);
+    const auto& sizes = std::get<1>(parameters);
+    const auto& strides = std::get<2>(parameters);
+    const auto& rates = std::get<3>(parameters);
+    const auto& padMode = std::get<4>(parameters);
+
+    EXPECT_ANY_THROW(transform(dataShape, sizes, strides, rates, padMode));
+}
+
+INSTANTIATE_TEST_CASE_P(smoke_NGraph, DoNotConvertEIPToReorgYoloOnNot4DInput, testing::Combine(
+        testing::Values(ngraph::PartialShape{1, 1, 64, 500, 500},
+                        ngraph::PartialShape{64, 500, 500},
+                        ngraph::PartialShape{500, 500},
+                        ngraph::PartialShape{500},
+                        ngraph::PartialShape::dynamic()),
+        testing::Values(ngraph::Shape{5, 5}),
+        testing::Values(ngraph::Strides{5, 5}),
+        testing::Values(ngraph::Shape{1, 1}),
+        testing::Values(ngraph::op::PadType::VALID)
+));
+
+class DoNotConvertEIPToReorgYoloOnNotStaticInput : public ConvertEIPToReorgYoloTest {};
+TEST_P(DoNotConvertEIPToReorgYoloOnNotStaticInput, CompareFunctions) {
+    const auto& parameters = GetParam();
+    const auto& dataShape  = std::get<0>(parameters);
+    const auto& sizes = std::get<1>(parameters);
+    const auto& strides = std::get<2>(parameters);
+    const auto& rates = std::get<3>(parameters);
+    const auto& padMode = std::get<4>(parameters);
+
+    const auto& function = transform(dataShape, sizes, strides, rates, padMode);
+    const auto& ops = function->get_ops();
+    const auto reorgIt = std::find_if(ops.begin(), ops.end(), [](const std::shared_ptr<ngraph::Node>& op) {
+        return ngraph::is_type<ngraph::opset5::ReorgYolo>(op); });
+    ASSERT_TRUE(reorgIt == ops.end());
+}
+
+INSTANTIATE_TEST_CASE_P(smoke_NGraph, DoNotConvertEIPToReorgYoloOnNotStaticInput, testing::Combine(
+        testing::Values(ngraph::PartialShape{1, 64, ngraph::Dimension::dynamic(), 500},
+                        ngraph::PartialShape{1, 64, 500, ngraph::Dimension::dynamic()}),
+        testing::Values(ngraph::Shape{5, 5}),
+        testing::Values(ngraph::Strides{5, 5}),
+        testing::Values(ngraph::Shape{1, 1}),
+        testing::Values(ngraph::op::PadType::VALID)
+));
+
+class DoNotConvertEIPToReorgYoloOnNonSingleRates : public ConvertEIPToReorgYoloTest {};
+TEST_P(DoNotConvertEIPToReorgYoloOnNonSingleRates, CompareFunctions) {
+    const auto res = compare();
+    ASSERT_FALSE(res.first) << res.second;
+}
+
+INSTANTIATE_TEST_CASE_P(smoke_NGraph, DoNotConvertEIPToReorgYoloOnNonSingleRates, testing::Combine(
+        testing::Values(ngraph::Shape{1, 64, 500, 500}),
+        testing::Values(ngraph::Shape{5, 5}),
+        testing::Values(ngraph::Strides{5, 5}),
+        testing::Values(ngraph::Shape{2, 2}),
+        testing::Values(ngraph::op::PadType::VALID)
+));
+
+}  // namespace