1 // Copyright (C) 2018-2020 Intel Corporation
2 // SPDX-License-Identifier: Apache-2.0
5 #include "ie_metric_helpers.hpp"
6 #include "mkldnn_plugin.h"
7 #include "mkldnn_extension_mngr.h"
8 #include "mkldnn_layers_dispatcher.hpp"
9 #include "mkldnn_weights_cache.hpp"
10 #include <cpp_interfaces/base/ie_plugin_base.hpp>
11 #include <threading/ie_executor_manager.hpp>
13 #include <ie_plugin_config.hpp>
16 #include <ie_system_conf.h>
17 #include <generic_ie.hpp>
19 #include "convert_function_to_cnn_network.hpp"
20 #include <transformations/common_optimizations/common_optimizations.hpp>
21 #include <transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.hpp>
22 #include <transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.hpp>
23 #include <transformations/convert_opset3_to_opset2/convert_opset3_to_opset2.hpp>
24 #include <ngraph/opsets/opset1.hpp>
25 #include <ngraph/opsets/opset2.hpp>
26 #include <ngraph/op/fused/gelu.hpp>
28 #if !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(_M_ARM64)
29 #if defined(_WIN32) || defined(WIN32)
38 using namespace MKLDNNPlugin;
39 using namespace InferenceEngine;
43 addDefaultExtensions(extensionManager);
47 ExecutorManager::getInstance()->clear("CPUStreamsExecutor");
48 ExecutorManager::getInstance()->clear("CPUCallbackExecutor");
51 InferenceEngine::ExecutableNetworkInternal::Ptr
52 Engine::LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network, const std::map<std::string, std::string> &config) {
53 // verification of supported input
54 InferenceEngine::InputsDataMap _networkInputs;
55 network.getInputsInfo(_networkInputs);
56 for (const auto &ii : _networkInputs) {
57 auto input_precision = ii.second->getPrecision();
58 if (input_precision != InferenceEngine::Precision::FP32 &&
59 input_precision != InferenceEngine::Precision::I32 &&
60 input_precision != InferenceEngine::Precision::U16 &&
61 input_precision != InferenceEngine::Precision::I16 &&
62 input_precision != InferenceEngine::Precision::I8 &&
63 input_precision != InferenceEngine::Precision::U8 &&
64 input_precision != InferenceEngine::Precision::BOOL) {
65 THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str
66 << "Input image format " << input_precision << " is not supported yet...";
70 // TODO: handle input precision differently - per input and not one per network...
72 // TODO: Clarify the behavior of SetConfig method. Skip eng_config or not?
73 Config conf = engConfig;
74 conf.readProperties(config);
76 if (conf.enableDynamicBatch) {
77 conf.batchLimit = static_cast<int>(network.getBatchSize());
80 std::shared_ptr<ICNNNetwork> clonedNetwork = cloneNetwork(network);
82 if (clonedNetwork->getFunction()) {
83 const auto transformations_callback = [](const std::shared_ptr<const ::ngraph::Node> &node) -> bool {
84 return std::dynamic_pointer_cast<const ::ngraph::opset2::Gelu>(node) ||
85 std::dynamic_pointer_cast<const ::ngraph::opset2::BatchToSpace>(node) ||
86 std::dynamic_pointer_cast<const ::ngraph::opset2::SpaceToBatch>(node);
88 auto nGraphFunc = clonedNetwork->getFunction();
89 // Disable shape inference (WA for generic operations)
90 ::ngraph::op::GenericIE::DisableReshape noReshape(nGraphFunc);
92 // Note: instead of running all Conversion Transformations you can make up your own transformation pipeline
93 ngraph::pass::CommonOptimizations().run_on_function(nGraphFunc);
94 ngraph::pass::ConvertOpSet3ToOpSet2(transformations_callback).run_on_function(nGraphFunc);
95 ngraph::pass::ConvertOpSet2ToOpSet1(transformations_callback).run_on_function(nGraphFunc);
96 ngraph::pass::ConvertOpSet1ToLegacy(transformations_callback).run_on_function(nGraphFunc);
97 clonedNetwork = InferenceEngine::details::convertFunctionToICNNNetwork(nGraphFunc, *clonedNetwork);
100 auto implNetwork = std::dynamic_pointer_cast<details::CNNNetworkImpl>(clonedNetwork);
102 // valid for CNNNetworkImpl only, while there's no API in ICNNNetwork to change network
103 ConstTransformer transformator(implNetwork.get());
104 transformator.fullTrim();
107 return std::make_shared<MKLDNNExecNetwork>(*clonedNetwork, conf, extensionManager, weightsSharing);
110 void Engine::SetConfig(const std::map<std::string, std::string> &config) {
111 // accumulate config parameters on engine level
112 engConfig.readProperties(config);
115 Parameter Engine::GetConfig(const std::string& name, const std::map<std::string, Parameter>& /*options*/) const {
117 auto option = engConfig._config.find(name);
118 if (option != engConfig._config.end()) {
119 result = option->second;
121 THROW_IE_EXCEPTION << "Unsupported config key " << name;
126 static bool hasAVX512() {
127 #if !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(_M_ARM64)
128 unsigned int regs[4] = {7, 0, 0, 0};
129 #if defined(_WIN32) || defined(WIN32)
130 __cpuid(reinterpret_cast<int*>(regs), regs[0]);
132 __cpuid_count(regs[0], regs[1], regs[0], regs[1], regs[2], regs[3]);
134 if (regs[1] & (1U << 16))
140 Parameter Engine::GetMetric(const std::string& name, const std::map<std::string, Parameter>& /*options*/) const {
141 if (name == METRIC_KEY(SUPPORTED_METRICS)) {
142 std::vector<std::string> metrics;
143 metrics.push_back(METRIC_KEY(AVAILABLE_DEVICES));
144 metrics.push_back(METRIC_KEY(SUPPORTED_METRICS));
145 metrics.push_back(METRIC_KEY(FULL_DEVICE_NAME));
146 metrics.push_back(METRIC_KEY(OPTIMIZATION_CAPABILITIES));
147 metrics.push_back(METRIC_KEY(SUPPORTED_CONFIG_KEYS));
148 metrics.push_back(METRIC_KEY(RANGE_FOR_ASYNC_INFER_REQUESTS));
149 metrics.push_back(METRIC_KEY(RANGE_FOR_STREAMS));
150 IE_SET_METRIC_RETURN(SUPPORTED_METRICS, metrics);
151 } else if (name == METRIC_KEY(FULL_DEVICE_NAME)) {
152 std::string brand_string;
153 #if !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(_M_ARM64)
154 unsigned int addr_list[3] = { 0x80000002, 0x80000003, 0x80000004 };
155 unsigned int regs[4];
156 for (auto addr : addr_list) {
158 #if defined(_WIN32) || defined(WIN32)
159 __cpuid(reinterpret_cast<int*>(regs), regs[0]);
161 __get_cpuid(regs[0], ®s[0], ®s[1], ®s[2], ®s[3]);
163 char *ch = reinterpret_cast<char*>(®s[0]);
164 for (size_t j = 0; j < sizeof(regs); j++)
165 brand_string += ch[j];
168 brand_string = "Non Intel Architecture";
170 IE_SET_METRIC_RETURN(FULL_DEVICE_NAME, brand_string);
171 } else if (name == METRIC_KEY(AVAILABLE_DEVICES)) {
172 std::vector<std::string> availableDevices = { "" };
173 IE_SET_METRIC_RETURN(AVAILABLE_DEVICES, availableDevices);
174 } else if (name == METRIC_KEY(OPTIMIZATION_CAPABILITIES)) {
175 std::vector<std::string> capabilities;
177 capabilities.push_back(METRIC_VALUE(WINOGRAD));
178 capabilities.push_back(METRIC_VALUE(FP32));
179 capabilities.push_back(METRIC_VALUE(FP16));
180 capabilities.push_back(METRIC_VALUE(INT8));
181 capabilities.push_back(METRIC_VALUE(BIN));
182 IE_SET_METRIC_RETURN(OPTIMIZATION_CAPABILITIES, capabilities);
183 } else if (name == METRIC_KEY(SUPPORTED_CONFIG_KEYS)) {
184 std::vector<std::string> configKeys;
185 for (auto && opt : engConfig._config)
186 configKeys.push_back(opt.first);
187 IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, configKeys);
188 } else if (name == METRIC_KEY(RANGE_FOR_ASYNC_INFER_REQUESTS)) {
189 std::tuple<unsigned int, unsigned int, unsigned int> range = std::make_tuple(1, 1, 1);
190 IE_SET_METRIC_RETURN(RANGE_FOR_ASYNC_INFER_REQUESTS, range);
191 } else if (name == METRIC_KEY(RANGE_FOR_STREAMS)) {
192 std::tuple<unsigned int, unsigned int> range = std::make_tuple(1, parallel_get_max_threads());
193 IE_SET_METRIC_RETURN(RANGE_FOR_STREAMS, range);
195 THROW_IE_EXCEPTION << "Unsupported metric key " << name;
199 void Engine::AddExtension(InferenceEngine::IExtensionPtr extension) {
200 extensionManager->AddExtension(extension);
203 void Engine::QueryNetwork(const ICNNNetwork& network, const std::map<std::string, std::string>& config, QueryNetworkResult& res) const {
204 details::CNNNetworkIterator i(&network);
205 while (i != details::CNNNetworkIterator()) {
207 mkldnn::engine eng(mkldnn::engine(mkldnn::engine::kind::cpu, 0));
208 MKLDNNWeightsSharing::Ptr fake_w_cache;
210 // if we can create and have not thrown exception, then layer is supported
211 std::unique_ptr <MKLDNNNode>(MKLDNNNode::CreateNode(*i, eng, extensionManager, fake_w_cache));
212 res.supportedLayersMap.insert({ (*i)->name, GetName() });
213 } catch (InferenceEngine::details::InferenceEngineException&) {
219 IE_SUPPRESS_DEPRECATED_START
221 INFERENCE_PLUGIN_API(StatusCode) CreatePluginEngine(IInferencePlugin*& plugin, ResponseDesc *resp) noexcept {
223 plugin = make_ie_compatible_plugin(
226 "MKLDNNPlugin"}, std::make_shared<Engine>());
229 catch (std::exception &ex) {
230 return DescriptionBuffer(GENERAL_ERROR, resp) << ex.what();
234 IE_SUPPRESS_DEPRECATED_END