inference-engine/src/mkldnn_plugin/mkldnn_plugin.cpp

   1 // Copyright (C) 2018-2020 Intel Corporation
   2 // SPDX-License-Identifier: Apache-2.0
   3 //
   4
   5 #include "ie_metric_helpers.hpp"
   6 #include "mkldnn_plugin.h"
   7 #include "mkldnn_extension_mngr.h"
   8 #include "mkldnn_layers_dispatcher.hpp"
   9 #include "mkldnn_weights_cache.hpp"
  10 #include <cpp_interfaces/base/ie_plugin_base.hpp>
  11 #include <threading/ie_executor_manager.hpp>
  12 #include <memory>
  13 #include <ie_plugin_config.hpp>
  14 #include <vector>
  15 #include <tuple>
  16 #include <ie_system_conf.h>
  17 #include <generic_ie.hpp>
  18
  19 #include "convert_function_to_cnn_network.hpp"
  20 #include <transformations/common_optimizations/common_optimizations.hpp>
  21 #include <transformations/convert_opset1_to_legacy/convert_opset1_to_legacy.hpp>
  22 #include <transformations/convert_opset2_to_opset1/convert_opset2_to_opset1.hpp>
  23 #include <transformations/convert_opset3_to_opset2/convert_opset3_to_opset2.hpp>
  24 #include <ngraph/opsets/opset1.hpp>
  25 #include <ngraph/opsets/opset2.hpp>
  26 #include <ngraph/op/fused/gelu.hpp>
  27
  28 #if !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(_M_ARM64)
  29 #if defined(_WIN32) || defined(WIN32)
  30 #include <intrin.h>
  31 #include <windows.h>
  32 #else
  33 #include <cpuid.h>
  34
  35 #endif
  36 #endif
  37
  38 using namespace MKLDNNPlugin;
  39 using namespace InferenceEngine;
  40
  41 Engine::Engine() {
  42     _pluginName = "CPU";
  43     addDefaultExtensions(extensionManager);
  44 }
  45
  46 Engine::~Engine() {
  47     ExecutorManager::getInstance()->clear("CPUStreamsExecutor");
  48     ExecutorManager::getInstance()->clear("CPUCallbackExecutor");
  49 }
  50
  51 InferenceEngine::ExecutableNetworkInternal::Ptr
  52 Engine::LoadExeNetworkImpl(const InferenceEngine::ICNNNetwork &network, const std::map<std::string, std::string> &config) {
  53     // verification of supported input
  54     InferenceEngine::InputsDataMap _networkInputs;
  55     network.getInputsInfo(_networkInputs);
  56     for (const auto &ii : _networkInputs) {
  57         auto input_precision = ii.second->getPrecision();
  58         if (input_precision != InferenceEngine::Precision::FP32 &&
  59             input_precision != InferenceEngine::Precision::I32 &&
  60             input_precision != InferenceEngine::Precision::U16 &&
  61             input_precision != InferenceEngine::Precision::I16 &&
  62             input_precision != InferenceEngine::Precision::I8 &&
  63             input_precision != InferenceEngine::Precision::U8 &&
  64             input_precision != InferenceEngine::Precision::BOOL) {
  65             THROW_IE_EXCEPTION << NOT_IMPLEMENTED_str
  66                                << "Input image format " << input_precision << " is not supported yet...";
  67         }
  68     }
  69
  70     // TODO: handle input precision differently - per input and not one per network...
  71
  72     // TODO: Clarify the behavior of SetConfig method. Skip eng_config or not?
  73     Config conf = engConfig;
  74     conf.readProperties(config);
  75
  76     if (conf.enableDynamicBatch) {
  77         conf.batchLimit = static_cast<int>(network.getBatchSize());
  78     }
  79
  80     std::shared_ptr<ICNNNetwork> clonedNetwork = cloneNetwork(network);
  81
  82     if (clonedNetwork->getFunction()) {
  83         const auto transformations_callback = [](const std::shared_ptr<const ::ngraph::Node> &node) -> bool {
  84             return std::dynamic_pointer_cast<const ::ngraph::opset2::Gelu>(node) ||
  85                 std::dynamic_pointer_cast<const ::ngraph::opset2::BatchToSpace>(node) ||
  86                 std::dynamic_pointer_cast<const ::ngraph::opset2::SpaceToBatch>(node);
  87         };
  88         auto nGraphFunc = clonedNetwork->getFunction();
  89         // Disable shape inference (WA for generic operations)
  90         ::ngraph::op::GenericIE::DisableReshape noReshape(nGraphFunc);
  91
  92         // Note: instead of running all Conversion Transformations you can make up your own transformation pipeline
  93         ngraph::pass::CommonOptimizations().run_on_function(nGraphFunc);
  94         ngraph::pass::ConvertOpSet3ToOpSet2(transformations_callback).run_on_function(nGraphFunc);
  95         ngraph::pass::ConvertOpSet2ToOpSet1(transformations_callback).run_on_function(nGraphFunc);
  96         ngraph::pass::ConvertOpSet1ToLegacy(transformations_callback).run_on_function(nGraphFunc);
  97         clonedNetwork = InferenceEngine::details::convertFunctionToICNNNetwork(nGraphFunc, *clonedNetwork);
  98     }
  99
 100     auto implNetwork = std::dynamic_pointer_cast<details::CNNNetworkImpl>(clonedNetwork);
 101     if (implNetwork) {
 102         // valid for CNNNetworkImpl only, while there's no API in ICNNNetwork to change network
 103         ConstTransformer transformator(implNetwork.get());
 104         transformator.fullTrim();
 105     }
 106
 107     return std::make_shared<MKLDNNExecNetwork>(*clonedNetwork, conf, extensionManager, weightsSharing);
 108 }
 109
 110 void Engine::SetConfig(const std::map<std::string, std::string> &config) {
 111     // accumulate config parameters on engine level
 112     engConfig.readProperties(config);
 113 }
 114
 115 Parameter Engine::GetConfig(const std::string& name, const std::map<std::string, Parameter>& /*options*/) const {
 116     Parameter result;
 117     auto option = engConfig._config.find(name);
 118     if (option != engConfig._config.end()) {
 119         result = option->second;
 120     } else {
 121         THROW_IE_EXCEPTION << "Unsupported config key " << name;
 122     }
 123     return result;
 124 }
 125
 126 static bool hasAVX512() {
 127 #if !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(_M_ARM64)
 128     unsigned int regs[4] = {7, 0, 0, 0};
 129 #if defined(_WIN32) || defined(WIN32)
 130     __cpuid(reinterpret_cast<int*>(regs), regs[0]);
 131 #else
 132     __cpuid_count(regs[0], regs[1], regs[0], regs[1], regs[2], regs[3]);
 133 #endif
 134     if (regs[1] & (1U << 16))
 135         return true;
 136 #endif
 137     return false;
 138 }
 139
 140 Parameter Engine::GetMetric(const std::string& name, const std::map<std::string, Parameter>& /*options*/) const {
 141     if (name == METRIC_KEY(SUPPORTED_METRICS)) {
 142         std::vector<std::string> metrics;
 143         metrics.push_back(METRIC_KEY(AVAILABLE_DEVICES));
 144         metrics.push_back(METRIC_KEY(SUPPORTED_METRICS));
 145         metrics.push_back(METRIC_KEY(FULL_DEVICE_NAME));
 146         metrics.push_back(METRIC_KEY(OPTIMIZATION_CAPABILITIES));
 147         metrics.push_back(METRIC_KEY(SUPPORTED_CONFIG_KEYS));
 148         metrics.push_back(METRIC_KEY(RANGE_FOR_ASYNC_INFER_REQUESTS));
 149         metrics.push_back(METRIC_KEY(RANGE_FOR_STREAMS));
 150         IE_SET_METRIC_RETURN(SUPPORTED_METRICS, metrics);
 151     } else if (name == METRIC_KEY(FULL_DEVICE_NAME)) {
 152         std::string brand_string;
 153 #if !defined(__arm__) && !defined(_M_ARM) && !defined(__aarch64__) && !defined(_M_ARM64)
 154         unsigned int addr_list[3] = { 0x80000002, 0x80000003, 0x80000004 };
 155         unsigned int regs[4];
 156         for (auto addr : addr_list) {
 157             regs[0] = addr;
 158 #if defined(_WIN32) || defined(WIN32)
 159             __cpuid(reinterpret_cast<int*>(regs), regs[0]);
 160 #else
 161             __get_cpuid(regs[0], &regs[0], &regs[1], &regs[2], &regs[3]);
 162 #endif
 163             char *ch = reinterpret_cast<char*>(&regs[0]);
 164             for (size_t j = 0; j < sizeof(regs); j++)
 165                 brand_string += ch[j];
 166         }
 167 #else
 168         brand_string = "Non Intel Architecture";
 169 #endif
 170         IE_SET_METRIC_RETURN(FULL_DEVICE_NAME, brand_string);
 171     } else if (name == METRIC_KEY(AVAILABLE_DEVICES)) {
 172         std::vector<std::string> availableDevices = { "" };
 173         IE_SET_METRIC_RETURN(AVAILABLE_DEVICES, availableDevices);
 174     } else if (name == METRIC_KEY(OPTIMIZATION_CAPABILITIES)) {
 175         std::vector<std::string> capabilities;
 176         if (hasAVX512())
 177             capabilities.push_back(METRIC_VALUE(WINOGRAD));
 178         capabilities.push_back(METRIC_VALUE(FP32));
 179         capabilities.push_back(METRIC_VALUE(FP16));
 180         capabilities.push_back(METRIC_VALUE(INT8));
 181         capabilities.push_back(METRIC_VALUE(BIN));
 182         IE_SET_METRIC_RETURN(OPTIMIZATION_CAPABILITIES, capabilities);
 183     } else if (name == METRIC_KEY(SUPPORTED_CONFIG_KEYS)) {
 184         std::vector<std::string> configKeys;
 185         for (auto && opt : engConfig._config)
 186             configKeys.push_back(opt.first);
 187         IE_SET_METRIC_RETURN(SUPPORTED_CONFIG_KEYS, configKeys);
 188     } else if (name == METRIC_KEY(RANGE_FOR_ASYNC_INFER_REQUESTS)) {
 189         std::tuple<unsigned int, unsigned int, unsigned int> range = std::make_tuple(1, 1, 1);
 190         IE_SET_METRIC_RETURN(RANGE_FOR_ASYNC_INFER_REQUESTS, range);
 191     } else if (name == METRIC_KEY(RANGE_FOR_STREAMS)) {
 192         std::tuple<unsigned int, unsigned int> range = std::make_tuple(1, parallel_get_max_threads());
 193         IE_SET_METRIC_RETURN(RANGE_FOR_STREAMS, range);
 194     } else {
 195         THROW_IE_EXCEPTION << "Unsupported metric key " << name;
 196     }
 197 }
 198
 199 void Engine::AddExtension(InferenceEngine::IExtensionPtr extension) {
 200     extensionManager->AddExtension(extension);
 201 }
 202
 203 void Engine::QueryNetwork(const ICNNNetwork& network, const std::map<std::string, std::string>& config, QueryNetworkResult& res) const {
 204     details::CNNNetworkIterator i(&network);
 205     while (i != details::CNNNetworkIterator()) {
 206         try {
 207             mkldnn::engine eng(mkldnn::engine(mkldnn::engine::kind::cpu, 0));
 208             MKLDNNWeightsSharing::Ptr fake_w_cache;
 209
 210             // if we can create and have not thrown exception, then layer is supported
 211             std::unique_ptr <MKLDNNNode>(MKLDNNNode::CreateNode(*i, eng, extensionManager, fake_w_cache));
 212             res.supportedLayersMap.insert({ (*i)->name, GetName() });
 213         } catch (InferenceEngine::details::InferenceEngineException&) {
 214         }
 215         i++;
 216     }
 217 }
 218
 219 IE_SUPPRESS_DEPRECATED_START
 220
 221 INFERENCE_PLUGIN_API(StatusCode) CreatePluginEngine(IInferencePlugin*& plugin, ResponseDesc *resp) noexcept {
 222     try {
 223         plugin = make_ie_compatible_plugin(
 224                 {{2, 1},
 225                  CI_BUILD_NUMBER,
 226                  "MKLDNNPlugin"}, std::make_shared<Engine>());
 227         return OK;
 228     }
 229     catch (std::exception &ex) {
 230         return DescriptionBuffer(GENERAL_ERROR, resp) << ex.what();
 231     }
 232 }
 233
 234 IE_SUPPRESS_DEPRECATED_END