1 // Copyright (C) 2018-2019 Intel Corporation
2 // SPDX-License-Identifier: Apache-2.0
14 #include "ie_plugin.hpp"
15 #include "cpp/ie_cnn_network.h"
16 #include "debug_options.h"
17 #include "inference_engine.hpp"
18 #include <CPP/network.hpp>
19 #include <CPP/memory.hpp>
20 #include <CPP/primitive.hpp>
21 #include <CPP/topology.hpp>
22 #include <CPP/pooling.hpp>
23 #include <CPP/eltwise.hpp>
24 #include <CPP/concatenation.hpp>
25 #include <CPP/detection_output.hpp>
26 #include <CPP/softmax.hpp>
27 #include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
28 #include <CPP/upsampling.hpp>
29 #include "cldnn_custom_layer.h"
31 namespace CLDNNPlugin {
34 InferenceEngine::InferenceEngineProfileInfo::LayerStatus status;
36 uint64_t realTime_uSec;
39 std::string layerType;
42 PerfCounter() : realTime_uSec(0), cpu_uSec(0), num(0),
43 status(InferenceEngine::InferenceEngineProfileInfo::NOT_RUN), isCPU(false) {}
45 long long realTime_avg() const { return (num == 0) ? 0 : realTime_uSec / num; }
46 long long cpu_avg() const { return (num == 0) ? 0 : cpu_uSec / num; }
50 std::shared_ptr<const cldnn::engine> engine;
51 std::shared_ptr<cldnn::network> network;
52 std::map<std::string, cldnn::primitive_id> primitiveIDs;
53 std::map<std::string, std::vector<cldnn::primitive_id>> prevPrimitiveIDs;
55 std::map<cldnn::primitive_id, std::pair<std::string, PerfCounter>> perfMap;
56 std::vector<cldnn::primitive_id> profilingIDs;
58 DebugOptions debugOptions;
60 std::map<std::string, InferenceEngine::SizeVector> outputDims;
61 std::map<std::string, cldnn::layout> inputLayouts;
63 std::vector<std::shared_ptr<cldnn::network>> batchNetworks;
68 class CLDNNGraph : public InferenceEngine::ExecutableNetworkThreadSafeDefault {
70 typedef std::shared_ptr<CLDNNGraph> Ptr;
72 Config() : useProfiling(false), dumpCustomKernels(false), exclusiveAsyncRequests(false),
74 enableDynamicBatch(false),
75 queuePriority(cldnn::priority_mode_types::disabled),
76 queueThrottle(cldnn::throttle_mode_types::disabled) {}
78 void LoadFromMap(const std::map<std::string, std::string>& configMap);
80 bool enableDynamicBatch;
82 bool dumpCustomKernels;
83 bool exclusiveAsyncRequests;
85 cldnn::priority_mode_types queuePriority;
86 cldnn::throttle_mode_types queueThrottle;
87 CLDNNCustomLayerMap customLayers;
88 cldnn::tuning_config_options tuningConfig;
89 std::string graph_dumps_dir;
90 std::string sources_dumps_dir;
92 explicit CLDNNGraph(InferenceEngine::ICNNNetwork &network, const Config& config = {}, int max_batch = -1);
94 InferenceEngine::InferRequestInternal::Ptr
95 CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override;
97 static bool IsLayerSupported(const std::string &type) {
98 return LayerTypeFromStr(type) != NO_TYPE;
103 std::shared_ptr<cldnn::topology> m_topology;
107 InferenceEngine::InputsDataMap* p_currentInputs;
108 InferenceEngine::OutputsDataMap* p_currentOutputs;
110 static const cldnn::primitive_id m_preProcessTag;
111 static const cldnn::primitive_id m_weightsTag;
112 static const cldnn::primitive_id m_biasesTag;
113 static const cldnn::primitive_id m_meanValuesTag;
114 static const cldnn::primitive_id m_postProcessTag;
115 static const cldnn::primitive_id m_scalesTag;
116 static const cldnn::primitive_id m_workaroundTag;
117 static const cldnn::primitive_id m_preCustomLayerTag;
118 static const cldnn::primitive_id m_postCustomLayerTag;
176 enum WeightRearrangeType {
182 cldnn::format m_defaultFormat;
183 void InitFormat(InferenceEngine::ICNNNetwork &network);
185 static cldnn::data_types DataTypeFromPrecision(InferenceEngine::Precision p);
186 static cldnn::format FormatFromLayout(InferenceEngine::Layout l);
187 static cldnn::upsampling_sample_type UpsamplingTypeFromString(const std::string& str);
189 void Load(InferenceEngine::ICNNNetwork &network);
190 static LayerType LayerTypeFromStr(const std::string& str);
191 static cldnn::pooling_mode PoolingModeFromIEPooling(InferenceEngine::PoolingLayer::PoolType pt, bool excludePadding = false);
192 static cldnn::eltwise_mode EltwiseModeFromIEEltwise(InferenceEngine::EltwiseLayer::eOperation op);
193 static cldnn::concatenation::concatenation_axis ConcatAxisFromIEAxis(unsigned axis);
194 static cldnn::prior_box_code_type PriorBoxCodeFromString(const std::string& str);
195 static cldnn::softmax::dimension_t SoftmaxDimensionFromIEAxis(const InferenceEngine::SoftMaxLayer* softmaxLayer, bool isPrevFC = false);
196 void CreatePrimitiveFromBlob(cldnn::primitive_id primID,
197 const InferenceEngine::Blob::Ptr pBlob,
198 cldnn::layout blobLayout,
199 size_t blobByteOffset = 0,
200 WeightRearrangeType rearrange = NO_REARRANGE);
201 void CreateWeightAndBiasPrimitives(const InferenceEngine::CNNLayerPtr& layer,
202 std::vector<cldnn::primitive_id>& weightsPrimID,
203 std::vector<cldnn::primitive_id>& biasesPrimID);
204 void CreateScaleWeightsAndBiasesFromBN(const InferenceEngine::BatchNormalizationLayer* bnLayer,
205 cldnn::primitive_id weightsPrimID,
206 cldnn::primitive_id biasesPrimID);
207 void AddPreProcessPrimitive(InferenceEngine::InputInfo::Ptr inputInfo);
208 void AddInputPrimitive(InferenceEngine::InputInfo::Ptr inputInfo, InferenceEngine::Precision inputPrecision);
209 void AddOutputPrimitive(std::string outputName, const InferenceEngine::DataPtr outputData,
210 InferenceEngine::Precision outputPrecision = InferenceEngine::Precision::UNSPECIFIED);
211 void CreateSingleLayerPrimitive(InferenceEngine::CNNLayerPtr& layer);
212 bool IsValidSplitConvMerge(const InferenceEngine::SplitLayer* splitLayer) const;
213 bool CanProcessDynBatch(InferenceEngine::ICNNNetwork &network) const;
214 static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::DataPtr data);
215 static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::CNNLayerPtr layer);
216 static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::DataPtr data);
217 static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::CNNLayerPtr layer);
218 std::vector<cldnn::primitive_id> GetPrevLayersPrimitives(const InferenceEngine::CNNLayerPtr layer) const;
219 void AddSingleValuePrimitive(cldnn::primitive_id valPrimID, cldnn::data_types dataType, float value);
221 void CreateGenericLayerBlobPrimitives(const InferenceEngine::GenericLayer* layer);
222 static void ValidateGenericLayerBlobs(const InferenceEngine::GenericLayer* layer, const std::vector<std::string>& blobNames);
223 static cldnn::tensor CldnnTensorFromIEDims(const InferenceEngine::SizeVector& dims);
224 static bool HasParam(const std::map<std::string, std::string>& layerParams, std::string paramName) {
225 auto p = layerParams.find(paramName);
226 return p != layerParams.end();
229 void InitProfileInfo(const std::string& layerName,
230 const std::string& layerType,
232 InferenceEngine::InferenceEngineProfileInfo::LayerStatus status
233 = InferenceEngine::InferenceEngineProfileInfo::EXECUTED);
234 void changeInputBatch(size_t batch);
235 void CompileNetwork();
237 // Layer Primitive Creators
238 void CreatePReLUPrimitive(InferenceEngine::CNNLayerPtr &layer);
239 void CreateBatchNormalizationPrimitive(InferenceEngine::CNNLayerPtr & layer);
240 void CreateFlattenPrimitive(InferenceEngine::CNNLayerPtr &layer);
241 void CreatePermutePrimitive(InferenceEngine::CNNLayerPtr &layer);
242 void CreateReshapePrimitive(InferenceEngine::CNNLayerPtr &layer);
243 void CreateNormalizePrimitive(InferenceEngine::CNNLayerPtr &layer);
244 void CreateDetectionOutputPrimitive(InferenceEngine::CNNLayerPtr &layer);
245 void CreatePriorBoxPrimitive(InferenceEngine::CNNLayerPtr &layer);
246 void CreateDeconvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
247 void CreateCropPrimitive(InferenceEngine::CNNLayerPtr &layer);
248 void CreateROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
249 void CreateSimplerNMSPrimitive(InferenceEngine::CNNLayerPtr &layer);
250 void CreateEltwisePrimitive(InferenceEngine::CNNLayerPtr &layer);
251 void CreateConcatenatePrimitive(InferenceEngine::CNNLayerPtr &layer);
252 void CreateSplitPrimitive(InferenceEngine::CNNLayerPtr &layer);
253 void CreateFusedSplitConvMergePrimitive(InferenceEngine::CNNLayerPtr &layer);
254 void CreatePowerPrimitive(InferenceEngine::CNNLayerPtr &layer);
255 void CreateSoftMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
256 void CreateFullyConnectedPrimitive(InferenceEngine::CNNLayerPtr &layer);
257 void CreatePoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
258 void CreateLRNPrimitive(InferenceEngine::CNNLayerPtr &layer);
259 void CreateActivationPrimitive(InferenceEngine::CNNLayerPtr &layer, const LayerType type);
260 void CreateConvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
261 void CreateScaleShiftPrimitive(InferenceEngine::CNNLayerPtr &layer);
262 void CreateProposalPrimitive(InferenceEngine::CNNLayerPtr &layer);
263 void CreatePSROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
264 void CreateCopyPrimitive(InferenceEngine::CNNLayerPtr &layer);
265 void CreateUpsamplingPrimitive(InferenceEngine::CNNLayerPtr &layer);
266 void CreateResamplePrimitive(InferenceEngine::CNNLayerPtr &layer);
267 void CreateYOLO2RegionPrimitive(InferenceEngine::CNNLayerPtr &layer);
268 void CreateYOLO2ReorgPrimitive(InferenceEngine::CNNLayerPtr &layer);
269 void CreateArgMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
270 void CreateMaxUnpoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
271 void CreateMVNPrimitive(InferenceEngine::CNNLayerPtr &layer);
272 void CreateTilePrimitive(InferenceEngine::CNNLayerPtr &layer);
273 void CreatePadPrimitive(InferenceEngine::CNNLayerPtr &layer);
274 void CreateRNNPrimitive(InferenceEngine::CNNLayerPtr &layer);
275 void CreateLSTMCellPrimitive(InferenceEngine::CNNLayerPtr &layer);
276 void AddConstantBlobInput(InferenceEngine::CNNLayerPtr &layer);
277 void CreateCustomLayerPrimitive(InferenceEngine::CNNLayerPtr &layer, CLDNNCustomLayerPtr customLayer);
278 void CreateGatherPrimitive(InferenceEngine::CNNLayerPtr &layer);
279 void CreateDepthToSpacePrimitive(InferenceEngine::CNNLayerPtr &layer);
280 void CreateShuffleChannelsPrimitive(InferenceEngine::CNNLayerPtr &layer);
281 void CreateStridedSlicePrimitive(InferenceEngine::CNNLayerPtr &layer);
282 void CreateReverseSequencePrimitive(InferenceEngine::CNNLayerPtr &layer);
285 }; // namespace CLDNNPlugin