1 // Copyright (C) 2018 Intel Corporation
3 // SPDX-License-Identifier: Apache-2.0
14 #include "ie_plugin.hpp"
15 #include "cpp/ie_cnn_network.h"
16 #include "debug_options.h"
17 #include "inference_engine.hpp"
18 #include <CPP/network.hpp>
19 #include <CPP/memory.hpp>
20 #include <CPP/primitive.hpp>
21 #include <CPP/topology.hpp>
22 #include <CPP/pooling.hpp>
23 #include <CPP/eltwise.hpp>
24 #include <CPP/concatenation.hpp>
25 #include <CPP/detection_output.hpp>
26 #include <CPP/softmax.hpp>
27 #include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
28 #include <CPP/upsampling.hpp>
29 #include "cldnn_custom_layer.h"
31 namespace CLDNNPlugin {
34 std::shared_ptr<const cldnn::engine> engine;
35 std::shared_ptr<cldnn::network> network;
36 std::map<std::string, cldnn::primitive_id> primitiveIDs;
37 std::map<std::string, std::vector<cldnn::primitive_id>> prevPrimitiveIDs;
38 std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> perfMap;
39 std::set<cldnn::primitive_id> profilingIDs;
41 DebugOptions debugOptions;
43 std::map<std::string, InferenceEngine::SizeVector> outputDims;
44 std::map<std::string, cldnn::layout> inputLayouts;
46 std::vector<std::shared_ptr<cldnn::network>> batchNetworks;
51 class CLDNNGraph : public InferenceEngine::ExecutableNetworkThreadSafeDefault {
53 typedef std::shared_ptr<CLDNNGraph> Ptr;
55 Config() : useProfiling(false), dumpCustomKernels(false), exclusiveAsyncRequests(false),
57 enableDynamicBatch(false),
58 queuePriority(cldnn::priority_mode_types::disabled),
59 queueThrottle(cldnn::throttle_mode_types::disabled) {}
61 void LoadFromMap(const std::map<std::string, std::string>& configMap);
63 bool enableDynamicBatch;
65 bool dumpCustomKernels;
66 bool exclusiveAsyncRequests;
68 cldnn::priority_mode_types queuePriority;
69 cldnn::throttle_mode_types queueThrottle;
70 CLDNNCustomLayerMap customLayers;
71 cldnn::tuning_config_options tuningConfig;
72 std::string graph_dumps_dir;
73 std::string sources_dumps_dir;
75 explicit CLDNNGraph(InferenceEngine::ICNNNetwork &network, const Config& config = {}, int max_batch = -1);
77 InferenceEngine::InferRequestInternal::Ptr
78 CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override;
80 static bool IsLayerSupported(const std::string &type) {
81 return LayerTypeFromStr(type) != NO_TYPE;
86 std::shared_ptr<cldnn::topology> m_topology;
90 InferenceEngine::InputsDataMap* p_currentInputs;
91 InferenceEngine::OutputsDataMap* p_currentOutputs;
93 static const cldnn::primitive_id m_preProcessTag;
94 static const cldnn::primitive_id m_weightsTag;
95 static const cldnn::primitive_id m_biasesTag;
96 static const cldnn::primitive_id m_meanValuesTag;
97 static const cldnn::primitive_id m_postProcessTag;
98 static const cldnn::primitive_id m_scalesTag;
99 static const cldnn::primitive_id m_workaroundTag;
100 static const cldnn::primitive_id m_preCustomLayerTag;
101 static const cldnn::primitive_id m_postCustomLayerTag;
150 enum WeightRearrangeType {
156 cldnn::format m_defaultFormat;
157 cldnn::data_types m_networkPrecision;
158 void InitFormat(InferenceEngine::ICNNNetwork &network);
160 static cldnn::data_types DataTypeFromPrecision(InferenceEngine::Precision p);
161 static cldnn::format FormatFromLayout(InferenceEngine::Layout l);
162 static cldnn::upsampling_sample_type UpsamplingTypeFromString(const std::string& str);
164 void Load(InferenceEngine::ICNNNetwork &network);
165 static LayerType LayerTypeFromStr(const std::string& str);
166 static cldnn::pooling_mode PoolingModeFromIEPooling(InferenceEngine::PoolingLayer::PoolType pt, bool excludePadding = false);
167 static cldnn::eltwise_mode EltwiseModeFromIEEltwise(InferenceEngine::EltwiseLayer::eOperation op);
168 static cldnn::concatenation::concatenation_axis ConcatAxisFromIEAxis(unsigned axis);
169 static cldnn::prior_box_code_type PriorBoxCodeFromString(const std::string& str);
170 static cldnn::softmax::dimension_t SoftmaxDimensionFromIEAxis(const InferenceEngine::SoftMaxLayer* softmaxLayer, bool isPrevFC = false);
171 void CreatePrimitiveFromBlob(cldnn::primitive_id primID,
172 const InferenceEngine::Blob::Ptr pBlob,
173 cldnn::layout blobLayout,
174 size_t blobByteOffset = 0,
175 WeightRearrangeType rearrange = NO_REARRANGE);
176 void CreateWeightAndBiasPrimitives(const InferenceEngine::CNNLayerPtr& layer,
177 std::vector<cldnn::primitive_id>& weightsPrimID,
178 std::vector<cldnn::primitive_id>& biasesPrimID);
179 void CreateScaleWeightsAndBiasesFromBN(const InferenceEngine::BatchNormalizationLayer* bnLayer,
180 cldnn::primitive_id weightsPrimID,
181 cldnn::primitive_id biasesPrimID);
182 void AddPreProcessPrimitive(InferenceEngine::InputInfo::Ptr inputInfo);
183 void AddInputPrimitive(InferenceEngine::InputInfo::Ptr inputInfo);
184 void AddOutputPrimitive(std::string outputName, const InferenceEngine::DataPtr outputData,
185 InferenceEngine::Precision outputPrecision = InferenceEngine::Precision::UNSPECIFIED);
186 void CreateSingleLayerPrimitive(InferenceEngine::CNNLayerPtr& layer);
187 bool IsValidSplitConvMerge(const InferenceEngine::SplitLayer* splitLayer) const;
188 bool CanProcessDynBatch(InferenceEngine::ICNNNetwork &network) const;
189 static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::DataPtr data);
190 static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::CNNLayerPtr layer);
191 static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::DataPtr data);
192 static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::CNNLayerPtr layer);
193 std::vector<cldnn::primitive_id> GetPrevLayersPrimitives(const InferenceEngine::CNNLayerPtr layer) const;
194 void AddSingleValuePrimitive(cldnn::primitive_id valPrimID, cldnn::data_types dataType, float value);
196 void CreateGenericLayerBlobPrimitives(const InferenceEngine::GenericLayer* layer);
197 static void ValidateGenericLayerBlobs(const InferenceEngine::GenericLayer* layer, const std::vector<std::string>& blobNames);
198 static cldnn::tensor CldnnTensorFromIEDims(const InferenceEngine::SizeVector& dims);
199 static bool HasParam(const std::map<std::string, std::string>& layerParams, std::string paramName) {
200 auto p = layerParams.find(paramName);
201 return p != layerParams.end();
204 void InitProfileInfo(const std::string& layerName,
205 const std::string& layerType,
206 const std::string& execType,
207 InferenceEngine::InferenceEngineProfileInfo::LayerStatus status);
208 void changeInputBatch(size_t batch);
209 void CompileNetwork();
211 // Layer Primitive Creators
212 void CreatePReLUPrimitive(InferenceEngine::CNNLayerPtr &layer);
213 void CreateBatchNormalizationPrimitive(InferenceEngine::CNNLayerPtr & layer);
214 void CreateFlattenPrimitive(InferenceEngine::CNNLayerPtr &layer);
215 void CreatePermutePrimitive(InferenceEngine::CNNLayerPtr &layer);
216 void CreateReshapePrimitive(InferenceEngine::CNNLayerPtr &layer);
217 void CreateNormalizePrimitive(InferenceEngine::CNNLayerPtr &layer);
218 void CreateDetectionOutputPrimitive(InferenceEngine::CNNLayerPtr &layer);
219 void CreatePriorBoxPrimitive(InferenceEngine::CNNLayerPtr &layer);
220 void CreateDeconvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
221 void CreateCropPrimitive(InferenceEngine::CNNLayerPtr &layer);
222 void CreateROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
223 void CreateSimplerNMSPrimitive(InferenceEngine::CNNLayerPtr &layer);
224 void CreateEltwisePrimitive(InferenceEngine::CNNLayerPtr &layer);
225 void CreateConcatenatePrimitive(InferenceEngine::CNNLayerPtr &layer);
226 void CreateSplitPrimitive(InferenceEngine::CNNLayerPtr &layer);
227 void CreateFusedSplitConvMergePrimitive(InferenceEngine::CNNLayerPtr &layer);
228 void CreatePowerPrimitive(InferenceEngine::CNNLayerPtr &layer);
229 void CreateSoftMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
230 void CreateFullyConnectedPrimitive(InferenceEngine::CNNLayerPtr &layer);
231 void CreatePoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
232 void CreateLRNPrimitive(InferenceEngine::CNNLayerPtr &layer);
233 void CreateActivationPrimitive(InferenceEngine::CNNLayerPtr &layer, const LayerType type);
234 void CreateConvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
235 void CreateScaleShiftPrimitive(InferenceEngine::CNNLayerPtr &layer);
236 void CreateProposalPrimitive(InferenceEngine::CNNLayerPtr &layer);
237 void CreatePSROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
238 void CreateCopyPrimitive(InferenceEngine::CNNLayerPtr &layer);
239 void CreateUpsamplingPrimitive(InferenceEngine::CNNLayerPtr &layer);
240 void CreateResamplePrimitive(InferenceEngine::CNNLayerPtr &layer);
241 void CreateYOLO2RegionPrimitive(InferenceEngine::CNNLayerPtr &layer);
242 void CreateYOLO2ReorgPrimitive(InferenceEngine::CNNLayerPtr &layer);
243 void CreateArgMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
244 void CreateMaxUnpoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
245 void CreateMVNPrimitive(InferenceEngine::CNNLayerPtr &layer);
246 void CreateTilePrimitive(InferenceEngine::CNNLayerPtr &layer);
247 void CreateRNNPrimitive(InferenceEngine::CNNLayerPtr &layer);
248 void AddConstantBlobInput(InferenceEngine::CNNLayerPtr &layer);
249 void CreateCustomLayerPrimitive(InferenceEngine::CNNLayerPtr &layer, CLDNNCustomLayerPtr customLayer);
252 }; // namespace CLDNNPlugin