inference-engine/src/cldnn_engine/cldnn_graph.h

   1 // Copyright (C) 2018 Intel Corporation
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4 //
   5
   6 #pragma once
   7
   8 #include <vector>
   9 #include <map>
  10 #include <set>
  11 #include <memory>
  12 #include <string>
  13 #include "ie_blob.h"
  14 #include "ie_plugin.hpp"
  15 #include "cpp/ie_cnn_network.h"
  16 #include "debug_options.h"
  17 #include "inference_engine.hpp"
  18 #include <CPP/network.hpp>
  19 #include <CPP/memory.hpp>
  20 #include <CPP/primitive.hpp>
  21 #include <CPP/topology.hpp>
  22 #include <CPP/pooling.hpp>
  23 #include <CPP/eltwise.hpp>
  24 #include <CPP/concatenation.hpp>
  25 #include <CPP/detection_output.hpp>
  26 #include <CPP/softmax.hpp>
  27 #include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
  28 #include <CPP/upsampling.hpp>
  29 #include "cldnn_custom_layer.h"
  30
  31 namespace CLDNNPlugin {
  32
  33 struct InferenceEnv {
  34     std::shared_ptr<const cldnn::engine> engine;
  35     std::shared_ptr<cldnn::network> network;
  36     std::map<std::string, cldnn::primitive_id> primitiveIDs;
  37     std::map<std::string, std::vector<cldnn::primitive_id>> prevPrimitiveIDs;
  38     std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> perfMap;
  39     std::set<cldnn::primitive_id> profilingIDs;
  40
  41     DebugOptions debugOptions;
  42
  43     std::map<std::string, InferenceEngine::SizeVector> outputDims;
  44     std::map<std::string, cldnn::layout> inputLayouts;
  45     std::map<cldnn::primitive_id, cldnn::memory> constBlobs;
  46
  47     std::vector<std::shared_ptr<cldnn::network>> batchNetworks;
  48     int m_max_batch;
  49     int m_bv_sz;
  50 };
  51
  52 class CLDNNGraph : public InferenceEngine::ExecutableNetworkThreadSafeDefault {
  53 public:
  54     typedef std::shared_ptr<CLDNNGraph> Ptr;
  55     struct Config {
  56         Config() : useProfiling(false), dumpCustomKernels(false), exclusiveAsyncRequests(false),
  57             memory_pool_on(false),
  58             enableDynamicBatch(false),
  59             queuePriority(cldnn::priority_mode_types::disabled),
  60             queueThrottle(cldnn::throttle_mode_types::disabled) {}
  61
  62         void LoadFromMap(const std::map<std::string, std::string>& configMap);
  63
  64         bool enableDynamicBatch;
  65         bool useProfiling;
  66         bool dumpCustomKernels;
  67         bool exclusiveAsyncRequests;
  68         bool memory_pool_on;
  69         cldnn::priority_mode_types queuePriority;
  70         cldnn::throttle_mode_types queueThrottle;
  71         CLDNNCustomLayerMap customLayers;
  72         cldnn::tuning_config_options tuningConfig;
  73         std::string graph_dumps_dir;
  74         std::string sources_dumps_dir;
  75     };
  76     explicit CLDNNGraph(InferenceEngine::ICNNNetwork &network, const Config& config = {}, int max_batch = -1);
  77
  78     InferenceEngine::InferRequestInternal::Ptr
  79     CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override;
  80
  81     static bool IsLayerSupported(const std::string &type) {
  82         return LayerTypeFromStr(type) != NO_TYPE;
  83     }
  84
  85 protected:
  86     // graph members
  87     std::shared_ptr<cldnn::topology> m_topology;
  88     InferenceEnv m_env;
  89     Config m_config;
  90
  91     InferenceEngine::InputsDataMap*  p_currentInputs;
  92     InferenceEngine::OutputsDataMap* p_currentOutputs;
  93     int m_curBatch;
  94     static const cldnn::primitive_id m_preProcessTag;
  95     static const cldnn::primitive_id m_weightsTag;
  96     static const cldnn::primitive_id m_biasesTag;
  97     static const cldnn::primitive_id m_meanValuesTag;
  98     static const cldnn::primitive_id m_postProcessTag;
  99     static const cldnn::primitive_id m_scalesTag;
 100     static const cldnn::primitive_id m_workaroundTag;
 101     static const cldnn::primitive_id m_preCustomLayerTag;
 102     static const cldnn::primitive_id m_postCustomLayerTag;
 103
 104     // internal types
 105     enum LayerType {
 106         Convolution,
 107         ReLU,
 108         ReLU6,
 109         Sigmoid,
 110         TanH,
 111         ELU,
 112         Activation,
 113         LRN,
 114         Pooling,
 115         FullyConnected,
 116         SoftMax,
 117         Power,
 118         Split,
 119         Concatenate,
 120         Eltwise,
 121         SimplerNMS,
 122         ROIPooling,
 123         Crop,
 124         Deconvolution,
 125         PriorBox,
 126         DetectionOutput,
 127         Normalize,
 128         Reshape,
 129         Permute,
 130         Flatten,
 131         BatchNormalization,
 132         PReLU,
 133         ScaleShift,
 134         Proposal,
 135         PSROIPooling,
 136         Clamp,
 137         Copy,
 138         Upsampling,
 139         Resample,
 140         RegionYolo,
 141         ReorgYolo,
 142         ConstantBlob,
 143         ArgMax,
 144         MVN,
 145         Unpooling,
 146         NO_TYPE
 147     };
 148
 149     enum WeightRearrangeType {
 150         BroadcastFeatures,
 151         FlipDeconvDims,
 152         NO_REARRANGE
 153     };
 154
 155     cldnn::format m_defaultFormat;
 156     cldnn::data_types m_networkPrecision;
 157     void InitFormat(InferenceEngine::ICNNNetwork &network);
 158
 159     static cldnn::data_types DataTypeFromPrecision(InferenceEngine::Precision p);
 160     static cldnn::format     FormatFromLayout(InferenceEngine::Layout l);
 161     static cldnn::upsampling_sample_type UpsamplingTypeFromString(const std::string& str);
 162
 163     void Load(InferenceEngine::ICNNNetwork &network);
 164     static LayerType LayerTypeFromStr(const std::string& str);
 165     static cldnn::pooling_mode PoolingModeFromIEPooling(InferenceEngine::PoolingLayer::PoolType pt, bool excludePadding = false);
 166     static cldnn::eltwise_mode EltwiseModeFromIEEltwise(InferenceEngine::EltwiseLayer::eOperation op);
 167     static cldnn::concatenation::concatenation_axis ConcatAxisFromIEAxis(unsigned axis);
 168     static cldnn::prior_box_code_type PriorBoxCodeFromString(const std::string& str);
 169     static cldnn::softmax::dimension_t SoftmaxDimensionFromIEAxis(const InferenceEngine::SoftMaxLayer* softmaxLayer, bool isPrevFC = false);
 170     void CreatePrimitiveFromBlob(cldnn::primitive_id primID,
 171                                  const InferenceEngine::Blob::Ptr pBlob,
 172                                  cldnn::layout blobLayout,
 173                                  size_t blobByteOffset = 0,
 174                                  WeightRearrangeType rearrange = NO_REARRANGE);
 175     void CreateWeightAndBiasPrimitives(const InferenceEngine::CNNLayerPtr& layer,
 176                                        std::vector<cldnn::primitive_id>& weightsPrimID,
 177                                        std::vector<cldnn::primitive_id>& biasesPrimID);
 178     void CreateScaleWeightsAndBiasesFromBN(const InferenceEngine::BatchNormalizationLayer* bnLayer,
 179                                            cldnn::primitive_id weightsPrimID,
 180                                            cldnn::primitive_id biasesPrimID);
 181     void AddPreProcessPrimitive(InferenceEngine::InputInfo::Ptr inputInfo);
 182     void AddInputPrimitive(InferenceEngine::InputInfo::Ptr inputInfo);
 183     void AddOutputPrimitive(std::string outputName, const InferenceEngine::DataPtr outputData,
 184                             InferenceEngine::Precision outputPrecision = InferenceEngine::Precision::UNSPECIFIED);
 185     void CreateSingleLayerPrimitive(InferenceEngine::CNNLayerPtr& layer);
 186     bool IsValidSplitConvMerge(const InferenceEngine::SplitLayer* splitLayer) const;
 187     bool CanProcessDynBatch(InferenceEngine::ICNNNetwork &network) const;
 188     static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::DataPtr data);
 189     static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::CNNLayerPtr layer);
 190     static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::DataPtr data);
 191     static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::CNNLayerPtr layer);
 192     std::vector<cldnn::primitive_id> GetPrevLayersPrimitives(const InferenceEngine::CNNLayerPtr layer) const;
 193     void AddSingleValuePrimitive(cldnn::primitive_id valPrimID, cldnn::data_types dataType, float value);
 194
 195     void CreateGenericLayerBlobPrimitives(const InferenceEngine::GenericLayer* layer);
 196     static void ValidateGenericLayerBlobs(const InferenceEngine::GenericLayer* layer, const std::vector<std::string>& blobNames);
 197     static cldnn::tensor CldnnTensorFromIEDims(const InferenceEngine::SizeVector& dims);
 198     static bool HasParam(const std::map<std::string, std::string>& layerParams, std::string paramName) {
 199         auto p = layerParams.find(paramName);
 200         return p != layerParams.end();
 201     }
 202
 203     void InitProfileInfo(const std::string& layerName,
 204                          const std::string& layerType,
 205                          const std::string& execType,
 206                          InferenceEngine::InferenceEngineProfileInfo::LayerStatus status);
 207     void changeInputBatch(size_t batch);
 208     void CompileNetwork();
 209
 210     // Layer Primitive Creators
 211     void CreatePReLUPrimitive(InferenceEngine::CNNLayerPtr &layer);
 212     void CreateBatchNormalizationPrimitive(InferenceEngine::CNNLayerPtr & layer);
 213     void CreateFlattenPrimitive(InferenceEngine::CNNLayerPtr &layer);
 214     void CreatePermutePrimitive(InferenceEngine::CNNLayerPtr &layer);
 215     void CreateReshapePrimitive(InferenceEngine::CNNLayerPtr &layer);
 216     void CreateNormalizePrimitive(InferenceEngine::CNNLayerPtr &layer);
 217     void CreateDetectionOutputPrimitive(InferenceEngine::CNNLayerPtr &layer);
 218     void CreatePriorBoxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 219     void CreateDeconvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 220     void CreateCropPrimitive(InferenceEngine::CNNLayerPtr &layer);
 221     void CreateROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 222     void CreateSimplerNMSPrimitive(InferenceEngine::CNNLayerPtr &layer);
 223     void CreateEltwisePrimitive(InferenceEngine::CNNLayerPtr &layer);
 224     void CreateConcatenatePrimitive(InferenceEngine::CNNLayerPtr &layer);
 225     void CreateSplitPrimitive(InferenceEngine::CNNLayerPtr &layer);
 226     void CreateFusedSplitConvMergePrimitive(InferenceEngine::CNNLayerPtr &layer);
 227     void CreatePowerPrimitive(InferenceEngine::CNNLayerPtr &layer);
 228     void CreateSoftMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 229     void CreateFullyConnectedPrimitive(InferenceEngine::CNNLayerPtr &layer);
 230     void CreatePoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 231     void CreateLRNPrimitive(InferenceEngine::CNNLayerPtr &layer);
 232     void CreateActivationPrimitive(InferenceEngine::CNNLayerPtr &layer, const LayerType type);
 233     void CreateConvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 234     void CreateScaleShiftPrimitive(InferenceEngine::CNNLayerPtr &layer);
 235     void CreateProposalPrimitive(InferenceEngine::CNNLayerPtr &layer);
 236     void CreatePSROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 237     void CreateCopyPrimitive(InferenceEngine::CNNLayerPtr &layer);
 238     void CreateUpsamplingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 239     void CreateResamplePrimitive(InferenceEngine::CNNLayerPtr &layer);
 240     void CreateYOLO2RegionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 241     void CreateYOLO2ReorgPrimitive(InferenceEngine::CNNLayerPtr &layer);
 242     void CreateArgMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 243     void CreateMaxUnpoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 244     void CreateMVNPrimitive(InferenceEngine::CNNLayerPtr &layer);
 245     void AddConstantBlobInput(InferenceEngine::CNNLayerPtr &layer);
 246     void CreateCustomLayerPrimitive(InferenceEngine::CNNLayerPtr &layer, CLDNNCustomLayerPtr customLayer);
 247 };
 248
 249 };  // namespace CLDNNPlugin