inference-engine/src/cldnn_engine/cldnn_graph.h

   1 // Copyright (C) 2018 Intel Corporation
   2 //
   3 // SPDX-License-Identifier: Apache-2.0
   4 //
   5
   6 #pragma once
   7
   8 #include <vector>
   9 #include <map>
  10 #include <set>
  11 #include <memory>
  12 #include <string>
  13 #include "ie_blob.h"
  14 #include "ie_plugin.hpp"
  15 #include "cpp/ie_cnn_network.h"
  16 #include "debug_options.h"
  17 #include "inference_engine.hpp"
  18 #include <CPP/network.hpp>
  19 #include <CPP/memory.hpp>
  20 #include <CPP/primitive.hpp>
  21 #include <CPP/topology.hpp>
  22 #include <CPP/pooling.hpp>
  23 #include <CPP/eltwise.hpp>
  24 #include <CPP/concatenation.hpp>
  25 #include <CPP/detection_output.hpp>
  26 #include <CPP/softmax.hpp>
  27 #include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
  28 #include <CPP/upsampling.hpp>
  29 #include "cldnn_custom_layer.h"
  30
  31 namespace CLDNNPlugin {
  32
  33 struct InferenceEnv {
  34     std::shared_ptr<const cldnn::engine> engine;
  35     std::shared_ptr<cldnn::network> network;
  36     std::map<std::string, cldnn::primitive_id> primitiveIDs;
  37     std::map<std::string, std::vector<cldnn::primitive_id>> prevPrimitiveIDs;
  38     std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> perfMap;
  39     std::set<cldnn::primitive_id> profilingIDs;
  40
  41     DebugOptions debugOptions;
  42
  43     std::map<std::string, InferenceEngine::SizeVector> outputDims;
  44     std::map<std::string, cldnn::layout> inputLayouts;
  45
  46     std::vector<std::shared_ptr<cldnn::network>> batchNetworks;
  47     int m_max_batch;
  48     int m_bv_sz;
  49 };
  50
  51 class CLDNNGraph : public InferenceEngine::ExecutableNetworkThreadSafeDefault {
  52 public:
  53     typedef std::shared_ptr<CLDNNGraph> Ptr;
  54     struct Config {
  55         Config() : useProfiling(false), dumpCustomKernels(false), exclusiveAsyncRequests(false),
  56             memory_pool_on(true),
  57             enableDynamicBatch(false),
  58             queuePriority(cldnn::priority_mode_types::disabled),
  59             queueThrottle(cldnn::throttle_mode_types::disabled) {}
  60
  61         void LoadFromMap(const std::map<std::string, std::string>& configMap);
  62
  63         bool enableDynamicBatch;
  64         bool useProfiling;
  65         bool dumpCustomKernels;
  66         bool exclusiveAsyncRequests;
  67         bool memory_pool_on;
  68         cldnn::priority_mode_types queuePriority;
  69         cldnn::throttle_mode_types queueThrottle;
  70         CLDNNCustomLayerMap customLayers;
  71         cldnn::tuning_config_options tuningConfig;
  72         std::string graph_dumps_dir;
  73         std::string sources_dumps_dir;
  74     };
  75     explicit CLDNNGraph(InferenceEngine::ICNNNetwork &network, const Config& config = {}, int max_batch = -1);
  76
  77     InferenceEngine::InferRequestInternal::Ptr
  78     CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override;
  79
  80     static bool IsLayerSupported(const std::string &type) {
  81         return LayerTypeFromStr(type) != NO_TYPE;
  82     }
  83
  84 protected:
  85     // graph members
  86     std::shared_ptr<cldnn::topology> m_topology;
  87     InferenceEnv m_env;
  88     Config m_config;
  89
  90     InferenceEngine::InputsDataMap*  p_currentInputs;
  91     InferenceEngine::OutputsDataMap* p_currentOutputs;
  92     int m_curBatch;
  93     static const cldnn::primitive_id m_preProcessTag;
  94     static const cldnn::primitive_id m_weightsTag;
  95     static const cldnn::primitive_id m_biasesTag;
  96     static const cldnn::primitive_id m_meanValuesTag;
  97     static const cldnn::primitive_id m_postProcessTag;
  98     static const cldnn::primitive_id m_scalesTag;
  99     static const cldnn::primitive_id m_workaroundTag;
 100     static const cldnn::primitive_id m_preCustomLayerTag;
 101     static const cldnn::primitive_id m_postCustomLayerTag;
 102
 103     // internal types
 104     enum LayerType {
 105         Convolution,
 106         ReLU,
 107         ReLU6,
 108         Sigmoid,
 109         TanH,
 110         ELU,
 111         Activation,
 112         LRN,
 113         Pooling,
 114         FullyConnected,
 115         SoftMax,
 116         Power,
 117         Split,
 118         Concatenate,
 119         Eltwise,
 120         SimplerNMS,
 121         ROIPooling,
 122         Crop,
 123         Deconvolution,
 124         PriorBox,
 125         DetectionOutput,
 126         Normalize,
 127         Reshape,
 128         Permute,
 129         Flatten,
 130         BatchNormalization,
 131         PReLU,
 132         ScaleShift,
 133         Proposal,
 134         PSROIPooling,
 135         Clamp,
 136         Copy,
 137         Upsampling,
 138         Resample,
 139         RegionYolo,
 140         ReorgYolo,
 141         ConstantBlob,
 142         ArgMax,
 143         MVN,
 144         Unpooling,
 145         Tile,
 146         RNN,
 147         NO_TYPE
 148     };
 149
 150     enum WeightRearrangeType {
 151         BroadcastFeatures,
 152         FlipDeconvDims,
 153         NO_REARRANGE
 154     };
 155
 156     cldnn::format m_defaultFormat;
 157     cldnn::data_types m_networkPrecision;
 158     void InitFormat(InferenceEngine::ICNNNetwork &network);
 159
 160     static cldnn::data_types DataTypeFromPrecision(InferenceEngine::Precision p);
 161     static cldnn::format     FormatFromLayout(InferenceEngine::Layout l);
 162     static cldnn::upsampling_sample_type UpsamplingTypeFromString(const std::string& str);
 163
 164     void Load(InferenceEngine::ICNNNetwork &network);
 165     static LayerType LayerTypeFromStr(const std::string& str);
 166     static cldnn::pooling_mode PoolingModeFromIEPooling(InferenceEngine::PoolingLayer::PoolType pt, bool excludePadding = false);
 167     static cldnn::eltwise_mode EltwiseModeFromIEEltwise(InferenceEngine::EltwiseLayer::eOperation op);
 168     static cldnn::concatenation::concatenation_axis ConcatAxisFromIEAxis(unsigned axis);
 169     static cldnn::prior_box_code_type PriorBoxCodeFromString(const std::string& str);
 170     static cldnn::softmax::dimension_t SoftmaxDimensionFromIEAxis(const InferenceEngine::SoftMaxLayer* softmaxLayer, bool isPrevFC = false);
 171     void CreatePrimitiveFromBlob(cldnn::primitive_id primID,
 172                                  const InferenceEngine::Blob::Ptr pBlob,
 173                                  cldnn::layout blobLayout,
 174                                  size_t blobByteOffset = 0,
 175                                  WeightRearrangeType rearrange = NO_REARRANGE);
 176     void CreateWeightAndBiasPrimitives(const InferenceEngine::CNNLayerPtr& layer,
 177                                        std::vector<cldnn::primitive_id>& weightsPrimID,
 178                                        std::vector<cldnn::primitive_id>& biasesPrimID);
 179     void CreateScaleWeightsAndBiasesFromBN(const InferenceEngine::BatchNormalizationLayer* bnLayer,
 180                                            cldnn::primitive_id weightsPrimID,
 181                                            cldnn::primitive_id biasesPrimID);
 182     void AddPreProcessPrimitive(InferenceEngine::InputInfo::Ptr inputInfo);
 183     void AddInputPrimitive(InferenceEngine::InputInfo::Ptr inputInfo);
 184     void AddOutputPrimitive(std::string outputName, const InferenceEngine::DataPtr outputData,
 185                             InferenceEngine::Precision outputPrecision = InferenceEngine::Precision::UNSPECIFIED);
 186     void CreateSingleLayerPrimitive(InferenceEngine::CNNLayerPtr& layer);
 187     bool IsValidSplitConvMerge(const InferenceEngine::SplitLayer* splitLayer) const;
 188     bool CanProcessDynBatch(InferenceEngine::ICNNNetwork &network) const;
 189     static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::DataPtr data);
 190     static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::CNNLayerPtr layer);
 191     static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::DataPtr data);
 192     static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::CNNLayerPtr layer);
 193     std::vector<cldnn::primitive_id> GetPrevLayersPrimitives(const InferenceEngine::CNNLayerPtr layer) const;
 194     void AddSingleValuePrimitive(cldnn::primitive_id valPrimID, cldnn::data_types dataType, float value);
 195
 196     void CreateGenericLayerBlobPrimitives(const InferenceEngine::GenericLayer* layer);
 197     static void ValidateGenericLayerBlobs(const InferenceEngine::GenericLayer* layer, const std::vector<std::string>& blobNames);
 198     static cldnn::tensor CldnnTensorFromIEDims(const InferenceEngine::SizeVector& dims);
 199     static bool HasParam(const std::map<std::string, std::string>& layerParams, std::string paramName) {
 200         auto p = layerParams.find(paramName);
 201         return p != layerParams.end();
 202     }
 203
 204     void InitProfileInfo(const std::string& layerName,
 205                          const std::string& layerType,
 206                          const std::string& execType,
 207                          InferenceEngine::InferenceEngineProfileInfo::LayerStatus status);
 208     void changeInputBatch(size_t batch);
 209     void CompileNetwork();
 210
 211     // Layer Primitive Creators
 212     void CreatePReLUPrimitive(InferenceEngine::CNNLayerPtr &layer);
 213     void CreateBatchNormalizationPrimitive(InferenceEngine::CNNLayerPtr & layer);
 214     void CreateFlattenPrimitive(InferenceEngine::CNNLayerPtr &layer);
 215     void CreatePermutePrimitive(InferenceEngine::CNNLayerPtr &layer);
 216     void CreateReshapePrimitive(InferenceEngine::CNNLayerPtr &layer);
 217     void CreateNormalizePrimitive(InferenceEngine::CNNLayerPtr &layer);
 218     void CreateDetectionOutputPrimitive(InferenceEngine::CNNLayerPtr &layer);
 219     void CreatePriorBoxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 220     void CreateDeconvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 221     void CreateCropPrimitive(InferenceEngine::CNNLayerPtr &layer);
 222     void CreateROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 223     void CreateSimplerNMSPrimitive(InferenceEngine::CNNLayerPtr &layer);
 224     void CreateEltwisePrimitive(InferenceEngine::CNNLayerPtr &layer);
 225     void CreateConcatenatePrimitive(InferenceEngine::CNNLayerPtr &layer);
 226     void CreateSplitPrimitive(InferenceEngine::CNNLayerPtr &layer);
 227     void CreateFusedSplitConvMergePrimitive(InferenceEngine::CNNLayerPtr &layer);
 228     void CreatePowerPrimitive(InferenceEngine::CNNLayerPtr &layer);
 229     void CreateSoftMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 230     void CreateFullyConnectedPrimitive(InferenceEngine::CNNLayerPtr &layer);
 231     void CreatePoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 232     void CreateLRNPrimitive(InferenceEngine::CNNLayerPtr &layer);
 233     void CreateActivationPrimitive(InferenceEngine::CNNLayerPtr &layer, const LayerType type);
 234     void CreateConvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 235     void CreateScaleShiftPrimitive(InferenceEngine::CNNLayerPtr &layer);
 236     void CreateProposalPrimitive(InferenceEngine::CNNLayerPtr &layer);
 237     void CreatePSROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 238     void CreateCopyPrimitive(InferenceEngine::CNNLayerPtr &layer);
 239     void CreateUpsamplingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 240     void CreateResamplePrimitive(InferenceEngine::CNNLayerPtr &layer);
 241     void CreateYOLO2RegionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 242     void CreateYOLO2ReorgPrimitive(InferenceEngine::CNNLayerPtr &layer);
 243     void CreateArgMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 244     void CreateMaxUnpoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 245     void CreateMVNPrimitive(InferenceEngine::CNNLayerPtr &layer);
 246     void CreateTilePrimitive(InferenceEngine::CNNLayerPtr &layer);
 247     void CreateRNNPrimitive(InferenceEngine::CNNLayerPtr &layer);
 248     void AddConstantBlobInput(InferenceEngine::CNNLayerPtr &layer);
 249     void CreateCustomLayerPrimitive(InferenceEngine::CNNLayerPtr &layer, CLDNNCustomLayerPtr customLayer);
 250 };
 251
 252 };  // namespace CLDNNPlugin