inference-engine/src/cldnn_engine/cldnn_graph.h

   1 // Copyright (C) 2018-2019 Intel Corporation
   2 // SPDX-License-Identifier: Apache-2.0
   3 //
   4
   5 #pragma once
   6
   7 #include <vector>
   8 #include <map>
   9 #include <set>
  10 #include <memory>
  11 #include <string>
  12 #include <utility>
  13 #include "ie_blob.h"
  14 #include "ie_plugin.hpp"
  15 #include "cpp/ie_cnn_network.h"
  16 #include "debug_options.h"
  17 #include "inference_engine.hpp"
  18 #include <CPP/network.hpp>
  19 #include <CPP/memory.hpp>
  20 #include <CPP/primitive.hpp>
  21 #include <CPP/topology.hpp>
  22 #include <CPP/pooling.hpp>
  23 #include <CPP/eltwise.hpp>
  24 #include <CPP/concatenation.hpp>
  25 #include <CPP/detection_output.hpp>
  26 #include <CPP/softmax.hpp>
  27 #include <cpp_interfaces/impl/ie_executable_network_thread_safe_default.hpp>
  28 #include <CPP/upsampling.hpp>
  29 #include "cldnn_custom_layer.h"
  30
  31 namespace CLDNNPlugin {
  32
  33 struct PerfCounter {
  34     InferenceEngine::InferenceEngineProfileInfo::LayerStatus status;
  35     bool isCPU;
  36     uint64_t realTime_uSec;
  37     uint64_t cpu_uSec;
  38     uint32_t num;
  39     std::string layerType;
  40
  41 public:
  42     PerfCounter() : realTime_uSec(0), cpu_uSec(0), num(0),
  43         status(InferenceEngine::InferenceEngineProfileInfo::NOT_RUN), isCPU(false) {}
  44
  45     long long realTime_avg() const { return (num == 0) ? 0 : realTime_uSec / num; }
  46     long long cpu_avg() const { return (num == 0) ? 0 : cpu_uSec / num; }
  47 };
  48
  49 struct InferenceEnv {
  50     std::shared_ptr<const cldnn::engine> engine;
  51     std::shared_ptr<cldnn::network> network;
  52     std::map<std::string, cldnn::primitive_id> primitiveIDs;
  53     std::map<std::string, std::vector<cldnn::primitive_id>> prevPrimitiveIDs;
  54
  55     std::map<cldnn::primitive_id, std::pair<std::string, PerfCounter>> perfMap;
  56     std::vector<cldnn::primitive_id> profilingIDs;
  57
  58     DebugOptions debugOptions;
  59
  60     std::map<std::string, InferenceEngine::SizeVector> outputDims;
  61     std::map<std::string, cldnn::layout> inputLayouts;
  62
  63     std::vector<std::shared_ptr<cldnn::network>> batchNetworks;
  64     int m_max_batch;
  65     int m_bv_sz;
  66 };
  67
  68 class CLDNNGraph : public InferenceEngine::ExecutableNetworkThreadSafeDefault {
  69 public:
  70     typedef std::shared_ptr<CLDNNGraph> Ptr;
  71     struct Config {
  72         Config() : useProfiling(false), dumpCustomKernels(false), exclusiveAsyncRequests(false),
  73             memory_pool_on(true),
  74             enableDynamicBatch(false),
  75             queuePriority(cldnn::priority_mode_types::disabled),
  76             queueThrottle(cldnn::throttle_mode_types::disabled) {}
  77
  78         void LoadFromMap(const std::map<std::string, std::string>& configMap);
  79
  80         bool enableDynamicBatch;
  81         bool useProfiling;
  82         bool dumpCustomKernels;
  83         bool exclusiveAsyncRequests;
  84         bool memory_pool_on;
  85         cldnn::priority_mode_types queuePriority;
  86         cldnn::throttle_mode_types queueThrottle;
  87         CLDNNCustomLayerMap customLayers;
  88         cldnn::tuning_config_options tuningConfig;
  89         std::string graph_dumps_dir;
  90         std::string sources_dumps_dir;
  91     };
  92     explicit CLDNNGraph(InferenceEngine::ICNNNetwork &network, const Config& config = {}, int max_batch = -1);
  93
  94     InferenceEngine::InferRequestInternal::Ptr
  95     CreateInferRequestImpl(InferenceEngine::InputsDataMap networkInputs, InferenceEngine::OutputsDataMap networkOutputs) override;
  96
  97     static bool IsLayerSupported(const std::string &type) {
  98         return LayerTypeFromStr(type) != NO_TYPE;
  99     }
 100
 101 protected:
 102     // graph members
 103     std::shared_ptr<cldnn::topology> m_topology;
 104     InferenceEnv m_env;
 105     Config m_config;
 106
 107     InferenceEngine::InputsDataMap*  p_currentInputs;
 108     InferenceEngine::OutputsDataMap* p_currentOutputs;
 109     int m_curBatch;
 110     static const cldnn::primitive_id m_preProcessTag;
 111     static const cldnn::primitive_id m_weightsTag;
 112     static const cldnn::primitive_id m_biasesTag;
 113     static const cldnn::primitive_id m_meanValuesTag;
 114     static const cldnn::primitive_id m_postProcessTag;
 115     static const cldnn::primitive_id m_scalesTag;
 116     static const cldnn::primitive_id m_workaroundTag;
 117     static const cldnn::primitive_id m_preCustomLayerTag;
 118     static const cldnn::primitive_id m_postCustomLayerTag;
 119
 120     // internal types
 121     enum LayerType {
 122         Convolution,
 123         ReLU,
 124         ReLU6,
 125         Sigmoid,
 126         TanH,
 127         ELU,
 128         Activation,
 129         Exp,
 130         Not,
 131         LRN,
 132         Pooling,
 133         FullyConnected,
 134         SoftMax,
 135         Power,
 136         Split,
 137         Concatenate,
 138         Eltwise,
 139         SimplerNMS,
 140         ROIPooling,
 141         Crop,
 142         Deconvolution,
 143         PriorBox,
 144         DetectionOutput,
 145         Normalize,
 146         Reshape,
 147         Permute,
 148         Flatten,
 149         BatchNormalization,
 150         PReLU,
 151         ScaleShift,
 152         Proposal,
 153         PSROIPooling,
 154         Clamp,
 155         Copy,
 156         Upsampling,
 157         Resample,
 158         RegionYolo,
 159         ReorgYolo,
 160         ConstantBlob,
 161         ArgMax,
 162         MVN,
 163         Unpooling,
 164         Tile,
 165         Pad,
 166         LSTMCell,
 167         RNN,
 168         Gather,
 169         DepthToSpace,
 170         ShuffleChannels,
 171         StridedSlice,
 172         ReverseSequence,
 173         NO_TYPE
 174     };
 175
 176     enum WeightRearrangeType {
 177         BroadcastFeatures,
 178         FlipDeconvDims,
 179         NO_REARRANGE
 180     };
 181
 182     cldnn::format m_defaultFormat;
 183     void InitFormat(InferenceEngine::ICNNNetwork &network);
 184
 185     static cldnn::data_types DataTypeFromPrecision(InferenceEngine::Precision p);
 186     static cldnn::format     FormatFromLayout(InferenceEngine::Layout l);
 187     static cldnn::upsampling_sample_type UpsamplingTypeFromString(const std::string& str);
 188
 189     void Load(InferenceEngine::ICNNNetwork &network);
 190     static LayerType LayerTypeFromStr(const std::string& str);
 191     static cldnn::pooling_mode PoolingModeFromIEPooling(InferenceEngine::PoolingLayer::PoolType pt, bool excludePadding = false);
 192     static cldnn::eltwise_mode EltwiseModeFromIEEltwise(InferenceEngine::EltwiseLayer::eOperation op);
 193     static cldnn::concatenation::concatenation_axis ConcatAxisFromIEAxis(unsigned axis);
 194     static cldnn::prior_box_code_type PriorBoxCodeFromString(const std::string& str);
 195     static cldnn::softmax::dimension_t SoftmaxDimensionFromIEAxis(const InferenceEngine::SoftMaxLayer* softmaxLayer, bool isPrevFC = false);
 196     void CreatePrimitiveFromBlob(cldnn::primitive_id primID,
 197                                  const InferenceEngine::Blob::Ptr pBlob,
 198                                  cldnn::layout blobLayout,
 199                                  size_t blobByteOffset = 0,
 200                                  WeightRearrangeType rearrange = NO_REARRANGE);
 201     void CreateWeightAndBiasPrimitives(const InferenceEngine::CNNLayerPtr& layer,
 202                                        std::vector<cldnn::primitive_id>& weightsPrimID,
 203                                        std::vector<cldnn::primitive_id>& biasesPrimID);
 204     void CreateScaleWeightsAndBiasesFromBN(const InferenceEngine::BatchNormalizationLayer* bnLayer,
 205                                            cldnn::primitive_id weightsPrimID,
 206                                            cldnn::primitive_id biasesPrimID);
 207     void AddPreProcessPrimitive(InferenceEngine::InputInfo::Ptr inputInfo);
 208     void AddInputPrimitive(InferenceEngine::InputInfo::Ptr inputInfo, InferenceEngine::Precision inputPrecision);
 209     void AddOutputPrimitive(std::string outputName, const InferenceEngine::DataPtr outputData,
 210                             InferenceEngine::Precision outputPrecision = InferenceEngine::Precision::UNSPECIFIED);
 211     void CreateSingleLayerPrimitive(InferenceEngine::CNNLayerPtr& layer);
 212     bool IsValidSplitConvMerge(const InferenceEngine::SplitLayer* splitLayer) const;
 213     bool CanProcessDynBatch(InferenceEngine::ICNNNetwork &network) const;
 214     static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::DataPtr data);
 215     static std::vector<InferenceEngine::CNNLayerPtr> GetNextLayers(const InferenceEngine::CNNLayerPtr layer);
 216     static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::DataPtr data);
 217     static InferenceEngine::CNNLayerPtr GetNextSingleLayer(const InferenceEngine::CNNLayerPtr layer);
 218     std::vector<cldnn::primitive_id> GetPrevLayersPrimitives(const InferenceEngine::CNNLayerPtr layer) const;
 219     void AddSingleValuePrimitive(cldnn::primitive_id valPrimID, cldnn::data_types dataType, float value);
 220
 221     void CreateGenericLayerBlobPrimitives(const InferenceEngine::GenericLayer* layer);
 222     static void ValidateGenericLayerBlobs(const InferenceEngine::GenericLayer* layer, const std::vector<std::string>& blobNames);
 223     static cldnn::tensor CldnnTensorFromIEDims(const InferenceEngine::SizeVector& dims);
 224     static bool HasParam(const std::map<std::string, std::string>& layerParams, std::string paramName) {
 225         auto p = layerParams.find(paramName);
 226         return p != layerParams.end();
 227     }
 228
 229     void InitProfileInfo(const std::string& layerName,
 230                          const std::string& layerType,
 231                          bool isCPU = false,
 232                          InferenceEngine::InferenceEngineProfileInfo::LayerStatus status
 233                                     = InferenceEngine::InferenceEngineProfileInfo::EXECUTED);
 234     void changeInputBatch(size_t batch);
 235     void CompileNetwork();
 236
 237     // Layer Primitive Creators
 238     void CreatePReLUPrimitive(InferenceEngine::CNNLayerPtr &layer);
 239     void CreateBatchNormalizationPrimitive(InferenceEngine::CNNLayerPtr & layer);
 240     void CreateFlattenPrimitive(InferenceEngine::CNNLayerPtr &layer);
 241     void CreatePermutePrimitive(InferenceEngine::CNNLayerPtr &layer);
 242     void CreateReshapePrimitive(InferenceEngine::CNNLayerPtr &layer);
 243     void CreateNormalizePrimitive(InferenceEngine::CNNLayerPtr &layer);
 244     void CreateDetectionOutputPrimitive(InferenceEngine::CNNLayerPtr &layer);
 245     void CreatePriorBoxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 246     void CreateDeconvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 247     void CreateCropPrimitive(InferenceEngine::CNNLayerPtr &layer);
 248     void CreateROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 249     void CreateSimplerNMSPrimitive(InferenceEngine::CNNLayerPtr &layer);
 250     void CreateEltwisePrimitive(InferenceEngine::CNNLayerPtr &layer);
 251     void CreateConcatenatePrimitive(InferenceEngine::CNNLayerPtr &layer);
 252     void CreateSplitPrimitive(InferenceEngine::CNNLayerPtr &layer);
 253     void CreateFusedSplitConvMergePrimitive(InferenceEngine::CNNLayerPtr &layer);
 254     void CreatePowerPrimitive(InferenceEngine::CNNLayerPtr &layer);
 255     void CreateSoftMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 256     void CreateFullyConnectedPrimitive(InferenceEngine::CNNLayerPtr &layer);
 257     void CreatePoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 258     void CreateLRNPrimitive(InferenceEngine::CNNLayerPtr &layer);
 259     void CreateActivationPrimitive(InferenceEngine::CNNLayerPtr &layer, const LayerType type);
 260     void CreateConvolutionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 261     void CreateScaleShiftPrimitive(InferenceEngine::CNNLayerPtr &layer);
 262     void CreateProposalPrimitive(InferenceEngine::CNNLayerPtr &layer);
 263     void CreatePSROIPoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 264     void CreateCopyPrimitive(InferenceEngine::CNNLayerPtr &layer);
 265     void CreateUpsamplingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 266     void CreateResamplePrimitive(InferenceEngine::CNNLayerPtr &layer);
 267     void CreateYOLO2RegionPrimitive(InferenceEngine::CNNLayerPtr &layer);
 268     void CreateYOLO2ReorgPrimitive(InferenceEngine::CNNLayerPtr &layer);
 269     void CreateArgMaxPrimitive(InferenceEngine::CNNLayerPtr &layer);
 270     void CreateMaxUnpoolingPrimitive(InferenceEngine::CNNLayerPtr &layer);
 271     void CreateMVNPrimitive(InferenceEngine::CNNLayerPtr &layer);
 272     void CreateTilePrimitive(InferenceEngine::CNNLayerPtr &layer);
 273     void CreatePadPrimitive(InferenceEngine::CNNLayerPtr &layer);
 274     void CreateRNNPrimitive(InferenceEngine::CNNLayerPtr &layer);
 275     void CreateLSTMCellPrimitive(InferenceEngine::CNNLayerPtr &layer);
 276     void AddConstantBlobInput(InferenceEngine::CNNLayerPtr &layer);
 277     void CreateCustomLayerPrimitive(InferenceEngine::CNNLayerPtr &layer, CLDNNCustomLayerPtr customLayer);
 278     void CreateGatherPrimitive(InferenceEngine::CNNLayerPtr &layer);
 279     void CreateDepthToSpacePrimitive(InferenceEngine::CNNLayerPtr &layer);
 280     void CreateShuffleChannelsPrimitive(InferenceEngine::CNNLayerPtr &layer);
 281     void CreateStridedSlicePrimitive(InferenceEngine::CNNLayerPtr &layer);
 282     void CreateReverseSequencePrimitive(InferenceEngine::CNNLayerPtr &layer);
 283 };
 284
 285 };  // namespace CLDNNPlugin