inference-engine/src/mkldnn_plugin/mkldnn_node.h

   1 // Copyright (C) 2018-2019 Intel Corporation
   2 // SPDX-License-Identifier: Apache-2.0
   3 //
   4
   5 #pragma once
   6
   7 #include <ie_api.h>
   8 #include <memory>
   9 #include <vector>
  10 #include <string>
  11 #include <map>
  12 #include <algorithm>
  13 #include <ie_common.h>
  14 #include <ie_profiling.hpp>
  15 #include "details/caseless.hpp"
  16 #include "mkldnn_dims.h"
  17 #include "mkldnn_memory.h"
  18 #include "mkldnn_edge.h"
  19 #include "mkldnn_descriptor.h"
  20 #include "mkldnn/iml_type_mapper.h"
  21 #include "mkldnn_extension_mngr.h"
  22 #include "mkldnn_primitive.h"
  23
  24 namespace MKLDNNPlugin {
  25
  26 using MKLDNNNodePtr = std::shared_ptr<MKLDNNNode>;
  27 using MKLDNNNodeWeakPtr = std::weak_ptr<MKLDNNNode>;
  28
  29 enum Type {
  30     Unknown,
  31     Generic,
  32     Reorder,
  33     Input,
  34     Output,
  35     Convolution,
  36     Deconvolution,
  37     Convolution_Sum,
  38     Convolution_Activation,
  39     Convolution_Depthwise,
  40     Convolution_Sum_Activation,
  41     Activation,
  42     Depthwise,
  43     Lrn,
  44     Pooling,
  45     FullyConnected,
  46     FullyConnected_Activation,
  47     SoftMax,
  48     Split,
  49     Concatenation,
  50     Power,
  51     Eltwise,
  52     Gemm,
  53     Crop,
  54     Reshape,
  55     Tile,
  56     SimplerNMS,
  57     ROIPooling,
  58     BatchNormalization,
  59     Flatten,
  60     Permute,
  61     Copy,
  62     MemoryOutput,
  63     MemoryInput,
  64     RNNCell,
  65     RNNSeq,
  66     Quantize,
  67     BinaryConvolution
  68 };
  69
  70 static Type TypeFromName(const std::string type) {
  71     static InferenceEngine::details::caseless_unordered_map<std::string, Type> type_to_name_tbl = {
  72             { "Unknown", Unknown },
  73             { "Input", Input },
  74             { "Const", Input },
  75             { "Output", Output },
  76             { "Reorder", Reorder },
  77             { "Convolution", Convolution },
  78             { "ReLU", Activation },
  79             { "ELU", Activation },
  80             { "Sigmoid", Activation },
  81             { "Logistic", Activation },
  82             { "TanH", Activation },
  83             { "ReLU6", Activation },
  84             { "Exp", Activation },
  85             { "Not", Activation },
  86             { "Activation", Activation },
  87             { "ScaleShift", Depthwise },
  88             { "PReLU", Depthwise },
  89             { "Clamp", Activation },
  90             { "Norm", Lrn },
  91             { "LRN", Lrn },
  92             { "Pooling", Pooling },
  93             { "FullyConnected", FullyConnected },
  94             { "InnerProduct", FullyConnected },
  95             { "Gemm", Gemm },
  96             { "Softmax", SoftMax },
  97             { "SoftMax", SoftMax },
  98             { "Split", Split },
  99             { "Slice", Split },
 100             { "Concat", Concatenation },
 101             { "Power", Power },
 102             { "Deconvolution", Deconvolution },
 103             { "Eltwise", Eltwise },
 104             { "Crop", Crop },
 105             { "Reshape", Reshape },
 106             { "Tile", Tile },
 107             { "SimplerNMS", SimplerNMS },
 108             { "ROIPooling", ROIPooling },
 109             { "BatchNormalization", BatchNormalization },
 110             { "Flatten", Flatten },
 111             { "Permute", Permute },
 112             { "Copy", Copy },
 113             { "LSTMCell", RNNCell },
 114             { "GRUCell", RNNCell },
 115             { "RNNCell", RNNCell },
 116             { "LSTMSequence", RNNSeq },
 117             { "GRUSequence", RNNSeq },
 118             { "RNNSequence", RNNSeq },
 119             { "Quantize", Quantize },
 120             { "BinaryConvolution", BinaryConvolution },
 121             { "MemoryInput", MemoryInput},  // for construction from name ctor, arbitrary name is used
 122             { "Memory", MemoryOutput },  // for construction from layer ctor
 123     };
 124
 125     if (type_to_name_tbl.find(type) != type_to_name_tbl.end()) {
 126         return type_to_name_tbl[type];
 127     } else {
 128         return Unknown;
 129     }
 130 }
 131
 132 class PrimitiveDescInfo {
 133 public:
 134     PrimitiveDescInfo(const InferenceEngine::LayerConfig conf, impl_desc_type type): config(conf) {
 135         implementationType = type;
 136     }
 137
 138     PrimitiveDescInfo(const PrimitiveDescInfo &descInfo) = default;
 139     PrimitiveDescInfo(PrimitiveDescInfo &&descInfo) = default;
 140
 141     PrimitiveDescInfo &operator=(const PrimitiveDescInfo &descInfo) = default;
 142
 143     const InferenceEngine::LayerConfig getConfig() const {
 144         return config;
 145     }
 146     InferenceEngine::LayerConfig& getConfig() {
 147         return config;
 148     }
 149
 150     impl_desc_type getImplementationType() const {
 151         return implementationType;
 152     }
 153
 154 private:
 155     InferenceEngine::LayerConfig config;
 156     impl_desc_type implementationType;
 157 };
 158
 159 class MKLDNNNode : public InferenceEngine::details::no_copy {
 160 public:
 161     static MKLDNNNode* CreateNode(const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng,
 162                                   const MKLDNNExtensionManager::Ptr& extMgr);
 163
 164     ~MKLDNNNode() override = default;
 165
 166     void addEdge(const MKLDNNEdgeWeakPtr& edge);
 167     void removeEdge(const MKLDNNEdgeWeakPtr& edge);
 168
 169     virtual void cleanup();
 170     void remove();
 171
 172     const std::vector<MKLDNNEdgeWeakPtr> &getParentEdges() const noexcept {
 173         return parentEdges;
 174     }
 175
 176     const std::vector<MKLDNNEdgeWeakPtr> &getChildEdges() const noexcept {
 177         return childEdges;
 178     }
 179
 180     const MKLDNNEdgePtr getParentEdgeAt(size_t idx) const;
 181     virtual const MKLDNNEdgePtr getChildEdgeAt(size_t idx) const;
 182
 183     const std::vector<MKLDNNEdgePtr> getParentEdgesAtPort(size_t idx) const;
 184     const std::vector<MKLDNNEdgePtr> getChildEdgesAtPort(size_t idx) const;
 185
 186     bool isDropped() {
 187         return (isEdgesEmpty(childEdges) && isEdgesEmpty(parentEdges));
 188     }
 189
 190     const mkldnn::engine& getEngine() const {
 191         return engine;
 192     }
 193
 194     bool isConstant();
 195
 196     bool isInplace() const;
 197
 198     void fuseWith(const MKLDNNNodePtr &fuse) {
 199         fusedWith.push_back(fuse);
 200     }
 201
 202     void mergeWith(const MKLDNNNodePtr &merge) {
 203         mergedWith.push_back(merge);
 204     }
 205
 206     void addOriginalLayer(const InferenceEngine::CNNLayerPtr &layer);
 207
 208     const std::vector <MKLDNNNodePtr> &getMergeWith() {
 209         return mergedWith;
 210     }
 211
 212     const std::vector <MKLDNNNodePtr> &getFusedWith() {
 213         return fusedWith;
 214     }
 215
 216     const std::string getName() const {
 217         return name;
 218     }
 219
 220     const std::string getOriginalLayers() const {
 221         return originalLayers;
 222     }
 223
 224     Type getType() const {
 225         return type;
 226     }
 227
 228     const InferenceEngine::CNNLayerPtr &getCnnLayer() const {
 229         return cnnLayer;
 230     }
 231
 232     const std::vector<PrimitiveDescInfo>& getSupportedPrimitiveDescriptors() const {
 233         return supportedPrimitiveDescriptors;
 234     }
 235
 236     inline const PrimitiveDescInfo* getSelectedPrimitiveDescriptor() const {
 237         if (selectedPrimitiveDescriptorIndex < 0 ||
 238             selectedPrimitiveDescriptorIndex >= supportedPrimitiveDescriptors.size())
 239             return nullptr;
 240         return &supportedPrimitiveDescriptors[selectedPrimitiveDescriptorIndex];
 241     }
 242
 243     inline PrimitiveDescInfo* getSelectedPrimitiveDescriptor() {
 244         if (selectedPrimitiveDescriptorIndex < 0 ||
 245             selectedPrimitiveDescriptorIndex >= supportedPrimitiveDescriptors.size())
 246             return nullptr;
 247         return &supportedPrimitiveDescriptors[selectedPrimitiveDescriptorIndex];
 248     }
 249
 250     void selectPrimitiveDescriptorByIndex(int index) {
 251         if (index < 0 || index >= supportedPrimitiveDescriptors.size())
 252             selectedPrimitiveDescriptorIndex = -1;
 253         else
 254             selectedPrimitiveDescriptorIndex = index;
 255     }
 256
 257     std::string getPrimitiveDescriptorType();
 258
 259     PerfCount &PerfCounter() { return perfCounter; }
 260
 261     virtual void setDynamicBatchLim(int lim);
 262
 263     void resolveNotAllocatedEdges();
 264     virtual void execute(mkldnn::stream strm);
 265     virtual void initSupportedPrimitiveDescriptors();
 266     virtual void createPrimitive() = 0;
 267
 268     virtual void selectOptimalPrimitiveDescriptor();
 269     virtual void initOptimalPrimitiveDescriptor();
 270
 271     virtual void getSupportedDescriptors() = 0;
 272     virtual void createDescriptor(const std::vector<InferenceEngine::TensorDesc>& inputDesc,
 273                                   const std::vector<InferenceEngine::TensorDesc>& outputDesc) {}
 274     virtual void initDescriptor(const InferenceEngine::LayerConfig& config);
 275     virtual bool created() const = 0;
 276     virtual bool created(const MKLDNNExtensionManager::Ptr& extMgr) {
 277         return created();
 278     }
 279
 280     template <class PD, class D, typename FPD = bool>
 281     PD createPrimitiveDescriptor(const mkldnn::primitive_attr &attr = mkldnn::primitive_attr()) {
 282         auto descsEqual = [](const std::vector<InferenceEngine::TensorDesc>& srcDescs,
 283                                const std::vector<InferenceEngine::DataConfig>& selectedDescs) {
 284             if (srcDescs.empty() && selectedDescs.empty())
 285                 return true;
 286             if (srcDescs.empty() || selectedDescs.empty())
 287                 return false;
 288             for (size_t i = 0; i < srcDescs.size() && i < selectedDescs.size(); i++) {
 289                 if (srcDescs[i] != selectedDescs[i].desc && srcDescs[i].getLayout() != InferenceEngine::Layout::ANY)
 290                     return false;
 291             }
 292             return true;
 293         };
 294
 295         const PrimitiveDescInfo *selected_pd = getSelectedPrimitiveDescriptor();
 296         if (selected_pd == nullptr)
 297             THROW_IE_EXCEPTION << "Preferable primitive descriptor does not set for node " << getName() << ".";
 298
 299         for (const auto& desc : descs) {
 300             try {
 301                 mkldnn::primitive_desc_iterator itpd = desc.createPrimitiveDescriptorIterator(engine, attr);
 302                 do {
 303                     std::vector<InferenceEngine::TensorDesc> srcDescs;
 304                     for (size_t i = 0; i < desc.inputNumbers(); i++)
 305                         srcDescs.push_back(getSrcMemDesc(itpd, i));
 306
 307                     std::vector<InferenceEngine::TensorDesc> dstDescs;
 308                     for (size_t i = 0; i < desc.outputNumbers(); i++)
 309                         dstDescs.push_back(getDstMemDesc(itpd, i));
 310
 311                     impl_desc_type impl_type = parse_impl_name(itpd.get_impl_info_str());
 312
 313                     if (impl_type == selected_pd->getImplementationType() &&
 314                         descsEqual(srcDescs, selected_pd->getConfig().inConfs) &&
 315                         descsEqual(dstDescs, selected_pd->getConfig().outConfs)) {
 316                         prepareMemory(selected_pd, itpd);
 317                         PD prim_desc = createPd<PD, D, FPD>(desc);
 318                         itpd.getPrimitiveDescriptor(prim_desc);
 319                         return prim_desc;
 320                     }
 321                 } while (itpd.next());
 322             } catch (std::exception& e) {
 323                 // it throw exception in case of no implementation found
 324                 continue;
 325             }
 326         }
 327
 328         THROW_IE_EXCEPTION << "Primitive descriptor was not found for node " << getName() << ".";
 329     }
 330
 331     static void invertVectorCopyUtoI(const InferenceEngine::PropertyVector<unsigned int>& src, std::vector<ptrdiff_t>& dst) {
 332         dst.clear();
 333         for (int i = 1; i <= src.size(); i++) {
 334             dst.push_back(static_cast<ptrdiff_t>(src[src.size() - i]));
 335         }
 336     }
 337
 338     std::vector<MKLDNNDims> inDims;
 339
 340
 341 protected:
 342     // TODO: It is necessary only in order to avoid modifications of cnnLayers and original topology
 343     std::vector<MKLDNNDims> outDims;
 344     void setType(Type type) {
 345         this->type = type;
 346     }
 347
 348     virtual int getMaxBatch();
 349
 350     virtual InferenceEngine::TensorDesc getConfiguredInputDesc(const InferenceEngine::LayerConfig& config, size_t idx) const;
 351     virtual InferenceEngine::TensorDesc getConfiguredOutputDesc(const InferenceEngine::LayerConfig& config, size_t idx) const;
 352     virtual MKLDNNMemoryDesc getSrcMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx);
 353     virtual MKLDNNMemoryDesc getDstMemDesc(mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx);
 354
 355     virtual std::shared_ptr<mkldnn::primitive_attr> initPrimitiveAttr() const { return nullptr; }
 356
 357     typedef std::function<MKLDNNMemoryDesc (mkldnn::primitive_desc_iterator &primitive_desc_it, size_t idx)>
 358             GetPrimitiveMemoryFormatFunc;
 359     std::vector<GetPrimitiveMemoryFormatFunc> internalBlobDesc;
 360
 361     std::vector <MKLDNNNodePtr> fusedWith;
 362     std::vector <MKLDNNNodePtr> mergedWith;
 363     std::vector <impl_desc_type> implPriorities;
 364
 365     std::string originalLayers;  // contains names of the original layers separated by comma
 366
 367     MKLDNNNode(const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng);
 368
 369     int selectedPrimitiveDescriptorIndex = -1;
 370     bool permanent = false;
 371     bool temporary = false;
 372     int dynBatchLim = 0;
 373     enum class ConstantType {
 374         Unknown,
 375         Const,
 376         NoConst
 377     };
 378     ConstantType constant = ConstantType::Unknown;
 379     std::vector<InferenceEngine::Blob::Ptr> internalBlobs;
 380     std::vector<MKLDNNMemoryPtr> internalBlobMemory;
 381     std::vector<PrimitiveDescInfo> supportedPrimitiveDescriptors;
 382     MKLDNNPrimitive prim;
 383     std::vector<MKLDNNDescriptor> descs;
 384
 385     InferenceEngine::Blob::Ptr ext_scales;
 386
 387     friend class MKLDNNEdge;
 388     friend class MKLDNNGraph;
 389     friend class MKLDNNGraphOptimizer;
 390
 391     bool isUninitTensorDesc(const InferenceEngine::TensorDesc& desc) const;
 392     bool isInitConfig(const InferenceEngine::LayerConfig& config) const;
 393     virtual void selectPreferPrimitiveDescriptor(const std::vector<impl_desc_type>& priority);
 394     virtual bool canBeInPlace() const;
 395
 396     virtual const std::vector<impl_desc_type>& getPrimitivesPriority();
 397
 398     std::vector<mkldnn::memory::format> getAvailableFormatsForDims(const MKLDNNDims& dims) const;
 399     int batchToProcess();
 400
 401     InferenceEngine::Blob::Ptr createInternalBlob(InferenceEngine::SizeVector dims, bool weights);
 402
 403     template<typename To>
 404     class Register {
 405     public:
 406         Register() {
 407             Registry::RegisterNode(
 408                 Registry::CreatorByLayerFunction(
 409                         [](const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng)
 410                         -> MKLDNNNode* { return new To(layer, eng); } ) );
 411         }
 412     };
 413
 414 private:
 415     std::vector<MKLDNNEdgeWeakPtr> parentEdges;
 416     std::vector<MKLDNNEdgeWeakPtr> childEdges;
 417
 418     InferenceEngine::CNNLayerPtr cnnLayer;
 419     mkldnn::engine engine;
 420
 421     std::string name;
 422     const std::string typeStr;
 423     Type type;
 424     int execIndex = -1;
 425
 426     std::string typeToStr(Type type);
 427
 428     PerfCount perfCounter;
 429     InferenceEngine::ProfilingTask profilingTask;
 430
 431     bool isEdgesEmpty(const std::vector<MKLDNNEdgeWeakPtr>& edges) const;
 432
 433     class Registry {
 434     public:
 435         typedef std::function<MKLDNNNode *(const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng)> CreatorByLayerFunction;
 436
 437         static MKLDNNNode *CreateNode(const InferenceEngine::CNNLayerPtr& layer, const mkldnn::engine& eng, const MKLDNNExtensionManager::Ptr& extMgr);
 438
 439         static void RegisterNode(CreatorByLayerFunction f);
 440     private:
 441         static std::vector<CreatorByLayerFunction> _dataByLayer;
 442     };
 443
 444     template <class PD, class D, typename FPD>
 445     typename std::enable_if<!std::is_same<FPD, bool>::value, PD>::type
 446     createPd(MKLDNNDescriptor desc) {
 447         std::shared_ptr<D> selected_desc_ptr = desc;
 448         std::shared_ptr<FPD> backward_prim_desc_ptr = desc;
 449         return PD(*selected_desc_ptr, engine, *backward_prim_desc_ptr);
 450     }
 451
 452     template <class PD, class D, typename FPD>
 453     typename std::enable_if<std::is_same<FPD, bool>::value, PD>::type
 454     createPd(MKLDNNDescriptor desc) {
 455         std::shared_ptr<D> selected_desc_ptr = desc;
 456         return PD(*selected_desc_ptr, engine);
 457     }
 458
 459     void prepareMemory(const PrimitiveDescInfo *selected_pd, mkldnn::primitive_desc_iterator& itpd);
 460     enum LOOK { LOOK_UP = 1, LOOK_DOWN = 2 };
 461     ConstantType checkConstant(LOOK look, std::vector<MKLDNNNodePtr>& checkNodes);
 462 };
 463
 464 template <typename T, typename U>
 465 inline T div_up(const T a, const U b) {
 466     assert(b);
 467     return (a + b - 1) / b;
 468 }
 469
 470 template <typename T, typename U>
 471 inline T rnd_up(const T a, const U b) {
 472     return div_up(a, b) * b;
 473 }
 474
 475 }  // namespace MKLDNNPlugin