inference-engine/tests/unit/engines/mkldnn/graph/layers/extensions/graph_generic_test.cpp

   1 // Copyright (C) 2018-2019 Intel Corporation
   2 // SPDX-License-Identifier: Apache-2.0
   3 //
   4
   5 #include <gtest/gtest.h>
   6 #include <gmock/gmock-spec-builders.h>
   7 #include "mkldnn_plugin/mkldnn_graph.h"
   8
   9 #include "test_graph.hpp"
  10
  11 #include <ie_iextension.h>
  12 #include <ie_plugin_config.hpp>
  13 #include <mock_error_listener.hpp>
  14 #include <mkldnn_plugin/mkldnn_extension_mngr.h>
  15 #include "tests_common.hpp"
  16
  17 #include <ext_list.hpp>
  18
  19 using namespace ::testing;
  20 using namespace std;
  21 using namespace mkldnn;
  22
  23 class FakeGenericPrimitiveImpl : public InferenceEngine::ILayerExecImpl {
  24 public:
  25     InferenceEngine::StatusCode getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig>& conf, InferenceEngine::ResponseDesc *resp) noexcept override {
  26         return InferenceEngine::OK;
  27     }
  28     InferenceEngine::StatusCode init(InferenceEngine::LayerConfig& config, InferenceEngine::ResponseDesc *resp) noexcept override {
  29         return InferenceEngine::OK;
  30     }
  31     InferenceEngine::StatusCode execute(std::vector<InferenceEngine::Blob::Ptr>& inputs, std::vector<InferenceEngine::Blob::Ptr>& outputs, InferenceEngine::ResponseDesc *resp) noexcept override {
  32         return InferenceEngine::NOT_IMPLEMENTED;
  33     }
  34 };
  35
  36 class FakeGenericPrimitiveFactory : public InferenceEngine::ILayerImplFactory {
  37 public:
  38     // set output shapes by input shapes.
  39     InferenceEngine::StatusCode getShapes(const std::vector<InferenceEngine::TensorDesc>& inShapes,
  40                                           std::vector<InferenceEngine::TensorDesc>& outShapes,
  41                                           InferenceEngine::ResponseDesc *resp) noexcept override {
  42         return InferenceEngine::NOT_IMPLEMENTED;
  43     }
  44     // First implementation has more priority than next
  45     InferenceEngine::StatusCode getImplementations(std::vector<InferenceEngine::ILayerImpl::Ptr>& impls, InferenceEngine::ResponseDesc *resp) noexcept override {
  46         impls.push_back(InferenceEngine::ILayerImpl::Ptr(new FakeGenericPrimitiveImpl()));
  47         return InferenceEngine::OK;
  48     }
  49 };
  50
  51 class DoublePrimitiveImpl : public InferenceEngine::ILayerExecImpl {
  52 public:
  53     DoublePrimitiveImpl(const InferenceEngine::CNNLayer *layer) {
  54         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
  55     }
  56     InferenceEngine::StatusCode getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig>& conf, InferenceEngine::ResponseDesc *resp) noexcept override {
  57         InferenceEngine::LayerConfig config;
  58         config.dynBatchSupport = true;
  59         if (cnnLayer->outData.size() != 1 && cnnLayer->insData.size() != 1)
  60             return InferenceEngine::GENERAL_ERROR;
  61         InferenceEngine::DataConfig cfg;
  62         cfg.constant = false;
  63         cfg.inPlace = 0;
  64         InferenceEngine::SizeVector order;
  65         for(size_t i = 0; i < cnnLayer->outData[0]->getTensorDesc().getDims().size(); i++) {
  66             order.push_back(i);
  67         }
  68         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->outData[0]->getTensorDesc().getPrecision(),
  69                               cnnLayer->outData[0]->getTensorDesc().getDims(),
  70                               {cnnLayer->outData[0]->getTensorDesc().getDims(), order});
  71         config.outConfs.push_back(cfg);
  72         config.inConfs.push_back(cfg);
  73         conf.push_back(config);
  74         return InferenceEngine::OK;
  75     }
  76     InferenceEngine::StatusCode init(InferenceEngine::LayerConfig& config, InferenceEngine::ResponseDesc *resp) noexcept override {
  77         for(auto input : config.inConfs) {
  78             if (input.inPlace < 0)
  79                 return InferenceEngine::GENERAL_ERROR;
  80             if (input.constant)
  81                 return InferenceEngine::GENERAL_ERROR;
  82         }
  83         for(auto output : config.outConfs) {
  84             if (output.constant)
  85                 return InferenceEngine::GENERAL_ERROR;
  86         }
  87         return InferenceEngine::OK;
  88     }
  89     InferenceEngine::StatusCode execute(std::vector<InferenceEngine::Blob::Ptr>& inputs, std::vector<InferenceEngine::Blob::Ptr>& outputs, InferenceEngine::ResponseDesc *resp) noexcept override {
  90         const float *src_data = inputs[0]->buffer();
  91         float *dst_data = outputs[0]->buffer();
  92         if (src_data != dst_data)
  93             return InferenceEngine::GENERAL_ERROR;
  94
  95         size_t data_size = inputs[0]->size();
  96         for (size_t i = 0; i < data_size; i++) {
  97             dst_data[i] = src_data[i]*2;
  98         }
  99         return InferenceEngine::OK;
 100     }
 101
 102 private:
 103     InferenceEngine::CNNLayer* cnnLayer;
 104 };
 105
 106 class ConstPrimitiveImpl : public InferenceEngine::ILayerExecImpl {
 107 public:
 108     ConstPrimitiveImpl(const InferenceEngine::CNNLayer *layer) {
 109         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 110     }
 111     InferenceEngine::StatusCode getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig>& conf, InferenceEngine::ResponseDesc *resp) noexcept override {
 112         InferenceEngine::LayerConfig config;
 113         config.dynBatchSupport = 0;
 114         if (cnnLayer->outData.size() != 1 && cnnLayer->insData.size() != 1)
 115             return InferenceEngine::GENERAL_ERROR;
 116         InferenceEngine::DataConfig cfg;
 117         cfg.constant = true;
 118         // Cannot be in-place because memory will change a memory.
 119         cfg.inPlace = -1;
 120         InferenceEngine::SizeVector order;
 121         for(size_t i = 0; i < cnnLayer->outData[0]->getTensorDesc().getDims().size(); i++) {
 122             order.push_back(i);
 123         }
 124         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->outData[0]->getTensorDesc().getPrecision(),
 125                                                cnnLayer->outData[0]->getTensorDesc().getDims(),
 126                                                {cnnLayer->outData[0]->getTensorDesc().getDims(), order});
 127         config.outConfs.push_back(cfg);
 128         config.inConfs.push_back(cfg);
 129         conf.push_back(config);
 130         return InferenceEngine::OK;
 131     }
 132     InferenceEngine::StatusCode init(InferenceEngine::LayerConfig& config, InferenceEngine::ResponseDesc *resp) noexcept override {
 133         if (config.dynBatchSupport)
 134             return InferenceEngine::NOT_IMPLEMENTED;
 135         for(auto input : config.inConfs) {
 136             if (input.inPlace >= 0)
 137                 return InferenceEngine::GENERAL_ERROR;
 138             if (!input.constant)
 139                 return InferenceEngine::GENERAL_ERROR;
 140         }
 141         for(auto output : config.outConfs) {
 142             if (output.inPlace >= 0)
 143                 return InferenceEngine::GENERAL_ERROR;
 144             if (!output.constant)
 145                 return InferenceEngine::GENERAL_ERROR;
 146         }
 147         return InferenceEngine::OK;
 148     }
 149     InferenceEngine::StatusCode execute(std::vector<InferenceEngine::Blob::Ptr>& inputs, std::vector<InferenceEngine::Blob::Ptr>& outputs, InferenceEngine::ResponseDesc *resp) noexcept override {
 150         float *dst_data = outputs[0]->buffer();
 151
 152         size_t data_size = outputs[0]->size();
 153         for (size_t i = 0; i < data_size; i++) {
 154             dst_data[i] = (dst_data[i] + 1)*2;
 155         }
 156         return InferenceEngine::OK;
 157     }
 158
 159 private:
 160     InferenceEngine::CNNLayer *cnnLayer;
 161 };
 162
 163 class ConstPrimitiveFactory : public InferenceEngine::ILayerImplFactory {
 164 public:
 165     ConstPrimitiveFactory(const InferenceEngine::CNNLayer *layer) {
 166         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 167     }
 168     // set output shapes by input shapes.
 169     InferenceEngine::StatusCode getShapes(const std::vector<InferenceEngine::TensorDesc>& inShapes, std::vector<InferenceEngine::TensorDesc>& outShapes, InferenceEngine::ResponseDesc *resp) noexcept override {
 170         outShapes.push_back(inShapes[0]);
 171         return InferenceEngine::OK;
 172     }
 173     // First implementation has more priority than next
 174     InferenceEngine::StatusCode getImplementations(std::vector<InferenceEngine::ILayerImpl::Ptr>& impls, InferenceEngine::ResponseDesc *resp) noexcept override {
 175         impls.push_back(InferenceEngine::ILayerImpl::Ptr(new ConstPrimitiveImpl(cnnLayer)));
 176         return InferenceEngine::OK;
 177     }
 178
 179 private:
 180     InferenceEngine::CNNLayer * cnnLayer;
 181 };
 182
 183 class DoublePrimitiveFactory : public InferenceEngine::ILayerImplFactory {
 184 public:
 185     DoublePrimitiveFactory(const InferenceEngine::CNNLayer *layer) {
 186         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 187     }
 188     // set output shapes by input shapes.
 189     InferenceEngine::StatusCode getShapes(const std::vector<InferenceEngine::TensorDesc>& inShapes, std::vector<InferenceEngine::TensorDesc>& outShapes, InferenceEngine::ResponseDesc *resp) noexcept override {
 190         outShapes.push_back(inShapes[0]);
 191         return InferenceEngine::OK;
 192     }
 193     // First implementation has more priority than next
 194     InferenceEngine::StatusCode getImplementations(std::vector<InferenceEngine::ILayerImpl::Ptr>& impls, InferenceEngine::ResponseDesc *resp) noexcept override {
 195         impls.push_back(InferenceEngine::ILayerImpl::Ptr(new DoublePrimitiveImpl(cnnLayer)));
 196         return InferenceEngine::OK;
 197     }
 198
 199 private:
 200     InferenceEngine::CNNLayer * cnnLayer;
 201 };
 202
 203 class TwoDifferentOutputsImpl : public InferenceEngine::ILayerExecImpl {
 204 public:
 205     TwoDifferentOutputsImpl(const InferenceEngine::CNNLayer *layer) {
 206         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 207     }
 208     InferenceEngine::StatusCode getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig>& conf, InferenceEngine::ResponseDesc *resp) noexcept override {
 209         InferenceEngine::LayerConfig config;
 210         config.dynBatchSupport = 0;
 211         if (cnnLayer->outData.size() != 2 && cnnLayer->insData.size() != 1)
 212             return InferenceEngine::GENERAL_ERROR;
 213         InferenceEngine::DataConfig cfg;
 214         cfg.constant = false;
 215         cfg.inPlace = -1;
 216         InferenceEngine::SizeVector order;
 217         for(size_t i = 0; i < cnnLayer->outData[0]->getTensorDesc().getDims().size(); i++) {
 218             order.push_back(i);
 219         }
 220         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->outData[0]->getTensorDesc().getPrecision(),
 221                                                cnnLayer->outData[0]->getTensorDesc().getDims(),
 222                                                {cnnLayer->outData[0]->getTensorDesc().getDims(), order});
 223         config.outConfs.push_back(cfg);
 224         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->outData[1]->getTensorDesc().getPrecision(),
 225                                                cnnLayer->outData[1]->getTensorDesc().getDims(),
 226                                                {cnnLayer->outData[1]->getTensorDesc().getDims(), order});
 227         config.outConfs.push_back(cfg);
 228         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->insData[0].lock()->getTensorDesc().getPrecision(),
 229                               cnnLayer->insData[0].lock()->getTensorDesc().getDims(),
 230                               {cnnLayer->insData[0].lock()->getTensorDesc().getDims(), order});
 231         config.inConfs.push_back(cfg);
 232         conf.push_back(config);
 233         return InferenceEngine::OK;
 234     }
 235     InferenceEngine::StatusCode init(InferenceEngine::LayerConfig& config, InferenceEngine::ResponseDesc *resp) noexcept override {
 236         if (config.dynBatchSupport)
 237             return InferenceEngine::NOT_IMPLEMENTED;
 238         for(auto input : config.inConfs) {
 239             if (input.inPlace >= 0)
 240                 return InferenceEngine::GENERAL_ERROR;
 241             if (input.constant)
 242                 return InferenceEngine::GENERAL_ERROR;
 243         }
 244         for(auto output : config.outConfs) {
 245             if (output.inPlace >= 0)
 246                 return InferenceEngine::GENERAL_ERROR;
 247             if (output.constant)
 248                 return InferenceEngine::GENERAL_ERROR;
 249         }
 250         return InferenceEngine::OK;
 251     }
 252     InferenceEngine::StatusCode execute(std::vector<InferenceEngine::Blob::Ptr>& inputs, std::vector<InferenceEngine::Blob::Ptr>& outputs, InferenceEngine::ResponseDesc *resp) noexcept override {
 253         const float *src_data = inputs[0]->buffer();
 254         float *dst_data0 = outputs[0]->buffer();
 255         float *dst_data1 = outputs[1]->buffer();
 256
 257         size_t out_data_size0 = outputs[0]->size();
 258         size_t out_data_size1 = outputs[1]->size();
 259         for (size_t i = 0; i < out_data_size0; i++) {
 260             dst_data0[i] = (*(src_data++))*2;
 261         }
 262
 263         for (size_t i = 0; i < out_data_size1; i++) {
 264             dst_data1[i] = (*(src_data++))*3;
 265         }
 266         return InferenceEngine::OK;
 267     }
 268
 269 private:
 270     InferenceEngine::CNNLayer* cnnLayer;
 271 };
 272
 273 class TwoDifferentOutputsFactory : public InferenceEngine::ILayerImplFactory {
 274 public:
 275     TwoDifferentOutputsFactory(const InferenceEngine::CNNLayer *layer) {
 276         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 277     }
 278     // set output shapes by input shapes.
 279     InferenceEngine::StatusCode getShapes(const std::vector<InferenceEngine::TensorDesc>& inShapes, std::vector<InferenceEngine::TensorDesc>& outShapes, InferenceEngine::ResponseDesc *resp) noexcept override {
 280         return InferenceEngine::NOT_IMPLEMENTED;
 281     }
 282     // First implementation has more priority than next
 283     InferenceEngine::StatusCode getImplementations(std::vector<InferenceEngine::ILayerImpl::Ptr>& impls, InferenceEngine::ResponseDesc *resp) noexcept override {
 284         impls.push_back(InferenceEngine::ILayerImpl::Ptr(new TwoDifferentOutputsImpl(cnnLayer)));
 285         return InferenceEngine::OK;
 286     }
 287
 288 private:
 289     InferenceEngine::CNNLayer * cnnLayer;
 290 };
 291
 292 class CustomConcatImpl : public InferenceEngine::ILayerImpl {
 293 public:
 294     CustomConcatImpl(const InferenceEngine::CNNLayer *layer) {
 295         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 296     }
 297     InferenceEngine::StatusCode getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig>& conf, InferenceEngine::ResponseDesc *resp) noexcept override {
 298         InferenceEngine::LayerConfig config;
 299         config.dynBatchSupport = 0;
 300         if (cnnLayer->outData.size() != 1 && cnnLayer->insData.size() != 2)
 301             return InferenceEngine::GENERAL_ERROR;
 302         InferenceEngine::DataConfig cfg;
 303         cfg.constant = false;
 304         cfg.inPlace = -1;
 305         InferenceEngine::SizeVector order;
 306         for(size_t i = 0; i < cnnLayer->outData[0]->getTensorDesc().getDims().size(); i++) {
 307             order.push_back(i);
 308         }
 309         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->outData[0]->getTensorDesc().getPrecision(),
 310                                                cnnLayer->outData[0]->getTensorDesc().getDims(),
 311                                                {cnnLayer->outData[0]->getTensorDesc().getDims(), order});
 312         config.outConfs.push_back(cfg);
 313         cfg.inPlace = 0;
 314         InferenceEngine::SizeVector dims = cnnLayer->insData[0].lock()->getTensorDesc().getDims();
 315         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->insData[0].lock()->getTensorDesc().getPrecision(),
 316                                                dims, {dims, order});
 317         size_t dataSize = std::accumulate(std::begin(dims), std::end(dims), (size_t) 1, std::multiplies<size_t>());
 318         config.inConfs.push_back(cfg);
 319         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->insData[1].lock()->getTensorDesc().getPrecision(),
 320                                                cnnLayer->insData[1].lock()->getTensorDesc().getDims(),
 321                                                {cnnLayer->insData[1].lock()->getTensorDesc().getDims(), order,
 322                                                 dataSize});
 323         config.inConfs.push_back(cfg);
 324         conf.push_back(config);
 325         return InferenceEngine::OK;
 326     }
 327     InferenceEngine::StatusCode init(InferenceEngine::LayerConfig& config, InferenceEngine::ResponseDesc *resp) noexcept override {
 328         if (config.dynBatchSupport)
 329             return InferenceEngine::NOT_IMPLEMENTED;
 330         for(auto input : config.inConfs) {
 331             if (input.inPlace < 0)
 332                 return InferenceEngine::GENERAL_ERROR;
 333             if (input.constant)
 334                 return InferenceEngine::GENERAL_ERROR;
 335         }
 336         for(auto output : config.outConfs) {
 337             if (output.inPlace >= 0)
 338                 return InferenceEngine::GENERAL_ERROR;
 339             if (output.constant)
 340                 return InferenceEngine::GENERAL_ERROR;
 341         }
 342         return InferenceEngine::OK;
 343     }
 344 private:
 345     InferenceEngine::CNNLayer * cnnLayer;
 346 };
 347
 348 class CustomConcatFactory : public InferenceEngine::ILayerImplFactory {
 349 public:
 350     CustomConcatFactory(const InferenceEngine::CNNLayer *layer) {
 351         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 352     }
 353     // set output shapes by input shapes.
 354     InferenceEngine::StatusCode getShapes(const std::vector<InferenceEngine::TensorDesc>& inShapes, std::vector<InferenceEngine::TensorDesc>& outShapes, InferenceEngine::ResponseDesc *resp) noexcept override {
 355         return InferenceEngine::NOT_IMPLEMENTED;
 356     }
 357     // First implementation has more priority than next
 358     InferenceEngine::StatusCode getImplementations(std::vector<InferenceEngine::ILayerImpl::Ptr>& impls, InferenceEngine::ResponseDesc *resp) noexcept override {
 359         impls.push_back(InferenceEngine::ILayerImpl::Ptr(new CustomConcatImpl(cnnLayer)));
 360         return InferenceEngine::OK;
 361     }
 362
 363 private:
 364     InferenceEngine::CNNLayer * cnnLayer;
 365 };
 366
 367 class CustomSplitImpl : public InferenceEngine::ILayerImpl {
 368 public:
 369     CustomSplitImpl(const InferenceEngine::CNNLayer *layer) {
 370         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 371     }
 372     InferenceEngine::StatusCode getSupportedConfigurations(std::vector<InferenceEngine::LayerConfig>& conf, InferenceEngine::ResponseDesc *resp) noexcept override {
 373         InferenceEngine::LayerConfig config;
 374         config.dynBatchSupport = 0;
 375         if (cnnLayer->outData.size() != 2 && cnnLayer->insData.size() != 1)
 376             return InferenceEngine::GENERAL_ERROR;
 377         InferenceEngine::DataConfig cfg;
 378         cfg.constant = false;
 379         cfg.inPlace = 0;
 380         InferenceEngine::SizeVector order;
 381         for(size_t i = 0; i < cnnLayer->outData[0]->getTensorDesc().getDims().size(); i++) {
 382             order.push_back(i);
 383         }
 384         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->outData[0]->getTensorDesc().getPrecision(),
 385                                                cnnLayer->outData[0]->getTensorDesc().getDims(),
 386                                                {cnnLayer->outData[0]->getTensorDesc().getDims(), order});
 387         config.outConfs.push_back(cfg);
 388         size_t dataSize = std::accumulate(std::begin(cnnLayer->outData[0]->getTensorDesc().getDims()),
 389                                           std::end(cnnLayer->outData[0]->getTensorDesc().getDims()),
 390                                           (size_t) 1, std::multiplies<size_t>());
 391         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->outData[1]->getTensorDesc().getPrecision(),
 392                                                cnnLayer->outData[1]->getTensorDesc().getDims(),
 393                                                {cnnLayer->outData[1]->getTensorDesc().getDims(), order, dataSize});
 394         config.outConfs.push_back(cfg);
 395         cfg.inPlace = -1;
 396         cfg.desc = InferenceEngine::TensorDesc(cnnLayer->insData[0].lock()->getTensorDesc().getPrecision(),
 397                                                cnnLayer->insData[0].lock()->getTensorDesc().getDims(),
 398                                                {cnnLayer->insData[0].lock()->getTensorDesc().getDims(), order});
 399         config.inConfs.push_back(cfg);
 400         conf.push_back(config);
 401         return InferenceEngine::OK;
 402     }
 403     InferenceEngine::StatusCode init(InferenceEngine::LayerConfig& config, InferenceEngine::ResponseDesc *resp) noexcept override {
 404         if (config.dynBatchSupport)
 405             return InferenceEngine::NOT_IMPLEMENTED;
 406         for(auto input : config.inConfs) {
 407             if (!input.inPlace)
 408                 return InferenceEngine::GENERAL_ERROR;
 409             if (input.constant)
 410                 return InferenceEngine::GENERAL_ERROR;
 411         }
 412         for(auto output : config.outConfs) {
 413             if (output.constant)
 414                 return InferenceEngine::GENERAL_ERROR;
 415         }
 416         return InferenceEngine::OK;
 417     }
 418 private:
 419     InferenceEngine::CNNLayer * cnnLayer;
 420 };
 421
 422 class CustomSplitFactory : public InferenceEngine::ILayerImplFactory {
 423 public:
 424     CustomSplitFactory(const InferenceEngine::CNNLayer *layer) {
 425         cnnLayer = const_cast<InferenceEngine::CNNLayer *>(layer);
 426     }
 427     // set output shapes by input shapes.
 428     InferenceEngine::StatusCode getShapes(const std::vector<InferenceEngine::TensorDesc>& inShapes, std::vector<InferenceEngine::TensorDesc>& outShapes, InferenceEngine::ResponseDesc *resp) noexcept override {
 429         return InferenceEngine::NOT_IMPLEMENTED;
 430     }
 431     // First implementation has more priority than next
 432     InferenceEngine::StatusCode getImplementations(std::vector<InferenceEngine::ILayerImpl::Ptr>& impls, InferenceEngine::ResponseDesc *resp) noexcept override {
 433         impls.push_back(InferenceEngine::ILayerImpl::Ptr(new CustomSplitImpl(cnnLayer)));
 434         return InferenceEngine::OK;
 435     }
 436
 437 private:
 438     InferenceEngine::CNNLayer * cnnLayer;
 439 };
 440 using fake_ext_factory = std::function<InferenceEngine::ILayerImplFactory*(const InferenceEngine::CNNLayer *)>;
 441
 442 class FakeExtensionFabric : public InferenceEngine::IExtension {
 443 public:
 444     FakeExtensionFabric() {
 445         factories["CustomNewConvolution"] = [](const InferenceEngine::CNNLayer * cnnLayer) -> InferenceEngine::ILayerImplFactory* { return new FakeGenericPrimitiveFactory(); };
 446         factories["NewDoubleLayer"] = [](const InferenceEngine::CNNLayer * cnnLayer) -> InferenceEngine::ILayerImplFactory* { return new DoublePrimitiveFactory(cnnLayer); };
 447         factories["NewTwoDifferentOutputs"] = [](const InferenceEngine::CNNLayer * cnnLayer) -> InferenceEngine::ILayerImplFactory* { return new TwoDifferentOutputsFactory(cnnLayer); };
 448         factories["ConstPrim"] = [](const InferenceEngine::CNNLayer * cnnLayer) -> InferenceEngine::ILayerImplFactory* { return new ConstPrimitiveFactory(cnnLayer); };
 449         factories["CustomInPlaceConcat"] = [](const InferenceEngine::CNNLayer * cnnLayer) -> InferenceEngine::ILayerImplFactory* { return new CustomConcatFactory(cnnLayer); };
 450         factories["CustomInPlaceSplit"] = [](const InferenceEngine::CNNLayer * cnnLayer) -> InferenceEngine::ILayerImplFactory* { return new CustomSplitFactory(cnnLayer); };
 451     }
 452
 453     virtual ~FakeExtensionFabric() {
 454         factories.clear();
 455     }
 456
 457     void GetVersion(const InferenceEngine::Version *&versionInfo) const noexcept override {}
 458     void SetLogCallback(InferenceEngine::IErrorListener &listener) noexcept override {}
 459     void Unload() noexcept override {}
 460     void Release() noexcept override {
 461         delete this;
 462     }
 463     InferenceEngine::StatusCode getPrimitiveTypes(char**& types, unsigned int& size, InferenceEngine::ResponseDesc* resp) noexcept override {
 464         types = new char *[factories.size()];
 465         size_t count = 0;
 466         for (auto it = factories.begin(); it != factories.end(); it++, count ++) {
 467             types[count] = new char[it->first.size() + 1];
 468             std::copy(it->first.begin(), it->first.end(), types[count]);
 469             types[count][it->first.size() ] = '\0';
 470         }
 471         return InferenceEngine::OK;
 472     };
 473     InferenceEngine::StatusCode getFactoryFor(InferenceEngine::ILayerImplFactory *&factory,
 474                                               const InferenceEngine::CNNLayer *cnnLayer,
 475                                               InferenceEngine::ResponseDesc *resp) noexcept override {
 476         if (factories.find(cnnLayer->type) == factories.end()) {
 477             std::string errorMsg = std::string("Factory for ") + cnnLayer->type + " wasn't found!";
 478             errorMsg.copy(resp->msg, sizeof(resp->msg) - 1);
 479             return InferenceEngine::NOT_FOUND;
 480         }
 481         factory = factories[cnnLayer->type](cnnLayer);
 482         return InferenceEngine::OK;
 483     }
 484
 485     InferenceEngine::StatusCode getShapeInferImpl(InferenceEngine::IShapeInferImpl::Ptr& impl, const char* type,
 486                                                   InferenceEngine::ResponseDesc* resp) noexcept override {
 487         return InferenceEngine::NOT_IMPLEMENTED;
 488     }
 489
 490 private:
 491     std::map<std::string, fake_ext_factory> factories;
 492 };
 493
 494 class MKLDNNGraphGenericTests: public TestsCommon {
 495 protected:
 496     virtual void SetUp() {
 497         TestsCommon::SetUp();
 498         extension.reset(new FakeExtensionFabric());
 499     }
 500     std::shared_ptr<InferenceEngine::IExtension> extension;
 501 };
 502
 503 TEST_F(MKLDNNGraphGenericTests, canGetPrimitiveDescriptorsList) {
 504     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
 505     extMgr->AddExtension(extension);
 506     std::shared_ptr<MKLDNNPlugin::MKLDNNNode> node;
 507     InferenceEngine::DataPtr dataPtr;
 508     dataPtr.reset(new InferenceEngine::Data("test", {1, 3, 4, 5}, InferenceEngine::Precision::FP32, InferenceEngine::Layout::NCHW));
 509     InferenceEngine::CNNLayerPtr layerPtr;
 510     layerPtr.reset(new InferenceEngine::CNNLayer({"name", "CustomNewConvolution", InferenceEngine::Precision::FP32}));
 511     layerPtr->outData.push_back(dataPtr);
 512
 513     mkldnn::engine eng(mkldnn::engine(mkldnn::engine::kind::cpu, 0));
 514     node.reset(MKLDNNPlugin::MKLDNNNode::CreateNode(layerPtr, eng, extMgr));
 515     ASSERT_EQ(MKLDNNPlugin::Type::Generic, node->getType());
 516
 517     ASSERT_NO_THROW(node->getSupportedDescriptors());
 518 }
 519
 520 template <typename data_t>
 521 void ref_double(const InferenceEngine::TBlob<data_t> &src, InferenceEngine::TBlob<data_t> &dst) {
 522     const data_t *src_data = src.readOnly();
 523     data_t *dst_data = dst.data();
 524
 525     for (int i=0; i < src.size(); i++)
 526         dst_data[i] = src_data[i]*2;
 527 }
 528
 529 template <typename data_t>
 530 void ref_double_batch1(const InferenceEngine::TBlob<data_t> &src, InferenceEngine::TBlob<data_t> &dst) {
 531     const data_t *src_data = src.readOnly();
 532     data_t *dst_data = dst.data();
 533
 534     for (int i= 0; i < src.size() / 2; i++)
 535         dst_data[i] = src_data[i]*2;
 536
 537     for (int i= src.size() / 2; i < src.size(); i++)
 538         dst_data[i] = 0;
 539 }
 540
 541 template <typename data_t>
 542 void ref_twoDifferent(const InferenceEngine::TBlob<data_t> &src, InferenceEngine::TBlob<data_t> &dst1, InferenceEngine::TBlob<data_t> &dst2) {
 543     const data_t *src_data = src.readOnly();
 544     data_t *dst_data1 = dst1.data();
 545     data_t *dst_data2 = dst2.data();
 546
 547     for (int i=0; i < dst1.size(); i++)
 548         dst_data1[i] = (*(src_data++))*2;
 549
 550     for (int i=0; i < dst2.size(); i++)
 551         dst_data2[i] = (*(src_data++))*6;
 552 }
 553
 554 TEST_F(MKLDNNGraphGenericTests, DontCreateGPUGenericPrimitive) {
 555     std::string model = R"V0G0N(
 556         <Net Name="DoubleLayer_Only" version="2" precision="FP32" batch="1">
 557             <layers>
 558                 <layer name="in1" type="Input" precision="FP32" id="0">
 559                     <output>
 560                         <port id="0">
 561                             <dim>1</dim>
 562                             <dim>3</dim>
 563                             <dim>5</dim>
 564                             <dim>5</dim>
 565                         </port>
 566                     </output>
 567                 </layer>
 568                 <layer name="gpulayer" id="1" type="CustomGPUConvolution" precision="FP32">
 569                     <input>
 570                         <port id="1">
 571                             <dim>1</dim>
 572                             <dim>3</dim>
 573                             <dim>5</dim>
 574                             <dim>5</dim>
 575                         </port>
 576                     </input>
 577                     <output>
 578                         <port id="2">
 579                             <dim>1</dim>
 580                             <dim>3</dim>
 581                             <dim>5</dim>
 582                             <dim>5</dim>
 583                         </port>
 584                     </output>
 585                 </layer>
 586             </layers>
 587             <edges>
 588                 <edge from-layer="0" from-port="0" to-layer="1" to-port="1"/>
 589             </edges>
 590         </Net>
 591         )V0G0N";
 592     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
 593     extMgr->AddExtension(extension);
 594
 595     InferenceEngine::CNNNetReader net_reader;
 596     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
 597
 598     MKLDNNGraphTestClass graph;
 599     ASSERT_THROW(graph.CreateGraph(net_reader.getNetwork(), extMgr), InferenceEngine::details::InferenceEngineException);
 600 }
 601
 602 TEST_F(MKLDNNGraphGenericTests, ExecuteConstGenericPrimitive) {
 603     std::string model = R"V0G0N(
 604         <Net Name="DoubleLayer_Only" version="2" precision="FP32" batch="1">
 605             <layers>
 606                 <layer name="in1" type="Input" precision="FP32" id="0">
 607                     <output>
 608                         <port id="0">
 609                             <dim>1</dim>
 610                             <dim>3</dim>
 611                             <dim>5</dim>
 612                             <dim>5</dim>
 613                         </port>
 614                     </output>
 615                 </layer>
 616                 <layer name="const_layer" id="1" type="ConstPrim" precision="FP32">
 617                     <input>
 618                         <port id="1">
 619                             <dim>1</dim>
 620                             <dim>3</dim>
 621                             <dim>5</dim>
 622                             <dim>5</dim>
 623                         </port>
 624                     </input>
 625                     <output>
 626                         <port id="2">
 627                             <dim>1</dim>
 628                             <dim>3</dim>
 629                             <dim>5</dim>
 630                             <dim>5</dim>
 631                         </port>
 632                     </output>
 633                 </layer>
 634             </layers>
 635             <edges>
 636                 <edge from-layer="0" from-port="0" to-layer="1" to-port="1"/>
 637             </edges>
 638         </Net>
 639         )V0G0N";
 640     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
 641     extMgr->AddExtension(extension);
 642
 643     InferenceEngine::CNNNetReader net_reader;
 644     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
 645
 646     MKLDNNGraphTestClass graph;
 647     graph.CreateGraph(net_reader.getNetwork(), extMgr);
 648
 649     InferenceEngine::SizeVector dims_src = {1, 3, 5, 5};
 650
 651     InferenceEngine::Blob::Ptr src =
 652             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src);
 653     src->allocate();
 654     fill_data(src->buffer(), src->size());
 655
 656     InferenceEngine::TBlob<float>* srcPtr = dynamic_cast<InferenceEngine::TBlob<float>*>(src.get());
 657
 658     if (srcPtr == nullptr)
 659         FAIL() << "Cannot cast blob to TBlob<float>.";
 660
 661     InferenceEngine::BlobMap srcs;
 662     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in1", src));
 663
 664     InferenceEngine::OutputsDataMap out;
 665     out = net_reader.getNetwork().getOutputsInfo();
 666     InferenceEngine::BlobMap outputBlobs;
 667
 668     std::pair<std::string, InferenceEngine::DataPtr> item = *out.begin();
 669
 670     InferenceEngine::TBlob<float>::Ptr output;
 671     output = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc());
 672     output->allocate();
 673     outputBlobs[item.first] = output;
 674
 675     graph.Infer(srcs, outputBlobs);
 676     graph.Infer(srcs, outputBlobs);
 677
 678     InferenceEngine::TBlob<float> dst_ref(item.second->getTensorDesc());
 679     dst_ref.allocate();
 680
 681     float * dst_data = dst_ref.buffer();
 682     for (size_t i = 0; i < dst_ref.size(); i++) {
 683         dst_data[i] = 2;
 684     }
 685
 686     compare(*output, dst_ref);
 687 }
 688
 689 TEST_F(MKLDNNGraphGenericTests, ExecuteGenericPrimitive) {
 690     std::string model = R"V0G0N(
 691         <Net Name="DoubleLayer_Only" version="2" precision="FP32" batch="1">
 692             <layers>
 693                 <layer name="in1" type="Input" precision="FP32" id="0">
 694                     <output>
 695                         <port id="0">
 696                             <dim>1</dim>
 697                             <dim>3</dim>
 698                             <dim>5</dim>
 699                             <dim>5</dim>
 700                         </port>
 701                     </output>
 702                 </layer>
 703                 <layer name="double_layer" id="1" type="NewDoubleLayer" precision="FP32">
 704                     <input>
 705                         <port id="1">
 706                             <dim>1</dim>
 707                             <dim>3</dim>
 708                             <dim>5</dim>
 709                             <dim>5</dim>
 710                         </port>
 711                     </input>
 712                     <output>
 713                         <port id="2">
 714                             <dim>1</dim>
 715                             <dim>3</dim>
 716                             <dim>5</dim>
 717                             <dim>5</dim>
 718                         </port>
 719                     </output>
 720                 </layer>
 721             </layers>
 722             <edges>
 723                 <edge from-layer="0" from-port="0" to-layer="1" to-port="1"/>
 724             </edges>
 725         </Net>
 726         )V0G0N";
 727     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
 728     extMgr->AddExtension(extension);
 729
 730     InferenceEngine::CNNNetReader net_reader;
 731     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
 732
 733     MKLDNNGraphTestClass graph;
 734     graph.CreateGraph(net_reader.getNetwork(), extMgr);
 735
 736     InferenceEngine::SizeVector dims_src = {1, 3, 5, 5};
 737
 738     InferenceEngine::Blob::Ptr src =
 739            InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src);
 740     src->allocate();
 741     fill_data(src->buffer(), src->size());
 742
 743     InferenceEngine::TBlob<float>* srcPtr = dynamic_cast<InferenceEngine::TBlob<float>*>(src.get());
 744
 745     if (srcPtr == nullptr)
 746         FAIL() << "Cannot cast blob to TBlob<float>.";
 747
 748     InferenceEngine::BlobMap srcs;
 749     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in1", src));
 750
 751     InferenceEngine::OutputsDataMap out;
 752     out = net_reader.getNetwork().getOutputsInfo();
 753     InferenceEngine::BlobMap outputBlobs;
 754
 755     std::pair<std::string, InferenceEngine::DataPtr> item = *out.begin();
 756
 757     InferenceEngine::TBlob<float>::Ptr output;
 758     output = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc());
 759     output->allocate();
 760     outputBlobs[item.first] = output;
 761
 762     graph.Infer(srcs, outputBlobs);
 763
 764     InferenceEngine::TBlob<float> dst_ref(item.second->getTensorDesc());
 765     dst_ref.allocate();
 766
 767     ref_double(*srcPtr, dst_ref);
 768
 769     compare(*output, dst_ref);
 770 }
 771
 772 TEST_F(MKLDNNGraphGenericTests, ExecuteGenericPrimitiveWithTwoOutputs) {
 773     std::string model = R"V0G0N(
 774         <Net Name="DoubleLayer_Only" version="2" precision="FP32" batch="1">
 775             <layers>
 776                 <layer name="in1" type="Input" precision="FP32" id="0">
 777                     <output>
 778                         <port id="0">
 779                             <dim>1</dim>
 780                             <dim>3</dim>
 781                             <dim>5</dim>
 782                             <dim>5</dim>
 783                         </port>
 784                     </output>
 785                 </layer>
 786                 <layer name="two_diff_layer" id="1" type="NewTwoDifferentOutputs" precision="FP32">
 787                     <input>
 788                         <port id="1">
 789                             <dim>1</dim>
 790                             <dim>3</dim>
 791                             <dim>5</dim>
 792                             <dim>5</dim>
 793                         </port>
 794                     </input>
 795                     <output>
 796                         <port id="2">
 797                             <dim>1</dim>
 798                             <dim>1</dim>
 799                             <dim>5</dim>
 800                             <dim>5</dim>
 801                         </port>
 802                         <port id="3">
 803                             <dim>1</dim>
 804                             <dim>2</dim>
 805                             <dim>5</dim>
 806                             <dim>5</dim>
 807                         </port>
 808                     </output>
 809                 </layer>
 810                 <layer name="power" id="3" type="Power" precision="FP32">
 811                     <power_data power="1" scale="2" shift="0"/>
 812                     <input>
 813                         <port id="1">
 814                             <dim>1</dim>
 815                             <dim>2</dim>
 816                             <dim>5</dim>
 817                             <dim>5</dim>
 818                         </port>
 819                     </input>
 820                     <output>
 821                         <port id="2">
 822                             <dim>1</dim>
 823                             <dim>2</dim>
 824                             <dim>5</dim>
 825                             <dim>5</dim>
 826                         </port>
 827                     </output>
 828                 </layer>
 829                 <layer name="con" id="2" type="Concat" precision="FP32">
 830                     <concat_data axis="1"/>
 831                     <input>
 832                         <port id="4">
 833                             <dim>1</dim>
 834                             <dim>1</dim>
 835                             <dim>5</dim>
 836                             <dim>5</dim>
 837                         </port>
 838                         <port id="5">
 839                             <dim>1</dim>
 840                             <dim>2</dim>
 841                             <dim>5</dim>
 842                             <dim>5</dim>
 843                         </port>
 844                     </input>
 845                     <output>
 846                         <port id="6">
 847                             <dim>1</dim>
 848                             <dim>3</dim>
 849                             <dim>5</dim>
 850                             <dim>5</dim>
 851                         </port>
 852                     </output>
 853                 </layer>
 854             </layers>
 855             <edges>
 856                 <edge from-layer="0" from-port="0" to-layer="1" to-port="1"/>
 857                 <edge from-layer="1" from-port="2" to-layer="2" to-port="4"/>
 858                 <edge from-layer="1" from-port="3" to-layer="3" to-port="1"/>
 859                 <edge from-layer="3" from-port="2" to-layer="2" to-port="5"/>
 860             </edges>
 861         </Net>
 862         )V0G0N";
 863     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
 864     extMgr->AddExtension(extension);
 865
 866     InferenceEngine::CNNNetReader net_reader;
 867     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
 868
 869     MKLDNNGraphTestClass graph;
 870     graph.CreateGraph(net_reader.getNetwork(), extMgr);
 871
 872     InferenceEngine::SizeVector dims_src = {1, 3, 5, 5};
 873
 874     InferenceEngine::Blob::Ptr src =
 875             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src);
 876     src->allocate();
 877
 878     float * data_src = src->buffer();
 879     for (size_t i = 0; i < src->size(); i++)
 880         data_src[i] = 1;
 881     auto * srcPtr = dynamic_cast<InferenceEngine::TBlob<float>*>(src.get());
 882
 883     if (srcPtr == nullptr)
 884         FAIL() << "Cannot cast blob to TBlob<float>.";
 885
 886     InferenceEngine::BlobMap srcs;
 887     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in1", src));
 888
 889     InferenceEngine::OutputsDataMap out;
 890     out = net_reader.getNetwork().getOutputsInfo();
 891     InferenceEngine::BlobMap outputBlobs;
 892
 893     auto it = out.begin();
 894
 895     std::pair<std::string, InferenceEngine::DataPtr> item = *it;
 896     InferenceEngine::DataPtr data1 = item.second;
 897
 898     InferenceEngine::TensorDesc outputDesc1 = item.second->getTensorDesc();
 899     InferenceEngine::TBlob<float>::Ptr output1;
 900     output1 = InferenceEngine::make_shared_blob<float>(outputDesc1);
 901     output1->allocate();
 902     outputBlobs[item.first] = output1;
 903
 904     graph.Infer(srcs, outputBlobs);
 905
 906     float * data = outputBlobs.begin()->second->buffer();
 907     for (size_t i = 0; i < 25; i++) {
 908         ASSERT_EQ(*data, 2);
 909         data++;
 910     }
 911     for (size_t i = 0; i < 50; i++) {
 912         ASSERT_EQ(*data, 6);
 913         data++;
 914     }
 915 }
 916
 917 TEST_F(MKLDNNGraphGenericTests, ExecuteGenericInPlaceConcat) {
 918     std::string model = R"V0G0N(
 919         <Net Name="CustomConcat_Only" version="2" precision="FP32" batch="1">
 920             <layers>
 921                 <layer name="in1" type="Input" precision="FP32" id="0">
 922                     <output>
 923                         <port id="0">
 924                             <dim>1</dim>
 925                             <dim>3</dim>
 926                             <dim>5</dim>
 927                             <dim>5</dim>
 928                         </port>
 929                     </output>
 930                 </layer>
 931                 <layer name="in2" type="Input" precision="FP32" id="1">
 932                     <output>
 933                         <port id="0">
 934                             <dim>1</dim>
 935                             <dim>2</dim>
 936                             <dim>5</dim>
 937                             <dim>5</dim>
 938                         </port>
 939                     </output>
 940                 </layer>
 941                 <layer name="con" id="2" type="CustomInPlaceConcat" precision="FP32">
 942                     <input>
 943                         <port id="1">
 944                             <dim>1</dim>
 945                             <dim>3</dim>
 946                             <dim>5</dim>
 947                             <dim>5</dim>
 948                         </port>
 949                         <port id="2">
 950                             <dim>1</dim>
 951                             <dim>2</dim>
 952                             <dim>5</dim>
 953                             <dim>5</dim>
 954                         </port>
 955                     </input>
 956                     <output>
 957                         <port id="3">
 958                             <dim>1</dim>
 959                             <dim>5</dim>
 960                             <dim>5</dim>
 961                             <dim>5</dim>
 962                         </port>
 963                     </output>
 964                 </layer>
 965             </layers>
 966             <edges>
 967                 <edge from-layer="0" from-port="0" to-layer="2" to-port="1"/>
 968                 <edge from-layer="1" from-port="0" to-layer="2" to-port="2"/>
 969             </edges>
 970         </Net>
 971         )V0G0N";
 972     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
 973     extMgr->AddExtension(extension);
 974
 975     InferenceEngine::CNNNetReader net_reader;
 976     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
 977
 978     MKLDNNGraphTestClass graph;
 979     graph.CreateGraph(net_reader.getNetwork(), extMgr);
 980
 981     InferenceEngine::SizeVector dims_src1 = {1, 3, 5, 5};
 982
 983     InferenceEngine::Blob::Ptr src1 =
 984             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src1);
 985     src1->allocate();
 986
 987     float * data_src1 = src1->buffer();
 988     for (size_t i = 0; i < src1->size(); i++)
 989         data_src1[i] = 1;
 990
 991     InferenceEngine::SizeVector dims_src2 = {1, 2, 5, 5};
 992
 993     InferenceEngine::Blob::Ptr src2 =
 994             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src2);
 995     src2->allocate();
 996
 997     float * data_src2 = src2->buffer();
 998     for (size_t i = 0; i < src2->size(); i++)
 999         data_src2[i] = 2;
1000
1001     InferenceEngine::BlobMap srcs;
1002     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in1", src1));
1003     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in2", src2));
1004
1005     InferenceEngine::OutputsDataMap out;
1006     out = net_reader.getNetwork().getOutputsInfo();
1007     InferenceEngine::BlobMap outputBlobs;
1008
1009     auto it = out.begin();
1010
1011     std::pair<std::string, InferenceEngine::DataPtr> item = *it;
1012
1013     InferenceEngine::TensorDesc outputDesc1 = item.second->getTensorDesc();
1014     InferenceEngine::TBlob<float>::Ptr output1;
1015     output1 = InferenceEngine::make_shared_blob<float>(outputDesc1);
1016     output1->allocate();
1017     outputBlobs[item.first] = output1;
1018
1019     graph.Infer(srcs, outputBlobs);
1020
1021     float * data = outputBlobs.begin()->second->buffer();
1022     for (size_t i = 0; i < 75; i++) {
1023         ASSERT_EQ(*data, 1);
1024         data++;
1025     }
1026     for (size_t i = 0; i < 50; i++) {
1027         ASSERT_EQ(*data, 2);
1028         data++;
1029     }
1030 }
1031
1032 TEST_F(MKLDNNGraphGenericTests, ExecuteGenericInPlaceSplit) {
1033     std::string model = R"V0G0N(
1034         <net name="ConcatOnly" version="2" precision="FP32" batch="1">
1035             <layers>
1036                 <layer name="in1" type="Input" precision="FP32" id="1">
1037                     <output>
1038                         <port id="1">
1039                             <dim>1</dim>
1040                             <dim>4</dim>
1041                             <dim>4</dim>
1042                             <dim>4</dim>
1043                         </port>
1044                     </output>
1045                 </layer>
1046                 <layer name="split" id="2" type="CustomInPlaceSplit" precision="FP32">
1047                     <input>
1048                         <port id="1">
1049                             <dim>1</dim>
1050                             <dim>4</dim>
1051                             <dim>4</dim>
1052                             <dim>4</dim>
1053                         </port>
1054                     </input>
1055                     <output>
1056                         <port id="2">
1057                             <dim>1</dim>
1058                             <dim>2</dim>
1059                             <dim>4</dim>
1060                             <dim>4</dim>
1061                         </port>
1062                         <port id="3">
1063                             <dim>1</dim>
1064                             <dim>2</dim>
1065                             <dim>4</dim>
1066                             <dim>4</dim>
1067                         </port>
1068                     </output>
1069                 </layer>
1070                 <layer name="power1" id="3" type="Power" precision="FP32">
1071                     <power_data power="1" scale="1" shift="3"/>
1072                     <input>
1073                         <port id="1">
1074                             <dim>1</dim>
1075                             <dim>2</dim>
1076                             <dim>4</dim>
1077                             <dim>4</dim>
1078                         </port>
1079                     </input>
1080                     <output>
1081                         <port id="2">
1082                             <dim>1</dim>
1083                             <dim>2</dim>
1084                             <dim>4</dim>
1085                             <dim>4</dim>
1086                         </port>
1087                     </output>
1088                 </layer>
1089                 <layer name="power2" id="4" type="Power" precision="FP32">
1090                     <power_data power="1" scale="1" shift="2"/>
1091                     <input>
1092                         <port id="1">
1093                             <dim>1</dim>
1094                             <dim>2</dim>
1095                             <dim>4</dim>
1096                             <dim>4</dim>
1097                         </port>
1098                     </input>
1099                     <output>
1100                         <port id="2">
1101                             <dim>1</dim>
1102                             <dim>2</dim>
1103                             <dim>4</dim>
1104                             <dim>4</dim>
1105                         </port>
1106                     </output>
1107                 </layer>
1108             </layers>
1109             <edges>
1110                 <edge from-layer="1" from-port="1" to-layer="2" to-port="1"/>
1111                 <edge from-layer="2" from-port="2" to-layer="3" to-port="1"/>
1112                 <edge from-layer="2" from-port="3" to-layer="4" to-port="1"/>
1113             </edges>
1114         </net>
1115         )V0G0N";
1116     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
1117     extMgr->AddExtension(extension);
1118
1119     InferenceEngine::CNNNetReader net_reader;
1120     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
1121
1122     MKLDNNGraphTestClass graph;
1123     graph.CreateGraph(net_reader.getNetwork(), extMgr);
1124
1125     InferenceEngine::SizeVector dims_src = {1, 4, 4, 4};
1126
1127     InferenceEngine::Blob::Ptr src =
1128             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src);
1129     src->allocate();
1130
1131     float * data_src = src->buffer();
1132     for (size_t i = 0; i < src->size(); i++) {
1133         if (i < src->size() / 2)
1134             data_src[i] = 1;
1135         else
1136             data_src[i] = 2;
1137     }
1138
1139     InferenceEngine::BlobMap srcs;
1140     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in1", src));
1141
1142     InferenceEngine::OutputsDataMap out;
1143     out = net_reader.getNetwork().getOutputsInfo();
1144     InferenceEngine::BlobMap outputBlobs;
1145     auto it = out.begin();
1146
1147     std::pair<std::string, InferenceEngine::DataPtr> item = *it;
1148
1149     InferenceEngine::TBlob<float>::Ptr output1;
1150     output1 = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc());
1151     output1->allocate();
1152     outputBlobs[item.first] = output1;
1153
1154     item = *(++it);
1155     InferenceEngine::TBlob<float>::Ptr output2;
1156     output2 = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc());
1157     output2->allocate();
1158     outputBlobs[item.first] = output2;
1159
1160     graph.Infer(srcs, outputBlobs);
1161
1162     float * data = output1->buffer();
1163     for (size_t i = 0; i < output1->size(); i++) {
1164         ASSERT_EQ(*data, 4);
1165         data++;
1166     }
1167     data = output2->buffer();
1168     for (size_t i = 0; i < output2->size(); i++) {
1169         ASSERT_EQ(*data, 4);
1170         data++;
1171     }
1172 }
1173
1174 TEST_F(MKLDNNGraphGenericTests, ExecuteGenericPrimitiveWithDynamicBatch) {
1175     std::string model = R"V0G0N(
1176         <Net Name="DoubleLayer_Only" version="2" precision="FP32" batch="2">
1177             <layers>
1178                 <layer name="in1" type="Input" precision="FP32" id="0">
1179                     <output>
1180                         <port id="0">
1181                             <dim>2</dim>
1182                             <dim>3</dim>
1183                             <dim>5</dim>
1184                             <dim>5</dim>
1185                         </port>
1186                     </output>
1187                 </layer>
1188                 <layer name="double_layer" id="1" type="NewDoubleLayer" precision="FP32">
1189                     <input>
1190                         <port id="1">
1191                             <dim>2</dim>
1192                             <dim>3</dim>
1193                             <dim>5</dim>
1194                             <dim>5</dim>
1195                         </port>
1196                     </input>
1197                     <output>
1198                         <port id="2">
1199                             <dim>2</dim>
1200                             <dim>3</dim>
1201                             <dim>5</dim>
1202                             <dim>5</dim>
1203                         </port>
1204                     </output>
1205                 </layer>
1206             </layers>
1207             <edges>
1208                 <edge from-layer="0" from-port="0" to-layer="1" to-port="1"/>
1209             </edges>
1210         </Net>
1211         )V0G0N";
1212     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
1213     extMgr->AddExtension(extension);
1214
1215     InferenceEngine::CNNNetReader net_reader;
1216     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
1217
1218     MKLDNNGraphTestClass graph;
1219     graph.CreateGraph(net_reader.getNetwork(), extMgr);
1220
1221     InferenceEngine::SizeVector dims_src = {2, 3, 5, 5};
1222
1223     InferenceEngine::Blob::Ptr src =
1224             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src);
1225     src->allocate();
1226     fill_data(src->buffer(), src->size());
1227
1228     InferenceEngine::TBlob<float>* srcPtr = dynamic_cast<InferenceEngine::TBlob<float>*>(src.get());
1229
1230     if (srcPtr == nullptr)
1231         FAIL() << "Cannot cast blob to TBlob<float>.";
1232
1233     InferenceEngine::BlobMap srcs;
1234     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("in1", src));
1235
1236     InferenceEngine::OutputsDataMap out;
1237     out = net_reader.getNetwork().getOutputsInfo();
1238     InferenceEngine::BlobMap outputBlobs;
1239
1240     std::pair<std::string, InferenceEngine::DataPtr> item = *out.begin();
1241
1242     InferenceEngine::TBlob<float>::Ptr output;
1243     output = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc());
1244     output->allocate();
1245     outputBlobs[item.first] = output;
1246
1247     float *dstData = output->data();
1248
1249     for (size_t i = 0; i < output->size(); i++) {
1250         dstData[i] = 0;
1251     }
1252
1253     graph.Infer(srcs, outputBlobs);
1254
1255     InferenceEngine::TBlob<float> dst_ref(item.second->getTensorDesc());
1256     dst_ref.allocate();
1257
1258     ref_double(*srcPtr, dst_ref);
1259
1260     compare(*output, dst_ref);
1261
1262     graph.setProperty({{InferenceEngine::PluginConfigParams::KEY_DYN_BATCH_LIMIT, "1"}});
1263
1264     for (size_t i = 0; i < output->size(); i++) {
1265         dstData[i] = 0;
1266     }
1267
1268     graph.Infer(srcs, outputBlobs);
1269
1270     InferenceEngine::TBlob<float> dst_ref2(item.second->getTensorDesc());
1271     dst_ref2.allocate();
1272
1273     ref_double_batch1(*srcPtr, dst_ref2);
1274
1275     compare(*output, dst_ref2);
1276 }
1277
1278 TEST_F(MKLDNNGraphGenericTests, ExecuteNotInLineGRN) {
1279     std::string model = R"V0G0N(
1280 <net name="default" version="2" batch="1">
1281     <layers>
1282         <layer name="data" type="Input" precision="FP32" id="0">
1283             <output>
1284                 <port id="0">
1285                     <dim>1</dim>
1286                     <dim>3</dim>
1287                     <dim>2</dim>
1288                     <dim>2</dim>
1289                 </port>
1290             </output>
1291         </layer>
1292         <layer name="norm_8x_grn" type="GRN" precision="FP32" id="1">
1293             <data bias="1"/>
1294             <input>
1295                 <port id="1">
1296                     <dim>1</dim>
1297                     <dim>3</dim>
1298                     <dim>2</dim>
1299                     <dim>2</dim>
1300                 </port>
1301             </input>
1302             <output>
1303                 <port id="2">
1304                     <dim>1</dim>
1305                     <dim>3</dim>
1306                     <dim>2</dim>
1307                     <dim>2</dim>
1308                 </port>
1309             </output>
1310         </layer>
1311         <layer name="norm_4x_grn" type="GRN" precision="FP32" id="2">
1312             <data bias="1"/>
1313             <input>
1314                 <port id="3">
1315                     <dim>1</dim>
1316                     <dim>3</dim>
1317                     <dim>2</dim>
1318                     <dim>2</dim>
1319                 </port>
1320             </input>
1321             <output>
1322                 <port id="4">
1323                     <dim>1</dim>
1324                     <dim>3</dim>
1325                     <dim>2</dim>
1326                     <dim>2</dim>
1327                 </port>
1328             </output>
1329         </layer>
1330         <layer name="merge_4x_8x_concat" type="Concat" precision="FP32" id="3">
1331             <concat_data axis="1"/>
1332             <input>
1333                 <port id="5">
1334                     <dim>1</dim>
1335                     <dim>3</dim>
1336                     <dim>2</dim>
1337                     <dim>2</dim>
1338                 </port>
1339                 <port id="6">
1340                     <dim>1</dim>
1341                     <dim>3</dim>
1342                     <dim>2</dim>
1343                     <dim>2</dim>
1344                 </port>
1345             </input>
1346             <output>
1347                 <port id="7">
1348                     <dim>1</dim>
1349                     <dim>6</dim>
1350                     <dim>2</dim>
1351                     <dim>2</dim>
1352                 </port>
1353             </output>
1354         </layer>
1355     </layers>
1356     <edges>
1357         <edge from-layer="0" from-port="0" to-layer="1" to-port="1"/>
1358         <edge from-layer="0" from-port="0" to-layer="2" to-port="3"/>
1359         <edge from-layer="1" from-port="2" to-layer="3" to-port="5"/>
1360         <edge from-layer="2" from-port="4" to-layer="3" to-port="6"/>
1361     </edges>
1362 </net>)V0G0N";
1363     InferenceEngine::Extension cpuExt(make_so_name("cpu_extension"));
1364     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
1365
1366     extMgr->AddExtension(InferenceEngine::IExtensionPtr(&cpuExt, [](InferenceEngine::IExtension*){}));
1367
1368     InferenceEngine::CNNNetReader net_reader;
1369     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
1370
1371     MKLDNNGraphTestClass graph;
1372     graph.CreateGraph(net_reader.getNetwork(), extMgr);
1373
1374     InferenceEngine::SizeVector dims_src = {1, 3, 2, 2};
1375
1376     InferenceEngine::Blob::Ptr src =
1377             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src);
1378     src->allocate();
1379     fill_data(src->buffer(), src->size());
1380
1381     InferenceEngine::TBlob<float>* srcPtr = dynamic_cast<InferenceEngine::TBlob<float>*>(src.get());
1382
1383     if (srcPtr == nullptr)
1384         FAIL() << "Cannot cast blob to TBlob<float>.";
1385
1386     InferenceEngine::BlobMap srcs;
1387     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("data", src));
1388
1389     InferenceEngine::OutputsDataMap out;
1390     out = net_reader.getNetwork().getOutputsInfo();
1391     InferenceEngine::BlobMap outputBlobs;
1392
1393     std::pair<std::string, InferenceEngine::DataPtr> item = *out.begin();
1394
1395     InferenceEngine::TBlob<float>::Ptr output;
1396     output = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc());
1397     output->allocate();
1398     outputBlobs[item.first] = output;
1399
1400     graph.Infer(srcs, outputBlobs);
1401
1402     std::vector<float> refDst = {0.000f, 0.503f, 0.659f, 0.117f, -0.474f, -0.573f, -0.202f, 0.545f, 0.619f, 0.246f,
1403                                  0.000f, 0.000f, 0.000f, 0.503f, 0.659f, 0.117f, -0.474f, -0.573f, -0.202f, 0.545f,
1404                                  0.619f, 0.246f, 0.000f, 0.000f};
1405
1406     InferenceEngine::TBlob<float>::Ptr dstOut = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc(), refDst.data());
1407
1408     compare(*output, *dstOut);
1409 }
1410
1411 TEST_F(MKLDNNGraphGenericTests, ExecuteInLineGRN) {
1412     std::string model = R"V0G0N(
1413 <net name="default" version="2" batch="1">
1414     <layers>
1415         <layer name="data1" type="Input" precision="FP32" id="0">
1416             <output>
1417                 <port id="0">
1418                     <dim>1</dim>
1419                     <dim>3</dim>
1420                     <dim>2</dim>
1421                     <dim>2</dim>
1422                 </port>
1423             </output>
1424         </layer>
1425         <layer name="data2" type="Input" precision="FP32" id="1">
1426             <output>
1427                 <port id="0">
1428                     <dim>1</dim>
1429                     <dim>3</dim>
1430                     <dim>2</dim>
1431                     <dim>2</dim>
1432                 </port>
1433             </output>
1434         </layer>
1435         <layer name="norm_8x_grn" type="GRN" precision="FP32" id="2">
1436             <data bias="1"/>
1437             <input>
1438                 <port id="1">
1439                     <dim>1</dim>
1440                     <dim>3</dim>
1441                     <dim>2</dim>
1442                     <dim>2</dim>
1443                 </port>
1444             </input>
1445             <output>
1446                 <port id="2">
1447                     <dim>1</dim>
1448                     <dim>3</dim>
1449                     <dim>2</dim>
1450                     <dim>2</dim>
1451                 </port>
1452             </output>
1453         </layer>
1454         <layer name="norm_4x_grn" type="GRN" precision="FP32" id="3">
1455             <data bias="1"/>
1456             <input>
1457                 <port id="3">
1458                     <dim>1</dim>
1459                     <dim>3</dim>
1460                     <dim>2</dim>
1461                     <dim>2</dim>
1462                 </port>
1463             </input>
1464             <output>
1465                 <port id="4">
1466                     <dim>1</dim>
1467                     <dim>3</dim>
1468                     <dim>2</dim>
1469                     <dim>2</dim>
1470                 </port>
1471             </output>
1472         </layer>
1473         <layer name="merge_4x_8x_concat" type="Concat" precision="FP32" id="4">
1474             <concat_data axis="1"/>
1475             <input>
1476                 <port id="5">
1477                     <dim>1</dim>
1478                     <dim>3</dim>
1479                     <dim>2</dim>
1480                     <dim>2</dim>
1481                 </port>
1482                 <port id="6">
1483                     <dim>1</dim>
1484                     <dim>3</dim>
1485                     <dim>2</dim>
1486                     <dim>2</dim>
1487                 </port>
1488             </input>
1489             <output>
1490                 <port id="7">
1491                     <dim>1</dim>
1492                     <dim>6</dim>
1493                     <dim>2</dim>
1494                     <dim>2</dim>
1495                 </port>
1496             </output>
1497         </layer>
1498     </layers>
1499     <edges>
1500         <edge from-layer="0" from-port="0" to-layer="2" to-port="1"/>
1501         <edge from-layer="1" from-port="0" to-layer="3" to-port="3"/>
1502         <edge from-layer="2" from-port="2" to-layer="4" to-port="5"/>
1503         <edge from-layer="3" from-port="4" to-layer="4" to-port="6"/>
1504     </edges>
1505 </net>)V0G0N";
1506
1507     InferenceEngine::Extension cpuExt(make_so_name("cpu_extension"));
1508     MKLDNNPlugin::MKLDNNExtensionManager::Ptr extMgr(new MKLDNNPlugin::MKLDNNExtensionManager());
1509     extMgr->AddExtension(InferenceEngine::IExtensionPtr(&cpuExt, [](InferenceEngine::IExtension*){}));
1510
1511     InferenceEngine::CNNNetReader net_reader;
1512     ASSERT_NO_THROW(net_reader.ReadNetwork(model.data(), model.length()));
1513
1514     MKLDNNGraphTestClass graph;
1515     graph.CreateGraph(net_reader.getNetwork(), extMgr);
1516
1517     InferenceEngine::SizeVector dims_src = {1, 3, 2, 2};
1518
1519     InferenceEngine::Blob::Ptr src1 =
1520             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src);
1521     src1->allocate();
1522     fill_data(src1->buffer(), src1->size());
1523
1524     InferenceEngine::Blob::Ptr src2 =
1525             InferenceEngine::make_shared_blob<float, const InferenceEngine::SizeVector>(InferenceEngine::Precision::FP32, InferenceEngine::NCHW, dims_src);
1526     src2->allocate();
1527     fill_data(src2->buffer(), src2->size());
1528
1529     InferenceEngine::BlobMap srcs;
1530     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("data1", src1));
1531     srcs.insert(std::pair<std::string, InferenceEngine::Blob::Ptr>("data2", src2));
1532
1533     InferenceEngine::OutputsDataMap out;
1534     out = net_reader.getNetwork().getOutputsInfo();
1535     InferenceEngine::BlobMap outputBlobs;
1536
1537     std::pair<std::string, InferenceEngine::DataPtr> item = *out.begin();
1538
1539     InferenceEngine::TBlob<float>::Ptr output;
1540     output = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc());
1541     output->allocate();
1542     outputBlobs[item.first] = output;
1543
1544     graph.Infer(srcs, outputBlobs);
1545
1546     std::vector<float> refDst = {0.000f, 0.503f, 0.659f, 0.117f, -0.474f, -0.573f, -0.202f, 0.545f, 0.619f, 0.246f,
1547                                  0.000f, 0.000f, 0.000f, 0.503f, 0.659f, 0.117f, -0.474f, -0.573f, -0.202f, 0.545f,
1548                                  0.619f, 0.246f, 0.000f, 0.000f};
1549
1550     InferenceEngine::TBlob<float>::Ptr dstOut = InferenceEngine::make_shared_blob<float>(item.second->getTensorDesc(), refDst.data());
1551
1552     compare(*output, *dstOut);
1553 }