modules/dnn/src/op_inf_engine.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4 //
   5 // Copyright (C) 2018, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 #include "precomp.hpp"
   9 #include "op_inf_engine.hpp"
  10 #include <opencv2/dnn/shape_utils.hpp>
  11
  12 #ifdef HAVE_INF_ENGINE
  13 #include <ie_extension.h>
  14 #include <ie_plugin_dispatcher.hpp>
  15 #endif  // HAVE_INF_ENGINE
  16
  17 namespace cv { namespace dnn {
  18
  19 #ifdef HAVE_INF_ENGINE
  20
  21 InfEngineBackendNode::InfEngineBackendNode(const InferenceEngine::CNNLayerPtr& _layer)
  22     : BackendNode(DNN_BACKEND_INFERENCE_ENGINE), layer(_layer) {}
  23
  24 void InfEngineBackendNode::connect(std::vector<Ptr<BackendWrapper> >& inputs,
  25                                    std::vector<Ptr<BackendWrapper> >& outputs)
  26 {
  27     layer->insData.resize(inputs.size());
  28     for (int i = 0; i < inputs.size(); ++i)
  29     {
  30         InferenceEngine::DataPtr dataPtr = infEngineDataNode(inputs[i]);
  31         layer->insData[i] = InferenceEngine::DataWeakPtr(dataPtr);
  32         dataPtr->inputTo[layer->name] = layer;
  33     }
  34
  35     CV_Assert(!outputs.empty());
  36
  37     layer->outData.resize(1);
  38     InferenceEngine::DataPtr dataPtr = infEngineDataNode(outputs[0]);
  39     dataPtr->name = layer->name;
  40     layer->outData[0] = dataPtr;
  41     dataPtr->creatorLayer = InferenceEngine::CNNLayerWeakPtr(layer);
  42 }
  43
  44 static std::vector<Ptr<InfEngineBackendWrapper> >
  45 infEngineWrappers(const std::vector<Ptr<BackendWrapper> >& ptrs)
  46 {
  47     std::vector<Ptr<InfEngineBackendWrapper> > wrappers(ptrs.size());
  48     for (int i = 0; i < ptrs.size(); ++i)
  49     {
  50         CV_Assert(!ptrs[i].empty());
  51         wrappers[i] = ptrs[i].dynamicCast<InfEngineBackendWrapper>();
  52         CV_Assert(!wrappers[i].empty());
  53     }
  54     return wrappers;
  55 }
  56
  57 static InferenceEngine::Layout estimateLayout(const Mat& m)
  58 {
  59     if (m.dims == 4)
  60         return InferenceEngine::Layout::NCHW;
  61     else if (m.dims == 2)
  62         return InferenceEngine::Layout::NC;
  63     else
  64         return InferenceEngine::Layout::ANY;
  65 }
  66
  67 static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "")
  68 {
  69     std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
  70     std::reverse(reversedShape.begin(), reversedShape.end());
  71     if (m.type() == CV_32F)
  72         return InferenceEngine::DataPtr(
  73             new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::FP32, estimateLayout(m))
  74         );
  75     else if (m.type() == CV_8U)
  76         return InferenceEngine::DataPtr(
  77             new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::U8, estimateLayout(m))
  78         );
  79     else
  80         CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
  81 }
  82
  83 InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape,
  84                                                InferenceEngine::Layout layout)
  85 {
  86     if (m.type() == CV_32F)
  87         return InferenceEngine::make_shared_blob<float>(InferenceEngine::Precision::FP32,
  88                                                         layout, shape, (float*)m.data);
  89     else if (m.type() == CV_8U)
  90         return InferenceEngine::make_shared_blob<uint8_t>(InferenceEngine::Precision::U8,
  91                                                           layout, shape, (uint8_t*)m.data);
  92     else
  93         CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
  94 }
  95
  96 InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, InferenceEngine::Layout layout)
  97 {
  98     std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
  99     std::reverse(reversedShape.begin(), reversedShape.end());
 100     return wrapToInfEngineBlob(m, reversedShape, layout);
 101 }
 102
 103 InferenceEngine::DataPtr infEngineDataNode(const Ptr<BackendWrapper>& ptr)
 104 {
 105     CV_Assert(!ptr.empty());
 106     Ptr<InfEngineBackendWrapper> p = ptr.dynamicCast<InfEngineBackendWrapper>();
 107     CV_Assert(!p.empty());
 108     return p->dataPtr;
 109 }
 110
 111 InfEngineBackendWrapper::InfEngineBackendWrapper(int targetId, const cv::Mat& m)
 112     : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, targetId)
 113 {
 114     dataPtr = wrapToInfEngineDataNode(m);
 115     blob = wrapToInfEngineBlob(m, estimateLayout(m));
 116 }
 117
 118 InfEngineBackendWrapper::InfEngineBackendWrapper(Ptr<BackendWrapper> wrapper)
 119     : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, wrapper->targetId)
 120 {
 121     Ptr<InfEngineBackendWrapper> ieWrapper = wrapper.dynamicCast<InfEngineBackendWrapper>();
 122     CV_Assert(!ieWrapper.empty());
 123     InferenceEngine::DataPtr srcData = ieWrapper->dataPtr;
 124     dataPtr = InferenceEngine::DataPtr(
 125         new InferenceEngine::Data(srcData->name, srcData->dims, srcData->precision,
 126                                   srcData->layout)
 127     );
 128     blob = ieWrapper->blob;
 129 }
 130
 131 Ptr<BackendWrapper> InfEngineBackendWrapper::create(Ptr<BackendWrapper> wrapper)
 132 {
 133     return Ptr<BackendWrapper>(new InfEngineBackendWrapper(wrapper));
 134 }
 135
 136 InfEngineBackendWrapper::~InfEngineBackendWrapper()
 137 {
 138
 139 }
 140
 141 void InfEngineBackendWrapper::copyToHost()
 142 {
 143
 144 }
 145
 146 void InfEngineBackendWrapper::setHostDirty()
 147 {
 148
 149 }
 150
 151 InfEngineBackendNet::InfEngineBackendNet()
 152 {
 153     targetDevice = InferenceEngine::TargetDevice::eCPU;
 154     precision = InferenceEngine::Precision::FP32;
 155 }
 156
 157 InfEngineBackendNet::InfEngineBackendNet(InferenceEngine::CNNNetwork& net)
 158 {
 159     targetDevice = InferenceEngine::TargetDevice::eCPU;
 160     precision = InferenceEngine::Precision::FP32;
 161     inputs = net.getInputsInfo();
 162     outputs = net.getOutputsInfo();
 163     layers.resize(net.layerCount());  // A hack to execute InfEngineBackendNet::layerCount correctly.
 164 }
 165
 166 void InfEngineBackendNet::Release() noexcept
 167 {
 168     layers.clear();
 169     inputs.clear();
 170     outputs.clear();
 171 }
 172
 173 void InfEngineBackendNet::setPrecision(InferenceEngine::Precision p) noexcept
 174 {
 175     precision = p;
 176 }
 177
 178 InferenceEngine::Precision InfEngineBackendNet::getPrecision() noexcept
 179 {
 180     return precision;
 181 }
 182
 183 InferenceEngine::Precision InfEngineBackendNet::getPrecision() const noexcept
 184 {
 185     return precision;
 186 }
 187
 188 // Assume that outputs of network is unconnected blobs.
 189 void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) noexcept
 190 {
 191     const_cast<const InfEngineBackendNet*>(this)->getOutputsInfo(outputs_);
 192 }
 193 void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) const noexcept
 194 {
 195     outputs_ = outputs;
 196 }
 197
 198 // Returns input references that aren't connected to internal outputs.
 199 void InfEngineBackendNet::getInputsInfo(InferenceEngine::InputsDataMap &inputs_) noexcept
 200 {
 201     const_cast<const InfEngineBackendNet*>(this)->getInputsInfo(inputs_);
 202 }
 203
 204 // Returns input references that aren't connected to internal outputs.
 205 void InfEngineBackendNet::getInputsInfo(InferenceEngine::InputsDataMap &inputs_) const noexcept
 206 {
 207     inputs_ = inputs;
 208 }
 209
 210 InferenceEngine::InputInfo::Ptr InfEngineBackendNet::getInput(const std::string &inputName) noexcept
 211 {
 212     return const_cast<const InfEngineBackendNet*>(this)->getInput(inputName);
 213 }
 214
 215 InferenceEngine::InputInfo::Ptr InfEngineBackendNet::getInput(const std::string &inputName) const noexcept
 216 {
 217     const auto& it = inputs.find(inputName);
 218     CV_Assert(it != inputs.end());
 219     return it->second;
 220 }
 221
 222 void InfEngineBackendNet::getName(char*, size_t) noexcept
 223 {
 224 }
 225
 226 void InfEngineBackendNet::getName(char*, size_t) const noexcept
 227 {
 228 }
 229
 230 const std::string& InfEngineBackendNet::getName() const noexcept
 231 {
 232     return name;
 233 }
 234
 235 size_t InfEngineBackendNet::layerCount() noexcept
 236 {
 237     return const_cast<const InfEngineBackendNet*>(this)->layerCount();
 238 }
 239
 240 size_t InfEngineBackendNet::layerCount() const noexcept
 241 {
 242     return layers.size();
 243 }
 244
 245 InferenceEngine::DataPtr& InfEngineBackendNet::getData(const char *dname) noexcept
 246 {
 247     CV_Error(Error::StsNotImplemented, "");
 248     return outputs.begin()->second;  // Just return something.
 249 }
 250
 251 void InfEngineBackendNet::addLayer(const InferenceEngine::CNNLayerPtr &layer) noexcept
 252 {
 253     layers.push_back(layer);
 254     inputs.clear();
 255     outputs.clear();
 256 }
 257
 258 InferenceEngine::StatusCode
 259 InfEngineBackendNet::addOutput(const std::string &layerName, size_t outputIndex,
 260                                InferenceEngine::ResponseDesc *resp) noexcept
 261 {
 262     for (const auto& l : layers)
 263     {
 264         for (const InferenceEngine::DataPtr& out : l->outData)
 265         {
 266             if (out->name == layerName)
 267             {
 268                 outputs[out->name] = out;
 269                 return InferenceEngine::StatusCode::OK;
 270             }
 271         }
 272     }
 273     CV_Error(Error::StsObjectNotFound, "Cannot find a layer " + layerName);
 274     return InferenceEngine::StatusCode::OK;
 275 }
 276
 277 InferenceEngine::StatusCode
 278 InfEngineBackendNet::getLayerByName(const char *layerName, InferenceEngine::CNNLayerPtr &out,
 279                                     InferenceEngine::ResponseDesc *resp) noexcept
 280 {
 281     return const_cast<const InfEngineBackendNet*>(this)->getLayerByName(layerName, out, resp);
 282 }
 283
 284 InferenceEngine::StatusCode InfEngineBackendNet::getLayerByName(const char *layerName,
 285                                                                 InferenceEngine::CNNLayerPtr &out,
 286                                                                 InferenceEngine::ResponseDesc *resp) const noexcept
 287 {
 288     for (auto& l : layers)
 289     {
 290         if (l->name == layerName)
 291         {
 292             out = l;
 293             return InferenceEngine::StatusCode::OK;
 294         }
 295     }
 296     CV_Error(Error::StsObjectNotFound, cv::format("Cannot find a layer %s", layerName));
 297     return InferenceEngine::StatusCode::NOT_FOUND;
 298 }
 299
 300 void InfEngineBackendNet::setTargetDevice(InferenceEngine::TargetDevice device) noexcept
 301 {
 302     if (device != InferenceEngine::TargetDevice::eCPU &&
 303         device != InferenceEngine::TargetDevice::eGPU &&
 304         device != InferenceEngine::TargetDevice::eMYRIAD)
 305         CV_Error(Error::StsNotImplemented, "");
 306     targetDevice = device;
 307 }
 308
 309 InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() noexcept
 310 {
 311     return targetDevice;
 312 }
 313
 314 InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() const noexcept
 315 {
 316     return targetDevice;
 317 }
 318
 319 InferenceEngine::StatusCode InfEngineBackendNet::setBatchSize(const size_t) noexcept
 320 {
 321     CV_Error(Error::StsNotImplemented, "");
 322     return InferenceEngine::StatusCode::OK;
 323 }
 324
 325 size_t InfEngineBackendNet::getBatchSize() const noexcept
 326 {
 327     CV_Error(Error::StsNotImplemented, "");
 328     return 0;
 329 }
 330
 331 void InfEngineBackendNet::init(int targetId)
 332 {
 333     if (inputs.empty())
 334     {
 335         // Collect all external input blobs.
 336         inputs.clear();
 337         std::map<std::string, InferenceEngine::DataPtr> internalOutputs;
 338         for (const auto& l : layers)
 339         {
 340             for (const InferenceEngine::DataWeakPtr& ptr : l->insData)
 341             {
 342                 InferenceEngine::DataPtr inp(ptr);
 343                 if (internalOutputs.find(inp->name) == internalOutputs.end())
 344                 {
 345                     InferenceEngine::InputInfo::Ptr inpInfo(new InferenceEngine::InputInfo());
 346                     inpInfo->setInputData(inp);
 347                     if (inputs.find(inp->name) == inputs.end())
 348                         inputs[inp->name] = inpInfo;
 349                 }
 350             }
 351             for (const InferenceEngine::DataPtr& out : l->outData)
 352             {
 353                 // TODO: Replace to uniqueness assertion.
 354                 if (internalOutputs.find(out->name) == internalOutputs.end())
 355                     internalOutputs[out->name] = out;
 356             }
 357         }
 358         CV_Assert(!inputs.empty());
 359     }
 360
 361     if (outputs.empty())
 362     {
 363         // Add all unconnected blobs to output blobs.
 364         InferenceEngine::OutputsDataMap unconnectedOuts;
 365         for (const auto& l : layers)
 366         {
 367             // Add all outputs.
 368             for (const InferenceEngine::DataPtr& out : l->outData)
 369             {
 370                 // TODO: Replace to uniqueness assertion.
 371                 if (unconnectedOuts.find(out->name) == unconnectedOuts.end())
 372                     unconnectedOuts[out->name] = out;
 373             }
 374             // Remove internally connected outputs.
 375             for (const InferenceEngine::DataWeakPtr& inp : l->insData)
 376             {
 377                 unconnectedOuts.erase(InferenceEngine::DataPtr(inp)->name);
 378             }
 379         }
 380         CV_Assert(!unconnectedOuts.empty());
 381
 382         for (auto it = unconnectedOuts.begin(); it != unconnectedOuts.end(); ++it)
 383         {
 384             outputs[it->first] = it->second;
 385         }
 386     }
 387
 388     // Set up input blobs.
 389     inpBlobs.clear();
 390     for (const auto& it : inputs)
 391     {
 392         CV_Assert(allBlobs.find(it.first) != allBlobs.end());
 393         inpBlobs[it.first] = allBlobs[it.first];
 394         it.second->setPrecision(inpBlobs[it.first]->precision());
 395     }
 396
 397     // Set up output blobs.
 398     outBlobs.clear();
 399     for (const auto& it : outputs)
 400     {
 401         CV_Assert(allBlobs.find(it.first) != allBlobs.end());
 402         outBlobs[it.first] = allBlobs[it.first];
 403     }
 404
 405     switch (targetId)
 406     {
 407     case DNN_TARGET_CPU: setTargetDevice(InferenceEngine::TargetDevice::eCPU); break;
 408     case DNN_TARGET_OPENCL_FP16:
 409         setPrecision(InferenceEngine::Precision::FP16);
 410         /* Falls through. */
 411     case DNN_TARGET_OPENCL: setTargetDevice(InferenceEngine::TargetDevice::eGPU); break;
 412     case DNN_TARGET_MYRIAD:
 413     {
 414         setPrecision(InferenceEngine::Precision::FP16);
 415         setTargetDevice(InferenceEngine::TargetDevice::eMYRIAD); break;
 416     }
 417     default:
 418         CV_Error(Error::StsError, format("Unknown target identifier: %d", targetId));
 419     }
 420
 421     if (!isInitialized())
 422         initPlugin(*this);
 423 }
 424
 425 void InfEngineBackendNet::initPlugin(InferenceEngine::ICNNNetwork& net)
 426 {
 427     CV_Assert(!isInitialized());
 428
 429     try
 430     {
 431         static std::map<std::string, InferenceEngine::InferenceEnginePluginPtr> sharedPlugins;
 432         std::string deviceName = InferenceEngine::getDeviceName(targetDevice);
 433         auto pluginIt = sharedPlugins.find(deviceName);
 434         if (pluginIt != sharedPlugins.end())
 435         {
 436             enginePtr = pluginIt->second;
 437         }
 438         else
 439         {
 440             enginePtr = InferenceEngine::PluginDispatcher({""}).getSuitablePlugin(targetDevice);
 441             sharedPlugins[deviceName] = enginePtr;
 442
 443             if (targetDevice == InferenceEngine::TargetDevice::eCPU)
 444             {
 445                 std::string suffixes[] = {"_avx2", "_sse4", ""};
 446                 bool haveFeature[] = {
 447                     checkHardwareSupport(CPU_AVX2),
 448                     checkHardwareSupport(CPU_SSE4_2),
 449                     true
 450                 };
 451                 for (int i = 0; i < 3; ++i)
 452                 {
 453                     if (!haveFeature[i])
 454                         continue;
 455     #ifdef _WIN32
 456                     std::string libName = "cpu_extension" + suffixes[i] + ".dll";
 457     #else
 458                     std::string libName = "libcpu_extension" + suffixes[i] + ".so";
 459     #endif  // _WIN32
 460                     try
 461                     {
 462                         InferenceEngine::IExtensionPtr extension =
 463                             InferenceEngine::make_so_pointer<InferenceEngine::IExtension>(libName);
 464                         enginePtr->AddExtension(extension, 0);
 465                         break;
 466                     }
 467                     catch(...) {}
 468                 }
 469                 // Some of networks can work without a library of extra layers.
 470             }
 471         }
 472         plugin = InferenceEngine::InferencePlugin(enginePtr);
 473
 474         netExec = plugin.LoadNetwork(net, {});
 475         infRequest = netExec.CreateInferRequest();
 476         infRequest.SetInput(inpBlobs);
 477         infRequest.SetOutput(outBlobs);
 478     }
 479     catch (const std::exception& ex)
 480     {
 481         CV_Error(Error::StsAssert, format("Failed to initialize Inference Engine backend: %s", ex.what()));
 482     }
 483 }
 484
 485 bool InfEngineBackendNet::isInitialized()
 486 {
 487     return (bool)enginePtr;
 488 }
 489
 490 void InfEngineBackendNet::addBlobs(const std::vector<Ptr<BackendWrapper> >& ptrs)
 491 {
 492     auto wrappers = infEngineWrappers(ptrs);
 493     for (const auto& wrapper : wrappers)
 494     {
 495         allBlobs.insert({wrapper->dataPtr->name, wrapper->blob});
 496     }
 497 }
 498
 499 void InfEngineBackendNet::forward()
 500 {
 501     infRequest.Infer();
 502 }
 503
 504 Mat infEngineBlobToMat(const InferenceEngine::Blob::Ptr& blob)
 505 {
 506     // NOTE: Inference Engine sizes are reversed.
 507     std::vector<size_t> dims = blob->dims();
 508     std::vector<int> size(dims.begin(), dims.end());
 509     std::reverse(size.begin(), size.end());
 510     return Mat(size, CV_32F, (void*)blob->buffer());
 511 }
 512
 513 InfEngineBackendLayer::InfEngineBackendLayer(const InferenceEngine::DataPtr& output_)
 514 {
 515     output = output_;
 516 }
 517
 518 bool InfEngineBackendLayer::getMemoryShapes(const std::vector<MatShape> &inputs,
 519                                             const int requiredOutputs,
 520                                             std::vector<MatShape> &outputs,
 521                                             std::vector<MatShape> &internals) const
 522 {
 523     std::vector<size_t> dims = output->dims;
 524     std::vector<int> shape(dims.begin(), dims.end());
 525     std::reverse(shape.begin(), shape.end());
 526     outputs.assign(1, shape);
 527     return false;
 528 }
 529
 530 bool InfEngineBackendLayer::supportBackend(int backendId)
 531 {
 532     return backendId == DNN_BACKEND_DEFAULT ||
 533            backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
 534 }
 535
 536 void InfEngineBackendLayer::forward(std::vector<Mat*> &input, std::vector<Mat> &output,
 537                                     std::vector<Mat> &internals)
 538 {
 539     CV_Error(Error::StsError, "Choose Inference Engine as a preferable backend.");
 540 }
 541
 542 void InfEngineBackendLayer::forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs,
 543                                     OutputArrayOfArrays internals)
 544 {
 545     CV_Error(Error::StsInternal, "Choose Inference Engine as a preferable backend.");
 546 }
 547
 548 InferenceEngine::TBlob<int16_t>::Ptr convertFp16(const InferenceEngine::Blob::Ptr& blob)
 549 {
 550     auto halfs = InferenceEngine::make_shared_blob<int16_t>(InferenceEngine::Precision::FP16, blob->layout(), blob->dims());
 551     halfs->allocate();
 552     Mat floatsData(1, blob->size(), CV_32F, blob->buffer());
 553     Mat halfsData(1, blob->size(), CV_16SC1, halfs->buffer());
 554     convertFp16(floatsData, halfsData);
 555     return halfs;
 556 }
 557
 558 #endif  // HAVE_INF_ENGINE
 559
 560 bool haveInfEngine()
 561 {
 562 #ifdef HAVE_INF_ENGINE
 563     return true;
 564 #else
 565     return false;
 566 #endif  // HAVE_INF_ENGINE
 567 }
 568
 569 void forwardInfEngine(Ptr<BackendNode>& node)
 570 {
 571     CV_Assert(haveInfEngine());
 572 #ifdef HAVE_INF_ENGINE
 573     CV_Assert(!node.empty());
 574     Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
 575     CV_Assert(!ieNode.empty());
 576     ieNode->net->forward();
 577 #endif  // HAVE_INF_ENGINE
 578 }
 579
 580 }}  // namespace dnn, namespace cv