modules/dnn/src/op_inf_engine.cpp

   1 // This file is part of OpenCV project.
   2 // It is subject to the license terms in the LICENSE file found in the top-level directory
   3 // of this distribution and at http://opencv.org/license.html.
   4 //
   5 // Copyright (C) 2018, Intel Corporation, all rights reserved.
   6 // Third party copyrights are property of their respective owners.
   7
   8 #include "precomp.hpp"
   9 #include "op_inf_engine.hpp"
  10 #include <opencv2/dnn/shape_utils.hpp>
  11
  12 #ifdef HAVE_INF_ENGINE
  13 #include <ie_extension.h>
  14 #include <ie_plugin_dispatcher.hpp>
  15 #endif  // HAVE_INF_ENGINE
  16
  17 namespace cv { namespace dnn {
  18
  19 #ifdef HAVE_INF_ENGINE
  20
  21 InfEngineBackendNode::InfEngineBackendNode(const InferenceEngine::CNNLayerPtr& _layer)
  22     : BackendNode(DNN_BACKEND_INFERENCE_ENGINE), layer(_layer) {}
  23
  24 void InfEngineBackendNode::connect(std::vector<Ptr<BackendWrapper> >& inputs,
  25                                    std::vector<Ptr<BackendWrapper> >& outputs)
  26 {
  27     layer->insData.resize(inputs.size());
  28     for (int i = 0; i < inputs.size(); ++i)
  29     {
  30         InferenceEngine::DataPtr dataPtr = infEngineDataNode(inputs[i]);
  31         layer->insData[i] = InferenceEngine::DataWeakPtr(dataPtr);
  32         dataPtr->inputTo[layer->name] = layer;
  33     }
  34
  35     CV_Assert(!outputs.empty());
  36
  37     layer->outData.resize(1);
  38     InferenceEngine::DataPtr dataPtr = infEngineDataNode(outputs[0]);
  39     dataPtr->name = layer->name;
  40     layer->outData[0] = dataPtr;
  41     dataPtr->creatorLayer = InferenceEngine::CNNLayerWeakPtr(layer);
  42 }
  43
  44 static std::vector<Ptr<InfEngineBackendWrapper> >
  45 infEngineWrappers(const std::vector<Ptr<BackendWrapper> >& ptrs)
  46 {
  47     std::vector<Ptr<InfEngineBackendWrapper> > wrappers(ptrs.size());
  48     for (int i = 0; i < ptrs.size(); ++i)
  49     {
  50         CV_Assert(!ptrs[i].empty());
  51         wrappers[i] = ptrs[i].dynamicCast<InfEngineBackendWrapper>();
  52         CV_Assert(!wrappers[i].empty());
  53     }
  54     return wrappers;
  55 }
  56
  57 static InferenceEngine::Layout estimateLayout(const Mat& m)
  58 {
  59     if (m.dims == 4)
  60         return InferenceEngine::Layout::NCHW;
  61     else if (m.dims == 2)
  62         return InferenceEngine::Layout::NC;
  63     else
  64         return InferenceEngine::Layout::ANY;
  65 }
  66
  67 static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "")
  68 {
  69     std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
  70     std::reverse(reversedShape.begin(), reversedShape.end());
  71     if (m.type() == CV_32F)
  72         return InferenceEngine::DataPtr(
  73             new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::FP32, estimateLayout(m))
  74         );
  75     else if (m.type() == CV_8U)
  76         return InferenceEngine::DataPtr(
  77             new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::U8, estimateLayout(m))
  78         );
  79     else
  80         CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
  81 }
  82
  83 InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape,
  84                                                InferenceEngine::Layout layout)
  85 {
  86     if (m.type() == CV_32F)
  87         return InferenceEngine::make_shared_blob<float>(InferenceEngine::Precision::FP32,
  88                                                         layout, shape, (float*)m.data);
  89     else if (m.type() == CV_8U)
  90         return InferenceEngine::make_shared_blob<uint8_t>(InferenceEngine::Precision::U8,
  91                                                           layout, shape, (uint8_t*)m.data);
  92     else
  93         CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
  94 }
  95
  96 InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, InferenceEngine::Layout layout)
  97 {
  98     std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
  99     std::reverse(reversedShape.begin(), reversedShape.end());
 100     return wrapToInfEngineBlob(m, reversedShape, layout);
 101 }
 102
 103 InferenceEngine::DataPtr infEngineDataNode(const Ptr<BackendWrapper>& ptr)
 104 {
 105     CV_Assert(!ptr.empty());
 106     Ptr<InfEngineBackendWrapper> p = ptr.dynamicCast<InfEngineBackendWrapper>();
 107     CV_Assert(!p.empty());
 108     return p->dataPtr;
 109 }
 110
 111 InfEngineBackendWrapper::InfEngineBackendWrapper(int targetId, const cv::Mat& m)
 112     : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, targetId)
 113 {
 114     dataPtr = wrapToInfEngineDataNode(m);
 115     blob = wrapToInfEngineBlob(m, estimateLayout(m));
 116 }
 117
 118 InfEngineBackendWrapper::InfEngineBackendWrapper(Ptr<BackendWrapper> wrapper)
 119     : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, wrapper->targetId)
 120 {
 121     Ptr<InfEngineBackendWrapper> ieWrapper = wrapper.dynamicCast<InfEngineBackendWrapper>();
 122     CV_Assert(!ieWrapper.empty());
 123     InferenceEngine::DataPtr srcData = ieWrapper->dataPtr;
 124     dataPtr = InferenceEngine::DataPtr(
 125         new InferenceEngine::Data(srcData->name, srcData->dims, srcData->precision,
 126                                   srcData->layout)
 127     );
 128     blob = ieWrapper->blob;
 129 }
 130
 131 Ptr<BackendWrapper> InfEngineBackendWrapper::create(Ptr<BackendWrapper> wrapper)
 132 {
 133     return Ptr<BackendWrapper>(new InfEngineBackendWrapper(wrapper));
 134 }
 135
 136 InfEngineBackendWrapper::~InfEngineBackendWrapper()
 137 {
 138
 139 }
 140
 141 void InfEngineBackendWrapper::copyToHost()
 142 {
 143
 144 }
 145
 146 void InfEngineBackendWrapper::setHostDirty()
 147 {
 148
 149 }
 150
 151 InfEngineBackendNet::InfEngineBackendNet()
 152 {
 153     targetDevice = InferenceEngine::TargetDevice::eCPU;
 154     precision = InferenceEngine::Precision::FP32;
 155 }
 156
 157 InfEngineBackendNet::InfEngineBackendNet(InferenceEngine::CNNNetwork& net)
 158 {
 159     targetDevice = InferenceEngine::TargetDevice::eCPU;
 160     precision = InferenceEngine::Precision::FP32;
 161     inputs = net.getInputsInfo();
 162     outputs = net.getOutputsInfo();
 163     layers.resize(net.layerCount());  // A hack to execute InfEngineBackendNet::layerCount correctly.
 164 }
 165
 166 void InfEngineBackendNet::Release() noexcept
 167 {
 168     layers.clear();
 169     inputs.clear();
 170     outputs.clear();
 171 }
 172
 173 void InfEngineBackendNet::setPrecision(InferenceEngine::Precision p) noexcept
 174 {
 175     precision = p;
 176 }
 177
 178 InferenceEngine::Precision InfEngineBackendNet::getPrecision() noexcept
 179 {
 180     return precision;
 181 }
 182
 183 // Assume that outputs of network is unconnected blobs.
 184 void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) noexcept
 185 {
 186     outputs_ = outputs;
 187 }
 188 void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) const noexcept
 189 {
 190     outputs_ = outputs;
 191 }
 192
 193 // Returns input references that aren't connected to internal outputs.
 194 void InfEngineBackendNet::getInputsInfo(InferenceEngine::InputsDataMap &inputs_) noexcept
 195 {
 196     inputs_ = inputs;
 197 }
 198
 199 // Returns input references that aren't connected to internal outputs.
 200 void InfEngineBackendNet::getInputsInfo(InferenceEngine::InputsDataMap &inputs_) const noexcept
 201 {
 202     inputs_ = inputs;
 203 }
 204
 205 InferenceEngine::InputInfo::Ptr InfEngineBackendNet::getInput(const std::string &inputName) noexcept
 206 {
 207     getInputsInfo(inputs);
 208     const auto& it = inputs.find(inputName);
 209     CV_Assert(it != inputs.end());
 210     return it->second;
 211 }
 212
 213 void InfEngineBackendNet::getName(char*, size_t) noexcept
 214 {
 215 }
 216
 217 void InfEngineBackendNet::getName(char*, size_t) const noexcept
 218 {
 219 }
 220
 221 size_t InfEngineBackendNet::layerCount() noexcept
 222 {
 223     return layers.size();
 224 }
 225
 226 InferenceEngine::DataPtr& InfEngineBackendNet::getData(const char *dname) noexcept
 227 {
 228     CV_Error(Error::StsNotImplemented, "");
 229     return outputs.begin()->second;  // Just return something.
 230 }
 231
 232 void InfEngineBackendNet::addLayer(const InferenceEngine::CNNLayerPtr &layer) noexcept
 233 {
 234     layers.push_back(layer);
 235     inputs.clear();
 236     outputs.clear();
 237 }
 238
 239 InferenceEngine::StatusCode
 240 InfEngineBackendNet::addOutput(const std::string &layerName, size_t outputIndex,
 241                                InferenceEngine::ResponseDesc *resp) noexcept
 242 {
 243     for (const auto& l : layers)
 244     {
 245         for (const InferenceEngine::DataPtr& out : l->outData)
 246         {
 247             if (out->name == layerName)
 248             {
 249                 outputs[out->name] = out;
 250                 return InferenceEngine::StatusCode::OK;
 251             }
 252         }
 253     }
 254     CV_Error(Error::StsObjectNotFound, "Cannot find a layer " + layerName);
 255     return InferenceEngine::StatusCode::OK;
 256 }
 257
 258 InferenceEngine::StatusCode
 259 InfEngineBackendNet::getLayerByName(const char *layerName, InferenceEngine::CNNLayerPtr &out,
 260                                     InferenceEngine::ResponseDesc *resp) noexcept
 261 {
 262     for (auto& l : layers)
 263     {
 264         if (l->name == layerName)
 265         {
 266             out = l;
 267             return InferenceEngine::StatusCode::OK;
 268         }
 269     }
 270     CV_Error(Error::StsObjectNotFound, cv::format("Cannot find a layer %s", layerName));
 271     return InferenceEngine::StatusCode::NOT_FOUND;
 272 }
 273
 274 void InfEngineBackendNet::setTargetDevice(InferenceEngine::TargetDevice device) noexcept
 275 {
 276     if (device != InferenceEngine::TargetDevice::eCPU &&
 277         device != InferenceEngine::TargetDevice::eGPU &&
 278         device != InferenceEngine::TargetDevice::eMYRIAD)
 279         CV_Error(Error::StsNotImplemented, "");
 280     targetDevice = device;
 281 }
 282
 283 InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() noexcept
 284 {
 285     return targetDevice;
 286 }
 287
 288 InferenceEngine::StatusCode InfEngineBackendNet::setBatchSize(const size_t size) noexcept
 289 {
 290     CV_Error(Error::StsNotImplemented, "");
 291     return InferenceEngine::StatusCode::OK;
 292 }
 293
 294 size_t InfEngineBackendNet::getBatchSize() const noexcept
 295 {
 296     CV_Error(Error::StsNotImplemented, "");
 297     return 0;
 298 }
 299
 300 void InfEngineBackendNet::init(int targetId)
 301 {
 302     if (inputs.empty())
 303     {
 304         // Collect all external input blobs.
 305         inputs.clear();
 306         std::map<std::string, InferenceEngine::DataPtr> internalOutputs;
 307         for (const auto& l : layers)
 308         {
 309             for (const InferenceEngine::DataWeakPtr& ptr : l->insData)
 310             {
 311                 InferenceEngine::DataPtr inp(ptr);
 312                 if (internalOutputs.find(inp->name) == internalOutputs.end())
 313                 {
 314                     InferenceEngine::InputInfo::Ptr inpInfo(new InferenceEngine::InputInfo());
 315                     inpInfo->setInputData(inp);
 316                     if (inputs.find(inp->name) == inputs.end())
 317                         inputs[inp->name] = inpInfo;
 318                 }
 319             }
 320             for (const InferenceEngine::DataPtr& out : l->outData)
 321             {
 322                 // TODO: Replace to uniqueness assertion.
 323                 if (internalOutputs.find(out->name) == internalOutputs.end())
 324                     internalOutputs[out->name] = out;
 325             }
 326         }
 327         CV_Assert(!inputs.empty());
 328     }
 329
 330     if (outputs.empty())
 331     {
 332         // Add all unconnected blobs to output blobs.
 333         InferenceEngine::OutputsDataMap unconnectedOuts;
 334         for (const auto& l : layers)
 335         {
 336             // Add all outputs.
 337             for (const InferenceEngine::DataPtr& out : l->outData)
 338             {
 339                 // TODO: Replace to uniqueness assertion.
 340                 if (unconnectedOuts.find(out->name) == unconnectedOuts.end())
 341                     unconnectedOuts[out->name] = out;
 342             }
 343             // Remove internally connected outputs.
 344             for (const InferenceEngine::DataWeakPtr& inp : l->insData)
 345             {
 346                 unconnectedOuts.erase(InferenceEngine::DataPtr(inp)->name);
 347             }
 348         }
 349         CV_Assert(!unconnectedOuts.empty());
 350
 351         for (auto it = unconnectedOuts.begin(); it != unconnectedOuts.end(); ++it)
 352         {
 353             outputs[it->first] = it->second;
 354         }
 355     }
 356
 357     // Set up input blobs.
 358     inpBlobs.clear();
 359     for (const auto& it : inputs)
 360     {
 361         CV_Assert(allBlobs.find(it.first) != allBlobs.end());
 362         inpBlobs[it.first] = allBlobs[it.first];
 363         it.second->setPrecision(inpBlobs[it.first]->precision());
 364     }
 365
 366     // Set up output blobs.
 367     outBlobs.clear();
 368     for (const auto& it : outputs)
 369     {
 370         CV_Assert(allBlobs.find(it.first) != allBlobs.end());
 371         outBlobs[it.first] = allBlobs[it.first];
 372     }
 373
 374     switch (targetId)
 375     {
 376     case DNN_TARGET_CPU: setTargetDevice(InferenceEngine::TargetDevice::eCPU); break;
 377     case DNN_TARGET_OPENCL_FP16: setPrecision(InferenceEngine::Precision::FP16);  // Fallback to the next.
 378     case DNN_TARGET_OPENCL: setTargetDevice(InferenceEngine::TargetDevice::eGPU); break;
 379     case DNN_TARGET_MYRIAD:
 380     {
 381         setPrecision(InferenceEngine::Precision::FP16);
 382         setTargetDevice(InferenceEngine::TargetDevice::eMYRIAD); break;
 383     }
 384     default:
 385         CV_Error(Error::StsError, format("Unknown target identifier: %d", targetId));
 386     }
 387
 388     if (!isInitialized())
 389         initPlugin(*this);
 390 }
 391
 392 void InfEngineBackendNet::initPlugin(InferenceEngine::ICNNNetwork& net)
 393 {
 394     CV_Assert(!isInitialized());
 395
 396     try
 397     {
 398         static std::map<std::string, InferenceEngine::InferenceEnginePluginPtr> sharedPlugins;
 399         std::string deviceName = InferenceEngine::getDeviceName(targetDevice);
 400         auto pluginIt = sharedPlugins.find(deviceName);
 401         if (pluginIt != sharedPlugins.end())
 402         {
 403             enginePtr = pluginIt->second;
 404         }
 405         else
 406         {
 407             enginePtr = InferenceEngine::PluginDispatcher({""}).getSuitablePlugin(targetDevice);
 408             sharedPlugins[deviceName] = enginePtr;
 409
 410             if (targetDevice == InferenceEngine::TargetDevice::eCPU)
 411             {
 412                 std::string suffixes[] = {"_avx2", "_sse4", ""};
 413                 bool haveFeature[] = {
 414                     checkHardwareSupport(CPU_AVX2),
 415                     checkHardwareSupport(CPU_SSE4_2),
 416                     true
 417                 };
 418                 for (int i = 0; i < 3; ++i)
 419                 {
 420                     if (!haveFeature[i])
 421                         continue;
 422     #ifdef _WIN32
 423                     std::string libName = "cpu_extension" + suffixes[i] + ".dll";
 424     #else
 425                     std::string libName = "libcpu_extension" + suffixes[i] + ".so";
 426     #endif  // _WIN32
 427                     try
 428                     {
 429                         InferenceEngine::IExtensionPtr extension =
 430                             InferenceEngine::make_so_pointer<InferenceEngine::IExtension>(libName);
 431                         enginePtr->AddExtension(extension, 0);
 432                         break;
 433                     }
 434                     catch(...) {}
 435                 }
 436                 // Some of networks can work without a library of extra layers.
 437             }
 438         }
 439         plugin = InferenceEngine::InferencePlugin(enginePtr);
 440
 441         netExec = plugin.LoadNetwork(net, {});
 442         infRequest = netExec.CreateInferRequest();
 443         infRequest.SetInput(inpBlobs);
 444         infRequest.SetOutput(outBlobs);
 445     }
 446     catch (const std::exception& ex)
 447     {
 448         CV_Error(Error::StsAssert, format("Failed to initialize Inference Engine backend: %s", ex.what()));
 449     }
 450 }
 451
 452 bool InfEngineBackendNet::isInitialized()
 453 {
 454     return (bool)enginePtr;
 455 }
 456
 457 void InfEngineBackendNet::addBlobs(const std::vector<Ptr<BackendWrapper> >& ptrs)
 458 {
 459     auto wrappers = infEngineWrappers(ptrs);
 460     for (const auto& wrapper : wrappers)
 461     {
 462         allBlobs.insert({wrapper->dataPtr->name, wrapper->blob});
 463     }
 464 }
 465
 466 void InfEngineBackendNet::forward()
 467 {
 468     infRequest.Infer();
 469 }
 470
 471 Mat infEngineBlobToMat(const InferenceEngine::Blob::Ptr& blob)
 472 {
 473     // NOTE: Inference Engine sizes are reversed.
 474     std::vector<size_t> dims = blob->dims();
 475     std::vector<int> size(dims.begin(), dims.end());
 476     std::reverse(size.begin(), size.end());
 477     return Mat(size, CV_32F, (void*)blob->buffer());
 478 }
 479
 480 InfEngineBackendLayer::InfEngineBackendLayer(const InferenceEngine::DataPtr& output_)
 481 {
 482     output = output_;
 483 }
 484
 485 bool InfEngineBackendLayer::getMemoryShapes(const std::vector<MatShape> &inputs,
 486                                             const int requiredOutputs,
 487                                             std::vector<MatShape> &outputs,
 488                                             std::vector<MatShape> &internals) const
 489 {
 490     std::vector<size_t> dims = output->dims;
 491     std::vector<int> shape(dims.begin(), dims.end());
 492     std::reverse(shape.begin(), shape.end());
 493     outputs.assign(1, shape);
 494     return false;
 495 }
 496
 497 bool InfEngineBackendLayer::supportBackend(int backendId)
 498 {
 499     return backendId == DNN_BACKEND_DEFAULT ||
 500            backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
 501 }
 502
 503 void InfEngineBackendLayer::forward(std::vector<Mat*> &input, std::vector<Mat> &output,
 504                                     std::vector<Mat> &internals)
 505 {
 506     CV_Error(Error::StsError, "Choose Inference Engine as a preferable backend.");
 507 }
 508
 509 void InfEngineBackendLayer::forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs,
 510                                     OutputArrayOfArrays internals)
 511 {
 512     CV_Error(Error::StsInternal, "Choose Inference Engine as a preferable backend.");
 513 }
 514
 515 InferenceEngine::TBlob<int16_t>::Ptr convertFp16(const InferenceEngine::Blob::Ptr& blob)
 516 {
 517     auto halfs = InferenceEngine::make_shared_blob<int16_t>(InferenceEngine::Precision::FP16, blob->layout(), blob->dims());
 518     halfs->allocate();
 519     Mat floatsData(1, blob->size(), CV_32F, blob->buffer());
 520     Mat halfsData(1, blob->size(), CV_16SC1, halfs->buffer());
 521     convertFp16(floatsData, halfsData);
 522     return halfs;
 523 }
 524
 525 #endif  // HAVE_INF_ENGINE
 526
 527 bool haveInfEngine()
 528 {
 529 #ifdef HAVE_INF_ENGINE
 530     return true;
 531 #else
 532     return false;
 533 #endif  // HAVE_INF_ENGINE
 534 }
 535
 536 void forwardInfEngine(Ptr<BackendNode>& node)
 537 {
 538     CV_Assert(haveInfEngine());
 539 #ifdef HAVE_INF_ENGINE
 540     CV_Assert(!node.empty());
 541     Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
 542     CV_Assert(!ieNode.empty());
 543     ieNode->net->forward();
 544 #endif  // HAVE_INF_ENGINE
 545 }
 546
 547 }}  // namespace dnn, namespace cv