1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
5 // Copyright (C) 2018, Intel Corporation, all rights reserved.
6 // Third party copyrights are property of their respective owners.
9 #include "op_inf_engine.hpp"
10 #include <opencv2/dnn/shape_utils.hpp>
12 #ifdef HAVE_INF_ENGINE
13 #include <ie_extension.h>
14 #include <ie_plugin_dispatcher.hpp>
15 #endif // HAVE_INF_ENGINE
17 namespace cv { namespace dnn {
19 #ifdef HAVE_INF_ENGINE
21 InfEngineBackendNode::InfEngineBackendNode(const InferenceEngine::CNNLayerPtr& _layer)
22 : BackendNode(DNN_BACKEND_INFERENCE_ENGINE), layer(_layer) {}
24 void InfEngineBackendNode::connect(std::vector<Ptr<BackendWrapper> >& inputs,
25 std::vector<Ptr<BackendWrapper> >& outputs)
27 layer->insData.resize(inputs.size());
28 for (int i = 0; i < inputs.size(); ++i)
30 InferenceEngine::DataPtr dataPtr = infEngineDataNode(inputs[i]);
31 layer->insData[i] = InferenceEngine::DataWeakPtr(dataPtr);
32 dataPtr->inputTo[layer->name] = layer;
35 CV_Assert(!outputs.empty());
37 layer->outData.resize(1);
38 InferenceEngine::DataPtr dataPtr = infEngineDataNode(outputs[0]);
39 dataPtr->name = layer->name;
40 layer->outData[0] = dataPtr;
41 dataPtr->creatorLayer = InferenceEngine::CNNLayerWeakPtr(layer);
44 static std::vector<Ptr<InfEngineBackendWrapper> >
45 infEngineWrappers(const std::vector<Ptr<BackendWrapper> >& ptrs)
47 std::vector<Ptr<InfEngineBackendWrapper> > wrappers(ptrs.size());
48 for (int i = 0; i < ptrs.size(); ++i)
50 CV_Assert(!ptrs[i].empty());
51 wrappers[i] = ptrs[i].dynamicCast<InfEngineBackendWrapper>();
52 CV_Assert(!wrappers[i].empty());
57 static InferenceEngine::Layout estimateLayout(const Mat& m)
60 return InferenceEngine::Layout::NCHW;
62 return InferenceEngine::Layout::NC;
64 return InferenceEngine::Layout::ANY;
67 static InferenceEngine::DataPtr wrapToInfEngineDataNode(const Mat& m, const std::string& name = "")
69 std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
70 std::reverse(reversedShape.begin(), reversedShape.end());
71 if (m.type() == CV_32F)
72 return InferenceEngine::DataPtr(
73 new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::FP32, estimateLayout(m))
75 else if (m.type() == CV_8U)
76 return InferenceEngine::DataPtr(
77 new InferenceEngine::Data(name, reversedShape, InferenceEngine::Precision::U8, estimateLayout(m))
80 CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
83 InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, const std::vector<size_t>& shape,
84 InferenceEngine::Layout layout)
86 if (m.type() == CV_32F)
87 return InferenceEngine::make_shared_blob<float>(InferenceEngine::Precision::FP32,
88 layout, shape, (float*)m.data);
89 else if (m.type() == CV_8U)
90 return InferenceEngine::make_shared_blob<uint8_t>(InferenceEngine::Precision::U8,
91 layout, shape, (uint8_t*)m.data);
93 CV_Error(Error::StsNotImplemented, format("Unsupported data type %d", m.type()));
96 InferenceEngine::Blob::Ptr wrapToInfEngineBlob(const Mat& m, InferenceEngine::Layout layout)
98 std::vector<size_t> reversedShape(&m.size[0], &m.size[0] + m.dims);
99 std::reverse(reversedShape.begin(), reversedShape.end());
100 return wrapToInfEngineBlob(m, reversedShape, layout);
103 InferenceEngine::DataPtr infEngineDataNode(const Ptr<BackendWrapper>& ptr)
105 CV_Assert(!ptr.empty());
106 Ptr<InfEngineBackendWrapper> p = ptr.dynamicCast<InfEngineBackendWrapper>();
107 CV_Assert(!p.empty());
111 InfEngineBackendWrapper::InfEngineBackendWrapper(int targetId, const cv::Mat& m)
112 : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, targetId)
114 dataPtr = wrapToInfEngineDataNode(m);
115 blob = wrapToInfEngineBlob(m, estimateLayout(m));
118 InfEngineBackendWrapper::InfEngineBackendWrapper(Ptr<BackendWrapper> wrapper)
119 : BackendWrapper(DNN_BACKEND_INFERENCE_ENGINE, wrapper->targetId)
121 Ptr<InfEngineBackendWrapper> ieWrapper = wrapper.dynamicCast<InfEngineBackendWrapper>();
122 CV_Assert(!ieWrapper.empty());
123 InferenceEngine::DataPtr srcData = ieWrapper->dataPtr;
124 dataPtr = InferenceEngine::DataPtr(
125 new InferenceEngine::Data(srcData->name, srcData->dims, srcData->precision,
128 blob = ieWrapper->blob;
131 Ptr<BackendWrapper> InfEngineBackendWrapper::create(Ptr<BackendWrapper> wrapper)
133 return Ptr<BackendWrapper>(new InfEngineBackendWrapper(wrapper));
136 InfEngineBackendWrapper::~InfEngineBackendWrapper()
141 void InfEngineBackendWrapper::copyToHost()
146 void InfEngineBackendWrapper::setHostDirty()
151 InfEngineBackendNet::InfEngineBackendNet()
153 targetDevice = InferenceEngine::TargetDevice::eCPU;
154 precision = InferenceEngine::Precision::FP32;
157 InfEngineBackendNet::InfEngineBackendNet(InferenceEngine::CNNNetwork& net)
159 targetDevice = InferenceEngine::TargetDevice::eCPU;
160 precision = InferenceEngine::Precision::FP32;
161 inputs = net.getInputsInfo();
162 outputs = net.getOutputsInfo();
163 layers.resize(net.layerCount()); // A hack to execute InfEngineBackendNet::layerCount correctly.
166 void InfEngineBackendNet::Release() noexcept
173 void InfEngineBackendNet::setPrecision(InferenceEngine::Precision p) noexcept
178 InferenceEngine::Precision InfEngineBackendNet::getPrecision() noexcept
183 InferenceEngine::Precision InfEngineBackendNet::getPrecision() const noexcept
188 // Assume that outputs of network is unconnected blobs.
189 void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) noexcept
191 const_cast<const InfEngineBackendNet*>(this)->getOutputsInfo(outputs_);
193 void InfEngineBackendNet::getOutputsInfo(InferenceEngine::OutputsDataMap &outputs_) const noexcept
198 // Returns input references that aren't connected to internal outputs.
199 void InfEngineBackendNet::getInputsInfo(InferenceEngine::InputsDataMap &inputs_) noexcept
201 const_cast<const InfEngineBackendNet*>(this)->getInputsInfo(inputs_);
204 // Returns input references that aren't connected to internal outputs.
205 void InfEngineBackendNet::getInputsInfo(InferenceEngine::InputsDataMap &inputs_) const noexcept
210 InferenceEngine::InputInfo::Ptr InfEngineBackendNet::getInput(const std::string &inputName) noexcept
212 return const_cast<const InfEngineBackendNet*>(this)->getInput(inputName);
215 InferenceEngine::InputInfo::Ptr InfEngineBackendNet::getInput(const std::string &inputName) const noexcept
217 const auto& it = inputs.find(inputName);
218 CV_Assert(it != inputs.end());
222 void InfEngineBackendNet::getName(char*, size_t) noexcept
226 void InfEngineBackendNet::getName(char*, size_t) const noexcept
230 const std::string& InfEngineBackendNet::getName() const noexcept
235 size_t InfEngineBackendNet::layerCount() noexcept
237 return const_cast<const InfEngineBackendNet*>(this)->layerCount();
240 size_t InfEngineBackendNet::layerCount() const noexcept
242 return layers.size();
245 InferenceEngine::DataPtr& InfEngineBackendNet::getData(const char *dname) noexcept
247 CV_Error(Error::StsNotImplemented, "");
248 return outputs.begin()->second; // Just return something.
251 void InfEngineBackendNet::addLayer(const InferenceEngine::CNNLayerPtr &layer) noexcept
253 layers.push_back(layer);
258 InferenceEngine::StatusCode
259 InfEngineBackendNet::addOutput(const std::string &layerName, size_t outputIndex,
260 InferenceEngine::ResponseDesc *resp) noexcept
262 for (const auto& l : layers)
264 for (const InferenceEngine::DataPtr& out : l->outData)
266 if (out->name == layerName)
268 outputs[out->name] = out;
269 return InferenceEngine::StatusCode::OK;
273 CV_Error(Error::StsObjectNotFound, "Cannot find a layer " + layerName);
274 return InferenceEngine::StatusCode::OK;
277 InferenceEngine::StatusCode
278 InfEngineBackendNet::getLayerByName(const char *layerName, InferenceEngine::CNNLayerPtr &out,
279 InferenceEngine::ResponseDesc *resp) noexcept
281 return const_cast<const InfEngineBackendNet*>(this)->getLayerByName(layerName, out, resp);
284 InferenceEngine::StatusCode InfEngineBackendNet::getLayerByName(const char *layerName,
285 InferenceEngine::CNNLayerPtr &out,
286 InferenceEngine::ResponseDesc *resp) const noexcept
288 for (auto& l : layers)
290 if (l->name == layerName)
293 return InferenceEngine::StatusCode::OK;
296 CV_Error(Error::StsObjectNotFound, cv::format("Cannot find a layer %s", layerName));
297 return InferenceEngine::StatusCode::NOT_FOUND;
300 void InfEngineBackendNet::setTargetDevice(InferenceEngine::TargetDevice device) noexcept
302 if (device != InferenceEngine::TargetDevice::eCPU &&
303 device != InferenceEngine::TargetDevice::eGPU &&
304 device != InferenceEngine::TargetDevice::eMYRIAD)
305 CV_Error(Error::StsNotImplemented, "");
306 targetDevice = device;
309 InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() noexcept
314 InferenceEngine::TargetDevice InfEngineBackendNet::getTargetDevice() const noexcept
319 InferenceEngine::StatusCode InfEngineBackendNet::setBatchSize(const size_t) noexcept
321 CV_Error(Error::StsNotImplemented, "");
322 return InferenceEngine::StatusCode::OK;
325 size_t InfEngineBackendNet::getBatchSize() const noexcept
327 CV_Error(Error::StsNotImplemented, "");
331 void InfEngineBackendNet::init(int targetId)
335 // Collect all external input blobs.
337 std::map<std::string, InferenceEngine::DataPtr> internalOutputs;
338 for (const auto& l : layers)
340 for (const InferenceEngine::DataWeakPtr& ptr : l->insData)
342 InferenceEngine::DataPtr inp(ptr);
343 if (internalOutputs.find(inp->name) == internalOutputs.end())
345 InferenceEngine::InputInfo::Ptr inpInfo(new InferenceEngine::InputInfo());
346 inpInfo->setInputData(inp);
347 if (inputs.find(inp->name) == inputs.end())
348 inputs[inp->name] = inpInfo;
351 for (const InferenceEngine::DataPtr& out : l->outData)
353 // TODO: Replace to uniqueness assertion.
354 if (internalOutputs.find(out->name) == internalOutputs.end())
355 internalOutputs[out->name] = out;
358 CV_Assert(!inputs.empty());
363 // Add all unconnected blobs to output blobs.
364 InferenceEngine::OutputsDataMap unconnectedOuts;
365 for (const auto& l : layers)
368 for (const InferenceEngine::DataPtr& out : l->outData)
370 // TODO: Replace to uniqueness assertion.
371 if (unconnectedOuts.find(out->name) == unconnectedOuts.end())
372 unconnectedOuts[out->name] = out;
374 // Remove internally connected outputs.
375 for (const InferenceEngine::DataWeakPtr& inp : l->insData)
377 unconnectedOuts.erase(InferenceEngine::DataPtr(inp)->name);
380 CV_Assert(!unconnectedOuts.empty());
382 for (auto it = unconnectedOuts.begin(); it != unconnectedOuts.end(); ++it)
384 outputs[it->first] = it->second;
388 // Set up input blobs.
390 for (const auto& it : inputs)
392 CV_Assert(allBlobs.find(it.first) != allBlobs.end());
393 inpBlobs[it.first] = allBlobs[it.first];
394 it.second->setPrecision(inpBlobs[it.first]->precision());
397 // Set up output blobs.
399 for (const auto& it : outputs)
401 CV_Assert(allBlobs.find(it.first) != allBlobs.end());
402 outBlobs[it.first] = allBlobs[it.first];
407 case DNN_TARGET_CPU: setTargetDevice(InferenceEngine::TargetDevice::eCPU); break;
408 case DNN_TARGET_OPENCL_FP16:
409 setPrecision(InferenceEngine::Precision::FP16);
411 case DNN_TARGET_OPENCL: setTargetDevice(InferenceEngine::TargetDevice::eGPU); break;
412 case DNN_TARGET_MYRIAD:
414 setPrecision(InferenceEngine::Precision::FP16);
415 setTargetDevice(InferenceEngine::TargetDevice::eMYRIAD); break;
418 CV_Error(Error::StsError, format("Unknown target identifier: %d", targetId));
421 if (!isInitialized())
425 void InfEngineBackendNet::initPlugin(InferenceEngine::ICNNNetwork& net)
427 CV_Assert(!isInitialized());
431 static std::map<std::string, InferenceEngine::InferenceEnginePluginPtr> sharedPlugins;
432 std::string deviceName = InferenceEngine::getDeviceName(targetDevice);
433 auto pluginIt = sharedPlugins.find(deviceName);
434 if (pluginIt != sharedPlugins.end())
436 enginePtr = pluginIt->second;
440 enginePtr = InferenceEngine::PluginDispatcher({""}).getSuitablePlugin(targetDevice);
441 sharedPlugins[deviceName] = enginePtr;
443 if (targetDevice == InferenceEngine::TargetDevice::eCPU)
445 std::string suffixes[] = {"_avx2", "_sse4", ""};
446 bool haveFeature[] = {
447 checkHardwareSupport(CPU_AVX2),
448 checkHardwareSupport(CPU_SSE4_2),
451 for (int i = 0; i < 3; ++i)
456 std::string libName = "cpu_extension" + suffixes[i] + ".dll";
458 std::string libName = "libcpu_extension" + suffixes[i] + ".so";
462 InferenceEngine::IExtensionPtr extension =
463 InferenceEngine::make_so_pointer<InferenceEngine::IExtension>(libName);
464 enginePtr->AddExtension(extension, 0);
469 // Some of networks can work without a library of extra layers.
472 plugin = InferenceEngine::InferencePlugin(enginePtr);
474 netExec = plugin.LoadNetwork(net, {});
475 infRequest = netExec.CreateInferRequest();
476 infRequest.SetInput(inpBlobs);
477 infRequest.SetOutput(outBlobs);
479 catch (const std::exception& ex)
481 CV_Error(Error::StsAssert, format("Failed to initialize Inference Engine backend: %s", ex.what()));
485 bool InfEngineBackendNet::isInitialized()
487 return (bool)enginePtr;
490 void InfEngineBackendNet::addBlobs(const std::vector<Ptr<BackendWrapper> >& ptrs)
492 auto wrappers = infEngineWrappers(ptrs);
493 for (const auto& wrapper : wrappers)
495 allBlobs.insert({wrapper->dataPtr->name, wrapper->blob});
499 void InfEngineBackendNet::forward()
504 Mat infEngineBlobToMat(const InferenceEngine::Blob::Ptr& blob)
506 // NOTE: Inference Engine sizes are reversed.
507 std::vector<size_t> dims = blob->dims();
508 std::vector<int> size(dims.begin(), dims.end());
509 std::reverse(size.begin(), size.end());
510 return Mat(size, CV_32F, (void*)blob->buffer());
513 InfEngineBackendLayer::InfEngineBackendLayer(const InferenceEngine::DataPtr& output_)
518 bool InfEngineBackendLayer::getMemoryShapes(const std::vector<MatShape> &inputs,
519 const int requiredOutputs,
520 std::vector<MatShape> &outputs,
521 std::vector<MatShape> &internals) const
523 std::vector<size_t> dims = output->dims;
524 std::vector<int> shape(dims.begin(), dims.end());
525 std::reverse(shape.begin(), shape.end());
526 outputs.assign(1, shape);
530 bool InfEngineBackendLayer::supportBackend(int backendId)
532 return backendId == DNN_BACKEND_DEFAULT ||
533 backendId == DNN_BACKEND_INFERENCE_ENGINE && haveInfEngine();
536 void InfEngineBackendLayer::forward(std::vector<Mat*> &input, std::vector<Mat> &output,
537 std::vector<Mat> &internals)
539 CV_Error(Error::StsError, "Choose Inference Engine as a preferable backend.");
542 void InfEngineBackendLayer::forward(InputArrayOfArrays inputs, OutputArrayOfArrays outputs,
543 OutputArrayOfArrays internals)
545 CV_Error(Error::StsInternal, "Choose Inference Engine as a preferable backend.");
548 InferenceEngine::TBlob<int16_t>::Ptr convertFp16(const InferenceEngine::Blob::Ptr& blob)
550 auto halfs = InferenceEngine::make_shared_blob<int16_t>(InferenceEngine::Precision::FP16, blob->layout(), blob->dims());
552 Mat floatsData(1, blob->size(), CV_32F, blob->buffer());
553 Mat halfsData(1, blob->size(), CV_16SC1, halfs->buffer());
554 convertFp16(floatsData, halfsData);
558 #endif // HAVE_INF_ENGINE
562 #ifdef HAVE_INF_ENGINE
566 #endif // HAVE_INF_ENGINE
569 void forwardInfEngine(Ptr<BackendNode>& node)
571 CV_Assert(haveInfEngine());
572 #ifdef HAVE_INF_ENGINE
573 CV_Assert(!node.empty());
574 Ptr<InfEngineBackendNode> ieNode = node.dynamicCast<InfEngineBackendNode>();
575 CV_Assert(!ieNode.empty());
576 ieNode->net->forward();
577 #endif // HAVE_INF_ENGINE
580 }} // namespace dnn, namespace cv