1 // Copyright (C) 2018-2020 Intel Corporation
2 // SPDX-License-Identifier: Apache-2.0
5 #include "low_precision_transformations/network_helper.hpp"
8 #include <blob_factory.hpp>
10 #include <details/caseless.hpp>
15 #include <unordered_set>
19 #include <details/ie_cnn_network_tools.h>
20 #include <ie_common.h>
21 #include <precision_utils.h>
22 #include "cnn_network_impl.hpp"
23 #include "ie_util_internal.hpp"
24 #include "ie_parallel.hpp"
25 #include "low_precision_transformations/common/ie_lpt_exception.hpp"
27 using namespace InferenceEngine;
28 using namespace InferenceEngine::details;
30 static const std::unordered_set<std::string> intermediateLayers{
35 bool Subgraph::fillSubgraphForQuantization(const CNNLayerPtr& fakeQuantize, std::unordered_set<std::string>& handledLayers) {
36 if (fakeQuantize->type != "FakeQuantize") {
37 THROW_IE_EXCEPTION << "unexpected layer type " << fakeQuantize->type;
40 if (!QuantizationDetails::outputLayoutIsSupported(*fakeQuantize)) {
44 quantizationLayers.push_back(fakeQuantize);
45 handledLayers.insert(fakeQuantize->name);
46 layers.emplace(fakeQuantize->name, fakeQuantize.get());
48 const std::vector<CNNLayerPtr> children = CNNNetworkHelper::getChildren(*fakeQuantize);
49 for (const CNNLayerPtr& child : children) {
50 if (handledLayers.find(child->name) != handledLayers.end()) {
54 if (child->type == "Concat") {
55 if (!fillSubgraphForConcat(child, handledLayers)) {
58 } else if (child->type == "FakeQuantize") {
60 } else if (intermediateLayers.find(child->type) != intermediateLayers.end()) {
61 if (!fillSubgraphForIntermediate(child, handledLayers)) {
70 bool Subgraph::fill(const CNNLayerPtr& layer, std::unordered_set<std::string>& handledLayers) {
71 const std::vector<CNNLayerPtr> parents = CNNNetworkHelper::getParents(*layer);
72 for (const CNNLayerPtr& parent : parents) {
73 if (handledLayers.find(parent->name) != handledLayers.end()) {
77 if (parent->type == "Concat") {
78 if (!fillSubgraphForConcat(parent, handledLayers)) {
81 } else if (parent->type == "FakeQuantize") {
82 if (!fillSubgraphForQuantization(parent, handledLayers)) {
85 } else if (intermediateLayers.find(parent->type) != intermediateLayers.end()) {
86 if (!fillSubgraphForIntermediate(parent, handledLayers)) {
94 const std::vector<CNNLayerPtr> children = CNNNetworkHelper::getChildren(*layer);
95 for (const CNNLayerPtr& child : children) {
96 if (handledLayers.find(child->name) != handledLayers.end()) {
100 if (child->type == "Concat") {
101 if (!fillSubgraphForConcat(child, handledLayers)) {
104 } else if (child->type == "FakeQuantize") {
106 } else if (intermediateLayers.find(child->type) != intermediateLayers.end()) {
107 if (!fillSubgraphForIntermediate(child, handledLayers)) {
116 bool Subgraph::fillSubgraphForIntermediate(const CNNLayerPtr& intermediate, std::unordered_set<std::string>& handledLayers) {
117 if (intermediateLayers.find(intermediate->type) == intermediateLayers.end()) {
118 THROW_IE_EXCEPTION << "unexpected layer type " << intermediate->type;
121 handledLayers.insert(intermediate->name);
122 layers.emplace(intermediate->name, intermediate.get());
124 return fill(intermediate, handledLayers);
127 bool Subgraph::empty() const {
128 return quantizationLayers.empty();
131 bool Subgraph::fillSubgraphForConcat(const CNNLayerPtr& concat, std::unordered_set<std::string>& handledLayers) {
132 if (concat->type != "Concat") {
133 THROW_IE_EXCEPTION << "unexpected layer type " << concat->type;
136 concatLayers.push_back(concat);
137 handledLayers.insert(concat->name);
138 layers.emplace(concat->name, concat.get());
140 return fill(concat, handledLayers);
143 Subgraph CNNNetworkHelper::getSubgraph(const CNNLayer& concat) {
144 if (concat.type != "Concat") {
145 THROW_IE_EXCEPTION << "unexpected layer type " << concat.type;
149 std::unordered_set<std::string> handledLayers;
150 if (!subgraph.fillSubgraphForConcat(std::make_shared<CNNLayer>(concat), handledLayers)) {
157 CNNLayerPtr CNNNetworkHelper::getLayer(const ICNNNetwork& network, const std::string& layerName) {
158 std::vector<CNNLayerPtr> layers = InferenceEngine::details::CNNNetSortTopologically(network);
159 for (CNNLayerPtr layer : layers) {
160 if (layer->name == layerName) {
168 Blob::Ptr CNNNetworkHelper::makeNewBlobPtr(const TensorDesc& desc) {
170 if (desc.getPrecision() == Precision::FP32)
171 newBlob = make_shared_blob<PrecisionTrait<Precision::FP32>::value_type>(desc);
172 else if (desc.getPrecision() == Precision::FP16)
173 newBlob = make_shared_blob<PrecisionTrait<Precision::FP16>::value_type>(desc);
174 else if (desc.getPrecision() == Precision::I8)
175 newBlob = make_shared_blob<PrecisionTrait<Precision::I8>::value_type>(desc);
176 else if (desc.getPrecision() == Precision::U8)
177 newBlob = make_shared_blob<PrecisionTrait<Precision::U8>::value_type>(desc);
178 else if (desc.getPrecision() == Precision::I32)
179 newBlob = make_shared_blob<PrecisionTrait<Precision::I32>::value_type>(desc);
181 THROW_IE_EXCEPTION << "Unsupported transformation precision: " << desc.getPrecision();
186 void CNNNetworkHelper::updateBlobs(CNNLayer& layer, const std::string& blobName, float value) {
187 const auto existingBlobIt = layer.blobs.find(blobName);
188 if (existingBlobIt == layer.blobs.end()) {
189 THROW_IE_EXCEPTION << "blob '" << blobName << "' was not found in layer " << layer.name;
191 const auto& existingBlobTensorDesc = existingBlobIt->second->getTensorDesc();
192 Blob::Ptr newBlob = makeNewBlobPtr(existingBlobTensorDesc);
195 fillBlobByFP32(newBlob, value);
196 layer.blobs[existingBlobIt->first] = newBlob;
199 void CNNNetworkHelper::invertFakeQuantize(const CNNLayer& fakeQuantize) {
200 if (fakeQuantize.type != "FakeQuantize") {
201 THROW_IE_EXCEPTION << "invalid layer type " << fakeQuantize.type;
203 const QuantizationDetails quantizationDetails = QuantizationDetails::getDetails(fakeQuantize);
204 const size_t valuesCount =
205 std::max(quantizationDetails.inputLowValues.size(), quantizationDetails.outputLowValues.size());
206 std::vector<float> inputLowValues(valuesCount);
207 std::vector<float> inputHightValues(valuesCount);
208 std::vector<float> outputLowValues(valuesCount);
209 std::vector<float> outputHighValues(valuesCount);
210 bool wasInverted = false;
211 for (size_t i = 0ul; i < valuesCount; ++i) {
212 if ((quantizationDetails.getInputLowValue(i) > quantizationDetails.getInputHighValue(i)) &&
213 (quantizationDetails.getOutputLowValue(i) > quantizationDetails.getOutputHighValue(i))) {
214 inputLowValues[i] = quantizationDetails.getInputHighValue(i);
215 inputHightValues[i] = quantizationDetails.getInputLowValue(i);
216 outputLowValues[i] = quantizationDetails.getOutputHighValue(i);
217 outputHighValues[i] = quantizationDetails.getOutputLowValue(i);
220 inputLowValues[i] = quantizationDetails.getInputLowValue(i);
221 inputHightValues[i] = quantizationDetails.getInputHighValue(i);
222 outputLowValues[i] = quantizationDetails.getOutputLowValue(i);
223 outputHighValues[i] = quantizationDetails.getOutputHighValue(i);
228 CNNNetworkHelper::updateBlobs(fakeQuantize, 1, inputLowValues);
229 CNNNetworkHelper::updateBlobs(fakeQuantize, 2, inputHightValues);
230 CNNNetworkHelper::updateBlobs(fakeQuantize, 3, outputLowValues);
231 CNNNetworkHelper::updateBlobs(fakeQuantize, 4, outputHighValues);
234 void CNNNetworkHelper::updateBlobs(const CNNLayer& quantizeLayer, int constLayerIndex,
235 const std::vector<float>& values) {
236 CNNLayerPtr blobLayer = CNNNetworkHelper::getParent(quantizeLayer, constLayerIndex);
237 if (blobLayer == nullptr) {
238 THROW_IE_EXCEPTION << "layer is absent";
241 const auto existingBlobIt = blobLayer->blobs.find("custom");
242 if (existingBlobIt == blobLayer->blobs.end()) {
243 THROW_IE_EXCEPTION << "custom blob was not found ";
246 TensorDesc newBlobTensorDesc;
248 const TensorDesc existingBlobTensorDesc = existingBlobIt->second->getTensorDesc();
249 if ((existingBlobIt->second->size() != values.size()) && (values.size() != 1)) {
250 if (existingBlobTensorDesc.getLayout() == Layout::SCALAR) {
252 } else if (existingBlobTensorDesc.getLayout() == Layout::C) {
253 if (existingBlobTensorDesc.getDims().size() != 1) {
254 THROW_IE_EXCEPTION << "temporary dimensions size " << existingBlobTensorDesc.getDims().size()
255 << " for layout " << existingBlobTensorDesc.getLayout() << " is not supported";
257 if (existingBlobTensorDesc.getDims()[0] != 1) {
258 THROW_IE_EXCEPTION << "temporary is not supported";
260 } else if (existingBlobTensorDesc.getLayout() == Layout::NCHW) {
261 if (existingBlobTensorDesc.getDims().size() != 4) {
262 THROW_IE_EXCEPTION << "temporary dimensions size " << existingBlobTensorDesc.getDims().size()
263 << " for layout " << existingBlobTensorDesc.getLayout() << " is not supported";
266 if (existingBlobTensorDesc.getDims()[0] != 1) {
267 THROW_IE_EXCEPTION << "temporary is not supported";
271 const std::vector<size_t> dims = {values.size()};
272 const Layout layout = Layout::C;
273 newBlobTensorDesc = TensorDesc(existingBlobTensorDesc.getPrecision(), dims, layout);
274 for (DataPtr data : blobLayer->outData) {
275 data->reshape(dims, layout);
278 newBlobTensorDesc = existingBlobTensorDesc;
281 Blob::Ptr newBlob = makeNewBlobPtr(newBlobTensorDesc);
283 blobLayer->blobs[existingBlobIt->first] = newBlob;
285 if (values.size() == 1)
286 fillBlobByFP32(newBlob, values[0]);
288 fillBlobByFP32(newBlob, values.data());
291 void CNNNetworkHelper::updateBlobs(CNNLayer& layer, const std::string& blobName, const std::vector<float>& values) {
292 const auto existingBlobIt = layer.blobs.find(blobName);
293 if (existingBlobIt == layer.blobs.end()) {
294 THROW_IE_EXCEPTION << "custom blob was not found ";
297 TensorDesc newBlobTensorDesc;
299 const TensorDesc existingBlobTensorDesc = existingBlobIt->second->getTensorDesc();
300 if ((existingBlobIt->second->size() != values.size()) && (values.size() != 1)) {
301 if (existingBlobTensorDesc.getLayout() == Layout::SCALAR) {
303 } else if (existingBlobTensorDesc.getLayout() == Layout::C) {
304 if (existingBlobTensorDesc.getDims().size() != 1) {
305 THROW_IE_EXCEPTION << "temporary dimensions size " << existingBlobTensorDesc.getDims().size()
306 << " for layout " << existingBlobTensorDesc.getLayout() << " is not supported";
308 if (existingBlobTensorDesc.getDims()[0] != 1) {
309 THROW_IE_EXCEPTION << "temporary is not supported";
311 } else if (existingBlobTensorDesc.getLayout() == Layout::NCHW) {
312 if (existingBlobTensorDesc.getDims().size() != 4) {
313 THROW_IE_EXCEPTION << "temporary dimensions size " << existingBlobTensorDesc.getDims().size()
314 << " for layout " << existingBlobTensorDesc.getLayout() << " is not supported";
317 if (existingBlobTensorDesc.getDims()[0] != 1) {
318 THROW_IE_EXCEPTION << "temporary is not supported";
322 const std::vector<size_t> dims = {values.size()};
323 const Layout layout = Layout::C;
324 newBlobTensorDesc = TensorDesc(existingBlobTensorDesc.getPrecision(), dims, layout);
325 for (DataPtr data : layer.outData) {
326 data->reshape(dims, layout);
329 newBlobTensorDesc = existingBlobTensorDesc;
332 Blob::Ptr newBlob = makeNewBlobPtr(newBlobTensorDesc);
334 layer.blobs[existingBlobIt->first] = newBlob;
336 if ((blobName == "weights") || (blobName == "biases")) {
337 WeightableLayer* weightableLayer = dynamic_cast<WeightableLayer*>(&layer);
338 if (weightableLayer == nullptr) {
339 THROW_IE_EXCEPTION << "layer '" << layer.name << "' with blob name '" << blobName << "' is not weightable";
341 if (blobName == "weights") {
342 weightableLayer->_weights = newBlob;
343 } else if (blobName == "biases") {
344 weightableLayer->_biases = newBlob;
346 THROW_IE_EXCEPTION << "unexpected blob name '" << blobName << "' for layer " << layer.name;
350 if (values.size() == 1)
351 fillBlobByFP32(newBlob, values[0]);
353 fillBlobByFP32(newBlob, values.data());
356 void CNNNetworkHelper::updateBlobs(const CNNLayer& quantizeLayer, int constLayerIndex, float value) {
357 auto inData = quantizeLayer.insData[constLayerIndex].lock();
358 if (inData == nullptr) {
359 THROW_IE_EXCEPTION << "data is absent";
362 CNNLayerPtr blobLayer = inData->getCreatorLayer().lock();
363 if (blobLayer == nullptr) {
364 THROW_IE_EXCEPTION << "layer is absent";
367 if (blobLayer->blobs.size() != 1) {
368 THROW_IE_EXCEPTION << "unexpected blobs size";
371 const auto existingBlobIt = blobLayer->blobs.begin();
372 const auto& existingBlobTensorDesc = existingBlobIt->second->getTensorDesc();
373 Blob::Ptr newBlob = makeNewBlobPtr(existingBlobTensorDesc);
376 fillBlobByFP32(newBlob, value);
377 blobLayer->blobs[existingBlobIt->first] = newBlob;
380 int CNNNetworkHelper::onWeightsInDepth(const CNNLayer& layer) {
381 const std::vector<CNNLayerPtr> children = getChildren(layer);
382 for (const CNNLayerPtr& child : children) {
383 if ((CaselessEq<std::string>()(child->type, "Convolution") ||
384 CaselessEq<std::string>()(child->type, "FullyConnected") ||
385 CaselessEq<std::string>()(child->type, "GEMM")) &&
386 (child->insData.size() >= 2lu)) {
387 const std::vector<CNNLayerPtr> parents = getParentsRecursivelyExceptTypes(*child, {}, 1);
388 for (const CNNLayerPtr& parent : parents) {
389 if (parent->name == layer.name) {
396 const int result = onWeightsInDepth(*child);
404 bool CNNNetworkHelper::onWeights(const CNNLayer& layer) {
405 const int result = onWeightsInDepth(layer);
409 size_t CNNNetworkHelper::getIndex(const CNNLayer& layer) {
410 const std::vector<CNNLayerPtr> children = CNNNetworkHelper::getChildren(layer);
411 if (children.size() != 1) {
412 THROW_IE_EXCEPTION << "not supported";
415 for (size_t i = 0; i < children[0]->insData.size(); ++i) {
416 const DataPtr insData = children[0]->insData[i].lock();
417 if (insData == nullptr) {
420 const CNNLayerPtr parent = insData->getCreatorLayer().lock();
421 if ((parent != nullptr) && (parent->name == layer.name)) {
426 THROW_IE_EXCEPTION << "not found";
429 std::vector<CNNLayerPtr> CNNNetworkHelper::transformFakeQuantizeToConst(TransformationContext& context,
430 const CNNLayerPtr fakeQuantize,
431 const Blob::Ptr weights,
432 const std::string& constLayerName) {
433 std::vector<CNNLayerPtr> constLayersToRemove;
434 constLayersToRemove.reserve(fakeQuantize->insData.size());
436 for (const DataWeakPtr& insDataWeak : fakeQuantize->insData) {
437 const DataPtr insData = insDataWeak.lock();
438 if (insData == nullptr) {
439 THROW_IE_EXCEPTION << "input data for FakeQuantize '" << fakeQuantize->name << "' is nullable";
441 const CNNLayerPtr parent = insData->getCreatorLayer().lock();
442 if (parent == nullptr) {
443 THROW_IE_EXCEPTION << "input layer for FakeQuantize '" << fakeQuantize->name << "' is nullable";
445 if (!CaselessEq<std::string>()(parent->type, "Const") || (parent->insData.size() != 0lu)) {
446 THROW_IE_EXCEPTION << "unexpected FakeQuantize input layer type " << parent->type << " for layer '"
447 << fakeQuantize->name << "' is nullable";
450 constLayersToRemove.push_back(parent);
453 for (const CNNLayerPtr& parent : constLayersToRemove) {
454 CNNNetworkHelper::removeLayer(context.network, parent);
455 context.removeLayer(*parent);
458 if (fakeQuantize->outData.size() != 1lu) {
459 THROW_IE_EXCEPTION << "FakeQuantize " << fakeQuantize->name << " has several outputs";
462 const DataPtr outData = fakeQuantize->outData[0];
463 if (outData == nullptr) {
464 THROW_IE_EXCEPTION << "FakeQuantize output data is nullable";
467 // const Precision precision = outData->getPrecision();
468 const auto inputTo = outData->getInputTo();
469 std::vector<CNNLayerPtr> constLayers;
470 for (auto it : inputTo) {
471 const CNNLayerPtr child = it.second;
472 if (child == nullptr) {
473 THROW_IE_EXCEPTION << "child layer for FakeQuantize " << fakeQuantize->name << " is nullable";
476 constLayers.push_back(
477 CNNNetworkHelper::addConstBetween(context.network, fakeQuantize, child, weights, constLayerName));
480 CNNNetworkHelper::removeLayer(context.network, fakeQuantize);
481 context.removeLayer(*fakeQuantize);
486 void CNNNetworkHelper::setOutDataPrecision(const CNNLayer& layer, const Precision& precision) {
487 for (const DataPtr& data : layer.outData) {
488 data->setPrecision(precision);
492 void CNNNetworkHelper::setOutDataPrecision(const std::vector<CNNLayerPtr>& layers, const Precision& precision) {
493 for (const CNNLayerPtr layer : layers) {
494 setOutDataPrecision(*layer, precision);
498 void CNNNetworkHelper::setOutDataPrecision(const CNNLayer& beginLayer, const size_t branchWithEndBeforeLayer,
499 const CNNLayer& endBeforeLayer, const Precision& precision) {
500 CNNLayerPtr child = std::make_shared<CNNLayer>(beginLayer);
501 while (child->name != endBeforeLayer.name) {
502 CNNNetworkHelper::setOutDataPrecision(*child, precision);
503 std::vector<CNNLayerPtr> children = CNNNetworkHelper::getChildren(*child);
504 if (child->name == beginLayer.name) {
505 if (branchWithEndBeforeLayer >= children.size()) {
506 THROW_IE_EXCEPTION << "branch with end before layer is out of children count " << children.size();
508 child = children[branchWithEndBeforeLayer];
510 if (children.size() != 1) {
511 THROW_IE_EXCEPTION << "not supported";
519 bool CNNNetworkHelper::IsChild(const std::vector<CNNLayerPtr>& children,
520 const std::unordered_set<std::string>& layerTypes,
521 const std::unordered_set<std::string>& ignoreLayerTypes) {
522 for (const CNNLayerPtr& child : children) {
523 if (layerTypes.find(child->type) != layerTypes.end()) {
526 if (ignoreLayerTypes.find(child->type) != ignoreLayerTypes.end()) {
527 if (child->outData.size() != 1) {
530 if (IsChild(CNNNetworkHelper::getChildren(*child), layerTypes, ignoreLayerTypes)) {
538 size_t CNNNetworkHelper::getOutputChannelsCount(const CNNLayer& layer, bool isOnWeights) {
539 if (layer.outData.empty()) {
540 THROW_IE_EXCEPTION << "Layer " << layer.name << " doesn't have output tensors";
543 auto& data = layer.outData[0];
545 if (data->getDims().empty()) {
546 THROW_IE_EXCEPTION << "Invalid dimensions count (0) in output of " << layer.name << " layer on weights";
548 return data->getDims()[0];
550 if (data->getDims().empty()) {
551 THROW_IE_EXCEPTION << "Invalid dimensions count (0) in output of " << layer.name << " layer on activations";
553 if (data->getDims().size() == 1ul) {
554 return data->getDims()[0];
556 return data->getDims()[1];
560 std::vector<CNNLayerPtr> CNNNetworkHelper::getLayers(const CNNLayer& parent, const CNNLayer& child) {
561 std::vector<CNNLayerPtr> layers;
562 CNNLayerPtr tmpChild = std::make_shared<CNNLayer>(child);
563 while (tmpChild != nullptr) {
564 const std::vector<CNNLayerPtr> parents = CNNNetworkHelper::getParents(*tmpChild);
565 for (const CNNLayerPtr tmpParent : parents) {
566 if (tmpParent->name == parent.name) {
571 if (parents.size() == 0) {
572 THROW_IE_EXCEPTION << "not found";
575 if (parents.size() != 1ul) {
576 THROW_IE_EXCEPTION << "not supported";
579 layers.push_back(parents[0]);
580 tmpChild = parents[0];
585 Blob::Ptr CNNNetworkHelper::getBlob(CNNLayer* layer, const std::string& blobName) {
586 if (layer == nullptr) {
587 THROW_IE_EXCEPTION << "layer is nullable";
589 if (layer->blobs.empty()) {
590 THROW_IE_EXCEPTION << "Layer '" << layer->name << "' does not have any blob";
592 if (blobName.empty() && (layer->blobs.size() != 1)) {
593 THROW_IE_EXCEPTION << "several blobs";
595 Blob::Ptr blob = blobName.empty() ? layer->blobs.begin()->second : layer->blobs[blobName];
599 Blob::Ptr CNNNetworkHelper::getBlob(CNNLayerPtr layer, const std::string& blobName) {
600 return getBlob(layer.get(), blobName);
603 std::shared_ptr<float> CNNNetworkHelper::getFloatData(const Blob::Ptr& srcBlob) {
604 if (srcBlob == nullptr) {
605 THROW_IE_EXCEPTION << "Invalid blob";
608 const auto& precision = srcBlob->getTensorDesc().getPrecision();
609 if (!isBlobPrecisionSupported(precision)) {
610 THROW_IE_EXCEPTION << "precision '" << precision << "' is not supported";
613 const size_t dataSize = srcBlob->size();
614 std::shared_ptr<float> floatPtr(new float[dataSize], std::default_delete<float[]>());
616 if (precision == Precision::FP32) {
617 const float* srcData = srcBlob->buffer().as<float*>();
618 std::copy(srcData, srcData + dataSize, floatPtr.get());
619 } else if (precision == Precision::FP16) {
620 const short* srcData = srcBlob->buffer().as<short*>();
621 PrecisionUtils::f16tof32Arrays(floatPtr.get(), srcData, dataSize, 1.f, 0.f);
622 } else if (precision == Precision::I8) {
623 const auto* srcData = srcBlob->buffer().as<PrecisionTrait<Precision::I8>::value_type*>();
624 std::copy(srcData, srcData + dataSize, floatPtr.get());
625 } else if (precision == Precision::U8) {
626 const auto* srcData = srcBlob->buffer().as<PrecisionTrait<Precision::U8>::value_type*>();
627 std::copy(srcData, srcData + dataSize, floatPtr.get());
628 } else if (precision == Precision::I32) {
629 const auto* srcData = srcBlob->buffer().as<PrecisionTrait<Precision::I32>::value_type*>();
630 std::copy(srcData, srcData + dataSize, floatPtr.get());
631 } else if (precision == Precision::I64) {
632 const auto* srcData = srcBlob->buffer().as<PrecisionTrait<Precision::I64>::value_type*>();
633 std::copy(srcData, srcData + dataSize, floatPtr.get());
634 } else if (precision == Precision::U64) {
635 const auto* srcData = srcBlob->buffer().as<PrecisionTrait<Precision::U64>::value_type*>();
636 std::copy(srcData, srcData + dataSize, floatPtr.get());
638 THROW_IE_EXCEPTION << "Unsupported transformation precision: " << precision;
644 bool CNNNetworkHelper::isBlobPrecisionSupported(const Precision precision) {
645 return (precision == Precision::FP32) ||
646 (precision == Precision::FP16) ||
647 (precision == Precision::I8) ||
648 (precision == Precision::U8) ||
649 (precision == Precision::I32) ||
650 (precision == Precision::I64) ||
651 (precision == Precision::U64);
654 std::shared_ptr<float> CNNNetworkHelper::getFloatData(const CNNLayerPtr& layer, const std::string& blobName) {
655 const Blob::Ptr blob = getBlob(layer, blobName);
656 if (blob == nullptr) THROW_IE_EXCEPTION << "Could not find blob '" << blobName << "' for layer " << layer->name;
658 return getFloatData(blob);
661 void CNNNetworkHelper::fillBlobByFP32(Blob::Ptr& dstBlob, const float* srcData) {
662 if (dstBlob == nullptr) THROW_IE_EXCEPTION << "Invalid blob";
664 const auto& precision = dstBlob->getTensorDesc().getPrecision();
665 const size_t dataSize = dstBlob->size();
667 if (precision == Precision::FP32) {
668 float* dstData = dstBlob->buffer().as<float*>();
669 std::copy(srcData, srcData + dataSize, dstData);
670 } else if (precision == Precision::FP16) {
671 short* dstData = dstBlob->buffer().as<short*>();
672 PrecisionUtils::f32tof16Arrays(dstData, srcData, dataSize, 1.f, 0.f);
673 } else if (precision == Precision::I8) {
674 auto* dstData = dstBlob->buffer().as<PrecisionTrait<Precision::I8>::value_type*>();
675 for (size_t i = 0ul; i < dataSize; ++i) {
676 dstData[i] = static_cast<PrecisionTrait<Precision::I8>::value_type>(std::roundf(srcData[i]));
678 } else if (precision == Precision::U8) {
679 auto* dstData = dstBlob->buffer().as<PrecisionTrait<Precision::U8>::value_type*>();
680 for (size_t i = 0ul; i < dataSize; ++i) {
681 dstData[i] = static_cast<PrecisionTrait<Precision::U8>::value_type>(std::roundf(srcData[i]));
683 } else if (precision == Precision::I32) {
684 auto* dstData = dstBlob->buffer().as<PrecisionTrait<Precision::I32>::value_type*>();
685 for (size_t i = 0ul; i < dataSize; ++i) {
686 dstData[i] = static_cast<PrecisionTrait<Precision::I32>::value_type>(std::roundf(srcData[i]));
689 THROW_IE_EXCEPTION << "Unsupported transformation precision: " << precision;
693 std::shared_ptr<float> CNNNetworkHelper::convertFloatData(const float* srcData, const size_t dataSize,
694 const Precision precision) {
695 std::shared_ptr<float> dstData(new float[dataSize], std::default_delete<float[]>());
697 if (precision == Precision::FP32) {
698 std::copy(srcData, srcData + dataSize, dstData.get());
699 } else if (precision == Precision::FP16) {
700 for (size_t i = 0ul; i < dataSize; ++i) {
701 dstData.get()[i] = PrecisionUtils::f16tof32(PrecisionUtils::f16tof32(srcData[i]));
703 } else if (precision == Precision::I8) {
704 for (size_t i = 0ul; i < dataSize; ++i) {
706 static_cast<float>(static_cast<PrecisionTrait<Precision::I8>::value_type>(std::roundf(srcData[i])));
708 } else if (precision == Precision::U8) {
709 for (size_t i = 0ul; i < dataSize; ++i) {
711 static_cast<float>(static_cast<PrecisionTrait<Precision::U8>::value_type>(std::roundf(srcData[i])));
713 } else if (precision == Precision::I32) {
714 for (size_t i = 0ul; i < dataSize; ++i) {
716 static_cast<float>(static_cast<PrecisionTrait<Precision::I32>::value_type>(std::roundf(srcData[i])));
719 THROW_IE_EXCEPTION << "Unsupported transformation precision: " << precision;
725 void CNNNetworkHelper::fillBlobByFP32(const CNNLayerPtr& layer, const std::string& blobName, const float* srcData) {
726 Blob::Ptr blob = getBlob(layer, blobName);
727 return fillBlobByFP32(blob, srcData);
730 void CNNNetworkHelper::fillBlobByFP32(Blob::Ptr& dstBlob, float value) {
731 const auto& precision = dstBlob->getTensorDesc().getPrecision();
732 const size_t dataSize = dstBlob->size();
734 if (precision == Precision::FP32) {
735 float* dstData = dstBlob->buffer().as<float*>();
736 std::fill(dstData, dstData + dataSize, value);
737 } else if (precision == Precision::FP16) {
738 short* dstData = dstBlob->buffer().as<short*>();
739 const short s_value = PrecisionUtils::f32tof16(value);
740 std::fill(dstData, dstData + dataSize, s_value);
741 } else if (precision == Precision::I8) {
742 auto* dstData = dstBlob->buffer().as<PrecisionTrait<Precision::I8>::value_type*>();
743 std::fill(dstData, dstData + dataSize, static_cast<PrecisionTrait<Precision::I8>::value_type>(value));
744 } else if (precision == Precision::U8) {
745 auto* dstData = dstBlob->buffer().as<PrecisionTrait<Precision::U8>::value_type*>();
746 std::fill(dstData, dstData + dataSize, static_cast<PrecisionTrait<Precision::U8>::value_type>(value));
747 } else if (precision == Precision::I32) {
748 auto* dstData = dstBlob->buffer().as<PrecisionTrait<Precision::I32>::value_type*>();
749 std::fill(dstData, dstData + dataSize, static_cast<PrecisionTrait<Precision::I32>::value_type>(value));
751 THROW_IE_EXCEPTION << "Unsupported transformation precision: " << precision;
755 CNNLayerPtr CNNNetworkHelper::getParent(const CNNLayer& layer, const size_t index, const std::string& ignoreLayerType) {
756 if (index >= layer.insData.size()) {
760 DataPtr inputLayerData = layer.insData[index].lock();
761 if (inputLayerData == nullptr) {
762 THROW_IE_EXCEPTION << "input data is absent";
765 CNNLayerPtr inputLayer;
767 inputLayer = inputLayerData->getCreatorLayer().lock();
769 THROW_IE_EXCEPTION << "input is absent";
772 if (inputLayer->type != ignoreLayerType) {
776 if (inputLayer->insData.size() == 0) {
777 inputLayer = nullptr;
781 if (inputLayer->insData.size() != 1) {
782 THROW_IE_EXCEPTION << "too much branches";
785 inputLayerData = inputLayer->insData[0].lock();
786 if (inputLayerData == nullptr) {
787 THROW_IE_EXCEPTION << "input data is absent";
794 std::vector<CNNLayerPtr> CNNNetworkHelper::getParents(const CNNLayer& layer, const std::string& exceptionLayerName) {
795 std::vector<CNNLayerPtr> parents;
796 for (const DataWeakPtr insDataWeak : layer.insData) {
797 const DataPtr insData = insDataWeak.lock();
798 if (insData == nullptr) {
799 THROW_IE_EXCEPTION << "input data is absent";
802 CNNLayerPtr parent = insData->getCreatorLayer().lock();
803 if (parent == nullptr) {
804 THROW_IE_EXCEPTION << "input layer is absent";
807 if (exceptionLayerName.empty() || parent->name != exceptionLayerName) {
808 parents.push_back(parent);
814 std::vector<CNNLayerPtr> CNNNetworkHelper::getParentsRecursivelyExceptTypes(
815 const CNNLayer& layer, const std::unordered_set<std::string>& exceptionLayerTypes, const int portIndex) {
816 std::vector<CNNLayerPtr> parents;
818 for (DataWeakPtr insDataWeak : layer.insData) {
819 if (insDataWeak.expired()) {
823 const DataPtr insData = insDataWeak.lock();
824 if (insData == nullptr) {
825 THROW_IE_EXCEPTION << "input data is absent";
828 CNNLayerWeakPtr parentWeak = insData->getCreatorLayer();
829 if (parentWeak.expired()) {
833 if ((portIndex == -1) || (portIndex == i)) {
834 CNNLayerPtr parent = parentWeak.lock();
835 if (parent == nullptr) {
836 THROW_IE_EXCEPTION << "input layer is absent";
839 if (exceptionLayerTypes.find(parent->type) != exceptionLayerTypes.end()) {
840 const std::vector<CNNLayerPtr> tmpParents = CNNNetworkHelper::getParentsRecursivelyExceptTypes(*parent, exceptionLayerTypes);
841 parents.insert(parents.end(), tmpParents.begin(), tmpParents.end());
843 parents.push_back(parent);
852 size_t CNNNetworkHelper::getInputChannelsCount(const CNNLayer& layer) {
853 if (layer.insData.size() == 0) {
854 THROW_IE_EXCEPTION << "There are no input layers";
857 const DataPtr insertData = layer.insData[0].lock();
858 if (insertData == nullptr) {
859 THROW_IE_EXCEPTION << "insert data is absent";
862 switch (insertData->getLayout()) {
865 case Layout::NCDHW: {
866 return insertData->getDims()[1];
869 if (insertData->getDims().size() != 3lu) {
870 THROW_IE_EXCEPTION << "Unexpected dimensions size " << insertData->getDims().size() << " for layer "
874 // Actually MO assumes NCH layout for 3D blobs, so we get channels count from dimension 1
875 return insertData->getDims()[1];
878 THROW_IE_EXCEPTION << "Not supported layout " << insertData->getLayout();
883 size_t CNNNetworkHelper::getParamOutput(const CNNLayer& layer) {
884 if (!layer.CheckParamPresence("output")) {
885 THROW_IE_EXCEPTION << "convolution parameter 'output' is absent";
887 return layer.GetParamAsUInt("output");
890 size_t CNNNetworkHelper::getKernelSize(const CNNLayer& layer) {
891 if (!layer.CheckParamPresence("kernel")) {
892 THROW_IE_EXCEPTION << "convolution parameter 'kernel' is absent";
894 const auto dims = layer.GetParamAsUInts("kernel");
895 if (dims.size() == 2) {
896 return dims[0] * dims[1];
897 } else if (dims.size() == 3) {
898 return dims[0] * dims[1] * dims[2];
900 THROW_IE_EXCEPTION << "kernel dimensions are not correct";
904 void CNNNetworkHelper::renameLayer(ICNNNetwork& net, const std::string& currentName, const std::string& newName) {
905 CNNNetworkImpl* netImpl = dynamic_cast<CNNNetworkImpl*>(&net);
906 if (netImpl == nullptr) {
907 THROW_IE_EXCEPTION << "unexpected network type";
910 netImpl->renameLayer(currentName, newName);
913 CNNLayerPtr CNNNetworkHelper::addLayer(
914 TransformationContext& context,
915 const CNNLayerPtr parent,
916 const CNNLayerPtr child,
917 const CNNLayerPtr newLayer) {
920 if (parent != nullptr) {
921 // Searching the connection between the layers
923 if (child != nullptr) {
924 for (; l1_out_i < parent->outData.size(); l1_out_i++) {
925 if (parent->outData[l1_out_i]->getInputTo().find(child->name) !=
926 parent->outData[l1_out_i]->getInputTo().end()) {
931 if (l1_out_i == parent->outData.size()) {
932 if (child != nullptr)
933 THROW_IE_EXCEPTION << "Can't find layer " << child->name << " among layer " << parent->name << " outputs";
935 THROW_IE_EXCEPTION << "Layer '" << parent->name << "' has invalid output";
938 outData = parent->outData[l1_out_i];
939 precision = context.getOriginalLayerPrecision(parent->name, outData->getName());
940 IE_SUPPRESS_DEPRECATED_START
941 if (precision == Precision::UNSPECIFIED) {
942 if (child != nullptr)
943 precision = child->precision;
944 else if (context.network.getPrecision() != Precision::MIXED)
945 precision = context.network.getPrecision();
947 precision = Precision::FP32;
949 IE_SUPPRESS_DEPRECATED_END
952 precision = Precision::FP32;
955 addLayerToCNNNetworkAfterData(outData, newLayer, child != nullptr ? child->name : "", context.network);
957 CNNNetworkHelper::setOutDataPrecision(*newLayer, precision);
961 void CNNNetworkHelper::replaceLayer(TransformationContext& context, const CNNLayerPtr source, const CNNLayerPtr target) {
962 CNNNetworkImpl* networkImpl = dynamic_cast<CNNNetworkImpl*>(&context.network);
963 networkImpl->removeLayer(source->name);
965 std::vector<CNNLayerPtr> parents = CNNNetworkHelper::getParents(*source);
966 for (CNNLayerPtr parent : parents) {
967 for (size_t outDataIndex = 0ul; outDataIndex < parent->outData.size(); ++outDataIndex) {
968 const DataPtr outData = parent->outData[outDataIndex];
969 std::map<std::string, CNNLayerPtr>& inputTo = outData->getInputTo();
970 inputTo[source->name] = target;
971 target->insData.push_back(outData);
975 const std::vector<CNNLayerPtr> children = CNNNetworkHelper::getChildren(*source);
977 target->outData.resize(source->outData.size());
978 for (size_t outDataIndex = 0ul; outDataIndex < source->outData.size(); ++outDataIndex) {
979 const DataPtr outData = source->outData[outDataIndex];
980 networkImpl->removeData(outData->getName());
982 DataPtr newOutData(new Data(outData->getName(), outData->getTensorDesc()));
983 newOutData->getCreatorLayer() = target;
984 target->outData[outDataIndex] = newOutData;
985 networkImpl->addData(newOutData->getName().c_str(), newOutData);
987 std::map<std::string, CNNLayerPtr> inputTo = outData->getInputTo();
988 for (const auto it : inputTo) {
989 const CNNLayerPtr child = it.second;
990 newOutData->getInputTo().emplace(it.first, child);
992 for (const CNNLayerPtr& child : children) {
993 for (size_t insDataIndex = 0ul; insDataIndex < child->insData.size(); ++insDataIndex) {
994 const DataPtr insData = child->insData[insDataIndex].lock();
995 if (insData == nullptr) {
996 THROW_IE_LPT_EXCEPTION(*child) << "insert data " << insDataIndex << " is absent";
999 const CNNLayerPtr parent = insData->getCreatorLayer().lock();
1000 if (parent == nullptr) {
1001 THROW_IE_LPT_EXCEPTION(*child) << "parent layer for insert data " << insDataIndex << " is absent";
1003 if (parent->name == source->name) {
1004 const auto it = target->outData[outDataIndex];
1005 child->insData[insDataIndex] = newOutData;
1010 outData->getInputTo().clear();
1013 IE_SUPPRESS_DEPRECATED_START
1014 context.network.addLayer(target);
1015 IE_SUPPRESS_DEPRECATED_END
1018 CNNLayerPtr CNNNetworkHelper::addScaleShiftBetween(TransformationContext& context, const CNNLayerPtr parent,
1019 const CNNLayerPtr child,
1020 const DequantizationDetails& dequantizationDetails,
1021 const std::string& name) {
1022 if (parent == nullptr)
1023 THROW_IE_EXCEPTION << "Parent layer is nullable";
1025 if (child && (child->type == "ScaleShift") && (CNNNetworkHelper::getParents(*child).size() == 1)) {
1026 auto scalesIt = child->blobs.find("weights");
1027 if (scalesIt == child->blobs.end()) {
1028 THROW_IE_EXCEPTION << "weights for layer " << child->name << " was not found";
1030 const std::shared_ptr<float> scales = CNNNetworkHelper::getFloatData(scalesIt->second);
1031 std::vector<float> updatedScales(scalesIt->second->size());
1032 for (size_t i = 0ul; i < updatedScales.size(); ++i) {
1033 updatedScales[i] = scales.get()[i] * dequantizationDetails.scales[i];
1035 CNNNetworkHelper::updateBlobs(*child, "weights", updatedScales);
1037 auto shiftsIt = child->blobs.find("biases");
1038 if (shiftsIt != child->blobs.end()) {
1039 const std::shared_ptr<float> shifts = CNNNetworkHelper::getFloatData(shiftsIt->second);
1040 std::vector<float> updatedShifts(shiftsIt->second->size());
1041 for (size_t i = 0ul; i < updatedShifts.size(); ++i) {
1042 updatedShifts[i] = scales.get()[i] * dequantizationDetails.shifts[i] + shifts.get()[i];
1044 CNNNetworkHelper::updateBlobs(*child, "biases", updatedShifts);
1050 // Searching the connection between the layers
1052 if (child != nullptr) {
1053 for (; l1_out_i < parent->outData.size(); l1_out_i++) {
1054 if (parent->outData[l1_out_i]->getInputTo().find(child->name) !=
1055 parent->outData[l1_out_i]->getInputTo().end()) {
1060 if (l1_out_i == parent->outData.size()) {
1061 if (child != nullptr)
1062 THROW_IE_EXCEPTION << "Can't find layer " << child->name << " among layer " << parent->name << " outputs";
1064 THROW_IE_EXCEPTION << "Layer '" << parent->name << "' has invalid output";
1067 DataPtr outData = parent->outData[l1_out_i];
1069 std::string layerName = name.empty() ? (child != nullptr ? (parent->name + "_ScaleShift_" + child->name)
1070 : (parent->name + "_ScaleShift"))
1073 Precision ssPrecision = context.getOriginalLayerPrecision(parent->name, outData->getName());
1074 IE_SUPPRESS_DEPRECATED_START
1075 if (ssPrecision == Precision::UNSPECIFIED) {
1076 if (child != nullptr)
1077 ssPrecision = child->precision;
1078 else if (context.network.getPrecision() != Precision::MIXED)
1079 ssPrecision = context.network.getPrecision();
1081 ssPrecision = Precision::FP32;
1083 IE_SUPPRESS_DEPRECATED_END
1085 LayerParams ssCnnLayerParams {layerName, "ScaleShift", ssPrecision};
1086 CNNLayerPtr ssCnnLayer(new ScaleShiftLayer(ssCnnLayerParams));
1088 const std::vector<size_t> dims = outData->getDims();
1089 if ((dims.size() > 1) && (dims[1] != dequantizationDetails.channelsCount)) {
1090 THROW_IE_EXCEPTION << "unexpected parent channels count " << dims[1];
1092 addLayerToCNNNetworkAfterData(outData, ssCnnLayer, child != nullptr ? child->name : "", context.network);
1095 ScaleShiftLayer* scshLayer = dynamic_cast<ScaleShiftLayer*>(ssCnnLayer.get());
1096 if (scshLayer == nullptr) {
1097 THROW_IE_EXCEPTION << "Layer " << ssCnnLayer->name << " is not instance of ScaleShiftLayer class";
1099 fillInScaleShift(scshLayer, dequantizationDetails.channelsCount, dequantizationDetails.scales.data(),
1100 dequantizationDetails.shifts.data());
1103 CNNNetworkHelper::setOutDataPrecision(*ssCnnLayer, ssPrecision);
1107 CNNLayerPtr CNNNetworkHelper::addConstBetween(ICNNNetwork& net, const CNNLayerPtr layer1, const CNNLayerPtr layer2,
1108 const Blob::Ptr customBlob, const std::string& name) {
1109 if (layer1 == nullptr)
1110 THROW_IE_EXCEPTION << "First layer is nullable";
1111 // Searching the connection between the layers
1113 if (layer2 != nullptr) {
1114 for (; l1_out_i < layer1->outData.size(); l1_out_i++) {
1115 if (layer1->outData[l1_out_i]->getInputTo().find(layer2->name) !=
1116 layer1->outData[l1_out_i]->getInputTo().end()) {
1122 if (l1_out_i == layer1->outData.size()) {
1123 if (layer2 != nullptr)
1124 THROW_IE_EXCEPTION << "Can't find layer " << layer2->name << " among layer " << layer1->name << " outputs";
1126 THROW_IE_EXCEPTION << "Layer " << layer1->name << " has invalid outputs";
1129 DataPtr outData = layer1->outData[l1_out_i];
1131 std::string layerName = name.empty() ? layer1->name + "_Const" : name;
1132 CNNLayerPtr layer(new CNNLayer({layerName, "Const", customBlob->getTensorDesc().getPrecision()}));
1134 addLayerToCNNNetworkAfterData(outData, layer, layer2 != nullptr ? layer2->name : "", net);
1135 layer->blobs.emplace("custom", customBlob);
1136 layer->outData[0]->setPrecision(customBlob->getTensorDesc().getPrecision());
1140 void CNNNetworkHelper::addLayerToCNNNetworkAfterData(
1141 DataPtr parentOutData,
1142 CNNLayer::Ptr layer,
1143 const std::string& nextLayerName,
1145 CNNNetworkImpl* netImpl = dynamic_cast<CNNNetworkImpl*>(&net);
1146 if (netImpl == nullptr) {
1147 THROW_IE_EXCEPTION << "unexpected network type";
1150 CNNLayerPtr nextLayer;
1151 if (!nextLayerName.empty()) {
1152 netImpl->getLayerByName(nextLayerName.c_str(), nextLayer, nullptr);
1155 if (layer && (nextLayerName.empty() || (parentOutData == nullptr) ||
1156 (parentOutData->getInputTo().find(nextLayerName) != parentOutData->getInputTo().end()))) {
1157 auto getTensorDesc = [](CNNLayerPtr& nextLayer) {
1158 const DataPtr insData = nextLayer->insData[0].lock();
1159 if (insData == nullptr) {
1160 THROW_IE_LPT_EXCEPTION(*nextLayer) << "insert data is absent";
1162 return insData->getTensorDesc();
1165 const TensorDesc& parentTensorDesc = parentOutData != nullptr ? parentOutData->getTensorDesc() : getTensorDesc(nextLayer);
1166 DataPtr newEdgeAfterLayer(new Data(layer->name, parentTensorDesc));
1167 newEdgeAfterLayer->setName(layer->name);
1168 newEdgeAfterLayer->getCreatorLayer() = layer;
1169 newEdgeAfterLayer->getInputTo().clear();
1171 CNNNetworkImpl* netImpl = dynamic_cast<CNNNetworkImpl*>(&net);
1172 if (netImpl == nullptr) {
1173 THROW_IE_EXCEPTION << "unexpected network type";
1175 netImpl->addData(layer->name.c_str(), newEdgeAfterLayer);
1176 IE_SUPPRESS_DEPRECATED_START
1177 netImpl->addLayer(layer);
1178 IE_SUPPRESS_DEPRECATED_END
1180 if (parentOutData != nullptr) {
1181 parentOutData->getInputTo()[layer->name] = layer;
1182 layer->insData.push_back(parentOutData);
1184 layer->outData.push_back(newEdgeAfterLayer);
1186 if (!nextLayerName.empty()) {
1187 // CNNLayerPtr nextLayer = parentOutData->getInputTo()[nextLayerName];
1188 newEdgeAfterLayer->getInputTo()[nextLayerName] = nextLayer;
1189 if (parentOutData != nullptr) {
1190 parentOutData->getInputTo().erase(nextLayerName);
1191 for (size_t i = 0; i < nextLayer->insData.size(); i++) {
1192 if (nextLayer->insData[i].lock() == parentOutData) {
1193 nextLayer->insData[i] = newEdgeAfterLayer;
1198 nextLayer->insData.push_back(newEdgeAfterLayer);
1201 CNNLayerPtr parent = parentOutData->getCreatorLayer().lock();
1202 if (parent == nullptr) {
1203 THROW_IE_EXCEPTION << "parent data is absent";
1205 netImpl->removeOutput(parent->name);
1206 netImpl->addData(layer->name.c_str(), newEdgeAfterLayer);
1207 netImpl->addOutput(layer->name);
1210 THROW_IE_EXCEPTION << "Invalid argument";
1214 void CNNNetworkHelper::fillInScaleShift(ScaleShiftLayer* layer, const size_t channels, const float* scales,
1215 const float* shifts) {
1216 if (layer == nullptr) {
1217 THROW_IE_EXCEPTION << "ScaleShiftLayer is nullable";
1220 layer->_weights = makeNewBlobPtr({layer->precision, {channels}, Layout::C});
1221 layer->_weights->allocate();
1222 fillBlobByFP32(layer->_weights, scales);
1223 layer->blobs["weights"] = layer->_weights;
1225 layer->_biases = makeNewBlobPtr({layer->precision, {channels}, Layout::C});
1226 layer->_biases->allocate();
1227 fillBlobByFP32(layer->_biases, shifts);
1228 layer->blobs["biases"] = layer->_biases;
1231 std::vector<CNNLayerPtr> CNNNetworkHelper::getChildren(const CNNLayer& layer, const std::string& exceptionLayerName) {
1232 std::vector<CNNLayerPtr> children;
1233 for (const DataPtr outData : layer.outData) {
1234 const std::map<std::string, CNNLayerPtr>& inputTo = outData->getInputTo();
1235 for (auto it = inputTo.begin(); it != inputTo.end(); ++it) {
1236 CNNLayerPtr child = it->second;
1237 if (exceptionLayerName.empty() || child->name != exceptionLayerName) {
1238 children.push_back(child);
1245 std::vector<CNNLayerPtr> CNNNetworkHelper::getChildrenRecursivelyExceptTypes(
1246 const CNNLayer& layer, const std::unordered_set<std::string>& exceptionLayerTypes) {
1247 std::vector<CNNLayerPtr> children;
1248 for (const DataPtr outData : layer.outData) {
1249 const std::map<std::string, CNNLayerPtr>& inputTo = outData->getInputTo();
1250 for (auto it = inputTo.begin(); it != inputTo.end(); ++it) {
1251 CNNLayerPtr child = it->second;
1252 if (exceptionLayerTypes.find(child->type) != exceptionLayerTypes.end()) {
1253 const std::vector<CNNLayerPtr> tmpChildren =
1254 getChildrenRecursivelyExceptTypes(*child, exceptionLayerTypes);
1255 children.insert(children.end(), tmpChildren.begin(), tmpChildren.end());
1259 children.push_back(child);
1265 void CNNNetworkHelper::checkConstWithBlobs(const CNNLayerPtr layer) {
1266 if (layer->type != "Const") {
1267 THROW_IE_EXCEPTION << "Unexpected layer type '" << layer->name << "'";
1269 if (layer->blobs.size() != 1) {
1270 THROW_IE_EXCEPTION << "Unexpected blobs count " << layer->blobs.size() << " for layer '" << layer->name << "'";
1272 if (layer->insData.size() != 0) {
1273 THROW_IE_EXCEPTION << "Unexpected inputs count " << layer->insData.size() << " for layer '" << layer->name
1276 if (layer->outData.size() != 1) {
1277 THROW_IE_EXCEPTION << "Unexpected outputs count " << layer->outData.size() << " for layer '" << layer->name
1282 void CNNNetworkHelper::checkQuantizeOnWeights(const CNNLayerPtr layer) {
1283 if (layer->type != "FakeQuantize") {
1284 THROW_IE_EXCEPTION << "Unexpected layer type '" << layer->name << "'";
1286 if (layer->blobs.size() != 0) {
1287 THROW_IE_EXCEPTION << "Unexpected blobs count " << layer->blobs.size() << " for layer '" << layer->name << "'";
1289 if (layer->insData.size() != 5) {
1290 THROW_IE_EXCEPTION << "Unexpected inputs count " << layer->insData.size() << " for layer '" << layer->name
1293 if (layer->outData.size() != 1) {
1294 THROW_IE_EXCEPTION << "Unexpected outputs count " << layer->outData.size() << " for layer '" << layer->name
1299 void CNNNetworkHelper::updateInput(CNNNetworkImpl* network, CNNLayerPtr& layer, DataPtr outData) {
1300 if (!CaselessEq<std::string>()(layer->type, "Input")) {
1304 InputInfo::Ptr inputInfo = network->getInput(layer->name);
1305 if (inputInfo->name() == layer->name) {
1306 inputInfo->setInputData(outData);
1310 size_t CNNNetworkHelper::disconnectLayers(CNNNetworkImpl* network, const CNNLayerPtr& parentLayer,
1311 const CNNLayerPtr& childLayer) {
1312 bool wasFound = false;
1313 for (auto dataIt = parentLayer->outData.begin(); dataIt != parentLayer->outData.end(); ++dataIt) {
1314 auto data = *dataIt;
1315 for (auto inputIt = data->getInputTo().begin(); inputIt != data->getInputTo().end(); ++inputIt) {
1316 auto currentChildLayer = inputIt->second;
1317 if (currentChildLayer == nullptr) {
1318 THROW_IE_EXCEPTION << "Output layer for '" << parentLayer->name << "'is absent";
1320 if (currentChildLayer->name == childLayer->name) {
1321 data->getInputTo().erase(inputIt);
1332 THROW_IE_EXCEPTION << "Output layer '" << childLayer->name << "' was not found for '" << parentLayer->name
1337 for (auto it = childLayer->insData.begin(); it != childLayer->insData.end(); ++it) {
1338 auto data = it->lock();
1339 if (data == nullptr) {
1340 THROW_IE_EXCEPTION << "Input layer data for '" << childLayer->name << "'is absent";
1342 auto currentParentLayer = data->getCreatorLayer().lock();
1343 if (currentParentLayer == nullptr) {
1344 THROW_IE_EXCEPTION << "Input layer for '" << childLayer->name << "'is absent";
1346 if (currentParentLayer->name == parentLayer->name) {
1347 childLayer->insData.erase(it);
1353 THROW_IE_EXCEPTION << "Input layer '" << parentLayer->name << "' was not found for '" << childLayer->name
1359 size_t CNNNetworkHelper::getInputIndex(const CNNLayerPtr& childLayer, const CNNLayerPtr& parentLayer) {
1360 for (size_t index = 0; index < childLayer->insData.size(); ++index) {
1361 DataPtr currentParenData = childLayer->insData[index].lock();
1362 if (currentParenData == nullptr) {
1363 THROW_IE_EXCEPTION << "parent layer data is absent";
1365 CNNLayerPtr currentParrentLayer = currentParenData->getCreatorLayer().lock();
1366 if (currentParrentLayer == nullptr) {
1367 THROW_IE_EXCEPTION << "parent layer is absent";
1369 if (currentParrentLayer->name == parentLayer->name) {
1374 THROW_IE_EXCEPTION << "parent layer was not found";
1377 void CNNNetworkHelper::removeLayer(ICNNNetwork& network, const CNNLayerPtr& layer) {
1378 details::CNNNetworkImpl* networkImpl = dynamic_cast<details::CNNNetworkImpl*>(&network);
1379 if (networkImpl == nullptr) {
1380 THROW_IE_EXCEPTION << "Unexpected network type";
1383 if (layer->outData.size() > 1) {
1384 THROW_IE_EXCEPTION << "Layer '" << layer->name << "' has too many outputs " << layer->outData.size();
1387 if (layer->insData.size() > 1) {
1389 DataPtr data = layer->insData[0].lock();
1390 if (data == nullptr) {
1391 THROW_IE_EXCEPTION << "Layer's inserted data is nullptr";
1393 CNNLayerPtr parentLayer = data->getCreatorLayer().lock();
1394 if (parentLayer == nullptr) {
1395 THROW_IE_EXCEPTION << "Layer's parent layer is nullptr";
1397 CNNNetworkHelper::removeLayer(network, parentLayer);
1398 } while (!layer->insData.empty());
1402 std::vector<CNNLayerPtr> children;
1403 std::vector<size_t> childrenIndexes;
1404 if (layer->outData.size() > 0) {
1405 childData = layer->outData[0];
1406 auto inputTo = childData->getInputTo();
1407 if (inputTo.size() == 0) {
1408 std::vector<CNNLayerPtr> parents = getParents(*layer);
1409 if (parents.size() != 1) {
1410 THROW_IE_EXCEPTION << "not possible remove output layer with several parents";
1412 networkImpl->addOutput(parents[0]->name);
1413 CNNNetworkImpl* networkImpl = dynamic_cast<CNNNetworkImpl*>(&network);
1414 networkImpl->removeOutput(layer->name);
1416 for (auto it = inputTo.begin(); it != inputTo.end(); ++it) {
1417 children.push_back(it->second);
1418 childrenIndexes.push_back(getInputIndex(it->second, layer));
1419 disconnectLayers(networkImpl, layer, it->second);
1424 if (layer->insData.size() > 1) {
1426 THROW_IE_EXCEPTION << "not implemented";
1430 CNNLayerPtr parentLayer;
1431 if (layer->insData.size() > 0) {
1432 // remove connections with parent layers
1433 parentData = layer->insData[0].lock();
1434 if (parentData == nullptr) {
1435 THROW_IE_EXCEPTION << "Input data is absent";
1437 parentLayer = parentData->getCreatorLayer().lock();
1438 if (parentLayer == nullptr) {
1439 THROW_IE_EXCEPTION << "Input layer for '" << layer->name << "' is absent";
1442 const size_t ouputLayerOutDataIndex = disconnectLayers(networkImpl, parentLayer, layer);
1443 if (ouputLayerOutDataIndex >= parentLayer->outData.size()) {
1444 THROW_IE_EXCEPTION << "Index " << ouputLayerOutDataIndex << " out of range output ports count "
1445 << parentLayer->outData.size() << " for layer " << parentLayer->name;
1448 for (size_t index = 0; index < children.size(); ++index) {
1449 CNNLayerPtr childLayer = children[index];
1450 const size_t childInputIndex = childrenIndexes[index];
1452 DataPtr outData = parentLayer->outData[ouputLayerOutDataIndex];
1453 outData->getInputTo().emplace(childLayer->name, childLayer);
1454 childLayer->insData.insert(childLayer->insData.begin() + childInputIndex, outData);
1456 updateInput(networkImpl, parentLayer, outData);
1460 networkImpl->removeData(layer->name);
1461 networkImpl->removeLayer(layer->name);
1464 bool CNNNetworkHelper::isWeightsSupported(const CNNLayer& layer) noexcept {
1465 if (layer.insData.size() > 1) {
1466 CNNLayerPtr weightsLayer = CNNNetworkHelper::getParent(layer, 1);
1467 if (weightsLayer == nullptr)
1469 if ((weightsLayer->type == "Const") || (weightsLayer->type == "FakeQuantize")) {
1473 if (weightsLayer->type == "ScaleShift") {
1474 const std::vector<CNNLayerPtr> parents = CNNNetworkHelper::getParents(*weightsLayer);
1475 if (parents.size() != 1ul) {
1479 return (parents[0]->type == "FakeQuantize") || (parents[0]->type == "Const");
1484 return layer.blobs.find("weights") != layer.blobs.end();
1488 Blob::Ptr CNNNetworkHelper::getWeights(
1489 const CNNLayer& layer,
1490 const bool roundQuantizedValues) {
1491 if (layer.insData.size() > 1) {
1492 CNNLayerPtr weightsLayer = CNNNetworkHelper::getParent(layer, 1);
1493 if (weightsLayer == nullptr) {
1494 THROW_IE_EXCEPTION << "Convolution weights const layer are absent";
1497 if (weightsLayer->type == "Const") {
1498 CNNNetworkHelper::checkConstWithBlobs(weightsLayer);
1499 return weightsLayer->blobs.find("custom")->second;
1500 } else if (weightsLayer->type == "FakeQuantize") {
1501 return CNNNetworkHelper::quantizeWeights(*weightsLayer, roundQuantizedValues, Precision::UNSPECIFIED);
1502 } else if (weightsLayer->type == "ScaleShift") {
1503 const CNNLayerPtr parent = CNNNetworkHelper::getParent(*weightsLayer);
1504 if (parent == nullptr)
1505 THROW_IE_EXCEPTION << "Layer '" << weightsLayer->name << "' does not have parent";
1506 if (parent->type == "FakeQuantize") {
1507 return CNNNetworkHelper::quantizeWeights(*parent, roundQuantizedValues, Precision::UNSPECIFIED);
1508 } else if (parent->type == "Const") {
1509 CNNNetworkHelper::checkConstWithBlobs(parent);
1510 return CNNNetworkHelper::getBlob(parent, "custom");
1512 THROW_IE_EXCEPTION << "Unexpected weights layer " << parent->type << " " << parent->name << " for " << layer.type << " " << layer.name;
1515 THROW_IE_EXCEPTION << "Unexpected weights layer type " << weightsLayer->type;
1518 if (layer.blobs.find("weights") == layer.blobs.end()) {
1519 THROW_IE_EXCEPTION << "Convolution weights are absent";
1521 return layer.blobs.find("weights")->second;
1525 Blob::Ptr CNNNetworkHelper::getBiases(const CNNLayer& layer) {
1526 if (layer.insData.size() > 1U) {
1527 if (layer.insData.size() > 2U) {
1528 CNNLayerPtr biasesLayer = CNNNetworkHelper::getParent(layer, 2U);
1529 if (biasesLayer == nullptr) {
1533 CNNNetworkHelper::checkConstWithBlobs(biasesLayer);
1534 return biasesLayer->blobs.find("custom")->second;
1539 const auto it = layer.blobs.find("biases");
1540 return (it != layer.blobs.end()) ? it->second : nullptr;
1544 Blob::Ptr CNNNetworkHelper::quantizeWeights(const CNNLayer& quantize, const bool roundValues, const Precision precision) {
1545 if (quantize.insData.size() != 5lu) {
1546 THROW_IE_EXCEPTION << "Unexpected inputs count: " << quantize.insData.size();
1548 for (int i = 0; i < quantize.insData.size(); i++)
1549 if (quantize.insData[i].lock() == nullptr)
1550 THROW_IE_EXCEPTION << "Invalid input data for layer '" << quantize.name << "' with index " << i;
1552 const Blob::Ptr sourceBlob = getQuantizeLayerBlob(quantize);
1553 if (sourceBlob == nullptr) {
1554 THROW_IE_EXCEPTION << "weights blob is empty for " << quantize.type << " layer " << quantize.name;
1557 const auto& sourceBlobTD = sourceBlob->getTensorDesc();
1558 const Precision blobPrecision = sourceBlobTD.getPrecision();
1560 auto targetBlobPrecision = precision == Precision::UNSPECIFIED ? blobPrecision : precision;
1561 if (targetBlobPrecision != Precision::FP32 && targetBlobPrecision != Precision::FP16 &&
1562 targetBlobPrecision != Precision::I8 && targetBlobPrecision != Precision::U8)
1563 THROW_IE_EXCEPTION << "Unexpected precision: " << precision;
1565 Blob::Ptr targetBlob = make_blob_with_precision(TensorDesc(targetBlobPrecision, sourceBlobTD.getDims(), sourceBlobTD.getLayout()));
1566 targetBlob->allocate();
1568 quantizeBlob(quantize, targetBlob, roundValues);
1573 int CNNNetworkHelper::getConstParentBranchID(const CNNLayer& layer) {
1574 int constBranchID = -1;
1575 for (int i = 0; i < layer.insData.size(); i++) {
1576 bool allConst = true;
1578 const DataPtr insData = layer.insData[i].lock();
1579 if (insData == nullptr) {
1580 THROW_IE_LPT_EXCEPTION(layer) << "invalid input data with index " << i;
1583 const CNNLayerPtr parent = insData->getCreatorLayer().lock();
1584 if (parent == nullptr) {
1585 THROW_IE_LPT_EXCEPTION(layer) << "parent layer is absent";
1588 if (!CaselessEq<std::string>()(parent->type, "FakeQuantize")) continue;
1589 for (const auto& p : parent->insData) {
1590 const DataPtr parentConstInsData = p.lock();
1591 if (parentConstInsData == nullptr) {
1592 THROW_IE_LPT_EXCEPTION(*parent) << "input data is absent";
1594 const CNNLayerPtr parentConst = parentConstInsData->getCreatorLayer().lock();
1595 if (parentConst == nullptr) {
1596 THROW_IE_LPT_EXCEPTION(*parent) << "input layer is absent";
1598 if (!CaselessEq<std::string>()(parentConst->type, "Const")) {
1609 return constBranchID;
1612 Precision CNNNetworkHelper::getPrecisionParent(const CNNLayer& layer) {
1613 return getPrecisionParent(layer, 0ul, false);
1616 Precision CNNNetworkHelper::getPrecisionParent(const CNNLayer& layer, const size_t parentIndex) {
1617 return getPrecisionParent(layer, parentIndex, true);
1620 Precision CNNNetworkHelper::getPrecisionParent(const CNNLayer& layer, const size_t parentIndex, const bool useParentIndex) {
1621 const std::vector<CNNLayerPtr> parents = CNNNetworkHelper::getParents(layer);
1622 if (parents.empty()) {
1623 THROW_IE_EXCEPTION << "parents for layer " << layer.type << " '" << layer.name << "' are absent";
1626 if (useParentIndex) {
1627 DataPtr parentOutData = getOutData(*parents[parentIndex], layer);
1628 if (parentOutData == nullptr) {
1629 THROW_IE_EXCEPTION <<
1630 "parent layer " << parents[parentIndex]->type << " '" << parents[parentIndex]->name <<
1631 "' output data was not found for child " << layer.type << " '" << layer.name << "'";
1633 return parentOutData->getTensorDesc().getPrecision();
1636 Precision parentOutDataPrecision = Precision::UNSPECIFIED;
1637 for (CNNLayerPtr parent : parents) {
1638 DataPtr parentOutData = getOutData(*parent, layer);
1639 if (parentOutData == nullptr) {
1640 THROW_IE_EXCEPTION <<
1641 "parent layer " << parent->type << " '" << parent->name <<
1642 "' output data was not found for child " << layer.type << " '" << layer.name << "'";
1645 if (parentOutDataPrecision == Precision::UNSPECIFIED) {
1646 parentOutDataPrecision = parentOutData->getTensorDesc().getPrecision();
1647 } else if (parentOutDataPrecision != parentOutData->getTensorDesc().getPrecision()) {
1648 THROW_IE_EXCEPTION <<
1649 "Parent layer " << parent->type << " '" << parent->name <<
1650 "' output port has unexpected precision " << parentOutData->getTensorDesc().getPrecision();
1654 return parentOutDataPrecision;
1657 DataPtr CNNNetworkHelper::getOutData(const CNNLayer& parentLayer, const CNNLayer& childLayer) {
1658 DataPtr parentOutData;
1659 for (DataPtr outData : parentLayer.outData) {
1660 const std::map<std::string, CNNLayerPtr> inputTo = outData->getInputTo();
1661 for (auto childIt : inputTo) {
1662 if (childIt.second->name == childLayer.name) {
1663 parentOutData = outData;
1668 if (parentOutData != nullptr) {
1672 return parentOutData;
1675 void CNNNetworkHelper::quantizeBlob(const CNNLayer& quantize, Blob::Ptr& targetBlob, bool roundValues) {
1676 const Blob::Ptr sourceBlob = getQuantizeLayerBlob(quantize);
1677 if (sourceBlob == nullptr) {
1678 THROW_IE_EXCEPTION << "quantized blob is empty for " << quantize.type << " layer " << quantize.name;
1681 auto srcData = getFloatData(sourceBlob);
1682 const std::vector<size_t>& outDims = quantize.outData[0]->getDims();
1683 if (outDims.empty() || outDims.size() > 5lu) {
1684 THROW_IE_EXCEPTION << "Unexpected dimensions count " << outDims.size() << " for layer '" << quantize.name << "'";
1688 const size_t OC = outDims[0];
1689 const size_t IC = outDims.size() > 1lu ? outDims[1] : 1;
1690 const size_t D = outDims.size() > 4lu ? outDims[outDims.size() - 3] : 1;
1691 const size_t H = outDims.size() > 2lu ? outDims[outDims.size() - 2] : 1;
1692 const size_t W = outDims.size() > 3lu ? outDims[outDims.size() - 1] : 1;
1694 // Const layer blob shape (sourceBlob->getTensorDesc().getDims()) can be different from output port shape
1695 // CVS-27850: [IE COMMON] Align Const layer blob shape with output port shape
1696 if (sourceBlob->size() != OC * IC * D * H * W) {
1697 THROW_IE_EXCEPTION << "Unexpected weights size for layer '" << quantize.name << "'";
1700 const QuantizationDetails quantizationDetails = QuantizationDetails::getDetails(quantize);
1702 const bool isInputLowBroadcasted = quantizationDetails.inputLowValues.size() != OC;
1703 if ((quantizationDetails.inputLowValues.size() != 1) && (quantizationDetails.inputLowValues.size() != OC)) {
1704 THROW_IE_EXCEPTION << "Unexpected input low values count " << quantizationDetails.inputLowValues.size() <<
1705 " for " << OC << " channels, layer '" << quantize.name << "'";
1708 const bool isInputHighBroadcasted = quantizationDetails.inputHighValues.size() != OC;
1709 if ((quantizationDetails.inputHighValues.size() != 1) && (quantizationDetails.inputHighValues.size() != OC)) {
1710 THROW_IE_EXCEPTION << "Unexpected input high values count " << quantizationDetails.inputHighValues.size() <<
1711 " for " << OC << " channels, layer '" << quantize.name << "'";
1714 const bool isOutputLowBroadcasted = quantizationDetails.outputLowValues.size() != OC;
1715 if ((quantizationDetails.outputLowValues.size() != 1) && (quantizationDetails.outputLowValues.size() != OC)) {
1716 THROW_IE_EXCEPTION << "Unexpected output low values count " << quantizationDetails.outputLowValues.size() <<
1717 " for " << OC << " channels, layer '" << quantize.name << "'";
1720 const bool isOutputHighBroadcasted = quantizationDetails.outputHighValues.size() != OC;
1721 if ((quantizationDetails.outputHighValues.size() != 1) && (quantizationDetails.outputHighValues.size() != OC)) {
1722 THROW_IE_EXCEPTION << "Unexpected output high values count " << quantizationDetails.outputHighValues.size() <<
1723 " for " << OC << " channels, layer '" << quantize.name << "'";
1726 auto levels_1 = static_cast<float>(quantize.GetParamAsUInt("levels")) - 1.f;
1728 const size_t DHW = D * H * W;
1729 const size_t IDHW = IC * DHW;
1731 std::vector<float> dstBuffer(targetBlob->size());
1733 auto srcPtr = srcData.get();
1734 auto dstPtr = &dstBuffer[0];
1736 parallel_for4d(OC, IC, D, H, [&](size_t oc, size_t ic, size_t d, size_t h) {
1737 const float inputLow = quantizationDetails.inputLowValues[isInputLowBroadcasted ? 0 : oc];
1738 const float inputHigh = quantizationDetails.inputHighValues[isInputHighBroadcasted ? 0 : oc];
1739 const float outputLow = quantizationDetails.outputLowValues[isOutputLowBroadcasted ? 0 : oc];
1740 const float outputHigh = quantizationDetails.outputHighValues[isOutputHighBroadcasted ? 0 : oc];
1742 for (size_t w = 0; w < W; w++) {
1743 const size_t idx = oc * IDHW + ic * DHW + d * H * W + h * W + w;
1745 if (srcPtr[idx] <= inputLow) {
1746 dstPtr[idx] = roundValues ? std::roundf(outputLow) : outputLow;
1747 } else if (srcPtr[idx] > inputHigh) {
1748 dstPtr[idx] = roundValues ? std::roundf(outputHigh) : outputHigh;
1750 const float value = std::roundf((srcPtr[idx] - inputLow) / (inputHigh - inputLow) * levels_1) /
1751 levels_1 * (outputHigh - outputLow) + outputLow;
1752 dstPtr[idx] = roundValues ? std::roundf(value) : value;
1757 fillBlobByFP32(targetBlob, dstPtr);