From 0b1ef99fd73a3da77e93eaf8db0864948313b1d9 Mon Sep 17 00:00:00 2001 From: Vladislav Vinogradov Date: Tue, 28 Jul 2020 11:26:38 +0300 Subject: [PATCH] [IE] Add Blob::createROI method (#882) * Add default implementation that throws exception. * Implement `createROI` for `TBlob` and existing compound blobs. * Use reference couting for TBlob memory buffer to prolong its life time for ROI blobs. * Add private extension for ND ROI and use it as implementation detail for now: * Add `DimSlice` and `TensorSlice` structures for generic ND ROI support. * Add `make_roi_desc` function to create `TensorDesc` for ROI. --- inference-engine/include/ie_blob.h | 62 +++++++---- inference-engine/include/ie_compound_blob.h | 6 ++ inference-engine/include/ie_layouts.h | 32 ++++++ .../src/inference_engine/CMakeLists.txt | 1 + .../src/inference_engine/ie_blob_common.cpp | 55 ++-------- .../src/inference_engine/ie_compound_blob.cpp | 41 ++++++++ .../src/inference_engine/ie_layouts.cpp | 114 +++++++++++++++++++++ .../tests/unit/inference_engine/ie_blob_test.cpp | 50 +++++++++ .../io_blob_tests/cropResize_tests.hpp | 48 +++++++-- 9 files changed, 329 insertions(+), 80 deletions(-) diff --git a/inference-engine/include/ie_blob.h b/inference-engine/include/ie_blob.h index 205edde..aa0fb21 100644 --- a/inference-engine/include/ie_blob.h +++ b/inference-engine/include/ie_blob.h @@ -29,6 +29,7 @@ #include "ie_precision.hpp" namespace InferenceEngine { + /** * @brief This class represents a universal container in the Inference Engine * @@ -199,6 +200,17 @@ public: */ virtual LockedMemory cbuffer() const noexcept = 0; + /** + * @brief Creates a blob describing given ROI object based on the current blob with memory sharing. + * + * Note: default implementation throws "not implemented" exception. + * + * @param roi A ROI object inside of the current blob. + * + * @return A shared pointer to the newly created ROI blob. + */ + virtual Blob::Ptr createROI(const ROI& roi) const; + protected: /** * @brief The tensor descriptor of the given blob. @@ -437,8 +449,6 @@ public: */ virtual LockedMemory wmap()noexcept = 0; - - protected: /** * @brief Gets the allocator for allocator-based blobs. @@ -594,10 +604,18 @@ public: * @brief Allocates or reallocates memory */ void allocate() noexcept override { - if (_handle != nullptr) { - getAllocator()->free(_handle); + const auto allocator = getAllocator(); + const auto rawHandle = allocator->alloc(size() * sizeof(T)); + + if (rawHandle == nullptr) { + return; } - _handle = getAllocator()->alloc(size() * sizeof(T)); + + _handle.reset( + rawHandle, + [allocator](void* rawHandle) { + allocator->free(rawHandle); + }); } /** @@ -636,6 +654,10 @@ public: return std::move(lockme()); } + Blob::Ptr createROI(const ROI& roi) const override { + return Blob::Ptr(new TBlob(*this, roi)); + } + /** * @brief Gets BlobIterator for the data. * @@ -689,7 +711,7 @@ protected: /** * @brief A handle for the stored memory returned from _allocator.alloc(). */ - void* _handle = nullptr; + std::shared_ptr _handle; /** * @brief Copies dimensions and data from the TBlob object. @@ -720,8 +742,8 @@ protected: * @brief Frees handler and cleans up the stored data. */ virtual bool free() { - bool bCanRelease = getAllocator()->free(_handle); - _handle = nullptr; + bool bCanRelease = _handle != nullptr; + _handle.reset(); return bCanRelease; } @@ -733,7 +755,7 @@ protected: */ template LockedMemory lockme() const { - return LockedMemory(_allocator.get(), _handle, 0); + return LockedMemory(_allocator.get(), getHandle(), 0); } /** @@ -754,7 +776,16 @@ protected: * @brief Returns handle to the stored data. */ void* getHandle() const noexcept override { - return _handle; + return _handle.get(); + } + + TBlob(const TBlob& origBlob, const ROI& roi) : + MemoryBlob(make_roi_desc(origBlob.getTensorDesc(), roi, true)), + _allocator(origBlob._allocator) { + IE_ASSERT(origBlob._handle != nullptr) + << "Original Blob must be allocated before ROI creation"; + + _handle = origBlob._handle; } }; @@ -847,17 +878,6 @@ std::shared_ptr make_shared_blob(Args&&... args) { } /** - * @brief This structure describes ROI data. - */ -struct ROI { - size_t id; //!< ID of a ROI - size_t posX; //!< W upper left coordinate of ROI - size_t posY; //!< H upper left coordinate of ROI - size_t sizeX; //!< W size of ROI - size_t sizeY; //!< H size of ROI -}; - -/** * @brief Creates a blob describing given ROI object based on the given blob with pre-allocated memory. * * @param inputBlob original blob with pre-allocated memory. diff --git a/inference-engine/include/ie_compound_blob.h b/inference-engine/include/ie_compound_blob.h index 3b0d7d2..5ccf8c7 100644 --- a/inference-engine/include/ie_compound_blob.h +++ b/inference-engine/include/ie_compound_blob.h @@ -117,6 +117,8 @@ public: */ virtual Blob::Ptr getBlob(size_t i) const noexcept; + Blob::Ptr createROI(const ROI& roi) const override; + protected: /** * @brief A default constructor @@ -219,6 +221,8 @@ public: * @brief Returns a shared pointer to UV plane */ virtual const Blob::Ptr& uv() const noexcept; + + Blob::Ptr createROI(const ROI& roi) const override; }; /** @@ -342,5 +346,7 @@ public: * @return constant reference to shared pointer object of V plane */ const Blob::Ptr& v() const noexcept; + + Blob::Ptr createROI(const ROI& roi) const override; }; } // namespace InferenceEngine diff --git a/inference-engine/include/ie_layouts.h b/inference-engine/include/ie_layouts.h index 6da977b..a544231 100644 --- a/inference-engine/include/ie_layouts.h +++ b/inference-engine/include/ie_layouts.h @@ -318,4 +318,36 @@ private: BlockingDesc blockingDesc; }; +/** + * @brief This structure describes ROI data for image-like tensors. + */ +struct ROI { + size_t id = 0; //!< ID of a ROI (offset over batch dimension) + size_t posX = 0; //!< W upper left coordinate of ROI + size_t posY = 0; //!< H upper left coordinate of ROI + size_t sizeX = 0; //!< W size of ROI + size_t sizeY = 0; //!< H size of ROI + + ROI() = default; + + ROI(size_t id, size_t posX, size_t posY, size_t sizeX, size_t sizeY) : + id(id), posX(posX), posY(posY), sizeX(sizeX), sizeY(sizeY) { + } +}; + +/** + * @brief Creates a TensorDesc object for ROI. + * + * @param origDesc original TensorDesc object. + * @param roi An image ROI object inside of the original object. + * @param useOrigMemDesc Flag to use original memory description (strides/offset). + * Should be set if the new TensorDesc describes shared memory. + * + * @return A newly created TensorDesc object representing ROI. + */ +INFERENCE_ENGINE_API_CPP(TensorDesc) make_roi_desc( + const TensorDesc& origDesc, + const ROI& roi, + bool useOrigMemDesc); + } // namespace InferenceEngine diff --git a/inference-engine/src/inference_engine/CMakeLists.txt b/inference-engine/src/inference_engine/CMakeLists.txt index bd5962e..8e6b46a 100644 --- a/inference-engine/src/inference_engine/CMakeLists.txt +++ b/inference-engine/src/inference_engine/CMakeLists.txt @@ -20,6 +20,7 @@ set(IE_BASE_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/cnn_network_ngraph_impl.cpp ${CMAKE_CURRENT_SOURCE_DIR}/generic_ie.cpp ${CMAKE_CURRENT_SOURCE_DIR}/blob_factory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ie_blob_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ie_data.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ie_layouts.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ie_memcpy.cpp diff --git a/inference-engine/src/inference_engine/ie_blob_common.cpp b/inference-engine/src/inference_engine/ie_blob_common.cpp index 0aa6176..876b4ec 100644 --- a/inference-engine/src/inference_engine/ie_blob_common.cpp +++ b/inference-engine/src/inference_engine/ie_blob_common.cpp @@ -2,61 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 // +#include "ie_blob.h" + #include #include #include -#include "blob_factory.hpp" -#include "ie_blob.h" -#include "ie_compound_blob.h" - namespace InferenceEngine { -Blob::Ptr make_shared_blob(const Blob::Ptr& inputBlob, const ROI& roi) { - // reject compound blobs - if (inputBlob->is()) { - THROW_IE_EXCEPTION << "Compound blobs do not support ROI"; - } - - size_t blkDimsH = roi.sizeY; - size_t blkDimsW = roi.sizeX; - size_t blkDimsC = inputBlob->getTensorDesc().getDims()[1]; - size_t blkOffset; - SizeVector blkOrder; - SizeVector blkDims; - - if (roi.posX + roi.sizeX > inputBlob->getTensorDesc().getDims()[3] || - roi.posY + roi.sizeY > inputBlob->getTensorDesc().getDims()[2]) { - THROW_IE_EXCEPTION << "passed ROI coordinates are inconsistent to input size"; - } - - Layout blobLayout = inputBlob->getTensorDesc().getLayout(); - switch (blobLayout) { - case NCHW: { - blkOffset = inputBlob->getTensorDesc().getDims()[3] * roi.posY + roi.posX; - blkOrder = {0, 1, 2, 3}; - blkDims = {1, blkDimsC, blkDimsH, blkDimsW}; // we use BlockingDesc for 1 cropped image only - } break; - case NHWC: { - blkOffset = blkDimsC * (inputBlob->getTensorDesc().getDims()[3] * roi.posY + roi.posX); - blkOrder = {0, 2, 3, 1}; - blkDims = {1, blkDimsH, blkDimsW, blkDimsC}; // we use BlockingDesc for 1 cropped image only - } break; - default: { - THROW_IE_EXCEPTION << "ROI could not be cropped due to inconsistent input layout: " << blobLayout; - } - } - - // the strides are the same because ROI blob uses the same memory buffer as original input blob. - SizeVector blkStrides(inputBlob->getTensorDesc().getBlockingDesc().getStrides()); - - SizeVector blkDimsOffsets = {0, 0, 0, 0}; // no offset per dims by default - - BlockingDesc blkDesc(blkDims, blkOrder, blkOffset, blkDimsOffsets, blkStrides); - TensorDesc tDesc(inputBlob->getTensorDesc().getPrecision(), {1, blkDimsC, blkDimsH, blkDimsW}, blkDesc); - tDesc.setLayout(blobLayout); +Blob::Ptr Blob::createROI(const ROI&) const { + THROW_IE_EXCEPTION << "[NOT_IMPLEMENTED] createROI is not implemented for current type of Blob"; +} - return make_blob_with_precision(tDesc, inputBlob->buffer()); +Blob::Ptr make_shared_blob(const Blob::Ptr& inputBlob, const ROI& roi) { + return inputBlob->createROI(roi); } } // namespace InferenceEngine diff --git a/inference-engine/src/inference_engine/ie_compound_blob.cpp b/inference-engine/src/inference_engine/ie_compound_blob.cpp index c04f6cf..5d35f53 100644 --- a/inference-engine/src/inference_engine/ie_compound_blob.cpp +++ b/inference-engine/src/inference_engine/ie_compound_blob.cpp @@ -15,7 +15,9 @@ #include namespace InferenceEngine { + namespace { + void verifyNV12BlobInput(const Blob::Ptr& y, const Blob::Ptr& uv) { // Y and UV must be valid pointers if (y == nullptr || uv == nullptr) { @@ -189,6 +191,7 @@ void verifyI420BlobInput(const Blob::Ptr& y, const Blob::Ptr& u, const Blob::Ptr << yDims[3] << "(Y plane) and " << vDims[3] << "(V plane)"; } } + } // anonymous namespace CompoundBlob::CompoundBlob(): Blob(TensorDesc(Precision::UNSPECIFIED, {}, Layout::ANY)) {} @@ -272,6 +275,17 @@ Blob::Ptr CompoundBlob::getBlob(size_t i) const noexcept { return _blobs[i]; } +Blob::Ptr CompoundBlob::createROI(const ROI& roi) const { + std::vector roiBlobs; + roiBlobs.reserve(_blobs.size()); + + for (const auto& blob : _blobs) { + roiBlobs.push_back(blob->createROI(roi)); + } + + return std::make_shared(std::move(roiBlobs)); +} + const std::shared_ptr& CompoundBlob::getAllocator() const noexcept { static std::shared_ptr _allocator = nullptr; return _allocator; @@ -319,6 +333,19 @@ const Blob::Ptr& NV12Blob::uv() const noexcept { return _blobs[1]; } +Blob::Ptr NV12Blob::createROI(const ROI& roi) const { + auto yROI = roi; + yROI.sizeX += yROI.sizeX % 2; + yROI.sizeY += yROI.sizeY % 2; + + const auto uvROI = ROI(yROI.id, yROI.posX / 2, yROI.posY / 2, yROI.sizeX / 2, yROI.sizeY / 2); + + const auto yRoiBlob = y()->createROI(yROI); + const auto uvRoiBlob = uv()->createROI(uvROI); + + return std::make_shared(yRoiBlob, uvRoiBlob); +} + I420Blob::I420Blob(const Blob::Ptr& y, const Blob::Ptr& u, const Blob::Ptr& v) { // verify data is correct verifyI420BlobInput(y, u, v); @@ -371,4 +398,18 @@ const Blob::Ptr& I420Blob::v() const noexcept { return _blobs[2]; } +Blob::Ptr I420Blob::createROI(const ROI& roi) const { + auto yROI = roi; + yROI.sizeX += yROI.sizeX % 2; + yROI.sizeY += yROI.sizeY % 2; + + const auto uvROI = ROI(yROI.id, yROI.posX / 2, yROI.posY / 2, yROI.sizeX / 2, yROI.sizeY / 2); + + const auto yRoiBlob = y()->createROI(yROI); + const auto uRoiBlob = u()->createROI(uvROI); + const auto vRoiBlob = v()->createROI(uvROI); + + return std::make_shared(yRoiBlob, uRoiBlob, vRoiBlob); +} + } // namespace InferenceEngine diff --git a/inference-engine/src/inference_engine/ie_layouts.cpp b/inference-engine/src/inference_engine/ie_layouts.cpp index a35b547..6f2cb1a 100644 --- a/inference-engine/src/inference_engine/ie_layouts.cpp +++ b/inference-engine/src/inference_engine/ie_layouts.cpp @@ -368,3 +368,117 @@ bool BlockingDesc::operator==(const BlockingDesc& rhs) const { bool BlockingDesc::operator!=(const BlockingDesc& rhs) const { return !(*this == rhs); } + +namespace { + +struct DimSlice { + size_t startInd = 0; + size_t size = 0; + + DimSlice() = default; + + DimSlice(size_t startInd, size_t size) : + startInd(startInd), size(size) { + } +}; + +using TensorSlice = std::vector; + +void checkROI( + const TensorDesc& origDesc, + const TensorSlice& roi) { + const auto numDims = origDesc.getDims().size(); + + if (roi.size() != numDims) { + THROW_IE_EXCEPTION + << "ROI num dims " << roi.size() << + " differs from original num dims " << numDims; + } + + // TensorDesc stores dimensions in standard layout, as well as roi vector + for (size_t dimInd = 0; dimInd < numDims; ++dimInd) { + const auto fullSize = origDesc.getDims()[dimInd]; + + const auto& roiSlice = roi[dimInd]; + const auto endInd = roiSlice.startInd + roiSlice.size; + + if (endInd > fullSize) { + THROW_IE_EXCEPTION + << "ROI [" << roiSlice.startInd << ", " << endInd << ")" + << " is out of range " << fullSize + << " for dimension " << dimInd; + } + } +} + +TensorDesc make_roi_desc( + const TensorDesc& origDesc, + const TensorSlice& roi, + bool useOrigMemDesc) { + const auto numDims = origDesc.getDims().size(); + + checkROI(origDesc, roi); + + const auto origPrecision = origDesc.getPrecision(); + + const auto& origBlkDesc = origDesc.getBlockingDesc(); + const auto& origBlkStrides = origBlkDesc.getStrides(); + const auto& origBlkOrder = origBlkDesc.getOrder(); + + SizeVector roiDims(numDims); + SizeVector roiBlkDims(numDims); + SizeVector roiBlkDimOffsets = origBlkDesc.getOffsetPaddingToData(); + size_t roiBlkOffset = origBlkDesc.getOffsetPadding(); + + IE_ASSERT(origBlkStrides.size() == numDims); + IE_ASSERT(origBlkOrder.size() == numDims); + IE_ASSERT(roiBlkDimOffsets.size() == numDims); + + // BlockingDesc stores dimensions in memory order, so we need to use origOrder array. + // Offsets in `roi` relates to `origDesc` dimensions, while offsets in `BlockingDesc` relates to top parent tensor dimensions. + for (size_t memInd = 0; memInd < numDims; ++memInd) { + const auto dimInd = origBlkOrder[memInd]; + const auto& roiSlice = roi[dimInd]; + + roiDims[dimInd] = roiSlice.size; + roiBlkDims[memInd] = roiSlice.size; + roiBlkDimOffsets[memInd] += roiSlice.startInd; + roiBlkOffset += roiSlice.startInd * origBlkStrides[memInd]; + } + + const auto roiBlkDesc = + useOrigMemDesc ? + BlockingDesc(roiBlkDims, origBlkOrder, roiBlkOffset, roiBlkDimOffsets, origBlkStrides) : + BlockingDesc(roiBlkDims, origBlkOrder); + + const auto roiDesc = TensorDesc(origPrecision, roiDims, roiBlkDesc); + + return roiDesc; +} + +TensorSlice make_roi_slice( + const TensorDesc& origDesc, + const ROI& roi) { + const auto layout = origDesc.getLayout(); + if (layout != Layout::NCHW && layout != Layout::NHWC) { + THROW_IE_EXCEPTION + << "Unsupported layout " << layout; + } + + TensorSlice roiSlice(4); + roiSlice[0] = DimSlice {roi.id, 1}; // N + roiSlice[1] = DimSlice {0, origDesc.getDims()[1]}; // C + roiSlice[2] = DimSlice {roi.posY, roi.sizeY}; // H + roiSlice[3] = DimSlice {roi.posX, roi.sizeX}; // W + + return roiSlice; +} + +} // namespace + +TensorDesc InferenceEngine::make_roi_desc( + const TensorDesc& origDesc, + const ROI& roi, + bool useOrigMemDesc) { + return make_roi_desc(origDesc, make_roi_slice(origDesc, roi), useOrigMemDesc); +} diff --git a/inference-engine/tests/unit/inference_engine/ie_blob_test.cpp b/inference-engine/tests/unit/inference_engine/ie_blob_test.cpp index 647b320..f687506 100644 --- a/inference-engine/tests/unit/inference_engine/ie_blob_test.cpp +++ b/inference-engine/tests/unit/inference_engine/ie_blob_test.cpp @@ -395,3 +395,53 @@ TEST_F(BlobTests, makeRoiBlobWrongSize) { InferenceEngine::ROI roi = {0, 1, 1, 4, 4}; // cropped picture with: id = 0, (x,y) = (1,1), sizeX (W) = 4, sizeY (H) = 4 ASSERT_THROW(make_shared_blob(blob, roi), InferenceEngine::details::InferenceEngineException); } + +TEST_F(BlobTests, readRoiBlob) { + // Create original Blob + + const auto origDesc = + InferenceEngine::TensorDesc( + InferenceEngine::Precision::I32, + {1, 3, 4, 8}, + InferenceEngine::NCHW); + + const auto origBlob = + InferenceEngine::make_shared_blob(origDesc); + origBlob->allocate(); + + // Fill the original Blob + + { + auto origMemory = origBlob->wmap(); + const auto origPtr = origMemory.as(); + ASSERT_NE(nullptr, origPtr); + + for (size_t i = 0; i < origBlob->size(); ++i) { + origPtr[i] = i; + } + } + + // Create ROI Blob + + const auto roi = InferenceEngine::ROI(0, 4, 2, 4, 2); + + const auto roiBlob = InferenceEngine::as(origBlob->createROI(roi)); + ASSERT_NE(nullptr, roiBlob); + + // Read ROI Blob + + { + const auto roiOffset = roiBlob->getTensorDesc().getBlockingDesc().getOffsetPadding(); + + auto roiMemory = roiBlob->rmap(); + auto roiPtr = roiMemory.as(); + ASSERT_NE(nullptr, roiPtr); + + // Blob::rmap returns pointer to the original blob start, we have to add ROI offset manually. + roiPtr += roiOffset; + + for (size_t i = 0; i < roiBlob->size(); ++i) { + ASSERT_EQ(roiPtr[i], i + roiOffset); + } + } +} diff --git a/inference-engine/tests_deprecated/functional/shared_tests/io_blob_tests/cropResize_tests.hpp b/inference-engine/tests_deprecated/functional/shared_tests/io_blob_tests/cropResize_tests.hpp index bdec8af..f878731 100644 --- a/inference-engine/tests_deprecated/functional/shared_tests/io_blob_tests/cropResize_tests.hpp +++ b/inference-engine/tests_deprecated/functional/shared_tests/io_blob_tests/cropResize_tests.hpp @@ -398,15 +398,27 @@ TEST_P(RandomROITest, PreprocRandomROITest) if (_colorFormat == NV12) { - roi.sizeX += roi.sizeX % 2; - roi.sizeY += roi.sizeY % 2; + if (i % 2) + { + // New way to create NV12 ROI + + auto nv12Blob = make_shared_blob(yBlob, uvBlob); + cropBlob = nv12Blob->createROI(roi); + } + else + { + // Old way to create NV12 ROI - auto roiUV = roi/2; + roi.sizeX += roi.sizeX % 2; + roi.sizeY += roi.sizeY % 2; - auto cropYBlob = make_shared_blob(yBlob, roi); - auto cropUvBlob = make_shared_blob(uvBlob, roiUV); + auto roiUV = roi/2; - cropBlob = make_shared_blob(cropYBlob, cropUvBlob); + auto cropYBlob = make_shared_blob(yBlob, roi); + auto cropUvBlob = make_shared_blob(uvBlob, roiUV); + + cropBlob = make_shared_blob(cropYBlob, cropUvBlob); + } } else { @@ -1110,13 +1122,27 @@ TEST_P(NV12ColorConvertTest, NV12Test) { cv::resize(refImg, refImg, cv::Size(_netDims[3], _netDims[2]), 0, 0, cv_interpolation); auto refBlob = img2Blob(refImg, Layout::NCHW); - // Note: Y and UV blobs for original data must always be "alive" until the end of the execution: - // ROI blobs do not own the data auto yBlob = img2Blob(yPlane, NHWC); auto uvBlob = img2Blob(uvPlane, NHWC); - auto croppedYBlob = make_shared_blob(yBlob, yRoi); - auto croppedUvBlob = make_shared_blob(uvBlob, uvRoi); - auto inputBlob = make_shared_blob(croppedYBlob, croppedUvBlob); + Blob::Ptr inputBlob; + + if (i % 2) + { + // New way to create NV12 ROI + + auto nv12Blob = make_shared_blob(yBlob, uvBlob); + inputBlob = nv12Blob->createROI(yRoi); + } + else + { + // Old way to create NV12 ROI + + // Note: Y and UV blobs for original data must always be "alive" until the end of the execution: + // ROI blobs do not own the data + auto croppedYBlob = make_shared_blob(yBlob, yRoi); + auto croppedUvBlob = make_shared_blob(uvBlob, uvRoi); + inputBlob = make_shared_blob(croppedYBlob, croppedUvBlob); + } req.SetBlob(net.getInputsInfo().begin()->first, inputBlob); -- 2.7.4