From ea4b491a7311d3ea8d04ade12da09b149d998b0f Mon Sep 17 00:00:00 2001 From: Ruslan Garnov Date: Thu, 17 Sep 2020 15:39:10 +0300 Subject: [PATCH] Merge pull request #18213 from rgarnov:rg/rmat_api Basic RMat implementation * Added basic RMat implementation * Fix typos in basic RMat implementation Co-authored-by: Anton Potapov --- modules/gapi/include/opencv2/gapi/rmat.hpp | 124 +++++++++++++++++++++ modules/gapi/test/rmat/rmat_tests.cpp | 171 +++++++++++++++++++++++++++++ modules/gapi/test/rmat/rmat_view_tests.cpp | 147 +++++++++++++++++++++++++ 3 files changed, 442 insertions(+) create mode 100644 modules/gapi/include/opencv2/gapi/rmat.hpp create mode 100644 modules/gapi/test/rmat/rmat_tests.cpp create mode 100644 modules/gapi/test/rmat/rmat_view_tests.cpp diff --git a/modules/gapi/include/opencv2/gapi/rmat.hpp b/modules/gapi/include/opencv2/gapi/rmat.hpp new file mode 100644 index 0000000..2fbdf03 --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/rmat.hpp @@ -0,0 +1,124 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#ifndef OPENCV_GAPI_RMAT_HPP +#define OPENCV_GAPI_RMAT_HPP + +#include + +namespace cv { + +// "Remote Mat", a general class which provides an abstraction layer over the data +// storage and placement (host, remote device etc) and allows to access this data. +// +// The device specific implementation is hidden in the RMat::Adapter class +// +// The basic flow is the following: +// * Backend which is aware of the remote device: +// - Implements own AdapterT class which is derived from RMat::Adapter +// - Wraps device memory into RMat via make_rmat utility function: +// cv::RMat rmat = cv::make_rmat(args); +// +// * End user: +// - Writes the code which works with RMats without any knowledge of the remote device: +// void func(const cv::RMat& in_rmat, cv::RMat& out_rmat) { +// // Fetch input data from the device, get mapped memory for output +// cv::RMat::View in_view = in_rmat.access(Access::R); +// cv::RMat::View out_view = out_rmat.access(Access::W); +// performCalculations(in_view, out_view); +// // data from out_view is transferred to the device when out_view is destroyed +// } +class RMat +{ +public: + // A lightweight wrapper on image data: + // - Doesn't own the memory; + // - Doesn't implement copy semantics (it's assumed that a view is created each time + // wrapped data is being accessed); + // - Has an optional callback which is called when the view is destroyed. + class View + { + public: + using DestroyCallback = std::function; + + View() = default; + View(const GMatDesc& desc, uchar* data, size_t step = 0u, DestroyCallback&& cb = nullptr) + : m_desc(desc), m_data(data), m_step(step == 0u ? elemSize()*cols() : step), m_cb(cb) + {} + + View(const View&) = delete; + View(View&&) = default; + View& operator=(const View&) = delete; + View& operator=(View&&) = default; + ~View() { if (m_cb) m_cb(); } + + cv::Size size() const { return m_desc.size; } + const std::vector& dims() const { return m_desc.dims; } + int cols() const { return m_desc.size.width; } + int rows() const { return m_desc.size.height; } + int type() const { return CV_MAKE_TYPE(depth(), chan()); } + int depth() const { return m_desc.depth; } + int chan() const { return m_desc.chan; } + size_t elemSize() const { return CV_ELEM_SIZE(type()); } + + template T* ptr(int y = 0, int x = 0) { + return reinterpret_cast(m_data + m_step*y + x*CV_ELEM_SIZE(type())); + } + template const T* ptr(int y = 0, int x = 0) const { + return reinterpret_cast(m_data + m_step*y + x*CV_ELEM_SIZE(type())); + } + size_t step() const { return m_step; } + + private: + GMatDesc m_desc; + uchar* m_data = nullptr; + size_t m_step = 0u; + DestroyCallback m_cb = nullptr; + }; + + enum class Access { R, W }; + class Adapter + { + public: + virtual ~Adapter() = default; + virtual GMatDesc desc() const = 0; + // Implementation is responsible for setting the appropriate callback to + // the view when accessed for writing, to ensure that the data from the view + // is transferred to the device when the view is destroyed + virtual View access(Access) const = 0; + }; + using AdapterP = std::shared_ptr; + + RMat() = default; + RMat(AdapterP&& a) : m_adapter(std::move(a)) {} + GMatDesc desc() const { return m_adapter->desc(); } + + // Note: When accessed for write there is no guarantee that returned view + // will contain actual snapshot of the mapped device memory + // (no guarantee that fetch from a device is performed). The only + // guaranty is that when the view is destroyed, its data will be + // transferred to the device + View access(Access a) const { return m_adapter->access(a); } + + // Cast underlying RMat adapter to the particular adapter type, + // return nullptr if underlying type is different + template T* get() const + { + static_assert(std::is_base_of::value, "T is not derived from Adapter!"); + GAPI_Assert(m_adapter != nullptr); + return dynamic_cast(m_adapter.get()); + } + +private: + AdapterP m_adapter = nullptr; +}; + +template +RMat make_rmat(Ts&&... args) { return { std::make_shared(std::forward(args)...) }; } + +} //namespace cv + +#endif /* OPENCV_GAPI_RMAT_HPP */ diff --git a/modules/gapi/test/rmat/rmat_tests.cpp b/modules/gapi/test/rmat/rmat_tests.cpp new file mode 100644 index 0000000..8b93fbe --- /dev/null +++ b/modules/gapi/test/rmat/rmat_tests.cpp @@ -0,0 +1,171 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "../test_precomp.hpp" +#include + +namespace opencv_test { +namespace { +class RMatAdapterRef : public RMat::Adapter { + cv::Mat& m_mat; + bool& m_callbackCalled; +public: + RMatAdapterRef(cv::Mat& m, bool& callbackCalled) + : m_mat(m), m_callbackCalled(callbackCalled) + {} + virtual RMat::View access(RMat::Access access) const override { + if (access == RMat::Access::W) { + return RMat::View(cv::descr_of(m_mat), m_mat.data, m_mat.step, + [this](){ + EXPECT_FALSE(m_callbackCalled); + m_callbackCalled = true; + }); + } else { + return RMat::View(cv::descr_of(m_mat), m_mat.data, m_mat.step); + } + } + virtual cv::GMatDesc desc() const override { return cv::descr_of(m_mat); } +}; + +class RMatAdapterCopy : public RMat::Adapter { + cv::Mat& m_deviceMat; + cv::Mat m_hostMat; + bool& m_callbackCalled; + +public: + RMatAdapterCopy(cv::Mat& m, bool& callbackCalled) + : m_deviceMat(m), m_hostMat(m.clone()), m_callbackCalled(callbackCalled) + {} + virtual RMat::View access(RMat::Access access) const override { + if (access == RMat::Access::W) { + return RMat::View(cv::descr_of(m_hostMat), m_hostMat.data, m_hostMat.step, + [this](){ + EXPECT_FALSE(m_callbackCalled); + m_callbackCalled = true; + m_hostMat.copyTo(m_deviceMat); + }); + } else { + m_deviceMat.copyTo(m_hostMat); + return RMat::View(cv::descr_of(m_hostMat), m_hostMat.data, m_hostMat.step); + } + } + virtual cv::GMatDesc desc() const override { return cv::descr_of(m_hostMat); } +}; + +void randomizeMat(cv::Mat& m) { + auto ref = m.clone(); + while (cv::norm(m, ref, cv::NORM_INF) == 0) { + cv::randu(m, cv::Scalar::all(127), cv::Scalar::all(40)); + } +} + +template +struct RMatTest { + using AdapterT = RMatAdapterT; + RMatTest() + : m_deviceMat(8,8,CV_8UC1) + , m_rmat(make_rmat(m_deviceMat, m_callbackCalled)) { + randomizeMat(m_deviceMat); + expectNoCallbackCalled(); + } + + RMat& rmat() { return m_rmat; } + cv::Mat cloneDeviceMat() { return m_deviceMat.clone(); } + void expectCallbackCalled() { EXPECT_TRUE(m_callbackCalled); } + void expectNoCallbackCalled() { EXPECT_FALSE(m_callbackCalled); } + + void expectDeviceDataEqual(const cv::Mat& mat) { + EXPECT_EQ(0, cv::norm(mat, m_deviceMat, NORM_INF)); + } + void expectDeviceDataNotEqual(const cv::Mat& mat) { + EXPECT_NE(0, cv::norm(mat, m_deviceMat, NORM_INF)); + } + +private: + cv::Mat m_deviceMat; + bool m_callbackCalled = false; + cv::RMat m_rmat; +}; +} // anonymous namespace + +template +struct RMatTypedTest : public ::testing::Test, public T { using Type = T; }; + +using RMatTestTypes = ::testing::Types< RMatTest + , RMatTest + >; + +TYPED_TEST_CASE(RMatTypedTest, RMatTestTypes); + +TYPED_TEST(RMatTypedTest, Smoke) { + auto view = this->rmat().access(RMat::Access::R); + auto matFromDevice = cv::Mat(view.size(), view.type(), view.ptr()); + EXPECT_TRUE(cv::descr_of(this->cloneDeviceMat()) == this->rmat().desc()); + this->expectDeviceDataEqual(matFromDevice); +} + +static Mat asMat(RMat::View& view) { + return Mat(view.size(), view.type(), view.ptr(), view.step()); +} + +TYPED_TEST(RMatTypedTest, BasicWorkflow) { + { + auto view = this->rmat().access(RMat::Access::R); + this->expectDeviceDataEqual(asMat(view)); + } + this->expectNoCallbackCalled(); + + cv::Mat dataToWrite = this->cloneDeviceMat(); + randomizeMat(dataToWrite); + this->expectDeviceDataNotEqual(dataToWrite); + { + auto view = this->rmat().access(RMat::Access::W); + dataToWrite.copyTo(asMat(view)); + } + this->expectCallbackCalled(); + this->expectDeviceDataEqual(dataToWrite); +} + +TEST(RMat, TestEmptyAdapter) { + RMat rmat; + EXPECT_ANY_THROW(rmat.get()); +} + +TYPED_TEST(RMatTypedTest, CorrectAdapterCast) { + using T = typename TestFixture::Type::AdapterT; + EXPECT_NE(nullptr, this->rmat().template get()); +} + +class DummyAdapter : public RMat::Adapter { + virtual RMat::View access(RMat::Access) const override { return {}; } + virtual cv::GMatDesc desc() const override { return {}; } +}; + +TYPED_TEST(RMatTypedTest, IncorrectAdapterCast) { + EXPECT_EQ(nullptr, this->rmat().template get()); +} + +class RMatAdapterForBackend : public RMat::Adapter { + int m_i; +public: + RMatAdapterForBackend(int i) : m_i(i) {} + virtual RMat::View access(RMat::Access) const override { return {}; } + virtual GMatDesc desc() const override { return {}; } + int deviceSpecificData() const { return m_i; } +}; + +// RMat's usage scenario in the backend: +// we have some specific data hidden under RMat, +// test that we can obtain it via RMat.as() method +TEST(RMat, UsageInBackend) { + int i = std::rand(); + auto rmat = cv::make_rmat(i); + + auto adapter = rmat.get(); + EXPECT_NE(nullptr, adapter); + EXPECT_EQ(i, adapter->deviceSpecificData()); +} +} // namespace opencv_test diff --git a/modules/gapi/test/rmat/rmat_view_tests.cpp b/modules/gapi/test/rmat/rmat_view_tests.cpp new file mode 100644 index 0000000..750f5a7 --- /dev/null +++ b/modules/gapi/test/rmat/rmat_view_tests.cpp @@ -0,0 +1,147 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020 Intel Corporation + +#include "../test_precomp.hpp" +#include + +#include +#include "../src/backends/common/gbackend.hpp" + +namespace opencv_test +{ +using cv::GMatDesc; +using View = cv::RMat::View; +using cv::Mat; +using namespace ::testing; + +static void expect_eq_desc(const View& view, const GMatDesc& desc) { + EXPECT_EQ(view.size(), desc.size); + EXPECT_EQ(view.dims(), desc.dims); + EXPECT_EQ(view.cols(), desc.size.width); + EXPECT_EQ(view.rows(), desc.size.height); + EXPECT_EQ(view.type(), CV_MAKE_TYPE(desc.depth,desc.chan)); + EXPECT_EQ(view.depth(), desc.depth); + EXPECT_EQ(view.chan(), desc.chan); +} + +TEST(RMatView, TestDefaultConstruction) { + View view; + GMatDesc desc{}; + expect_eq_desc(view, desc); + EXPECT_EQ(view.ptr(), nullptr); + EXPECT_EQ(view.step(), 0u); +} + +struct RMatViewTest : public TestWithParam{}; +TEST_P(RMatViewTest, ConstructionFromMat) { + auto type = GetParam(); + Mat mat(8,8,type); + const auto desc = cv::descr_of(mat); + View view(cv::descr_of(mat), mat.ptr(), mat.step1()); + expect_eq_desc(view, desc); + EXPECT_EQ(view.ptr(), mat.ptr()); + EXPECT_EQ(view.step(), mat.step1()); +} + +TEST(RMatView, TestConstructionFromMatND) { + std::vector dims(4, 8); + Mat mat(dims, CV_8UC1); + const auto desc = cv::descr_of(mat); + View view(cv::descr_of(mat), mat.ptr()); + expect_eq_desc(view, desc); + EXPECT_EQ(view.ptr(), mat.ptr()); +} + +TEST_P(RMatViewTest, DefaultStep) { + auto type = GetParam(); + GMatDesc desc; + desc.chan = CV_MAT_CN(type); + desc.depth = CV_MAT_DEPTH(type); + desc.size = {8,8}; + std::vector data(desc.size.width*desc.size.height*CV_ELEM_SIZE(type)); + View view(desc, data.data()); + EXPECT_EQ(view.step(), static_cast(desc.size.width)*CV_ELEM_SIZE(type)); +} + +static Mat asMat(View& view) { + return Mat(view.size(), view.type(), view.ptr(), view.step()); +} + +TEST_P(RMatViewTest, NonDefaultStepInput) { + auto type = GetParam(); + Mat bigMat(16,16,type); + cv::randn(bigMat, cv::Scalar::all(127), cv::Scalar::all(40)); + Mat mat = bigMat(cv::Rect{4,4,8,8}); + View view(cv::descr_of(mat), mat.data, mat.step); + const auto viewMat = asMat(view); + Mat ref, out; + cv::Size ksize{1,1}; + cv::blur(viewMat, out, ksize); + cv::blur( mat, ref, ksize); + EXPECT_EQ(0, cvtest::norm(ref, out, NORM_INF)); +} + +TEST_P(RMatViewTest, NonDefaultStepOutput) { + auto type = GetParam(); + Mat mat(8,8,type); + cv::randn(mat, cv::Scalar::all(127), cv::Scalar::all(40)); + Mat bigMat = Mat::zeros(16,16,type); + Mat out = bigMat(cv::Rect{4,4,8,8}); + View view(cv::descr_of(out), out.ptr(), out.step); + auto viewMat = asMat(view); + Mat ref; + cv::Size ksize{1,1}; + cv::blur(mat, viewMat, ksize); + cv::blur(mat, ref, ksize); + EXPECT_EQ(0, cvtest::norm(ref, out, NORM_INF)); +} + +INSTANTIATE_TEST_CASE_P(Test, RMatViewTest, + Values(CV_8UC1, CV_8UC3, CV_32FC1)); + +struct RMatViewCallbackTest : public ::testing::Test { + RMatViewCallbackTest() + : mat(8,8,CV_8UC1), view(cv::descr_of(mat), mat.ptr(), mat.step1(), [this](){ callbackCalls++; }) { + cv::randn(mat, cv::Scalar::all(127), cv::Scalar::all(40)); + } + int callbackCalls = 0; + Mat mat; + View view; +}; + +TEST_F(RMatViewCallbackTest, MoveCopy) { + { + View copy(std::move(view)); + cv::util::suppress_unused_warning(copy); + EXPECT_EQ(callbackCalls, 0); + } + EXPECT_EQ(callbackCalls, 1); +} + +static int firstElement(const View& view) { return *view.ptr(); } +static void setFirstElement(View& view, uchar value) { *view.ptr() = value; } + +TEST_F(RMatViewCallbackTest, MagazineInteraction) { + cv::gimpl::magazine::Class mag; + constexpr int rc = 1; + constexpr uchar value = 11; + mag.slot()[rc] = std::move(view); + { + auto& mag_view = mag.slot()[rc]; + setFirstElement(mag_view, value); + auto mag_el = firstElement(mag_view); + EXPECT_EQ(mag_el, value); + } + { + const auto& mag_view = mag.slot()[rc]; + auto mag_el = firstElement(mag_view); + EXPECT_EQ(mag_el, value); + } + EXPECT_EQ(callbackCalls, 0); + mag.slot().erase(rc); + EXPECT_EQ(callbackCalls, 1); +} +} // namespace opencv_test -- 2.7.4