From f9c8b692fbfd7d07dfeeab3c9035c7eba0d78a3f Mon Sep 17 00:00:00 2001 From: Anton Potapov Date: Tue, 22 Sep 2020 17:40:01 +0300 Subject: [PATCH] [PP GAPI] Extended preprocessing graph to support precision conversions (#2290) - not yet visible via plugin interface - for resize non U8 input is converted to FP32 - tests --- .../src/preprocessing/ie_preprocess_gapi.cpp | 63 ++++++++++---- .../src/preprocessing/ie_preprocess_gapi.hpp | 2 +- .../fluid_preproc/common/fluid_tests.cpp | 96 +++++++++++++++------- .../fluid_preproc/common/fluid_tests.hpp | 9 +- .../fluid_preproc/cpu/fluid_tests_cpu.cpp | 43 ++++++++-- 5 files changed, 155 insertions(+), 58 deletions(-) diff --git a/inference-engine/src/preprocessing/ie_preprocess_gapi.cpp b/inference-engine/src/preprocessing/ie_preprocess_gapi.cpp index eba3d59..f05e656 100644 --- a/inference-engine/src/preprocessing/ie_preprocess_gapi.cpp +++ b/inference-engine/src/preprocessing/ie_preprocess_gapi.cpp @@ -30,10 +30,12 @@ namespace InferenceEngine { namespace { +int get_cv_depth(const TensorDesc &ie_desc); + namespace G { struct Strides {int N; int C; int H; int W;}; struct Dims {int N; int C; int H; int W;}; - struct Desc {Dims d; Strides s;}; + struct Desc {Dims d; Strides s; int prec;}; void fix_strides_nhwc(const Dims &d, Strides &s) { if (s.W > d.C) { @@ -66,7 +68,7 @@ namespace G { if (nhwc_layout) fix_strides_nhwc(d, s); - return Desc{d, s}; + return Desc{d, s, get_cv_depth(ie_desc)}; } Desc decompose(const Blob::Ptr& blob) { @@ -78,6 +80,8 @@ inline int get_cv_depth(const TensorDesc &ie_desc) { switch (ie_desc.getPrecision()) { case Precision::U8: return CV_8U; case Precision::FP32: return CV_32F; + case Precision::U16: return CV_16U; + default: THROW_IE_EXCEPTION << "Unsupported data type"; } } @@ -371,6 +375,7 @@ G::Desc getGDesc(G::Desc in_desc_y, const NV12Blob::Ptr &) { auto nv12_desc = G::Desc{}; nv12_desc.d = in_desc_y.d; nv12_desc.d.C = 2; + nv12_desc.prec = in_desc_y.prec; return nv12_desc; } @@ -379,6 +384,7 @@ G::Desc getGDesc(G::Desc in_desc_y, const I420Blob::Ptr &) { auto i420_desc = G::Desc{}; i420_desc.d = in_desc_y.d; i420_desc.d.C = 3; + i420_desc.prec = in_desc_y.prec; return i420_desc; } @@ -543,8 +549,7 @@ cv::GComputation buildGraph(const G::Desc &in_desc, Layout out_layout, ResizeAlgorithm algorithm, ColorFormat input_color_format, - ColorFormat output_color_format, - int precision) { + ColorFormat output_color_format) { // perform basic validation to ensure our assumptions about input and output are correct validateColorFormats(in_desc, out_desc, in_layout, out_layout, input_color_format, output_color_format); @@ -569,7 +574,7 @@ cv::GComputation buildGraph(const G::Desc &in_desc, (io_color_formats == std::make_tuple(ColorFormat::BGRX, ColorFormat::BGR)); const bool specific_case_of_preproc = ((in_layout == NHWC || specific_yuv420_input_handling) && (in_desc.d.C == 3 || specific_yuv420_input_handling || drop_channel) - && (precision == CV_8U) + && ((in_desc.prec == CV_8U) && (in_desc.prec == out_desc.prec)) && (algorithm == RESIZE_BILINEAR) && (input_color_format == ColorFormat::RAW || input_color_format == output_color_format @@ -591,9 +596,9 @@ cv::GComputation buildGraph(const G::Desc &in_desc, auto planes = drop_channel ? to_vec(gapi::ScalePlanes4:: on( - color_converted_input[0], precision, input_sz, scale_sz, cv::INTER_LINEAR)) + color_converted_input[0], in_desc.prec, input_sz, scale_sz, cv::INTER_LINEAR)) : to_vec(gapi::ScalePlanes ::on( - color_converted_input[0], precision, input_sz, scale_sz, cv::INTER_LINEAR)); + color_converted_input[0], in_desc.prec, input_sz, scale_sz, cv::INTER_LINEAR)); if (drop_channel) { planes.pop_back(); @@ -623,8 +628,13 @@ cv::GComputation buildGraph(const G::Desc &in_desc, << number_of_planes << " != " << out_desc.d.C; } + const int tmp_prec = CV_32F; + std::vector outputs; - if (algorithm != NO_RESIZE) { + const bool resize_needed = (algorithm != NO_RESIZE); + const bool need_tmp_prec_conv = resize_needed && (in_desc.prec != CV_8U) && (in_desc.prec != CV_32F); + + if (resize_needed) { // resize every plane std::vector out_planes; out_planes.reserve(planes.size()); @@ -635,18 +645,36 @@ cv::GComputation buildGraph(const G::Desc &in_desc, default: THROW_IE_EXCEPTION << "Unsupported resize operation"; } } (algorithm); - const auto input_sz = cv::gapi::own::Size(in_desc.d.W, in_desc.d.H); - const auto scale_sz = cv::gapi::own::Size(out_desc.d.W, out_desc.d.H); - const auto scale_fcn = std::bind(&gapi::ScalePlane::on, - std::placeholders::_1, - precision, - input_sz, scale_sz, interp_type); - std::transform(planes.begin(), planes.end(), std::back_inserter(out_planes), scale_fcn); + + std::transform(planes.begin(), planes.end(), std::back_inserter(out_planes), [&](const cv::GMat& m) { + const auto input_sz = cv::gapi::own::Size(in_desc.d.W, in_desc.d.H); + const auto scale_sz = cv::gapi::own::Size(out_desc.d.W, out_desc.d.H); + + cv::GMat converted = m; + int prec = in_desc.prec; + + if (need_tmp_prec_conv) { + std::tie(converted, prec) = std::make_tuple(gapi::ConvertDepth::on(m, tmp_prec), tmp_prec); + } + + return gapi::ScalePlane::on(converted, prec, input_sz, scale_sz, interp_type); + }); outputs = out_planes; } else { outputs = planes; } + if ((in_desc.prec != out_desc.prec) || need_tmp_prec_conv) { + auto convert_prec = [](const std::vector & src_gmats, int dst_precision) { + std::vector dst_gmats; + std::transform(src_gmats.begin(), src_gmats.end(), std::back_inserter(dst_gmats), [&](cv::GMat const& m){ + return gapi::ConvertDepth::on(m, dst_precision); + }); + return dst_gmats; + }; + + outputs = convert_prec(outputs, out_desc.prec); + } // convert to interleaved if NHWC is required as output if (out_layout == NHWC) { outputs = merge(outputs, out_desc.d.C); @@ -939,8 +967,7 @@ bool PreprocEngine::preprocessBlob(const BlobTypePtr &inBlob, MemoryBlob::Ptr &o out_layout, algorithm, in_fmt, - out_fmt, - get_cv_depth(in_desc_ie))); + out_fmt)); } } @@ -953,7 +980,7 @@ bool PreprocEngine::preprocessBlob(const BlobTypePtr &inBlob, MemoryBlob::Ptr &o return true; } -bool PreprocEngine::preprocessWithGAPI(Blob::Ptr &inBlob, Blob::Ptr &outBlob, +bool PreprocEngine::preprocessWithGAPI(const Blob::Ptr &inBlob, Blob::Ptr &outBlob, const ResizeAlgorithm& algorithm, ColorFormat in_fmt, bool omp_serial, int batch_size) { if (!useGAPI()) { return false; diff --git a/inference-engine/src/preprocessing/ie_preprocess_gapi.hpp b/inference-engine/src/preprocessing/ie_preprocess_gapi.hpp index 81a6014..418047a 100644 --- a/inference-engine/src/preprocessing/ie_preprocess_gapi.hpp +++ b/inference-engine/src/preprocessing/ie_preprocess_gapi.hpp @@ -54,7 +54,7 @@ public: static bool useGAPI(); static void checkApplicabilityGAPI(const Blob::Ptr &src, const Blob::Ptr &dst); static int getCorrectBatchSize(int batch_size, const Blob::Ptr& roiBlob); - bool preprocessWithGAPI(Blob::Ptr &inBlob, Blob::Ptr &outBlob, const ResizeAlgorithm &algorithm, + bool preprocessWithGAPI(const Blob::Ptr &inBlob, Blob::Ptr &outBlob, const ResizeAlgorithm &algorithm, ColorFormat in_fmt, bool omp_serial, int batch_size = -1); }; diff --git a/inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.cpp b/inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.cpp index 8f5a2f0..2f5ebd3 100644 --- a/inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.cpp +++ b/inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.cpp @@ -23,6 +23,8 @@ #include +#include + #include // Can be set externally (via CMake) if built with -DGAPI_TEST_PERF=ON @@ -82,6 +84,7 @@ cv::String depthToString(int depth) { case CV_8U : return "CV_8U"; case CV_32F : return "CV_32F"; + case CV_16U : return "CV_16U"; } CV_Assert(!"ERROR: unsupported depth!"); return nullptr; @@ -1078,14 +1081,17 @@ TEST_P(MergeTestIE, AccuracyTest) TEST_P(PreprocTest, Performance) { using namespace InferenceEngine; - Precision prec; + std::pair precisions; ResizeAlgorithm interp; Layout in_layout, out_layout; std::pair ocv_channels{-1, -1}; std::pair sizes; ColorFormat in_fmt = ColorFormat::RAW; ColorFormat out_fmt = ColorFormat::BGR; - std::tie(prec, interp, in_fmt, in_layout, out_layout, ocv_channels, sizes) = GetParam(); + std::tie(precisions, interp, in_fmt, in_layout, out_layout, ocv_channels, sizes) = GetParam(); + + Precision in_prec, out_prec; + std::tie(in_prec, out_prec) = precisions; cv::Size in_size, out_size; std::tie(in_size, out_size) = sizes; int in_ocv_chan = -1, out_ocv_chan = -1; @@ -1096,11 +1102,21 @@ TEST_P(PreprocTest, Performance) double tolerance = Precision::U8 ? 1 : 0.015; #endif - const int ocv_depth = prec == Precision::U8 ? CV_8U : - prec == Precision::FP32 ? CV_32F : -1; - const int in_ocv_type = CV_MAKETYPE(ocv_depth, in_ocv_chan); - const int out_ocv_type = CV_MAKETYPE(ocv_depth, out_ocv_chan); - initMatrixRandU(in_ocv_type, in_size, in_ocv_type, false); + auto precision_to_depth = [](Precision::ePrecision prec) -> int{ + switch (prec) + { + case Precision::U8: return CV_8U; + case Precision::U16: return CV_16U; + case Precision::FP32: return CV_32F; + default: + throw std::logic_error("Unsupported configuration"); + } + return -1; + }; + + const int in_ocv_type = CV_MAKETYPE(precision_to_depth(in_prec), in_ocv_chan); + const int out_ocv_type = CV_MAKETYPE(precision_to_depth(out_prec), out_ocv_chan); + initMatrixRandU(in_ocv_type, in_size, out_ocv_type, false); cv::Mat out_mat(out_size, out_ocv_type); @@ -1115,28 +1131,37 @@ TEST_P(PreprocTest, Performance) cv::randu(in_mat2, cv::Scalar::all(0), cv::Scalar::all(255)); } - Blob::Ptr in_blob, out_blob; - switch (prec) - { - case Precision::U8: - if (in_fmt == ColorFormat::NV12) { - auto y_blob = img2Blob(in_mat1, Layout::NHWC); - auto uv_blob = img2Blob(in_mat2, Layout::NHWC); - in_blob = make_shared_blob(y_blob, uv_blob); - } else { - in_blob = img2Blob(in_mat1, in_layout); - } - out_blob = img2Blob(out_mat, out_layout); - break; - case Precision::FP32: - in_blob = img2Blob(in_mat1, in_layout); - out_blob = img2Blob(out_mat, out_layout); - break; + auto create_blob = [](Precision::ePrecision prec, ColorFormat fmt, Layout layout, cv::Mat& m1, cv::util::optional m2 = {}){ + Blob::Ptr blob; + switch (prec) + { + case Precision::U8: + if (fmt == ColorFormat::NV12) { + auto y_blob = img2Blob(m1, Layout::NHWC); + auto uv_blob = img2Blob(m2.value(), Layout::NHWC); + blob = make_shared_blob(y_blob, uv_blob); + } else { + blob = img2Blob(m1, layout); + } + break; - default: - FAIL() << "Unsupported configuration"; - } + case Precision::U16: + blob = img2Blob(m1, layout); + break; + + case Precision::FP32: + blob = img2Blob(m1, layout); + break; + + default: + throw std::logic_error("Unsupported configuration"); + } + return blob; + }; + + auto in_blob = create_blob(in_prec, in_fmt, in_layout, in_mat1, cv::util::make_optional(in_mat2)); + auto out_blob = create_blob(out_prec, out_fmt, out_layout, out_mat); PreProcessDataPtr preprocess = CreatePreprocDataHelper(); preprocess->setRoiBlob(in_blob); @@ -1148,10 +1173,11 @@ TEST_P(PreprocTest, Performance) // test once to warm-up cache preprocess->execute(out_blob, info, false); - switch (prec) + switch (out_prec) { case Precision::U8: Blob2Img (out_blob, out_mat, out_layout); break; case Precision::FP32: Blob2Img(out_blob, out_mat, out_layout); break; + case Precision::U16: Blob2Img(out_blob, out_mat, out_layout); break; default: FAIL() << "Unsupported configuration"; } @@ -1166,11 +1192,18 @@ TEST_P(PreprocTest, Performance) auto cv_interp = interp == RESIZE_AREA ? cv::INTER_AREA : cv::INTER_LINEAR; cv::resize(ocv_out_mat, ocv_out_mat, out_size, 0, 0, cv_interp); + if (in_prec != out_prec) { + cv::Mat ocv_converted; + ocv_out_mat.convertTo(ocv_converted, out_ocv_type); + ocv_out_mat = ocv_converted; + } + EXPECT_LE(cv::norm(ocv_out_mat, out_mat, cv::NORM_INF), tolerance); #if PERF_TEST // iterate testing, and print performance - const auto type_str = depthToString(ocv_depth); + const auto in_type_str = depthToString(precision_to_depth(in_prec)); + const auto out_type_str = depthToString(precision_to_depth(out_prec)); const auto interp_str = interp == RESIZE_AREA ? "AREA" : interp == RESIZE_BILINEAR ? "BILINEAR" : "?"; const auto in_layout_str = layoutToString(in_layout); @@ -1178,8 +1211,9 @@ TEST_P(PreprocTest, Performance) test_ms([&]() { preprocess->execute(out_blob, info, false); }, 300, - "Preproc %s %s %d %s %dx%d %d %s %dx%d %s->%s", - type_str.c_str(), + "Preproc %s %s %s %d %s %dx%d %d %s %dx%d %s->%s", + in_type_str.c_str(), + out_type_str.c_str(), interp_str, in_ocv_chan, in_layout_str.c_str(), in_size.width, in_size.height, diff --git a/inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.hpp b/inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.hpp index 012a281..8bca5f3 100644 --- a/inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.hpp +++ b/inference-engine/tests_deprecated/fluid_preproc/common/fluid_tests.hpp @@ -48,9 +48,16 @@ struct ColorConvertYUV420TestIE: double>> // tolerance {}; +struct PrecisionConvertTestIE: public TestParams> // tolerance +{}; + //------------------------------------------------------------------------------ -using PreprocParams = std::tuple< InferenceEngine::Precision // input-output data type +using PreprocParams = std::tuple< std::pair // output data type , InferenceEngine::ResizeAlgorithm // resize algorithm, if needed , InferenceEngine::ColorFormat // input color format, if needed , InferenceEngine::Layout // input tensor layout diff --git a/inference-engine/tests_deprecated/fluid_preproc/cpu/fluid_tests_cpu.cpp b/inference-engine/tests_deprecated/fluid_preproc/cpu/fluid_tests_cpu.cpp index 1e21638..acdac01 100644 --- a/inference-engine/tests_deprecated/fluid_preproc/cpu/fluid_tests_cpu.cpp +++ b/inference-engine/tests_deprecated/fluid_preproc/cpu/fluid_tests_cpu.cpp @@ -323,8 +323,12 @@ static const auto PATCH_SIZES = cv::Size(80, 160))); // 80 - person-attributes-recognition-crossroad-0031 // 64 - person-reidentification-retail-0079 +static const auto U8toU8 = std::make_pair(IE::Precision::U8,IE::Precision::U8); + +static const auto PRECISIONS = Values(U8toU8, std::make_pair(IE::Precision::FP32,IE::Precision::FP32)); + INSTANTIATE_TEST_CASE_P(ReorderResize_Frame, PreprocTest, - Combine(Values(IE::Precision::U8, IE::Precision::FP32), + Combine(PRECISIONS, Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet Values(IE::ColorFormat::RAW), Values(IE::Layout::NHWC), @@ -333,7 +337,7 @@ INSTANTIATE_TEST_CASE_P(ReorderResize_Frame, PreprocTest, FRAME_SIZES)); INSTANTIATE_TEST_CASE_P(Scale3ch_Frame, PreprocTest, - Combine(Values(IE::Precision::U8, IE::Precision::FP32), + Combine(PRECISIONS, Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet Values(IE::ColorFormat::RAW), Values(IE::Layout::NHWC), @@ -342,7 +346,7 @@ INSTANTIATE_TEST_CASE_P(Scale3ch_Frame, PreprocTest, FRAME_SIZES)); INSTANTIATE_TEST_CASE_P(ReorderResize_Patch, PreprocTest, - Combine(Values(IE::Precision::U8, IE::Precision::FP32), + Combine(PRECISIONS, Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), // AREA is not there yet Values(IE::ColorFormat::RAW), Values(IE::Layout::NHWC), @@ -351,7 +355,7 @@ INSTANTIATE_TEST_CASE_P(ReorderResize_Patch, PreprocTest, PATCH_SIZES)); INSTANTIATE_TEST_CASE_P(Everything_Resize, PreprocTest, - Combine(Values(IE::Precision::U8, IE::Precision::FP32), + Combine(PRECISIONS, Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA), Values(IE::ColorFormat::RAW), Values(IE::Layout::NHWC, IE::Layout::NCHW), @@ -363,7 +367,7 @@ INSTANTIATE_TEST_CASE_P(Everything_Resize, PreprocTest, Values(TEST_SIZES_PREPROC))); INSTANTIATE_TEST_CASE_P(ColorFormats_3ch, PreprocTest, - Combine(Values(IE::Precision::U8, IE::Precision::FP32), + Combine(PRECISIONS, Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA), Values(IE::ColorFormat::RGB), Values(IE::Layout::NHWC, IE::Layout::NCHW), @@ -372,7 +376,7 @@ INSTANTIATE_TEST_CASE_P(ColorFormats_3ch, PreprocTest, Values(TEST_SIZES_PREPROC))); INSTANTIATE_TEST_CASE_P(ColorFormats_4ch, PreprocTest, - Combine(Values(IE::Precision::U8, IE::Precision::FP32), + Combine(PRECISIONS, Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA), Values(IE::ColorFormat::BGRX, IE::ColorFormat::RGBX), Values(IE::Layout::NHWC), @@ -381,10 +385,35 @@ INSTANTIATE_TEST_CASE_P(ColorFormats_4ch, PreprocTest, Values(TEST_SIZES_PREPROC))); INSTANTIATE_TEST_CASE_P(ColorFormat_NV12, PreprocTest, - Combine(Values(IE::Precision::U8), + Combine(Values(U8toU8), Values(IE::ResizeAlgorithm::RESIZE_BILINEAR, IE::ResizeAlgorithm::RESIZE_AREA), Values(IE::ColorFormat::NV12), Values(IE::Layout::NCHW), Values(IE::Layout::NHWC, IE::Layout::NCHW), Values(std::make_pair(1, 3)), Values(TEST_SIZES_PREPROC))); + + +INSTANTIATE_TEST_CASE_P(DISABLED_PlainPrecisionConversions, PreprocTest, + Combine(Values(std::make_pair(IE::Precision::U16,IE::Precision::FP32), + std::make_pair(IE::Precision::FP32,IE::Precision::U16) + ), + Values(IE::ResizeAlgorithm::NO_RESIZE), + Values(IE::ColorFormat::RAW), + Values(IE::Layout::NHWC), + Values(IE::Layout::NHWC), + Values(std::make_pair(1, 1)), + Values(std::make_pair(cv::Size(640,480), cv::Size(640,480))))); + + +INSTANTIATE_TEST_CASE_P(PrecisionConversionsPipelines, PreprocTest, + Combine(Values(std::make_pair(IE::Precision::U16, IE::Precision::FP32), + std::make_pair(IE::Precision::FP32,IE::Precision::U8), + std::make_pair(IE::Precision::U8, IE::Precision::FP32) + ), + Values(IE::ResizeAlgorithm::RESIZE_BILINEAR), + Values(IE::ColorFormat::RAW), + Values(IE::Layout::NHWC, IE::Layout::NCHW), + Values(IE::Layout::NHWC, IE::Layout::NCHW), + Values(std::make_pair(1, 1)/*, std::make_pair(3, 3)*/), //U16 Split and Merge are not there + Values(TEST_SIZES_PREPROC))); -- 2.7.4