From: Alexey Smirnov Date: Fri, 23 Sep 2022 11:08:29 +0000 (+0100) Subject: Copy mpashchenkov's changes X-Git-Tag: accepted/tizen/unified/20230127.161057~1^2~86^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=4c74e6d89d35dc45a3889d96158989352f73b1db;p=platform%2Fupstream%2Fopencv.git Copy mpashchenkov's changes Minor refactoring Partially address review comments Move DX-related stuff from the sample to a default source Simplify the default OneVPL config Address minor review comments Add class for the default VPL source WIP: Add initial stub for tests with description Removing default vpl source and minor refactoring Refactor default files Fix build and application crash Address review comments Add test on VPL + OCL interaction compared to CPU behavior Fix test --- diff --git a/modules/gapi/CMakeLists.txt b/modules/gapi/CMakeLists.txt index 2a9d598bb8..a25af7a5c2 100644 --- a/modules/gapi/CMakeLists.txt +++ b/modules/gapi/CMakeLists.txt @@ -188,6 +188,7 @@ set(gapi_srcs src/streaming/onevpl/cfg_params.cpp src/streaming/onevpl/cfg_params_parser.cpp src/streaming/onevpl/utils.cpp + src/streaming/onevpl/default.cpp src/streaming/onevpl/data_provider_interface_exception.cpp src/streaming/onevpl/accelerators/surface/base_frame_adapter.cpp src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -367,20 +368,20 @@ ocv_add_samples() # Required for sample with inference on host -if(TARGET example_gapi_onevpl_infer_single_roi) +if(TARGET example_gapi_onevpl_infer_with_advanced_device_selection) if(TARGET ocv.3rdparty.openvino AND OPENCV_GAPI_WITH_OPENVINO) - ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE ocv.3rdparty.openvino) + ocv_target_link_libraries(example_gapi_onevpl_infer_with_advanced_device_selection PRIVATE ocv.3rdparty.openvino) endif() if(HAVE_DIRECTX AND HAVE_D3D11) - ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE d3d11 dxgi) + ocv_target_link_libraries(example_gapi_onevpl_infer_with_advanced_device_selection PRIVATE d3d11 dxgi) endif() if(HAVE_D3D11 AND HAVE_OPENCL) - ocv_target_include_directories(example_gapi_onevpl_infer_single_roi SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) + ocv_target_include_directories(example_gapi_onevpl_infer_with_advanced_device_selection SYSTEM PRIVATE ${OPENCL_INCLUDE_DIRS}) endif() if(UNIX AND HAVE_VA) message ("GAPI VPL samples with VAAPI") - ocv_target_include_directories(example_gapi_onevpl_infer_single_roi SYSTEM PRIVATE ${VA_INCLUDE_DIR}) - ocv_target_link_libraries(example_gapi_onevpl_infer_single_roi PRIVATE ${VA_LIBRARIES}) + ocv_target_include_directories(example_gapi_onevpl_infer_with_advanced_device_selection SYSTEM PRIVATE ${VA_INCLUDE_DIR}) + ocv_target_link_libraries(example_gapi_onevpl_infer_with_advanced_device_selection PRIVATE ${VA_LIBRARIES}) endif() endif() diff --git a/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp b/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp index 6a2f1df769..b70b54267d 100644 --- a/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp +++ b/modules/gapi/include/opencv2/gapi/ocl/goclkernel.hpp @@ -119,6 +119,10 @@ template struct ocl_get_in > { static const std::vector& get(GOCLContext &ctx, int idx) { return ctx.inArg(idx).rref(); } }; +template<> struct ocl_get_in +{ + static cv::MediaFrame get(GOCLContext &ctx, int idx) { return ctx.inArg(idx); } +}; template struct ocl_get_in > { static const U& get(GOCLContext &ctx, int idx) { return ctx.inArg(idx).rref(); } diff --git a/modules/gapi/include/opencv2/gapi/streaming/onevpl/default.hpp b/modules/gapi/include/opencv2/gapi/streaming/onevpl/default.hpp new file mode 100644 index 0000000000..8b547e1aba --- /dev/null +++ b/modules/gapi/include/opencv2/gapi/streaming/onevpl/default.hpp @@ -0,0 +1,29 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + +#ifndef OPENCV_GAPI_STREAMING_ONEVPL_UTILS_HPP +#define OPENCV_GAPI_STREAMING_ONEVPL_UTILS_HPP + +#include // GAPI_EXPORTS +#include +#include + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +/** + * @brief Provides default device selector based on config. + */ +GAPI_EXPORTS std::shared_ptr getDefaultDeviceSelector(const std::vector& cfg_params); + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // OPENCV_GAPI_STREAMING_ONEVPL_UTILS_HPP diff --git a/modules/gapi/samples/onevpl_infer_single_roi.cpp b/modules/gapi/samples/onevpl_infer_single_roi.cpp deleted file mode 100644 index de1d233ae5..0000000000 --- a/modules/gapi/samples/onevpl_infer_single_roi.cpp +++ /dev/null @@ -1,704 +0,0 @@ -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include // CommandLineParser -#include - -#ifdef HAVE_INF_ENGINE -#include // ParamMap -#endif // HAVE_INF_ENGINE - -#ifdef HAVE_DIRECTX -#ifdef HAVE_D3D11 - -// get rid of generate macro max/min/etc from DX side -#define D3D11_NO_HELPERS -#define NOMINMAX -#include -#undef NOMINMAX -#undef D3D11_NO_HELPERS -#endif // HAVE_D3D11 -#endif // HAVE_DIRECTX - -#ifdef __linux__ -#if defined(HAVE_VA) || defined(HAVE_VA_INTEL) -#include "va/va.h" -#include "va/va_drm.h" - -#include -#include -#endif // defined(HAVE_VA) || defined(HAVE_VA_INTEL) -#endif // __linux__ - - -const std::string about = - "This is an OpenCV-based version of oneVPLSource decoder example"; -const std::string keys = - "{ h help | | Print this help message }" - "{ input | | Path to the input demultiplexed video file }" - "{ output | | Path to the output RAW video file. Use .avi extension }" - "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" - "{ faced | GPU | Target device for face detection model (e.g. AUTO, GPU, VPU, ...) }" - "{ cfg_params | | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }" - "{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automatically if 0 }" - "{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}" - "{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}" - "{ roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }" - "{ source_device | CPU | choose device for decoding }" - "{ preproc_device | | choose device for preprocessing }"; - - -namespace { -bool is_gpu(const std::string &device_name) { - return device_name.find("GPU") != std::string::npos; -} - -std::string get_weights_path(const std::string &model_path) { - const auto EXT_LEN = 4u; - const auto sz = model_path.size(); - GAPI_Assert(sz > EXT_LEN); - - auto ext = model_path.substr(sz - EXT_LEN); - std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ - return static_cast(std::tolower(c)); - }); - GAPI_Assert(ext == ".xml"); - return model_path.substr(0u, sz - EXT_LEN) + ".bin"; -} - -// TODO: It duplicates infer_single_roi sample -cv::util::optional parse_roi(const std::string &rc) { - cv::Rect rv; - char delim[3]; - - std::stringstream is(rc); - is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height; - if (is.bad()) { - return cv::util::optional(); // empty value - } - const auto is_delim = [](char c) { - return c == ','; - }; - if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) { - return cv::util::optional(); // empty value - } - if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) { - return cv::util::optional(); // empty value - } - return cv::util::make_optional(std::move(rv)); -} - -#ifdef HAVE_DIRECTX -#ifdef HAVE_D3D11 - -// Since ATL headers might not be available on specific MSVS Build Tools -// we use simple `CComPtr` implementation like as `ComPtrGuard` -// which is not supposed to be the full functional replacement of `CComPtr` -// and it uses as RAII to make sure utilization is correct -template -void release(COMNonManageableType *ptr) { - if (ptr) { - ptr->Release(); - } -} - -template -using ComPtrGuard = std::unique_ptr)>; - -template -ComPtrGuard createCOMPtrGuard(COMNonManageableType *ptr = nullptr) { - return ComPtrGuard {ptr, &release}; -} - - -using AccelParamsType = std::tuple, ComPtrGuard>; - -AccelParamsType create_device_with_ctx(IDXGIAdapter* adapter) { - UINT flags = 0; - D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1, - D3D_FEATURE_LEVEL_11_0, - }; - D3D_FEATURE_LEVEL featureLevel; - ID3D11Device* ret_device_ptr = nullptr; - ID3D11DeviceContext* ret_ctx_ptr = nullptr; - HRESULT err = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, - nullptr, flags, - feature_levels, - ARRAYSIZE(feature_levels), - D3D11_SDK_VERSION, &ret_device_ptr, - &featureLevel, &ret_ctx_ptr); - if (FAILED(err)) { - throw std::runtime_error("Cannot create D3D11CreateDevice, error: " + - std::to_string(HRESULT_CODE(err))); - } - - return std::make_tuple(createCOMPtrGuard(ret_device_ptr), - createCOMPtrGuard(ret_ctx_ptr)); -} -#endif // HAVE_D3D11 -#endif // HAVE_DIRECTX -} // anonymous namespace - -namespace custom { -G_API_NET(FaceDetector, , "face-detector"); - -using GDetections = cv::GArray; -using GRect = cv::GOpaque; -using GSize = cv::GOpaque; -using GPrims = cv::GArray; - -G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { - static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { - return cv::empty_array_desc(); - } -}; - -// TODO: It duplicates infer_single_roi sample -G_API_OP(LocateROI, , "sample.custom.locate-roi") { - static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { - return cv::empty_gopaque_desc(); - } -}; - -G_API_OP(BBoxes, , "sample.custom.b-boxes") { - static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { - return cv::empty_array_desc(); - } -}; - -GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { - // This is the place where we can run extra analytics - // on the input image frame and select the ROI (region - // of interest) where we want to detect our objects (or - // run any other inference). - // - // Currently it doesn't do anything intelligent, - // but only crops the input image to square (this is - // the most convenient aspect ratio for detectors to use) - - static void run(const cv::Size& in_size, - cv::Rect &out_rect) { - - // Identify the central point & square size (- some padding) - const auto center = cv::Point{in_size.width/2, in_size.height/2}; - auto sqside = std::min(in_size.width, in_size.height); - - // Now build the central square ROI - out_rect = cv::Rect{ center.x - sqside/2 - , center.y - sqside/2 - , sqside - , sqside - }; - } -}; - -GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { - // This kernel converts the rectangles into G-API's - // rendering primitives - static void run(const std::vector &in_face_rcs, - const cv::Rect &in_roi, - std::vector &out_prims) { - out_prims.clear(); - const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { - return cv::gapi::wip::draw::Rect(rc, clr, 2); - }; - out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan - for (auto &&rc : in_face_rcs) { - out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green - } - } -}; - -GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { - static void run(const cv::Mat &in_ssd_result, - const cv::Rect &in_roi, - const cv::Size &in_parent_size, - std::vector &out_objects) { - const auto &in_ssd_dims = in_ssd_result.size; - GAPI_Assert(in_ssd_dims.dims() == 4u); - - const int MAX_PROPOSALS = in_ssd_dims[2]; - const int OBJECT_SIZE = in_ssd_dims[3]; - GAPI_Assert(OBJECT_SIZE == 7); // fixed SSD object size - - const cv::Size up_roi = in_roi.size(); - const cv::Rect surface({0,0}, in_parent_size); - - out_objects.clear(); - - const float *data = in_ssd_result.ptr(); - for (int i = 0; i < MAX_PROPOSALS; i++) { - const float image_id = data[i * OBJECT_SIZE + 0]; - const float label = data[i * OBJECT_SIZE + 1]; - const float confidence = data[i * OBJECT_SIZE + 2]; - const float rc_left = data[i * OBJECT_SIZE + 3]; - const float rc_top = data[i * OBJECT_SIZE + 4]; - const float rc_right = data[i * OBJECT_SIZE + 5]; - const float rc_bottom = data[i * OBJECT_SIZE + 6]; - (void) label; // unused - - if (image_id < 0.f) { - break; // marks end-of-detections - } - if (confidence < 0.5f) { - continue; // skip objects with low confidence - } - - // map relative coordinates to the original image scale - // taking the ROI into account - cv::Rect rc; - rc.x = static_cast(rc_left * up_roi.width); - rc.y = static_cast(rc_top * up_roi.height); - rc.width = static_cast(rc_right * up_roi.width) - rc.x; - rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; - rc.x += in_roi.x; - rc.y += in_roi.y; - out_objects.emplace_back(rc & surface); - } - } -}; - -} // namespace custom - -namespace cfg { -typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line); - -struct flow { - flow(bool preproc, bool rctx) : - vpl_preproc_enable(preproc), - ie_remote_ctx_enable(rctx) { - } - bool vpl_preproc_enable = false; - bool ie_remote_ctx_enable = false; -}; - -using support_matrix = - std::map >>>; -support_matrix resolved_conf{{ - {"GPU", {{ - {"", {{ "CPU", std::make_shared(false, false)}, - { "GPU", {/* unsupported: - * ie GPU preproc isn't available */}} - }}, - - {"CPU", {{ "CPU", {/* unsupported: preproc mix */}}, - { "GPU", {/* unsupported: preproc mix */}} - }}, -#if defined(HAVE_DIRECTX) && defined(HAVE_D3D11) - {"GPU", {{ "CPU", std::make_shared(true, false)}, - { "GPU", std::make_shared(true, true)}}} -#else // TODO VAAPI under linux doesn't support GPU IE remote context - {"GPU", {{ "CPU", std::make_shared(true, false)}, - { "GPU", std::make_shared(true, false)}}} -#endif - }} - }, - {"CPU", {{ - {"", {{ "CPU", std::make_shared(false, false)}, - { "GPU", std::make_shared(false, false)} - }}, - - {"CPU", {{ "CPU", std::make_shared(true, false)}, - { "GPU", std::make_shared(true, false)} - }}, - - {"GPU", {{ "CPU", {/* unsupported: preproc mix */}}, - { "GPU", {/* unsupported: preproc mix */}}}} - }} - } - }}; - -static void print_available_cfg(std::ostream &out, - const std::string &source_device, - const std::string &preproc_device, - const std::string &ie_device_id) { - const std::string source_device_cfg_name("--source_device="); - const std::string preproc_device_cfg_name("--preproc_device="); - const std::string ie_cfg_name("--faced="); - out << "unsupported acceleration param combinations:\n" - << source_device_cfg_name << source_device << " " - << preproc_device_cfg_name << preproc_device << " " - << ie_cfg_name << ie_device_id << - "\n\nSupported matrix:\n\n" << std::endl; - for (const auto &s_d : cfg::resolved_conf) { - std::string prefix = source_device_cfg_name + s_d.first; - for (const auto &p_d : s_d.second) { - std::string mid_prefix = prefix + +"\t" + preproc_device_cfg_name + - (p_d.first.empty() ? "" : p_d.first); - for (const auto &i_d : p_d.second) { - if (i_d.second) { - std::cerr << mid_prefix << "\t" << ie_cfg_name <("input"); - const auto output = cmd.get("output"); - const auto opt_roi = parse_roi(cmd.get("roi")); - const auto face_model_path = cmd.get("facem"); - const auto streaming_queue_capacity = cmd.get("streaming_queue_capacity"); - const auto source_decode_queue_capacity = cmd.get("frames_pool_size"); - const auto source_vpp_queue_capacity = cmd.get("vpp_frames_pool_size"); - const auto device_id = cmd.get("faced"); - const auto source_device = cmd.get("source_device"); - const auto preproc_device = cmd.get("preproc_device"); - - // validate support matrix - std::shared_ptr flow_settings = cfg::resolved_conf[source_device][preproc_device][device_id]; - if (!flow_settings) { - cfg::print_available_cfg(std::cerr, source_device, preproc_device, device_id); - return -1; - } - - // check output file extension - if (!output.empty()) { - auto ext = output.find_last_of("."); - if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) { - std::cerr << "Output file should have *.avi extension for output video" << std::endl; - return -1; - } - } - - // get oneVPL cfg params from cmd - std::stringstream params_list(cmd.get("cfg_params")); - std::vector source_cfgs; - try { - std::string line; - while (std::getline(params_list, line, ';')) { - source_cfgs.push_back(cfg::create_from_string(line)); - } - } catch (const std::exception& ex) { - std::cerr << "Invalid cfg parameter: " << ex.what() << std::endl; - return -1; - } - - // apply VPL source optimization params - if (source_decode_queue_capacity != 0) { - source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_frames_pool_size(source_decode_queue_capacity)); - } - if (source_vpp_queue_capacity != 0) { - source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_vpp_frames_pool_size(source_vpp_queue_capacity)); - } - - auto face_net = cv::gapi::ie::Params { - face_model_path, // path to topology IR - get_weights_path(face_model_path), // path to weights - device_id - }; - - // It is allowed (and highly recommended) to reuse predefined device_ptr & context_ptr objects - // received from user application. Current sample demonstrate how to deal with this situation. - // - // But if you do not need this fine-grained acceleration devices configuration then - // just use default constructors for onevpl::GSource, IE and preprocessing module. - // But please pay attention that default pipeline construction in this case will be - // very inefficient and carries out multiple CPU-GPU memory copies - // - // If you want to reach max performance and seize copy-free approach for specific - // device & context selection then follow the steps below. - // The situation is complicated a little bit in comparison with default configuration, thus - // let's focusing this: - // - // - all component-participants (Source, Preprocessing, Inference) - // must share the same device & context instances - // - // - you must wrapping your available device & context instancs into thin - // `cv::gapi::wip::Device` & `cv::gapi::wip::Context`. - // !!! Please pay attention that both objects are weak wrapper so you must ensure - // that device & context would be alived before full pipeline created !!! - // - // - you should pass such wrappers as constructor arguments for each component in pipeline: - // a) use extended constructor for `onevpl::GSource` for activating predefined device & context - // b) use `cfgContextParams` method of `cv::gapi::ie::Params` to enable `PreprocesingEngine` - // for predefined device & context - // c) use `InferenceEngine::ParamMap` to activate remote ctx in Inference Engine for given - // device & context - // - // - //// P.S. the current sample supports heterogenous pipeline construction also. - //// It is possible to make up mixed device approach. - //// Please feel free to explore different configurations! - - cv::util::optional gpu_accel_device; - cv::util::optional gpu_accel_ctx; - cv::gapi::wip::onevpl::Device cpu_accel_device = cv::gapi::wip::onevpl::create_host_device(); - cv::gapi::wip::onevpl::Context cpu_accel_ctx = cv::gapi::wip::onevpl::create_host_context(); - // create GPU device if requested - if (is_gpu(device_id) - || is_gpu(source_device) - || is_gpu(preproc_device)) { -#ifdef HAVE_DIRECTX -#ifdef HAVE_D3D11 - // create DX11 device & context owning handles. - // wip::Device & wip::Context provide non-owning semantic of resources and act - // as weak references API wrappers in order to carry type-erased resources type - // into appropriate modules: onevpl::GSource, PreprocEngine and InferenceEngine - // Until modules are not created owner handles must stay alive - auto dx11_dev = createCOMPtrGuard(); - auto dx11_ctx = createCOMPtrGuard(); - - auto adapter_factory = createCOMPtrGuard(); - { - IDXGIFactory* out_factory = nullptr; - HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), - reinterpret_cast(&out_factory)); - if (FAILED(err)) { - std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl; - return -1; - } - adapter_factory = createCOMPtrGuard(out_factory); - } - - auto intel_adapter = createCOMPtrGuard(); - UINT adapter_index = 0; - const unsigned int refIntelVendorID = 0x8086; - IDXGIAdapter* out_adapter = nullptr; - - while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { - DXGI_ADAPTER_DESC desc{}; - out_adapter->GetDesc(&desc); - if (desc.VendorId == refIntelVendorID) { - intel_adapter = createCOMPtrGuard(out_adapter); - break; - } - ++adapter_index; - } - - if (!intel_adapter) { - std::cerr << "No Intel GPU adapter on aboard. Exit" << std::endl; - return -1; - } - - std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get()); - gpu_accel_device = cv::util::make_optional( - cv::gapi::wip::onevpl::create_dx11_device( - reinterpret_cast(dx11_dev.release()), - "GPU")); - gpu_accel_ctx = cv::util::make_optional( - cv::gapi::wip::onevpl::create_dx11_context( - reinterpret_cast(dx11_ctx.release()))); -#endif // HAVE_D3D11 -#endif // HAVE_DIRECTX -#ifdef __linux__ -#if defined(HAVE_VA) || defined(HAVE_VA_INTEL) - static const char *predefined_vaapi_devices_list[] {"/dev/dri/renderD128", - "/dev/dri/renderD129", - "/dev/dri/card0", - "/dev/dri/card1", - nullptr}; - std::stringstream ss; - int device_fd = -1; - VADisplay va_handle = nullptr; - for (const char **device_path = predefined_vaapi_devices_list; - *device_path != nullptr; device_path++) { - device_fd = open(*device_path, O_RDWR); - if (device_fd < 0) { - std::string info("Cannot open GPU file: \""); - info = info + *device_path + "\", error: " + strerror(errno); - ss << info << std::endl; - continue; - } - va_handle = vaGetDisplayDRM(device_fd); - if (!va_handle) { - close(device_fd); - std::string info("VAAPI device vaGetDisplayDRM failed, error: "); - info += strerror(errno); - ss << info << std::endl; - continue; - } - int major_version = 0, minor_version = 0; - VAStatus status {}; - status = vaInitialize(va_handle, &major_version, &minor_version); - if (VA_STATUS_SUCCESS != status) { - close(device_fd); - va_handle = nullptr; - - std::string info("Cannot initialize VAAPI device, error: "); - info += vaErrorStr(status); - ss << info << std::endl; - continue; - } - std::cout << "VAAPI created for device: " << *device_path << ", version: " - << major_version << "." << minor_version << std::endl; - break; - } - - // check device creation - if (!va_handle) { - std::cerr << "Cannot create VAAPI device. Log:\n" << ss.str() << std::endl; - return -1; - } - gpu_accel_device = cv::util::make_optional( - cv::gapi::wip::onevpl::create_vaapi_device(reinterpret_cast(va_handle), - "GPU")); - gpu_accel_ctx = cv::util::make_optional( - cv::gapi::wip::onevpl::create_vaapi_context(nullptr)); -#endif // defined(HAVE_VA) || defined(HAVE_VA_INTEL) -#endif // #ifdef __linux__ - } - -#ifdef HAVE_INF_ENGINE - // activate remote ctx in Inference Engine for GPU device - // when other pipeline component use the GPU device too - if (flow_settings->ie_remote_ctx_enable) { - InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, - {"VA_DEVICE", gpu_accel_device.value().get_ptr()} }); - face_net.cfgContextParams(ctx_config); - std::cout << "enforce InferenceEngine remote context on device: " << device_id << std::endl; - - // NB: consider NV12 surface because it's one of native GPU image format - face_net.pluginConfig({{"GPU_NV12_TWO_INPUTS", "YES" }}); - std::cout << "enforce InferenceEngine NV12 blob" << std::endl; - } -#endif // HAVE_INF_ENGINE - - // turn on VPP PreprocesingEngine if available & requested - if (flow_settings->vpl_preproc_enable) { - if (is_gpu(preproc_device)) { - // activate VPP PreprocesingEngine on GPU - face_net.cfgPreprocessingParams(gpu_accel_device.value(), - gpu_accel_ctx.value()); - } else { - // activate VPP PreprocesingEngine on CPU - face_net.cfgPreprocessingParams(cpu_accel_device, - cpu_accel_ctx); - } - std::cout << "enforce VPP preprocessing on device: " << preproc_device << std::endl; - } else { - std::cout << "use InferenceEngine default preprocessing" << std::endl; - } - - auto kernels = cv::gapi::kernels - < custom::OCVLocateROI - , custom::OCVParseSSD - , custom::OCVBBoxes>(); - auto networks = cv::gapi::networks(face_net); - auto face_detection_args = cv::compile_args(networks, kernels); - if (streaming_queue_capacity != 0) { - face_detection_args += cv::compile_args(cv::gapi::streaming::queue_capacity{ streaming_queue_capacity }); - } - - // Create source - cv::gapi::wip::IStreamSource::Ptr cap; - try { - if (is_gpu(source_device)) { - std::cout << "enforce VPL Source deconding on device: " << source_device << std::endl; - // use special 'Device' constructor for `onevpl::GSource` - cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, - gpu_accel_device.value(), - gpu_accel_ctx.value()); - } else { - cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); - } - std::cout << "oneVPL source description: " << cap->descr_of() << std::endl; - } catch (const std::exception& ex) { - std::cerr << "Cannot create source: " << ex.what() << std::endl; - return -1; - } - - cv::GMetaArg descr = cap->descr_of(); - auto frame_descr = cv::util::get(descr); - cv::GOpaque in_roi; - auto inputs = cv::gin(cap); - - // Now build the graph - cv::GFrame in; - auto size = cv::gapi::streaming::size(in); - auto graph_inputs = cv::GIn(in); - if (!opt_roi.has_value()) { - // Automatically detect ROI to infer. Make it output parameter - std::cout << "ROI is not set or invalid. Locating it automatically" - << std::endl; - in_roi = custom::LocateROI::on(size); - } else { - // Use the value provided by user - std::cout << "Will run inference for static region " - << opt_roi.value() - << " only" - << std::endl; - graph_inputs += cv::GIn(in_roi); - inputs += cv::gin(opt_roi.value()); - } - auto blob = cv::gapi::infer(in_roi, in); - cv::GArray rcs = custom::ParseSSD::on(blob, in_roi, size); - auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, in_roi)); - auto out = cv::gapi::streaming::BGR(out_frame); - cv::GStreamingCompiled pipeline = cv::GComputation(std::move(graph_inputs), cv::GOut(out)) // and move here - .compileStreaming(std::move(face_detection_args)); - // The execution part - pipeline.setSource(std::move(inputs)); - pipeline.start(); - - size_t frames = 0u; - cv::TickMeter tm; - cv::VideoWriter writer; - if (!output.empty() && !writer.isOpened()) { - const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; - writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); - GAPI_Assert(writer.isOpened()); - } - - cv::Mat outMat; - tm.start(); - while (pipeline.pull(cv::gout(outMat))) { - cv::imshow("Out", outMat); - cv::waitKey(1); - if (!output.empty()) { - writer << outMat; - } - ++frames; - } - tm.stop(); - std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; - - return 0; -} - - -namespace cfg { -typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) { - using namespace cv::gapi::wip; - - if (line.empty()) { - throw std::runtime_error("Cannot parse CfgParam from emply line"); - } - - std::string::size_type name_endline_pos = line.find(':'); - if (name_endline_pos == std::string::npos) { - throw std::runtime_error("Cannot parse CfgParam from: " + line + - "\nExpected separator \":\""); - } - - std::string name = line.substr(0, name_endline_pos); - std::string value = line.substr(name_endline_pos + 1); - - return cv::gapi::wip::onevpl::CfgParam::create(name, value, - /* vpp params strongly optional */ - name.find("vpp.") == std::string::npos); -} -} diff --git a/modules/gapi/samples/onevpl_infer_with_advanced_device_selection.cpp b/modules/gapi/samples/onevpl_infer_with_advanced_device_selection.cpp new file mode 100644 index 0000000000..de1d233ae5 --- /dev/null +++ b/modules/gapi/samples/onevpl_infer_with_advanced_device_selection.cpp @@ -0,0 +1,704 @@ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include // CommandLineParser +#include + +#ifdef HAVE_INF_ENGINE +#include // ParamMap +#endif // HAVE_INF_ENGINE + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#undef NOMINMAX +#undef D3D11_NO_HELPERS +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#ifdef __linux__ +#if defined(HAVE_VA) || defined(HAVE_VA_INTEL) +#include "va/va.h" +#include "va/va_drm.h" + +#include +#include +#endif // defined(HAVE_VA) || defined(HAVE_VA_INTEL) +#endif // __linux__ + + +const std::string about = + "This is an OpenCV-based version of oneVPLSource decoder example"; +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input demultiplexed video file }" + "{ output | | Path to the output RAW video file. Use .avi extension }" + "{ facem | face-detection-adas-0001.xml | Path to OpenVINO IE face detection model (.xml) }" + "{ faced | GPU | Target device for face detection model (e.g. AUTO, GPU, VPU, ...) }" + "{ cfg_params | | Semicolon separated list of oneVPL mfxVariants which is used for configuring source (see `MFXSetConfigFilterProperty` by https://spec.oneapi.io/versions/latest/elements/oneVPL/source/index.html) }" + "{ streaming_queue_capacity | 1 | Streaming executor queue capacity. Calculated automatically if 0 }" + "{ frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size}" + "{ vpp_frames_pool_size | 0 | OneVPL source applies this parameter as preallocated frames pool size for VPP preprocessing results}" + "{ roi | -1,-1,-1,-1 | Region of interest (ROI) to use for inference. Identified automatically when not set }" + "{ source_device | CPU | choose device for decoding }" + "{ preproc_device | | choose device for preprocessing }"; + + +namespace { +bool is_gpu(const std::string &device_name) { + return device_name.find("GPU") != std::string::npos; +} + +std::string get_weights_path(const std::string &model_path) { + const auto EXT_LEN = 4u; + const auto sz = model_path.size(); + GAPI_Assert(sz > EXT_LEN); + + auto ext = model_path.substr(sz - EXT_LEN); + std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c){ + return static_cast(std::tolower(c)); + }); + GAPI_Assert(ext == ".xml"); + return model_path.substr(0u, sz - EXT_LEN) + ".bin"; +} + +// TODO: It duplicates infer_single_roi sample +cv::util::optional parse_roi(const std::string &rc) { + cv::Rect rv; + char delim[3]; + + std::stringstream is(rc); + is >> rv.x >> delim[0] >> rv.y >> delim[1] >> rv.width >> delim[2] >> rv.height; + if (is.bad()) { + return cv::util::optional(); // empty value + } + const auto is_delim = [](char c) { + return c == ','; + }; + if (!std::all_of(std::begin(delim), std::end(delim), is_delim)) { + return cv::util::optional(); // empty value + } + if (rv.x < 0 || rv.y < 0 || rv.width <= 0 || rv.height <= 0) { + return cv::util::optional(); // empty value + } + return cv::util::make_optional(std::move(rv)); +} + +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + +// Since ATL headers might not be available on specific MSVS Build Tools +// we use simple `CComPtr` implementation like as `ComPtrGuard` +// which is not supposed to be the full functional replacement of `CComPtr` +// and it uses as RAII to make sure utilization is correct +template +void release(COMNonManageableType *ptr) { + if (ptr) { + ptr->Release(); + } +} + +template +using ComPtrGuard = std::unique_ptr)>; + +template +ComPtrGuard createCOMPtrGuard(COMNonManageableType *ptr = nullptr) { + return ComPtrGuard {ptr, &release}; +} + + +using AccelParamsType = std::tuple, ComPtrGuard>; + +AccelParamsType create_device_with_ctx(IDXGIAdapter* adapter) { + UINT flags = 0; + D3D_FEATURE_LEVEL feature_levels[] = { D3D_FEATURE_LEVEL_11_1, + D3D_FEATURE_LEVEL_11_0, + }; + D3D_FEATURE_LEVEL featureLevel; + ID3D11Device* ret_device_ptr = nullptr; + ID3D11DeviceContext* ret_ctx_ptr = nullptr; + HRESULT err = D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, + nullptr, flags, + feature_levels, + ARRAYSIZE(feature_levels), + D3D11_SDK_VERSION, &ret_device_ptr, + &featureLevel, &ret_ctx_ptr); + if (FAILED(err)) { + throw std::runtime_error("Cannot create D3D11CreateDevice, error: " + + std::to_string(HRESULT_CODE(err))); + } + + return std::make_tuple(createCOMPtrGuard(ret_device_ptr), + createCOMPtrGuard(ret_ctx_ptr)); +} +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +} // anonymous namespace + +namespace custom { +G_API_NET(FaceDetector, , "face-detector"); + +using GDetections = cv::GArray; +using GRect = cv::GOpaque; +using GSize = cv::GOpaque; +using GPrims = cv::GArray; + +G_API_OP(ParseSSD, , "sample.custom.parse-ssd") { + static cv::GArrayDesc outMeta(const cv::GMatDesc &, const cv::GOpaqueDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +// TODO: It duplicates infer_single_roi sample +G_API_OP(LocateROI, , "sample.custom.locate-roi") { + static cv::GOpaqueDesc outMeta(const cv::GOpaqueDesc &) { + return cv::empty_gopaque_desc(); + } +}; + +G_API_OP(BBoxes, , "sample.custom.b-boxes") { + static cv::GArrayDesc outMeta(const cv::GArrayDesc &, const cv::GOpaqueDesc &) { + return cv::empty_array_desc(); + } +}; + +GAPI_OCV_KERNEL(OCVLocateROI, LocateROI) { + // This is the place where we can run extra analytics + // on the input image frame and select the ROI (region + // of interest) where we want to detect our objects (or + // run any other inference). + // + // Currently it doesn't do anything intelligent, + // but only crops the input image to square (this is + // the most convenient aspect ratio for detectors to use) + + static void run(const cv::Size& in_size, + cv::Rect &out_rect) { + + // Identify the central point & square size (- some padding) + const auto center = cv::Point{in_size.width/2, in_size.height/2}; + auto sqside = std::min(in_size.width, in_size.height); + + // Now build the central square ROI + out_rect = cv::Rect{ center.x - sqside/2 + , center.y - sqside/2 + , sqside + , sqside + }; + } +}; + +GAPI_OCV_KERNEL(OCVBBoxes, BBoxes) { + // This kernel converts the rectangles into G-API's + // rendering primitives + static void run(const std::vector &in_face_rcs, + const cv::Rect &in_roi, + std::vector &out_prims) { + out_prims.clear(); + const auto cvt = [](const cv::Rect &rc, const cv::Scalar &clr) { + return cv::gapi::wip::draw::Rect(rc, clr, 2); + }; + out_prims.emplace_back(cvt(in_roi, CV_RGB(0,255,255))); // cyan + for (auto &&rc : in_face_rcs) { + out_prims.emplace_back(cvt(rc, CV_RGB(0,255,0))); // green + } + } +}; + +GAPI_OCV_KERNEL(OCVParseSSD, ParseSSD) { + static void run(const cv::Mat &in_ssd_result, + const cv::Rect &in_roi, + const cv::Size &in_parent_size, + std::vector &out_objects) { + const auto &in_ssd_dims = in_ssd_result.size; + GAPI_Assert(in_ssd_dims.dims() == 4u); + + const int MAX_PROPOSALS = in_ssd_dims[2]; + const int OBJECT_SIZE = in_ssd_dims[3]; + GAPI_Assert(OBJECT_SIZE == 7); // fixed SSD object size + + const cv::Size up_roi = in_roi.size(); + const cv::Rect surface({0,0}, in_parent_size); + + out_objects.clear(); + + const float *data = in_ssd_result.ptr(); + for (int i = 0; i < MAX_PROPOSALS; i++) { + const float image_id = data[i * OBJECT_SIZE + 0]; + const float label = data[i * OBJECT_SIZE + 1]; + const float confidence = data[i * OBJECT_SIZE + 2]; + const float rc_left = data[i * OBJECT_SIZE + 3]; + const float rc_top = data[i * OBJECT_SIZE + 4]; + const float rc_right = data[i * OBJECT_SIZE + 5]; + const float rc_bottom = data[i * OBJECT_SIZE + 6]; + (void) label; // unused + + if (image_id < 0.f) { + break; // marks end-of-detections + } + if (confidence < 0.5f) { + continue; // skip objects with low confidence + } + + // map relative coordinates to the original image scale + // taking the ROI into account + cv::Rect rc; + rc.x = static_cast(rc_left * up_roi.width); + rc.y = static_cast(rc_top * up_roi.height); + rc.width = static_cast(rc_right * up_roi.width) - rc.x; + rc.height = static_cast(rc_bottom * up_roi.height) - rc.y; + rc.x += in_roi.x; + rc.y += in_roi.y; + out_objects.emplace_back(rc & surface); + } + } +}; + +} // namespace custom + +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line); + +struct flow { + flow(bool preproc, bool rctx) : + vpl_preproc_enable(preproc), + ie_remote_ctx_enable(rctx) { + } + bool vpl_preproc_enable = false; + bool ie_remote_ctx_enable = false; +}; + +using support_matrix = + std::map >>>; +support_matrix resolved_conf{{ + {"GPU", {{ + {"", {{ "CPU", std::make_shared(false, false)}, + { "GPU", {/* unsupported: + * ie GPU preproc isn't available */}} + }}, + + {"CPU", {{ "CPU", {/* unsupported: preproc mix */}}, + { "GPU", {/* unsupported: preproc mix */}} + }}, +#if defined(HAVE_DIRECTX) && defined(HAVE_D3D11) + {"GPU", {{ "CPU", std::make_shared(true, false)}, + { "GPU", std::make_shared(true, true)}}} +#else // TODO VAAPI under linux doesn't support GPU IE remote context + {"GPU", {{ "CPU", std::make_shared(true, false)}, + { "GPU", std::make_shared(true, false)}}} +#endif + }} + }, + {"CPU", {{ + {"", {{ "CPU", std::make_shared(false, false)}, + { "GPU", std::make_shared(false, false)} + }}, + + {"CPU", {{ "CPU", std::make_shared(true, false)}, + { "GPU", std::make_shared(true, false)} + }}, + + {"GPU", {{ "CPU", {/* unsupported: preproc mix */}}, + { "GPU", {/* unsupported: preproc mix */}}}} + }} + } + }}; + +static void print_available_cfg(std::ostream &out, + const std::string &source_device, + const std::string &preproc_device, + const std::string &ie_device_id) { + const std::string source_device_cfg_name("--source_device="); + const std::string preproc_device_cfg_name("--preproc_device="); + const std::string ie_cfg_name("--faced="); + out << "unsupported acceleration param combinations:\n" + << source_device_cfg_name << source_device << " " + << preproc_device_cfg_name << preproc_device << " " + << ie_cfg_name << ie_device_id << + "\n\nSupported matrix:\n\n" << std::endl; + for (const auto &s_d : cfg::resolved_conf) { + std::string prefix = source_device_cfg_name + s_d.first; + for (const auto &p_d : s_d.second) { + std::string mid_prefix = prefix + +"\t" + preproc_device_cfg_name + + (p_d.first.empty() ? "" : p_d.first); + for (const auto &i_d : p_d.second) { + if (i_d.second) { + std::cerr << mid_prefix << "\t" << ie_cfg_name <("input"); + const auto output = cmd.get("output"); + const auto opt_roi = parse_roi(cmd.get("roi")); + const auto face_model_path = cmd.get("facem"); + const auto streaming_queue_capacity = cmd.get("streaming_queue_capacity"); + const auto source_decode_queue_capacity = cmd.get("frames_pool_size"); + const auto source_vpp_queue_capacity = cmd.get("vpp_frames_pool_size"); + const auto device_id = cmd.get("faced"); + const auto source_device = cmd.get("source_device"); + const auto preproc_device = cmd.get("preproc_device"); + + // validate support matrix + std::shared_ptr flow_settings = cfg::resolved_conf[source_device][preproc_device][device_id]; + if (!flow_settings) { + cfg::print_available_cfg(std::cerr, source_device, preproc_device, device_id); + return -1; + } + + // check output file extension + if (!output.empty()) { + auto ext = output.find_last_of("."); + if (ext == std::string::npos || (output.substr(ext + 1) != "avi")) { + std::cerr << "Output file should have *.avi extension for output video" << std::endl; + return -1; + } + } + + // get oneVPL cfg params from cmd + std::stringstream params_list(cmd.get("cfg_params")); + std::vector source_cfgs; + try { + std::string line; + while (std::getline(params_list, line, ';')) { + source_cfgs.push_back(cfg::create_from_string(line)); + } + } catch (const std::exception& ex) { + std::cerr << "Invalid cfg parameter: " << ex.what() << std::endl; + return -1; + } + + // apply VPL source optimization params + if (source_decode_queue_capacity != 0) { + source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_frames_pool_size(source_decode_queue_capacity)); + } + if (source_vpp_queue_capacity != 0) { + source_cfgs.push_back(cv::gapi::wip::onevpl::CfgParam::create_vpp_frames_pool_size(source_vpp_queue_capacity)); + } + + auto face_net = cv::gapi::ie::Params { + face_model_path, // path to topology IR + get_weights_path(face_model_path), // path to weights + device_id + }; + + // It is allowed (and highly recommended) to reuse predefined device_ptr & context_ptr objects + // received from user application. Current sample demonstrate how to deal with this situation. + // + // But if you do not need this fine-grained acceleration devices configuration then + // just use default constructors for onevpl::GSource, IE and preprocessing module. + // But please pay attention that default pipeline construction in this case will be + // very inefficient and carries out multiple CPU-GPU memory copies + // + // If you want to reach max performance and seize copy-free approach for specific + // device & context selection then follow the steps below. + // The situation is complicated a little bit in comparison with default configuration, thus + // let's focusing this: + // + // - all component-participants (Source, Preprocessing, Inference) + // must share the same device & context instances + // + // - you must wrapping your available device & context instancs into thin + // `cv::gapi::wip::Device` & `cv::gapi::wip::Context`. + // !!! Please pay attention that both objects are weak wrapper so you must ensure + // that device & context would be alived before full pipeline created !!! + // + // - you should pass such wrappers as constructor arguments for each component in pipeline: + // a) use extended constructor for `onevpl::GSource` for activating predefined device & context + // b) use `cfgContextParams` method of `cv::gapi::ie::Params` to enable `PreprocesingEngine` + // for predefined device & context + // c) use `InferenceEngine::ParamMap` to activate remote ctx in Inference Engine for given + // device & context + // + // + //// P.S. the current sample supports heterogenous pipeline construction also. + //// It is possible to make up mixed device approach. + //// Please feel free to explore different configurations! + + cv::util::optional gpu_accel_device; + cv::util::optional gpu_accel_ctx; + cv::gapi::wip::onevpl::Device cpu_accel_device = cv::gapi::wip::onevpl::create_host_device(); + cv::gapi::wip::onevpl::Context cpu_accel_ctx = cv::gapi::wip::onevpl::create_host_context(); + // create GPU device if requested + if (is_gpu(device_id) + || is_gpu(source_device) + || is_gpu(preproc_device)) { +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 + // create DX11 device & context owning handles. + // wip::Device & wip::Context provide non-owning semantic of resources and act + // as weak references API wrappers in order to carry type-erased resources type + // into appropriate modules: onevpl::GSource, PreprocEngine and InferenceEngine + // Until modules are not created owner handles must stay alive + auto dx11_dev = createCOMPtrGuard(); + auto dx11_ctx = createCOMPtrGuard(); + + auto adapter_factory = createCOMPtrGuard(); + { + IDXGIFactory* out_factory = nullptr; + HRESULT err = CreateDXGIFactory(__uuidof(IDXGIFactory), + reinterpret_cast(&out_factory)); + if (FAILED(err)) { + std::cerr << "Cannot create CreateDXGIFactory, error: " << HRESULT_CODE(err) << std::endl; + return -1; + } + adapter_factory = createCOMPtrGuard(out_factory); + } + + auto intel_adapter = createCOMPtrGuard(); + UINT adapter_index = 0; + const unsigned int refIntelVendorID = 0x8086; + IDXGIAdapter* out_adapter = nullptr; + + while (adapter_factory->EnumAdapters(adapter_index, &out_adapter) != DXGI_ERROR_NOT_FOUND) { + DXGI_ADAPTER_DESC desc{}; + out_adapter->GetDesc(&desc); + if (desc.VendorId == refIntelVendorID) { + intel_adapter = createCOMPtrGuard(out_adapter); + break; + } + ++adapter_index; + } + + if (!intel_adapter) { + std::cerr << "No Intel GPU adapter on aboard. Exit" << std::endl; + return -1; + } + + std::tie(dx11_dev, dx11_ctx) = create_device_with_ctx(intel_adapter.get()); + gpu_accel_device = cv::util::make_optional( + cv::gapi::wip::onevpl::create_dx11_device( + reinterpret_cast(dx11_dev.release()), + "GPU")); + gpu_accel_ctx = cv::util::make_optional( + cv::gapi::wip::onevpl::create_dx11_context( + reinterpret_cast(dx11_ctx.release()))); +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX +#ifdef __linux__ +#if defined(HAVE_VA) || defined(HAVE_VA_INTEL) + static const char *predefined_vaapi_devices_list[] {"/dev/dri/renderD128", + "/dev/dri/renderD129", + "/dev/dri/card0", + "/dev/dri/card1", + nullptr}; + std::stringstream ss; + int device_fd = -1; + VADisplay va_handle = nullptr; + for (const char **device_path = predefined_vaapi_devices_list; + *device_path != nullptr; device_path++) { + device_fd = open(*device_path, O_RDWR); + if (device_fd < 0) { + std::string info("Cannot open GPU file: \""); + info = info + *device_path + "\", error: " + strerror(errno); + ss << info << std::endl; + continue; + } + va_handle = vaGetDisplayDRM(device_fd); + if (!va_handle) { + close(device_fd); + std::string info("VAAPI device vaGetDisplayDRM failed, error: "); + info += strerror(errno); + ss << info << std::endl; + continue; + } + int major_version = 0, minor_version = 0; + VAStatus status {}; + status = vaInitialize(va_handle, &major_version, &minor_version); + if (VA_STATUS_SUCCESS != status) { + close(device_fd); + va_handle = nullptr; + + std::string info("Cannot initialize VAAPI device, error: "); + info += vaErrorStr(status); + ss << info << std::endl; + continue; + } + std::cout << "VAAPI created for device: " << *device_path << ", version: " + << major_version << "." << minor_version << std::endl; + break; + } + + // check device creation + if (!va_handle) { + std::cerr << "Cannot create VAAPI device. Log:\n" << ss.str() << std::endl; + return -1; + } + gpu_accel_device = cv::util::make_optional( + cv::gapi::wip::onevpl::create_vaapi_device(reinterpret_cast(va_handle), + "GPU")); + gpu_accel_ctx = cv::util::make_optional( + cv::gapi::wip::onevpl::create_vaapi_context(nullptr)); +#endif // defined(HAVE_VA) || defined(HAVE_VA_INTEL) +#endif // #ifdef __linux__ + } + +#ifdef HAVE_INF_ENGINE + // activate remote ctx in Inference Engine for GPU device + // when other pipeline component use the GPU device too + if (flow_settings->ie_remote_ctx_enable) { + InferenceEngine::ParamMap ctx_config({{"CONTEXT_TYPE", "VA_SHARED"}, + {"VA_DEVICE", gpu_accel_device.value().get_ptr()} }); + face_net.cfgContextParams(ctx_config); + std::cout << "enforce InferenceEngine remote context on device: " << device_id << std::endl; + + // NB: consider NV12 surface because it's one of native GPU image format + face_net.pluginConfig({{"GPU_NV12_TWO_INPUTS", "YES" }}); + std::cout << "enforce InferenceEngine NV12 blob" << std::endl; + } +#endif // HAVE_INF_ENGINE + + // turn on VPP PreprocesingEngine if available & requested + if (flow_settings->vpl_preproc_enable) { + if (is_gpu(preproc_device)) { + // activate VPP PreprocesingEngine on GPU + face_net.cfgPreprocessingParams(gpu_accel_device.value(), + gpu_accel_ctx.value()); + } else { + // activate VPP PreprocesingEngine on CPU + face_net.cfgPreprocessingParams(cpu_accel_device, + cpu_accel_ctx); + } + std::cout << "enforce VPP preprocessing on device: " << preproc_device << std::endl; + } else { + std::cout << "use InferenceEngine default preprocessing" << std::endl; + } + + auto kernels = cv::gapi::kernels + < custom::OCVLocateROI + , custom::OCVParseSSD + , custom::OCVBBoxes>(); + auto networks = cv::gapi::networks(face_net); + auto face_detection_args = cv::compile_args(networks, kernels); + if (streaming_queue_capacity != 0) { + face_detection_args += cv::compile_args(cv::gapi::streaming::queue_capacity{ streaming_queue_capacity }); + } + + // Create source + cv::gapi::wip::IStreamSource::Ptr cap; + try { + if (is_gpu(source_device)) { + std::cout << "enforce VPL Source deconding on device: " << source_device << std::endl; + // use special 'Device' constructor for `onevpl::GSource` + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs, + gpu_accel_device.value(), + gpu_accel_ctx.value()); + } else { + cap = cv::gapi::wip::make_onevpl_src(file_path, source_cfgs); + } + std::cout << "oneVPL source description: " << cap->descr_of() << std::endl; + } catch (const std::exception& ex) { + std::cerr << "Cannot create source: " << ex.what() << std::endl; + return -1; + } + + cv::GMetaArg descr = cap->descr_of(); + auto frame_descr = cv::util::get(descr); + cv::GOpaque in_roi; + auto inputs = cv::gin(cap); + + // Now build the graph + cv::GFrame in; + auto size = cv::gapi::streaming::size(in); + auto graph_inputs = cv::GIn(in); + if (!opt_roi.has_value()) { + // Automatically detect ROI to infer. Make it output parameter + std::cout << "ROI is not set or invalid. Locating it automatically" + << std::endl; + in_roi = custom::LocateROI::on(size); + } else { + // Use the value provided by user + std::cout << "Will run inference for static region " + << opt_roi.value() + << " only" + << std::endl; + graph_inputs += cv::GIn(in_roi); + inputs += cv::gin(opt_roi.value()); + } + auto blob = cv::gapi::infer(in_roi, in); + cv::GArray rcs = custom::ParseSSD::on(blob, in_roi, size); + auto out_frame = cv::gapi::wip::draw::renderFrame(in, custom::BBoxes::on(rcs, in_roi)); + auto out = cv::gapi::streaming::BGR(out_frame); + cv::GStreamingCompiled pipeline = cv::GComputation(std::move(graph_inputs), cv::GOut(out)) // and move here + .compileStreaming(std::move(face_detection_args)); + // The execution part + pipeline.setSource(std::move(inputs)); + pipeline.start(); + + size_t frames = 0u; + cv::TickMeter tm; + cv::VideoWriter writer; + if (!output.empty() && !writer.isOpened()) { + const auto sz = cv::Size{frame_descr.size.width, frame_descr.size.height}; + writer.open(output, cv::VideoWriter::fourcc('M','J','P','G'), 25.0, sz); + GAPI_Assert(writer.isOpened()); + } + + cv::Mat outMat; + tm.start(); + while (pipeline.pull(cv::gout(outMat))) { + cv::imshow("Out", outMat); + cv::waitKey(1); + if (!output.empty()) { + writer << outMat; + } + ++frames; + } + tm.stop(); + std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; + + return 0; +} + + +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) { + using namespace cv::gapi::wip; + + if (line.empty()) { + throw std::runtime_error("Cannot parse CfgParam from emply line"); + } + + std::string::size_type name_endline_pos = line.find(':'); + if (name_endline_pos == std::string::npos) { + throw std::runtime_error("Cannot parse CfgParam from: " + line + + "\nExpected separator \":\""); + } + + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 1); + + return cv::gapi::wip::onevpl::CfgParam::create(name, value, + /* vpp params strongly optional */ + name.find("vpp.") == std::string::npos); +} +} diff --git a/modules/gapi/samples/onevpl_source_to_bgr_conv.cpp b/modules/gapi/samples/onevpl_source_to_bgr_conv.cpp new file mode 100644 index 0000000000..660d7ed9a8 --- /dev/null +++ b/modules/gapi/samples/onevpl_source_to_bgr_conv.cpp @@ -0,0 +1,106 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include // CommandLineParser +#include + +const std::string about = + "This is an example presents decoding on GPU using VPL Source and passing it to OpenCL backend"; +const std::string keys = + "{ h help | | Print this help message }" + "{ input | | Path to the input video file. Use .avi extension }" + "{ accel_mode | mfxImplDescription.AccelerationMode:MFX_ACCEL_MODE_VIA_D3D11 | Acceleration mode for VPL }"; + +namespace { +namespace cfg { +// FIXME: Move OneVPL arguments parser to a single place +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line); +} // namespace cfg +} // anonymous namespace + +int main(int argc, char *argv[]) { + cv::CommandLineParser cmd(argc, argv, keys); + cmd.about(about); + if (cmd.has("help")) { + cmd.printMessage(); + return 0; + } + + // Get file name + const auto input = cmd.get("input"); + const auto accel_mode = cmd.get("accel_mode"); + + // Create VPL config + std::vector source_cfgs; + source_cfgs.push_back(cfg::create_from_string(accel_mode)); + + // Create VPL-based source + std::shared_ptr default_device_selector = + cv::gapi::wip::onevpl::getDefaultDeviceSelector(source_cfgs); + + cv::gapi::wip::IStreamSource::Ptr source = cv::gapi::wip::make_onevpl_src(input, source_cfgs, + default_device_selector); + + // Build the graph + cv::GFrame in; // input frame from VPL source + auto bgr_gmat = cv::gapi::streaming::BGR(in); // conversion from VPL source frame to BGR UMat + auto out = cv::gapi::blur(bgr_gmat, cv::Size(4,4)); // ocl kernel of blur operation + + cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(cv::compile_args(cv::gapi::core::ocl::kernels())); + pipeline.setSource(std::move(source)); + + // The execution part + size_t frames = 0u; + cv::TickMeter tm; + cv::Mat outMat; + + pipeline.start(); + tm.start(); + + while (pipeline.pull(cv::gout(outMat))) { + cv::imshow("OutVideo", outMat); + cv::waitKey(1); + ++frames; + } + tm.stop(); + std::cout << "Processed " << frames << " frames" << " (" << frames / tm.getTimeSec() << " FPS)" << std::endl; + + return 0; +} + +namespace { +namespace cfg { +typename cv::gapi::wip::onevpl::CfgParam create_from_string(const std::string &line) { + using namespace cv::gapi::wip; + + if (line.empty()) { + throw std::runtime_error("Cannot parse CfgParam from emply line"); + } + + std::string::size_type name_endline_pos = line.find(':'); + if (name_endline_pos == std::string::npos) { + throw std::runtime_error("Cannot parse CfgParam from: " + line + + "\nExpected separator \":\""); + } + + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 1); + + return cv::gapi::wip::onevpl::CfgParam::create(name, value, + /* vpp params strongly optional */ + name.find("vpp.") == std::string::npos); +} +} // namespace cfg +} // anonymous namespace diff --git a/modules/gapi/src/backends/ocl/goclbackend.cpp b/modules/gapi/src/backends/ocl/goclbackend.cpp index dba2b27b59..9c6d7154e4 100644 --- a/modules/gapi/src/backends/ocl/goclbackend.cpp +++ b/modules/gapi/src/backends/ocl/goclbackend.cpp @@ -114,7 +114,8 @@ cv::GArg cv::gimpl::GOCLExecutable::packArg(const GArg &arg) GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT && arg.kind != cv::detail::ArgKind::GSCALAR && arg.kind != cv::detail::ArgKind::GARRAY - && arg.kind != cv::detail::ArgKind::GOPAQUE); + && arg.kind != cv::detail::ArgKind::GOPAQUE + && arg.kind != cv::detail::ArgKind::GFRAME); if (arg.kind != cv::detail::ArgKind::GOBJREF) { @@ -136,6 +137,7 @@ cv::GArg cv::gimpl::GOCLExecutable::packArg(const GArg &arg) // Note: .at() is intentional for GOpaque as object MUST be already there // (and constructed by either bindIn/Out or resetInternal) case GShape::GOPAQUE: return GArg(m_res.slot().at(ref.id)); + case GShape::GFRAME: return GArg(m_res.slot().at(ref.id)); default: util::throw_error(std::logic_error("Unsupported GShape type")); break; diff --git a/modules/gapi/src/backends/ocl/goclcore.cpp b/modules/gapi/src/backends/ocl/goclcore.cpp index f3c5aa32bc..19fa54a40a 100644 --- a/modules/gapi/src/backends/ocl/goclcore.cpp +++ b/modules/gapi/src/backends/ocl/goclcore.cpp @@ -6,11 +6,32 @@ #include "precomp.hpp" +#include "logger.hpp" #include #include +#include + #include "backends/ocl/goclcore.hpp" +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#pragma comment(lib,"d3d11.lib") + +// get rid of generate macro max/min/etc from DX side +#define D3D11_NO_HELPERS +#define NOMINMAX +#include +#pragma comment(lib, "dxgi") +#undef NOMINMAX +#undef D3D11_NO_HELPERS +#include +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + +#include +#include "streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp" + GAPI_OCL_KERNEL(GOCLAdd, cv::gapi::core::GAdd) { static void run(const cv::UMat& a, const cv::UMat& b, int dtype, cv::UMat& out) @@ -523,6 +544,79 @@ GAPI_OCL_KERNEL(GOCLTranspose, cv::gapi::core::GTranspose) } }; +GAPI_OCL_KERNEL(GOCLBGR, cv::gapi::streaming::GBGR) +{ + static void run(const cv::MediaFrame& in, cv::UMat& out) + { + cv::util::suppress_unused_warning(in); + cv::util::suppress_unused_warning(out); +#ifdef HAVE_DIRECTX +#ifdef HAVE_D3D11 +#ifdef HAVE_ONEVPL + auto d = in.desc(); + if (d.fmt != cv::MediaFormat::NV12) + { + GAPI_LOG_FATAL(nullptr, "Unsupported format provided: " << static_cast(d.fmt) << + ". Expected cv::MediaFormat::NV12."); + cv::util::throw_error(std::logic_error("Unsupported MediaFrame format provided")); + } + + // FIXME: consider a better solution. + // Current approach cannot be easily extended for other adapters (getHandle). + auto adapterPtr = in.get(); + if (adapterPtr == nullptr) + { + GAPI_LOG_FATAL(nullptr, "Unsupported adapter type. Only VPLMediaFrameDX11Adapter is supported"); + cv::util::throw_error(std::logic_error("Unsupported adapter type. Only VPLMediaFrameDX11Adapter is supported")); + } + + auto params = adapterPtr->getHandle(); + auto handle = cv::util::any_cast(params); + ID3D11Texture2D* texture = reinterpret_cast(handle.first); + if (texture == nullptr) + { + GAPI_LOG_FATAL(nullptr, "mfxHDLPair contains ID3D11Texture2D that is nullptr. Handle address" << + reinterpret_cast(handle.first)); + cv::util::throw_error(std::logic_error("mfxHDLPair contains ID3D11Texture2D that is nullptr")); + } + + // FIXME: Assuming here that we only have 1 device + // TODO: Textures are reusable, so to improve the peroformance here + // consider creating a hash map texture <-> device/ctx + static thread_local ID3D11Device* pD3D11Device = nullptr; + if (pD3D11Device == nullptr) + { + texture->GetDevice(&pD3D11Device); + } + if (pD3D11Device == nullptr) + { + GAPI_LOG_FATAL(nullptr, "D3D11Texture2D::GetDevice returns pD3D11Device that is nullptr"); + cv::util::throw_error(std::logic_error("D3D11Texture2D::GetDevice returns pD3D11Device that is nullptr")); + } + + // FIXME: assuming here that the context is always the same + // TODO: Textures are reusable, so to improve the peroformance here + // consider creating a hash map texture <-> device/ctx + static thread_local cv::ocl::Context ctx = cv::directx::ocl::initializeContextFromD3D11Device(pD3D11Device); + if (ctx.ptr() == nullptr) + { + GAPI_LOG_FATAL(nullptr, "initializeContextFromD3D11Device returned null context"); + cv::util::throw_error(std::logic_error("initializeContextFromD3D11Device returned null context")); + } + + cv::directx::convertFromD3D11Texture2D(texture, out); +#else + GAPI_LOG_FATAL(nullptr, "HAVE_ONEVPL is not set. Please, check your cmake flags"); + cv::util::throw_error(std::logic_error("HAVE_ONEVPL is not set. Please, check your cmake flags")); +#endif // HAVE_ONEVPL +#else + GAPI_LOG_FATAL(nullptr, "HAVE_D3D11 or HAVE_DIRECTX is not set. Please, check your cmake flags"); + cv::util::throw_error(std::logic_error("HAVE_D3D11 or HAVE_DIRECTX is not set. Please, check your cmake flags")); +#endif // HAVE_D3D11 +#endif // HAVE_DIRECTX + } +}; + cv::GKernelPackage cv::gapi::core::ocl::kernels() { static auto pkg = cv::gapi::kernels @@ -587,6 +681,7 @@ cv::GKernelPackage cv::gapi::core::ocl::kernels() , GOCLLUT , GOCLConvertTo , GOCLTranspose + , GOCLBGR >(); return pkg; } diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp index 24a5b9fb7f..f78d97d571 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/cpu_frame_adapter.cpp @@ -65,7 +65,6 @@ MediaFrame::View VPLMediaFrameCPUAdapter::access(MediaFrame::Access) { cv::util::any VPLMediaFrameCPUAdapter::blobParams() const { throw std::runtime_error("VPLMediaFrameCPUAdapter::blobParams() is not implemented"); - return {}; } void VPLMediaFrameCPUAdapter::serialize(cv::gapi::s11n::IOStream&) { diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp index fad26e50a8..3744ddc8ce 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.cpp @@ -114,17 +114,24 @@ MediaFrame::View VPLMediaFrameDX11Adapter::access(MediaFrame::Access mode) { } } +mfxHDLPair VPLMediaFrameDX11Adapter::getHandle() const { + auto surface_ptr_copy = get_surface(); + const Surface::data_t& data = surface_ptr_copy->get_data(); + NativeHandleAdapter* native_handle_getter = reinterpret_cast(data.MemId); + + mfxHDLPair handle{}; + native_handle_getter->get_handle(data.MemId, reinterpret_cast(handle)); + return handle; +} + cv::util::any VPLMediaFrameDX11Adapter::blobParams() const { /*GAPI_Assert(false && "VPLMediaFrameDX11Adapter::blobParams() is not fully integrated" "in OpenVINO InferenceEngine and would be temporary disable.");*/ #ifdef HAVE_INF_ENGINE + mfxHDLPair handle = getHandle(); + auto surface_ptr_copy = get_surface(); - Surface::data_t& data = surface_ptr_copy->get_data(); const Surface::info_t& info = surface_ptr_copy->get_info(); - NativeHandleAdapter* native_handle_getter = reinterpret_cast(data.MemId); - - mfxHDLPair handle{}; - native_handle_getter->get_handle(data.MemId, reinterpret_cast(handle)); GAPI_Assert(frame_desc.fmt == MediaFormat::NV12 && "blobParams() for VPLMediaFrameDX11Adapter supports NV12 only"); diff --git a/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp index 39528ca6a5..a5eddbb407 100644 --- a/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp +++ b/modules/gapi/src/streaming/onevpl/accelerators/surface/dx11_frame_adapter.hpp @@ -37,6 +37,11 @@ public: GAPI_EXPORTS ~VPLMediaFrameDX11Adapter(); MediaFrame::View access(MediaFrame::Access) override; + // FIXME: Consider a better solution since this approach + // is not easily extendable for other adapters (oclcore.cpp) + // FIXME: Use with caution since the handle might become invalid + // due to reference counting + mfxHDLPair getHandle() const; // The default implementation does nothing cv::util::any blobParams() const override; void serialize(cv::gapi::s11n::IOStream&) override; diff --git a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp index f7672ce924..18b468fd86 100644 --- a/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp +++ b/modules/gapi/src/streaming/onevpl/cfg_param_device_selector.hpp @@ -20,7 +20,7 @@ namespace gapi { namespace wip { namespace onevpl { -class PlatformSpecificParams; +struct PlatformSpecificParams; std::vector update_param_with_accel_type(std::vector &¶m_array, AccelType type); struct GAPI_EXPORTS CfgParamDeviceSelector final: public IDeviceSelector { diff --git a/modules/gapi/src/streaming/onevpl/default.cpp b/modules/gapi/src/streaming/onevpl/default.cpp new file mode 100644 index 0000000000..0456915a00 --- /dev/null +++ b/modules/gapi/src/streaming/onevpl/default.cpp @@ -0,0 +1,52 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2022 Intel Corporation + +#include +#include + +#include +#include +#include + +#include "cfg_param_device_selector.hpp" + +#ifdef HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +std::shared_ptr getDefaultDeviceSelector(const std::vector& cfg_params) { + std::shared_ptr default_accel_contex(new CfgParamDeviceSelector(cfg_params)); + + return default_accel_contex; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#else // HAVE_ONEVPL + +namespace cv { +namespace gapi { +namespace wip { +namespace onevpl { + +std::shared_ptr getDefaultDeviceSelector(const std::vector&) { + std::cerr << "Cannot utilize getDefaultVPLDeviceAndCtx without HAVE_ONEVPL enabled" << std::endl; + util::throw_error(std::logic_error("Cannot utilize getDefaultVPLDeviceAndCtx without HAVE_ONEVPL enabled")); + return nullptr; +} + +} // namespace onevpl +} // namespace wip +} // namespace gapi +} // namespace cv + +#endif // HAVE_ONEVPL diff --git a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp index 0ab8301799..8a35cca063 100644 --- a/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/decode/decode_engine_legacy.cpp @@ -251,7 +251,8 @@ VPLLegacyDecodeEngine::SessionParam VPLLegacyDecodeEngine::prepare_session_param } - decRequest.Type |= MFX_MEMTYPE_EXTERNAL_FRAME | MFX_MEMTYPE_FROM_DECODE | MFX_MEMTYPE_FROM_VPPIN; + decRequest.Type |= MFX_MEMTYPE_EXTERNAL_FRAME | MFX_MEMTYPE_FROM_DECODE | + MFX_MEMTYPE_FROM_VPPIN | MFX_MEMTYPE_SHARED_RESOURCE; VPLAccelerationPolicy::pool_key_t decode_pool_key = acceleration_policy->create_surface_pool(decRequest, mfxDecParams.mfx.FrameInfo); diff --git a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp index 10ce92e20a..d0c4a85f38 100644 --- a/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp +++ b/modules/gapi/src/streaming/onevpl/engine/preproc/preproc_engine.cpp @@ -281,7 +281,7 @@ pp_session VPPPreprocEngine::initialize_preproc(const pp_params& initial_frame_p vppRequests[1].AllocId = std::numeric_limits::max() - request_id++; GAPI_Assert(request_id != std::numeric_limits::max() && "Something wrong"); - vppRequests[1].Type |= MFX_MEMTYPE_FROM_VPPIN; + vppRequests[1].Type |= MFX_MEMTYPE_FROM_VPPIN | MFX_MEMTYPE_SHARED_RESOURCE; vpp_out_pool_key = acceleration_policy->create_surface_pool(vppRequests[1], mfxVPPParams.vpp.Out); diff --git a/modules/gapi/src/streaming/onevpl/utils.cpp b/modules/gapi/src/streaming/onevpl/utils.cpp index efd1618b71..a5513f5377 100644 --- a/modules/gapi/src/streaming/onevpl/utils.cpp +++ b/modules/gapi/src/streaming/onevpl/utils.cpp @@ -429,4 +429,5 @@ std::string ext_mem_frame_type_to_cstr(int type) { } // namespace wip } // namespace gapi } // namespace cv + #endif // HAVE_ONEVPL diff --git a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp index d83b09d7d3..cabfe15d7e 100644 --- a/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp +++ b/modules/gapi/test/streaming/gapi_streaming_vpl_core_test.cpp @@ -8,6 +8,7 @@ #include "../test_precomp.hpp" #include "../common/gapi_streaming_tests_common.hpp" +#include "../common/gapi_tests_common.hpp" #include #include @@ -29,6 +30,7 @@ #ifdef HAVE_ONEVPL #include +#include #include "streaming/onevpl/file_data_provider.hpp" #include "streaming/onevpl/cfg_param_device_selector.hpp" @@ -327,6 +329,72 @@ TEST(OneVPL_Source_CPU_FrameAdapter, InitFrameAdapter) EXPECT_TRUE(0 == surf->get_locks_count()); } +TEST(OneVPL_Source_Default_Source_With_OCL_Backend, Accuracy) +{ + using namespace cv::gapi::wip::onevpl; + + auto create_from_string = [](const std::string& line){ + std::string::size_type name_endline_pos = line.find(':'); + std::string name = line.substr(0, name_endline_pos); + std::string value = line.substr(name_endline_pos + 1); + return CfgParam::create(name, value); + }; + + std::vector source_cfgs; + source_cfgs.push_back(create_from_string("mfxImplDescription.AccelerationMode:MFX_ACCEL_MODE_VIA_D3D11")); + + // Create VPL-based source + std::shared_ptr default_device_selector = getDefaultDeviceSelector(source_cfgs); + + cv::gapi::wip::IStreamSource::Ptr source; + cv::gapi::wip::IStreamSource::Ptr source_cpu; + + auto input = findDataFile("cv/video/768x576.avi"); + try { + source = cv::gapi::wip::make_onevpl_src(input, source_cfgs, default_device_selector); + source_cpu = cv::gapi::wip::make_onevpl_src(input, source_cfgs, default_device_selector); + } catch(...) { + throw SkipTestException("Video file can not be opened"); + } + + // Build the graph w/ OCL backend + cv::GFrame in; // input frame from VPL source + auto bgr_gmat = cv::gapi::streaming::BGR(in); // conversion from VPL source frame to BGR UMat + auto out = cv::gapi::blur(bgr_gmat, cv::Size(4,4)); // ocl kernel of blur operation + + cv::GStreamingCompiled pipeline = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(std::move(cv::compile_args(cv::gapi::core::ocl::kernels()))); + pipeline.setSource(std::move(source)); + + cv::GStreamingCompiled pipeline_cpu = cv::GComputation(cv::GIn(in), cv::GOut(out)) + .compileStreaming(std::move(cv::compile_args(cv::gapi::core::cpu::kernels()))); + pipeline_cpu.setSource(std::move(source_cpu)); + + // The execution part + cv::Mat out_mat; + std::vector ocl_mats, cpu_mats; + + // Run the pipelines + pipeline.start(); + while (pipeline.pull(cv::gout(out_mat))) + { + ocl_mats.push_back(out_mat); + } + + pipeline_cpu.start(); + while (pipeline_cpu.pull(cv::gout(out_mat))) + { + cpu_mats.push_back(out_mat); + } + + // Compare results + // FIXME: investigate why 2 sources produce different number of frames sometimes + for (size_t i = 0; i < std::min(ocl_mats.size(), cpu_mats.size()); ++i) + { + EXPECT_TRUE(AbsTolerance(1).to_compare_obj()(ocl_mats[i], cpu_mats[i])); + } +} + TEST(OneVPL_Source_CPU_Accelerator, InitDestroy) { using cv::gapi::wip::onevpl::VPLCPUAccelerationPolicy;