From: Mikhail Nikolskii Date: Mon, 1 Mar 2021 15:51:04 +0000 (-0800) Subject: Merge pull request #19460 from mikhail-nikolskiy:videoio-hw X-Git-Tag: accepted/tizen/unified/20220125.121719~1^2~175 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=7bcb51eded04cb0acb10e7eb92eb0f2c3a0329ca;p=platform%2Fupstream%2Fopencv.git Merge pull request #19460 from mikhail-nikolskiy:videoio-hw videoio: HW decode/encode in FFMPEG backend; new properties with support in FFMPEG/GST/MSMF * HW acceleration in FFMPEG backend * fixes on Windows, remove D3D9 * HW acceleration in FFMPEG backend * fixes on Windows, remove D3D9 * improve va test * Copyright * check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+ * CAP_MSMF test on .mp4 * .mp4 in test * improve va test * Copyright * check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+ * CAP_MSMF test on .mp4 * .mp4 in test * .avi for GStreamer test * revert changes around seek() * cv_writer_open_with_params * params.warnUnusedParameters * VideoCaptureParameters in GStreamer * open_with_params * params->getUnused * Reduce PSNR threshold 33->32 (other tests use 30) * require FFMPEG 4.0+; PSNR 30 as in other tests * GStreamer AVI-demux plugin not installed in Ubuntu test environment? * fix build on very old ffmpeg * fix build on very old ffmpeg * fix build issues * fix build issues (static_cast) * FFMPEG built on Windows without H264 encoder? * fix for write_nothing test on VAAPI * fix warnings * fix cv_writer_get_prop in plugins * use avcodec_get_hw_frames_parameters; more robust fallback to SW codecs * internal function hw_check_device() for device check/logging * two separate tests for HW read and write * image size 640x480 in encode test * WITH_VA=ON (only .h headers used in OpenCV, no linkage dependency) * exception on VP9 SW encoder? * rebase master; refine info message * videoio: fix FFmpeg standalone plugin build * videoio(ffmpeg): eliminate MSVC build warnings * address review comments * videoio(hw): update videocapture_acceleration.read test - remove parallel decoding by SW code path - check PSNR against the original generated image * videoio: minor fixes * videoio(test): disable unsupported MSMF cases (SW and HW) * videoio(test): update PSNR thresholds for HW acceleration read * videoio(test): update debug messages * "hw_acceleration" whitelisting parameter * little optimization in test * D3D11VA supports decoders, doesn't support encoders * videoio(test): adjust PSNR threshold in write_read_position tests * videoio(ffmpeg): fix rejecting on acceleration device name mismatch * videoio(ffmpeg): fix compilation USE_AV_HW_CODECS=0, add more debug logging * videoio: rework VideoAccelerationType behavior - enum is not a bitset - default value is backend specific - only '_NONE' and '_ANY' may fallback on software processing - specific H/W acceleration doesn't fallback on software processing. It fails if there is no support for specified H/W acceleration. * videoio(test): fix for current FFmpeg wrapper Co-authored-by: Alexander Alekhin --- diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index c8067e9..ad0017f 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -78,7 +78,7 @@ namespace cv //! @{ -/** @brief %VideoCapture API backends identifier. +/** @brief cv::VideoCapture API backends identifier. Select preferred API for a capture object. To be used in the VideoCapture::VideoCapture() constructor or VideoCapture::open() @@ -124,7 +124,7 @@ enum VideoCaptureAPIs { CAP_UEYE = 2500, //!< uEye Camera API }; -/** @brief %VideoCapture generic properties identifier. +/** @brief cv::VideoCapture generic properties identifier. Reading / writing properties involves many layers. Some unexpected result might happens along this chain. Effective behaviour depends from device hardware, driver and API Backend. @@ -182,12 +182,14 @@ enum VideoCaptureProperties { CAP_PROP_BITRATE =47, //!< (read-only) Video bitrate in kbits/s CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only) CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499) + CAP_PROP_HW_ACCELERATION=50, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in cv::VideoCapture constructor / .open() method. Default value is backend-specific. + CAP_PROP_HW_DEVICE =51, //!< (**open-only**) Hardware device index (select GPU if multiple available) #ifndef CV_DOXYGEN CV__CAP_PROP_LATEST #endif }; -/** @brief %VideoWriter generic properties identifier. +/** @brief cv::VideoWriter generic properties identifier. @sa VideoWriter::get(), VideoWriter::set() */ enum VideoWriterProperties { @@ -196,7 +198,12 @@ enum VideoWriterProperties { VIDEOWRITER_PROP_NSTRIPES = 3, //!< Number of stripes for parallel encoding. -1 for auto detection. VIDEOWRITER_PROP_IS_COLOR = 4, //!< If it is not zero, the encoder will expect and encode color frames, otherwise it //!< will work with grayscale frames. - VIDEOWRITER_PROP_DEPTH = 5 //!< Defaults to CV_8U. + VIDEOWRITER_PROP_DEPTH = 5, //!< Defaults to CV_8U. + VIDEOWRITER_PROP_HW_ACCELERATION = 6, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in VideoWriter constructor / .open() method. Default value is backend-specific. + VIDEOWRITER_PROP_HW_DEVICE = 7, //!< (**open-only**) Hardware device index (select GPU if multiple available) +#ifndef CV_DOXYGEN + CV__VIDEOWRITER_PROP_LATEST +#endif }; //! @} videoio_flags_base @@ -204,6 +211,26 @@ enum VideoWriterProperties { //! @addtogroup videoio_flags_others //! @{ +/** @brief Video Acceleration type + * + * Used as value in #CAP_PROP_HW_ACCELERATION and #VIDEOWRITER_PROP_HW_ACCELERATION + * + * @note In case of FFmpeg backend, it translated to enum AVHWDeviceType (https://github.com/FFmpeg/FFmpeg/blob/master/libavutil/hwcontext.h) + */ +enum VideoAccelerationType +{ + VIDEO_ACCELERATION_NONE = 0, //!< Do not require any specific H/W acceleration, prefer software processing. + //!< Reading of this value means that special H/W accelerated handling is not added or not detected by OpenCV. + + VIDEO_ACCELERATION_ANY = 1, //!< Prefer to use H/W acceleration. If no one supported, then fallback to software processing. + //!< @note H/W acceleration may require special configuration of used environment. + //!< @note Results in encoding scenario may differ between software and hardware accelerated encoders. + + VIDEO_ACCELERATION_D3D11 = 2, //!< DirectX 11 + VIDEO_ACCELERATION_VAAPI = 3, //!< VAAPI + VIDEO_ACCELERATION_MFX = 4, //!< libmfx (Intel MediaSDK/oneVPL) +}; + /** @name IEEE 1394 drivers @{ */ diff --git a/modules/videoio/src/backend_plugin.cpp b/modules/videoio/src/backend_plugin.cpp index 2f6389c..ad34602 100644 --- a/modules/videoio/src/backend_plugin.cpp +++ b/modules/videoio/src/backend_plugin.cpp @@ -415,7 +415,7 @@ public: if (plugin_api->api_header.api_version >= 1 && plugin_api->v1.Capture_open_with_params) { std::vector vint_params = params.getIntVector(); - int* c_params = &vint_params[0]; + int* c_params = vint_params.data(); unsigned n_params = (unsigned)(vint_params.size() / 2); if (CV_ERROR_OK == plugin_api->v1.Capture_open_with_params( @@ -547,7 +547,7 @@ public: } if (params.warnUnusedParameters()) { - CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in VideoWriter, see logger INFO channel for details"); + CV_LOG_ERROR(NULL, "VIDEOIO: unsupported parameters in VideoWriter, see logger INFO channel for details"); return Ptr(); } if (CV_ERROR_OK == plugin_api->v0.Writer_open(filename.c_str(), fourcc, fps, sz.width, sz.height, isColor, &writer)) diff --git a/modules/videoio/src/backend_static.cpp b/modules/videoio/src/backend_static.cpp index 1d23c28..2e0088f 100644 --- a/modules/videoio/src/backend_static.cpp +++ b/modules/videoio/src/backend_static.cpp @@ -19,16 +19,17 @@ void applyParametersFallback(const Ptr& cap, const VideoCapturePa { double value = params.get(prop, -1); CV_LOG_INFO(NULL, "VIDEOIO: apply parameter: [" << prop << "]=" << - cv::format("%g / %lld / 0x%16llx", value, (long long)value, (long long)value)); + cv::format("%g / %lld / 0x%016llx", value, (long long)value, (long long)value)); if (!cap->setProperty(prop, value)) { - CV_Error_(cv::Error::StsNotImplemented, ("VIDEOIO: Failed to apply invalid or unsupported parameter: [%d]=%g / %lld / 0x%08llx", prop, value, (long long)value, (long long)value)); + if (prop != CAP_PROP_HW_ACCELERATION && prop != CAP_PROP_HW_DEVICE) { // optional parameters + CV_Error_(cv::Error::StsNotImplemented, ("VIDEOIO: Failed to apply invalid or unsupported parameter: [%d]=%g / %lld / 0x%08llx", prop, value, (long long)value, (long long)value)); + } } } // NB: there is no dedicated "commit" parameters event, implementations should commit after each property automatically } - // Legacy API. Modern API with parameters is below class StaticBackend: public IBackend { diff --git a/modules/videoio/src/cap_ffmpeg.cpp b/modules/videoio/src/cap_ffmpeg.cpp index 9ce48f5..bd3600e 100644 --- a/modules/videoio/src/cap_ffmpeg.cpp +++ b/modules/videoio/src/cap_ffmpeg.cpp @@ -165,7 +165,7 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL : { public: CvVideoWriter_FFMPEG_proxy() { ffmpegWriter = 0; } - CvVideoWriter_FFMPEG_proxy(const cv::String& filename, int fourcc, double fps, cv::Size frameSize, bool isColor) { ffmpegWriter = 0; open(filename, fourcc, fps, frameSize, isColor); } + CvVideoWriter_FFMPEG_proxy(const cv::String& filename, int fourcc, double fps, cv::Size frameSize, const VideoWriterParameters& params) { ffmpegWriter = 0; open(filename, fourcc, fps, frameSize, params); } virtual ~CvVideoWriter_FFMPEG_proxy() { close(); } int getCaptureDomain() const CV_OVERRIDE { return cv::CAP_FFMPEG; } @@ -178,10 +178,10 @@ public: icvWriteFrame_FFMPEG_p(ffmpegWriter, (const uchar*)image.getMat().ptr(), (int)image.step(), image.cols(), image.rows(), image.channels(), 0); } - virtual bool open( const cv::String& filename, int fourcc, double fps, cv::Size frameSize, bool isColor ) + virtual bool open( const cv::String& filename, int fourcc, double fps, cv::Size frameSize, const VideoWriterParameters& params ) { close(); - ffmpegWriter = icvCreateVideoWriter_FFMPEG_p( filename.c_str(), fourcc, fps, frameSize.width, frameSize.height, isColor ); + ffmpegWriter = cvCreateVideoWriterWithParams_FFMPEG( filename.c_str(), fourcc, fps, frameSize.width, frameSize.height, params ); return ffmpegWriter != 0; } @@ -193,7 +193,12 @@ public: ffmpegWriter = 0; } - virtual double getProperty(int) const CV_OVERRIDE { return 0; } + virtual double getProperty(int propId) const CV_OVERRIDE { + if(!ffmpegWriter) + return 0; + return ffmpegWriter->getProperty(propId); + } + virtual bool setProperty(int, double) CV_OVERRIDE { return false; } virtual bool isOpened() const CV_OVERRIDE { return ffmpegWriter != 0; } @@ -207,8 +212,7 @@ cv::Ptr cvCreateVideoWriter_FFMPEG_proxy(const std::string& fi double fps, const cv::Size& frameSize, const VideoWriterParameters& params) { - const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true); - cv::Ptr writer = cv::makePtr(filename, fourcc, fps, frameSize, isColor); + cv::Ptr writer = cv::makePtr(filename, fourcc, fps, frameSize, params); if (writer && writer->isOpened()) return writer; return cv::Ptr(); @@ -233,7 +237,7 @@ cv::Ptr cvCreateVideoWriter_FFMPEG_proxy(const std::string& fi #define CAPTURE_API_VERSION 1 #include "plugin_capture_api.hpp" #define WRITER_ABI_VERSION 1 -#define WRITER_API_VERSION 0 +#define WRITER_API_VERSION 1 #include "plugin_writer_api.hpp" #endif @@ -400,7 +404,7 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx, Mat img; // TODO: avoid unnecessary copying if (instance->retrieveFrame(stream_idx, img)) - return callback(stream_idx, img.data, img.step, img.cols, img.rows, img.channels(), userdata); + return callback(stream_idx, img.data, (int)img.step, img.cols, img.rows, img.channels(), userdata); return CV_ERROR_FAIL; } catch (const std::exception& e) @@ -426,7 +430,7 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx, Mat img; // TODO: avoid unnecessary copying if (instance->retrieveFrame(stream_idx, img)) - return callback(stream_idx, img.data, img.step, img.cols, img.rows, img.type(), userdata); + return callback(stream_idx, img.data, (int)img.step, img.cols, img.rows, img.type(), userdata); return CV_ERROR_FAIL; } catch (const std::exception& e) @@ -443,14 +447,17 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx, #endif static -CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor, - CV_OUT CvPluginWriter* handle) +CvResult CV_API_CALL cv_writer_open_with_params( + const char* filename, int fourcc, double fps, int width, int height, + int* params, unsigned n_params, + CV_OUT CvPluginWriter* handle) { Size sz(width, height); CvVideoWriter_FFMPEG_proxy* wrt = 0; try { - wrt = new CvVideoWriter_FFMPEG_proxy(filename, fourcc, fps, sz, isColor != 0); + VideoWriterParameters parameters(params, n_params); + wrt = new CvVideoWriter_FFMPEG_proxy(filename, fourcc, fps, sz, parameters); if(wrt && wrt->isOpened()) { *handle = (CvPluginWriter)wrt; @@ -471,6 +478,14 @@ CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps } static +CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor, + CV_OUT CvPluginWriter* handle) +{ + int params[2] = { VIDEOWRITER_PROP_IS_COLOR, isColor }; + return cv_writer_open_with_params(filename, fourcc, fps, width, height, params, 1, handle); +} + +static CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle) { if (!handle) @@ -481,9 +496,22 @@ CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle) } static -CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter /*handle*/, int /*prop*/, CV_OUT double* /*val*/) +CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter handle, int prop, CV_OUT double* val) { - return CV_ERROR_FAIL; + if (!handle) + return CV_ERROR_FAIL; + if (!val) + return CV_ERROR_FAIL; + try + { + CvVideoWriter_FFMPEG_proxy* instance = (CvVideoWriter_FFMPEG_proxy*)handle; + *val = instance->getProperty(prop); + return CV_ERROR_OK; + } + catch (...) + { + return CV_ERROR_FAIL; + } } static @@ -594,6 +622,9 @@ static const OpenCV_VideoIO_Writer_Plugin_API writer_plugin_api = /* 4*/cv_writer_get_prop, /* 5*/cv_writer_set_prop, /* 6*/cv_writer_write + }, + { + /* 7*/cv_writer_open_with_params } }; diff --git a/modules/videoio/src/cap_ffmpeg_hw.hpp b/modules/videoio/src/cap_ffmpeg_hw.hpp new file mode 100644 index 0000000..405e381 --- /dev/null +++ b/modules/videoio/src/cap_ffmpeg_hw.hpp @@ -0,0 +1,555 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. +// +// Copyright (C) 2020-2021 Intel Corporation + +#include "opencv2/videoio.hpp" +#if defined(__OPENCV_BUILD) || defined(OPENCV_HAVE_CVCONFIG_H) // TODO Properly detect and add D3D11 / LIBVA dependencies for standalone plugins +#include "cvconfig.h" +#endif +#include + +#ifdef HAVE_D3D11 +#define D3D11_NO_HELPERS +#include +#include +#endif + +#ifdef HAVE_VA +#include +#endif + +extern "C" { +#include +#include + +#include +#ifdef HAVE_D3D11 +#include +#endif +#ifdef HAVE_VA +#include +#endif +} + +static +const char* getVideoAccelerationName(VideoAccelerationType va_type) +{ + switch (va_type) + { + case VIDEO_ACCELERATION_NONE: return "none"; + case VIDEO_ACCELERATION_ANY: return "any"; + case VIDEO_ACCELERATION_D3D11: return "d3d11"; + case VIDEO_ACCELERATION_VAAPI: return "vaapi"; + case VIDEO_ACCELERATION_MFX: return "mfx"; + } + return "unknown"; +} + +static +std::string getDecoderConfiguration(VideoAccelerationType va_type, AVDictionary *dict) +{ + std::string va_name = getVideoAccelerationName(va_type); + std::string key_name = std::string("hw_decoders_") + va_name; + const char *hw_acceleration = NULL; + if (dict) + { + AVDictionaryEntry* entry = av_dict_get(dict, key_name.c_str(), NULL, 0); + if (entry) + hw_acceleration = entry->value; + } + if (hw_acceleration) + return hw_acceleration; + + // some default values (FFMPEG_DECODE_ACCELERATION_TYPES) +#ifdef _WIN32 + switch (va_type) + { + case VIDEO_ACCELERATION_NONE: return ""; + case VIDEO_ACCELERATION_ANY: return "d3d11va"; + case VIDEO_ACCELERATION_D3D11: return "d3d11va"; + case VIDEO_ACCELERATION_VAAPI: return ""; + case VIDEO_ACCELERATION_MFX: return ""; + } + return ""; +#else + switch (va_type) + { + case VIDEO_ACCELERATION_NONE: return ""; + case VIDEO_ACCELERATION_ANY: return "vaapi.iHD"; + case VIDEO_ACCELERATION_D3D11: return ""; + case VIDEO_ACCELERATION_VAAPI: return "vaapi.iHD"; + case VIDEO_ACCELERATION_MFX: return ""; + } + return ""; +#endif +} + +static +std::string getEncoderConfiguration(VideoAccelerationType va_type, AVDictionary *dict) +{ + std::string va_name = getVideoAccelerationName(va_type); + std::string key_name = std::string("hw_encoders_") + va_name; + const char *hw_acceleration = NULL; + if (dict) + { + AVDictionaryEntry* entry = av_dict_get(dict, key_name.c_str(), NULL, 0); + if (entry) + hw_acceleration = entry->value; + } + if (hw_acceleration) + return hw_acceleration; + + // some default values (FFMPEG_ENCODE_ACCELERATION_TYPES) +#ifdef _WIN32 + switch (va_type) + { + case VIDEO_ACCELERATION_NONE: return ""; + case VIDEO_ACCELERATION_ANY: return "qsv"; + case VIDEO_ACCELERATION_D3D11: return ""; + case VIDEO_ACCELERATION_VAAPI: return ""; + case VIDEO_ACCELERATION_MFX: return "qsv"; + } + return ""; +#else + switch (va_type) + { + case VIDEO_ACCELERATION_NONE: return ""; + case VIDEO_ACCELERATION_ANY: return "qsv.iHD,vaapi.iHD"; + case VIDEO_ACCELERATION_D3D11: return ""; + case VIDEO_ACCELERATION_VAAPI: return "vaapi.iHD"; + case VIDEO_ACCELERATION_MFX: return "qsv.iHD"; + } + return "unknown"; +#endif +} + + +static +std::string getDecoderDisabledCodecs(AVDictionary *dict) +{ + std::string key_name = std::string("hw_disable_decoders"); + const char *disabled_codecs = NULL; + if (dict) + { + AVDictionaryEntry* entry = av_dict_get(dict, key_name.c_str(), NULL, 0); + if (entry) + disabled_codecs = entry->value; + } + if (disabled_codecs) + return disabled_codecs; + + // some default values (FFMPEG_DECODE_DISABLE_CODECS) +#ifdef _WIN32 + return "none"; +#else + return "av1.vaapi,av1_qsv,vp8.vaapi,vp8_qsv"; // "vp9_qsv" +#endif +} + +static +std::string getEncoderDisabledCodecs(AVDictionary *dict) +{ + std::string key_name = std::string("hw_disabled_encoders"); + const char *disabled_codecs = NULL; + if (dict) + { + AVDictionaryEntry* entry = av_dict_get(dict, key_name.c_str(), NULL, 0); + if (entry) + disabled_codecs = entry->value; + } + if (disabled_codecs) + return disabled_codecs; + + // some default values (FFMPEG_ENCODE_DISABLE_CODECS) +#ifdef _WIN32 + return "mjpeg_qsv"; +#else + return "mjpeg_vaapi,mjpeg_qsv,vp8_vaapi"; +#endif +} + + +#define HW_DEFAULT_POOL_SIZE 32 +#define HW_DEFAULT_SW_FORMAT AV_PIX_FMT_NV12 + +using namespace cv; + +static AVCodec *hw_find_codec(AVCodecID id, AVHWDeviceType hw_type, int (*check_category)(const AVCodec *), + const char *disabled_codecs, AVPixelFormat *hw_pix_fmt); +static AVBufferRef* hw_create_device(AVHWDeviceType hw_type, int hw_device, const std::string& device_subname); +static AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device_ctx, int width, int height, AVPixelFormat hw_format); +static AVPixelFormat hw_get_format_callback(struct AVCodecContext *ctx, const enum AVPixelFormat * fmt); +static VideoAccelerationType hw_type_to_va_type(AVHWDeviceType hw_type); + +static +bool hw_check_device(AVBufferRef* ctx, AVHWDeviceType hw_type, const std::string& device_subname) { + if (!ctx) + return false; + AVHWDeviceContext* hw_device_ctx = (AVHWDeviceContext*)ctx->data; + if (!hw_device_ctx->hwctx) + return false; + const char *hw_name = av_hwdevice_get_type_name(hw_type); + if (hw_type == AV_HWDEVICE_TYPE_QSV) + hw_name = "MFX"; + bool ret = true; + std::string device_name; +#if defined(HAVE_D3D11) + if (hw_device_ctx->type == AV_HWDEVICE_TYPE_D3D11VA) { + ID3D11Device* device = ((AVD3D11VADeviceContext*)hw_device_ctx->hwctx)->device; + IDXGIDevice* dxgiDevice = nullptr; + if (device && SUCCEEDED(device->QueryInterface(__uuidof(IDXGIDevice), reinterpret_cast(&dxgiDevice)))) { + IDXGIAdapter* adapter = nullptr; + if (SUCCEEDED(dxgiDevice->GetAdapter(&adapter))) { + DXGI_ADAPTER_DESC desc; + if (SUCCEEDED(adapter->GetDesc(&desc))) { + std::wstring_convert> conv; + device_name = conv.to_bytes(desc.Description); + } + adapter->Release(); + } + dxgiDevice->Release(); + } + } +#endif + if (hw_device_ctx->type == AV_HWDEVICE_TYPE_VAAPI) { +#if defined(HAVE_VA) && (VA_MAJOR_VERSION >= 1) + VADisplay display = ((AVVAAPIDeviceContext *) hw_device_ctx->hwctx)->display; + if (display) { + VADriverContext *va_ctx = ((VADisplayContext *) display)->pDriverContext; + device_name = va_ctx->str_vendor; + if (hw_type == AV_HWDEVICE_TYPE_QSV) { + // Workaround for issue fixed in MediaSDK 21.x https://github.com/Intel-Media-SDK/MediaSDK/issues/2595 + // Checks VAAPI driver for support of VideoProc operation required by MediaSDK + ret = false; + int n_entrypoints = va_ctx->max_entrypoints; + std::vector entrypoints(n_entrypoints); + if (va_ctx->vtable->vaQueryConfigEntrypoints(va_ctx, VAProfileNone, entrypoints.data(), &n_entrypoints) == VA_STATUS_SUCCESS) { + for (int i = 0; i < n_entrypoints; i++) { + if (entrypoints[i] == VAEntrypointVideoProc) { + ret = true; + break; + } + } + } + if (!ret) + CV_LOG_INFO(NULL, "FFMPEG: Skipping MFX video acceleration as entrypoint VideoProc not found in: " << device_name); + } + } +#else + ret = (hw_type != AV_HWDEVICE_TYPE_QSV); // disable MFX if we can't check VAAPI for VideoProc entrypoint +#endif + } + if (ret && !device_subname.empty() && device_name.find(device_subname) == std::string::npos) + { + CV_LOG_INFO(NULL, "FFMPEG: Skipping '" << hw_name << + "' video acceleration on the following device name as not matching substring '" << device_subname << "': " << device_name); + ret = false; // reject configuration + } + if (ret) + { + if (!device_name.empty()) { + CV_LOG_INFO(NULL, "FFMPEG: Using " << hw_name << " video acceleration on device: " << device_name); + } else { + CV_LOG_INFO(NULL, "FFMPEG: Using " << hw_name << " video acceleration"); + } + } + return ret; +} + +static +AVBufferRef* hw_create_device(AVHWDeviceType hw_type, int hw_device, const std::string& device_subname) { + if (AV_HWDEVICE_TYPE_NONE == hw_type) + return NULL; + + AVHWDeviceType child_type = hw_type; + if (hw_type == AV_HWDEVICE_TYPE_QSV) { +#ifdef _WIN32 + child_type = AV_HWDEVICE_TYPE_DXVA2; +#else + child_type = AV_HWDEVICE_TYPE_VAAPI; +#endif + } + + AVBufferRef* hw_device_ctx = NULL; + char device[128] = ""; + char* pdevice = NULL; + if (hw_device >= 0 && hw_device < 100000) { + if (child_type == AV_HWDEVICE_TYPE_VAAPI) { + snprintf(device, sizeof(device), "/dev/dri/renderD%d", 128 + hw_device); + } else { + snprintf(device, sizeof(device), "%d", hw_device); + } + pdevice = device; + } + const char *hw_child_name = av_hwdevice_get_type_name(child_type); + const char *device_name = pdevice ? pdevice : "'default'"; + int err = av_hwdevice_ctx_create(&hw_device_ctx, child_type, pdevice, NULL, 0); + if (hw_device_ctx && err >= 0) + { + CV_LOG_DEBUG(NULL, "FFMPEG: Created video acceleration context (av_hwdevice_ctx_create) for " << hw_child_name << " on device " << device_name); + if (!hw_check_device(hw_device_ctx, hw_type, device_subname)) { + av_buffer_unref(&hw_device_ctx); + return NULL; + } + if (hw_type != child_type) { + AVBufferRef *derived_ctx = NULL; + const char *hw_name = av_hwdevice_get_type_name(hw_type); + err = av_hwdevice_ctx_create_derived(&derived_ctx, hw_type, hw_device_ctx, 0); + if (!derived_ctx || err < 0) + { + if (derived_ctx) + av_buffer_unref(&derived_ctx); + CV_LOG_INFO(NULL, "FFMPEG: Failed to create derived video acceleration (av_hwdevice_ctx_create_derived) for " << hw_name << ". Error=" << err); + } + else + { + CV_LOG_DEBUG(NULL, "FFMPEG: Created derived video acceleration context (av_hwdevice_ctx_create_derived) for " << hw_name); + } + av_buffer_unref(&hw_device_ctx); + return derived_ctx; + } else { + return hw_device_ctx; + } + } + else + { + const char *hw_name = hw_child_name; + CV_LOG_INFO(NULL, "FFMPEG: Failed to create " << hw_name << " video acceleration (av_hwdevice_ctx_create) on device " << device_name); + return NULL; + } +} + +static +AVBufferRef* hw_create_frames(struct AVCodecContext* ctx, AVBufferRef *hw_device_ctx, int width, int height, AVPixelFormat hw_format) +{ + AVBufferRef *hw_frames_ref = nullptr; + if (ctx) + { + int res = avcodec_get_hw_frames_parameters(ctx, hw_device_ctx, hw_format, &hw_frames_ref); + if (res < 0) + { + CV_LOG_DEBUG(NULL, "FFMPEG: avcodec_get_hw_frames_parameters() call failed: " << res) + } + } + if (!hw_frames_ref) + { + hw_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx); + } + if (!hw_frames_ref) + { + CV_LOG_INFO(NULL, "FFMPEG: Failed to create HW frame context (av_hwframe_ctx_alloc)"); + return NULL; + } + AVHWFramesContext *frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data); + frames_ctx->width = width; + frames_ctx->height = height; + if (frames_ctx->format == AV_PIX_FMT_NONE) + frames_ctx->format = hw_format; + if (frames_ctx->sw_format == AV_PIX_FMT_NONE) + frames_ctx->sw_format = HW_DEFAULT_SW_FORMAT; + if (frames_ctx->initial_pool_size == 0) + frames_ctx->initial_pool_size = HW_DEFAULT_POOL_SIZE; + int res = av_hwframe_ctx_init(hw_frames_ref); + if (res < 0) + { + CV_LOG_INFO(NULL, "FFMPEG: Failed to initialize HW frame context (av_hwframe_ctx_init): " << res); + av_buffer_unref(&hw_frames_ref); + return NULL; + } + return hw_frames_ref; +} + +static +bool hw_check_codec(AVCodec* codec, AVHWDeviceType hw_type, const char *disabled_codecs) +{ + CV_Assert(disabled_codecs); + std::string hw_name = std::string(".") + av_hwdevice_get_type_name(hw_type); + std::stringstream s_stream(disabled_codecs); + while (s_stream.good()) { + std::string name; + getline(s_stream, name, ','); + if (name == codec->name || name == hw_name || name == codec->name + hw_name || name == "hw") { + CV_LOG_INFO(NULL, "FFMPEG: skipping codec " << codec->name << hw_name); + return false; + } + } + return true; +} + +static +AVCodec *hw_find_codec(AVCodecID id, AVHWDeviceType hw_type, int (*check_category)(const AVCodec *), const char *disabled_codecs, AVPixelFormat *hw_pix_fmt) { + AVCodec *c = 0; + void *opaque = 0; + + while (NULL != (c = (AVCodec*)av_codec_iterate(&opaque))) + { + if (!check_category(c)) + continue; + if (c->id != id) + continue; + if (c->capabilities & AV_CODEC_CAP_EXPERIMENTAL) + continue; + if (hw_type != AV_HWDEVICE_TYPE_NONE) { + AVPixelFormat hw_native_fmt = AV_PIX_FMT_NONE; +#if LIBAVUTIL_BUILD < AV_VERSION_INT(56, 51, 100) // VAAPI encoders support avcodec_get_hw_config() starting ffmpeg 4.3 + if (hw_type == AV_HWDEVICE_TYPE_VAAPI) + hw_native_fmt = AV_PIX_FMT_VAAPI_VLD; +#endif + if (hw_type == AV_HWDEVICE_TYPE_CUDA) // CUDA encoders don't support avcodec_get_hw_config() + hw_native_fmt = AV_PIX_FMT_CUDA; + if (av_codec_is_encoder(c) && hw_native_fmt != AV_PIX_FMT_NONE && c->pix_fmts) { + for (int i = 0; c->pix_fmts[i] != AV_PIX_FMT_NONE; i++) { + if (c->pix_fmts[i] == hw_native_fmt) { + *hw_pix_fmt = hw_native_fmt; + if (hw_check_codec(c, hw_type, disabled_codecs)) + return c; + } + } + } + for (int i = 0;; i++) { + const AVCodecHWConfig *hw_config = avcodec_get_hw_config(c, i); + if (!hw_config) + break; + if (hw_config->device_type == hw_type) { + *hw_pix_fmt = hw_config->pix_fmt; + if (hw_check_codec(c, hw_type, disabled_codecs)) + return c; + } + } + } else { + return c; + } + } + + return NULL; +} + +// Callback to select hardware pixel format (not software format) and allocate frame pool (hw_frames_ctx) +static +AVPixelFormat hw_get_format_callback(struct AVCodecContext *ctx, const enum AVPixelFormat * fmt) { + if (!ctx->hw_device_ctx) + return fmt[0]; + AVHWDeviceType hw_type = ((AVHWDeviceContext*)ctx->hw_device_ctx->data)->type; + for (int j = 0;; j++) { + const AVCodecHWConfig *hw_config = avcodec_get_hw_config(ctx->codec, j); + if (!hw_config) + break; + if (hw_config->device_type == hw_type) { + for (int i = 0; fmt[i] != AV_PIX_FMT_NONE; i++) { + if (fmt[i] == hw_config->pix_fmt) { + if (hw_config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) { + ctx->sw_pix_fmt = HW_DEFAULT_SW_FORMAT; + ctx->hw_frames_ctx = hw_create_frames(ctx, ctx->hw_device_ctx, ctx->width, ctx->height, fmt[i]); + if (ctx->hw_frames_ctx) { + //ctx->sw_pix_fmt = ((AVHWFramesContext *)(ctx->hw_frames_ctx->data))->sw_format; + return fmt[i]; + } + } + } + } + } + } + CV_LOG_DEBUG(NULL, "FFMPEG: Can't select HW format in 'get_format()' callback, use default"); + return fmt[0]; +} + +static +VideoAccelerationType hw_type_to_va_type(AVHWDeviceType hw_type) { + struct HWTypeFFMPEG { + AVHWDeviceType hw_type; + VideoAccelerationType va_type; + } known_hw_types[] = { + { AV_HWDEVICE_TYPE_D3D11VA, VIDEO_ACCELERATION_D3D11 }, + { AV_HWDEVICE_TYPE_VAAPI, VIDEO_ACCELERATION_VAAPI }, + { AV_HWDEVICE_TYPE_QSV, VIDEO_ACCELERATION_MFX }, + { AV_HWDEVICE_TYPE_CUDA, (VideoAccelerationType)(1 << 11) }, + }; + for (const HWTypeFFMPEG& hw : known_hw_types) { + if (hw_type == hw.hw_type) + return hw.va_type; + } + return VIDEO_ACCELERATION_NONE; +} + +class HWAccelIterator { +public: + HWAccelIterator(VideoAccelerationType va_type, bool isEncoder, AVDictionary *dict) + : hw_type_(AV_HWDEVICE_TYPE_NONE) + { + std::string accel_list; + if (va_type != VIDEO_ACCELERATION_NONE) + { + updateAccelList_(accel_list, va_type, isEncoder, dict); + } + if (va_type == VIDEO_ACCELERATION_ANY) + { + if (!accel_list.empty()) + accel_list = ","; // add no-acceleration case to the end of the list + } + CV_LOG_DEBUG(NULL, "FFMPEG: allowed acceleration types (" << getVideoAccelerationName(va_type) << "): '" << accel_list << "'"); + + if (accel_list.empty() && va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + // broke stream + std::string tmp; + s_stream_ >> tmp; + } + else + { + s_stream_ = std::istringstream(accel_list); + } + + if (va_type != VIDEO_ACCELERATION_NONE) + { + disabled_codecs_ = isEncoder + ? getEncoderDisabledCodecs(dict) + : getDecoderDisabledCodecs(dict); + CV_LOG_DEBUG(NULL, "FFMPEG: disabled codecs: '" << disabled_codecs_ << "'"); + } + } + bool good() const + { + return s_stream_.good(); + } + void parse_next() + { + getline(s_stream_, hw_type_device_string_, ','); + size_t index = hw_type_device_string_.find('.'); + if (index != std::string::npos) { + device_subname_ = hw_type_device_string_.substr(index + 1); + hw_type_string_ = hw_type_device_string_.substr(0, index); + } else { + device_subname_.clear(); + hw_type_string_ = hw_type_device_string_; + } + hw_type_ = av_hwdevice_find_type_by_name(hw_type_string_.c_str()); + } + const std::string& hw_type_device_string() const { return hw_type_device_string_; } + const std::string& hw_type_string() const { return hw_type_string_; } + AVHWDeviceType hw_type() const { return hw_type_; } + const std::string& device_subname() const { return device_subname_; } + const std::string& disabled_codecs() const { return disabled_codecs_; } +private: + bool updateAccelList_(std::string& accel_list, VideoAccelerationType va_type, bool isEncoder, AVDictionary *dict) + { + std::string new_accels = isEncoder + ? getEncoderConfiguration(va_type, dict) + : getDecoderConfiguration(va_type, dict); + if (new_accels.empty()) + return false; + if (accel_list.empty()) + accel_list = new_accels; + else + accel_list = accel_list + "," + new_accels; + return true; + } + std::istringstream s_stream_; + std::string hw_type_device_string_; + std::string hw_type_string_; + AVHWDeviceType hw_type_; + std::string device_subname_; + + std::string disabled_codecs_; +}; diff --git a/modules/videoio/src/cap_ffmpeg_impl.hpp b/modules/videoio/src/cap_ffmpeg_impl.hpp index 7f400c6..c0e3b4a 100644 --- a/modules/videoio/src/cap_ffmpeg_impl.hpp +++ b/modules/videoio/src/cap_ffmpeg_impl.hpp @@ -64,6 +64,9 @@ using namespace cv; #ifdef __GNUC__ # pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif +#ifdef _MSC_VER +#pragma warning(disable: 4996) // was declared deprecated +#endif #ifndef CV_UNUSED // Required for standalone compilation mode (OpenCV defines this in base.hpp) #define CV_UNUSED(name) (void)name @@ -90,6 +93,16 @@ extern "C" { } #endif +//#define USE_AV_HW_CODECS 0 +#ifndef USE_AV_HW_CODECS +#if LIBAVUTIL_VERSION_MAJOR >= 56 // FFMPEG 4.0+ +#define USE_AV_HW_CODECS 1 +#include "cap_ffmpeg_hw.hpp" +#else +#define USE_AV_HW_CODECS 0 +#endif +#endif + #if defined _MSC_VER && _MSC_VER >= 1200 #pragma warning( default: 4244 4510 4610 ) #endif @@ -237,7 +250,7 @@ inline void get_monotonic_time(timespec *tv) t.QuadPart -= offset.QuadPart; microseconds = (double)t.QuadPart / frequencyToMicroseconds; - t.QuadPart = microseconds; + t.QuadPart = (LONGLONG)microseconds; tv->tv_sec = t.QuadPart / 1000000; tv->tv_nsec = (t.QuadPart % 1000000) * 1000; } @@ -522,6 +535,8 @@ struct CvCapture_FFMPEG #else AVBitStreamFilterContext* bsfc; #endif + VideoAccelerationType va_type; + int hw_device; }; void CvCapture_FFMPEG::init() @@ -557,6 +572,8 @@ void CvCapture_FFMPEG::init() memset(&packet_filtered, 0, sizeof(packet_filtered)); av_init_packet(&packet_filtered); bsfc = NULL; + va_type = cv::VIDEO_ACCELERATION_ANY; + hw_device = -1; } @@ -882,9 +899,34 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& return false; } } + if (params.has(CAP_PROP_HW_ACCELERATION)) + { + va_type = params.get(CAP_PROP_HW_ACCELERATION); +#if !USE_AV_HW_CODECS + if (va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: FFmpeg backend is build without acceleration support. Can't handle CAP_PROP_HW_ACCELERATION parameter. Bailout"); + return false; + } +#endif + } + if (params.has(CAP_PROP_HW_DEVICE)) + { + hw_device = params.get(CAP_PROP_HW_DEVICE); + if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: Invalid usage of CAP_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); + return false; + } + if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: Invalid usage of CAP_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); + return false; + } + } if (params.warnUnusedParameters()) { - CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in .open(), see logger INFO channel for details"); + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in .open(), see logger INFO channel for details. Bailout"); return false; } } @@ -973,22 +1015,102 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& if( AVMEDIA_TYPE_VIDEO == enc->codec_type && video_stream < 0) { + CV_LOG_DEBUG(NULL, "FFMPEG: stream[" << i << "] is video stream with codecID=" << (int)enc->codec_id + << " width=" << enc->width + << " height=" << enc->height + ); + // backup encoder' width/height int enc_width = enc->width; int enc_height = enc->height; - AVCodec *codec; - if(av_dict_get(dict, "video_codec", NULL, 0) == NULL) { - codec = avcodec_find_decoder(enc->codec_id); - } else { - codec = avcodec_find_decoder_by_name(av_dict_get(dict, "video_codec", NULL, 0)->value); - } - if (!codec || avcodec_open2(enc, codec, NULL) < 0) +#if !USE_AV_HW_CODECS + va_type = VIDEO_ACCELERATION_NONE; +#endif + + // find and open decoder, try HW acceleration types specified in 'hw_acceleration' list (in order) + AVCodec *codec = NULL; + err = -1; +#if USE_AV_HW_CODECS + HWAccelIterator accel_iter(va_type, false/*isEncoder*/, dict); + while (accel_iter.good()) + { +#else + do { +#endif +#if USE_AV_HW_CODECS + accel_iter.parse_next(); + AVHWDeviceType hw_type = accel_iter.hw_type(); + enc->get_format = avcodec_default_get_format; + if (enc->hw_device_ctx) { + av_buffer_unref(&enc->hw_device_ctx); + } + if (hw_type != AV_HWDEVICE_TYPE_NONE) + { + CV_LOG_DEBUG(NULL, "FFMPEG: trying to configure H/W acceleration: '" << accel_iter.hw_type_device_string() << "'"); + AVPixelFormat hw_pix_fmt = AV_PIX_FMT_NONE; + codec = hw_find_codec(enc->codec_id, hw_type, av_codec_is_decoder, accel_iter.disabled_codecs().c_str(), &hw_pix_fmt); + if (codec) { + if (hw_pix_fmt != AV_PIX_FMT_NONE) + enc->get_format = hw_get_format_callback; // set callback to select HW pixel format, not SW format + enc->hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname()); + if (!enc->hw_device_ctx) + { + CV_LOG_DEBUG(NULL, "FFMPEG: ... can't create H/W device: '" << accel_iter.hw_type_device_string() << "'"); + codec = NULL; + } + } + } + else if (hw_type == AV_HWDEVICE_TYPE_NONE) +#endif // USE_AV_HW_CODECS + { + AVDictionaryEntry* video_codec_param = av_dict_get(dict, "video_codec", NULL, 0); + if (video_codec_param == NULL) + { + codec = avcodec_find_decoder(enc->codec_id); + if (!codec) + { + CV_LOG_ERROR(NULL, "Could not find decoder for codec_id=" << (int)enc->codec_id); + } + } + else + { + CV_LOG_DEBUG(NULL, "FFMPEG: Using video_codec='" << video_codec_param->value << "'"); + codec = avcodec_find_decoder_by_name(video_codec_param->value); + if (!codec) + { + CV_LOG_ERROR(NULL, "Could not find decoder '" << video_codec_param->value << "'"); + } + } + } + if (!codec) + continue; + err = avcodec_open2(enc, codec, NULL); + if (err >= 0) { +#if USE_AV_HW_CODECS + va_type = hw_type_to_va_type(hw_type); + if (hw_type != AV_HWDEVICE_TYPE_NONE && hw_device < 0) + hw_device = 0; +#endif + break; + } else { + CV_LOG_ERROR(NULL, "Could not open codec " << codec->name << ", error: " << err); + } +#if USE_AV_HW_CODECS + } // while (accel_iter.good()) +#else + } while (0); +#endif + if (err < 0) { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: Failed to initialize VideoCapture"); goto exit_func; + } // checking width/height (since decoder can sometimes alter it, eg. vp6f) - if (enc_width && (enc->width != enc_width)) { enc->width = enc_width; } - if (enc_height && (enc->height != enc_height)) { enc->height = enc_height; } + if (enc_width && (enc->width != enc_width)) + enc->width = enc_width; + if (enc_height && (enc->height != enc_height)) + enc->height = enc_height; video_stream = i; video_st = ic->streams[i]; @@ -1009,7 +1131,8 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters& } } - if(video_stream >= 0) valid = true; + if (video_stream >= 0) + valid = true; exit_func: @@ -1140,7 +1263,6 @@ bool CvCapture_FFMPEG::processRawPacket() bool CvCapture_FFMPEG::grabFrame() { bool valid = false; - int got_picture; int count_errs = 0; const int max_number_of_attempts = 1 << 9; @@ -1159,6 +1281,11 @@ bool CvCapture_FFMPEG::grabFrame() interrupt_metadata.timeout_after_ms = LIBAVFORMAT_INTERRUPT_READ_TIMEOUT_MS; #endif +#if USE_AV_SEND_FRAME_API + // check if we can receive frame from previously decoded packet + valid = avcodec_receive_frame(video_st->codec, picture) >= 0; +#endif + // get the next frame while (!valid) { @@ -1205,16 +1332,24 @@ bool CvCapture_FFMPEG::grabFrame() } // Decode video frame +#if USE_AV_SEND_FRAME_API + if (avcodec_send_packet(video_st->codec, &packet) < 0) { + break; + } + ret = avcodec_receive_frame(video_st->codec, picture); +#else + int got_picture = 0; avcodec_decode_video2(video_st->codec, picture, &got_picture, &packet); - - // Did we get a video frame? - if(got_picture) - { + ret = got_picture ? 0 : -1; +#endif + if (ret >= 0) { //picture_pts = picture->best_effort_timestamp; if( picture_pts == AV_NOPTS_VALUE_ ) picture_pts = picture->pkt_pts != AV_NOPTS_VALUE_ && picture->pkt_pts != 0 ? picture->pkt_pts : picture->pkt_dts; valid = true; + } else if (ret == AVERROR(EAGAIN)) { + continue; } else { @@ -1255,7 +1390,20 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* return p.data != NULL; } - if (!picture->data[0]) + AVFrame* sw_picture = picture; +#if USE_AV_HW_CODECS + // if hardware frame, copy it to system memory + if (picture && picture->hw_frames_ctx) { + sw_picture = av_frame_alloc(); + //if (av_hwframe_map(sw_picture, picture, AV_HWFRAME_MAP_READ) < 0) { + if (av_hwframe_transfer_data(sw_picture, picture, 0) < 0) { + CV_LOG_ERROR(NULL, "Error copying data from GPU to CPU (av_hwframe_transfer_data)"); + return false; + } + } +#endif + + if (!sw_picture || !sw_picture->data[0]) return false; if( img_convert_ctx == NULL || @@ -1270,7 +1418,7 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* img_convert_ctx = sws_getCachedContext( img_convert_ctx, buffer_width, buffer_height, - video_st->codec->pix_fmt, + (AVPixelFormat)sw_picture->format, buffer_width, buffer_height, AV_PIX_FMT_BGR24, SWS_BICUBIC, @@ -1308,8 +1456,8 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* sws_scale( img_convert_ctx, - picture->data, - picture->linesize, + sw_picture->data, + sw_picture->linesize, 0, video_st->codec->coded_height, rgb_picture.data, rgb_picture.linesize @@ -1321,6 +1469,9 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* *height = frame.height; *cn = frame.cn; + if (sw_picture != picture) { + av_frame_unref(sw_picture); + } return true; } @@ -1392,6 +1543,12 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const #else return 0; #endif +#if USE_AV_HW_CODECS + case CAP_PROP_HW_ACCELERATION: + return static_cast(va_type); + case CAP_PROP_HW_DEVICE: + return static_cast(hw_device); +#endif // USE_AV_HW_CODECS default: break; } @@ -1573,7 +1730,7 @@ bool CvCapture_FFMPEG::setProperty( int property_id, double value ) return false; case CAP_PROP_ORIENTATION_AUTO: #if LIBAVUTIL_BUILD >= CALC_FFMPEG_VERSION(52, 94, 100) - rotation_auto = static_cast(value); + rotation_auto = value ? true : false; return true; #else rotation_auto = 0; @@ -1591,9 +1748,10 @@ bool CvCapture_FFMPEG::setProperty( int property_id, double value ) struct CvVideoWriter_FFMPEG { bool open( const char* filename, int fourcc, - double fps, int width, int height, bool isColor ); + double fps, int width, int height, const VideoWriterParameters& params ); void close(); bool writeFrame( const unsigned char* data, int step, int width, int height, int cn, int origin ); + double getProperty(int propId) const; void init(); @@ -1606,13 +1764,15 @@ struct CvVideoWriter_FFMPEG AVFrame * input_picture; uint8_t * picbuf; AVStream * video_st; - int input_pix_fmt; + AVPixelFormat input_pix_fmt; unsigned char * aligned_input; size_t aligned_input_size; int frame_width, frame_height; int frame_idx; bool ok; struct SwsContext *img_convert_ctx; + VideoAccelerationType va_type; + int hw_device; }; static const char * icvFFMPEGErrStr(int err) @@ -1667,12 +1827,14 @@ void CvVideoWriter_FFMPEG::init() input_picture = 0; picbuf = 0; video_st = 0; - input_pix_fmt = 0; + input_pix_fmt = AV_PIX_FMT_NONE; aligned_input = NULL; aligned_input_size = 0; img_convert_ctx = 0; frame_width = frame_height = 0; frame_idx = 0; + va_type = VIDEO_ACCELERATION_NONE; + hw_device = -1; ok = false; } @@ -1714,35 +1876,17 @@ static AVFrame * icv_alloc_picture_FFMPEG(int pix_fmt, int width, int height, bo return picture; } -/* add a video output stream to the container */ -static AVStream *icv_add_video_stream_FFMPEG(AVFormatContext *oc, - CV_CODEC_ID codec_id, - int w, int h, int bitrate, - double fps, int pixel_format) +/* configure video stream */ +static bool icv_configure_video_stream_FFMPEG(AVFormatContext *oc, + AVStream *st, + const AVCodec* codec, + int w, int h, int bitrate, + double fps, AVPixelFormat pixel_format) { - AVCodecContext *c; - AVStream *st; + AVCodecContext *c = st->codec; int frame_rate, frame_rate_base; - AVCodec *codec; - - st = avformat_new_stream(oc, 0); - - if (!st) { - CV_WARN("Could not allocate stream"); - return NULL; - } - - c = st->codec; - - c->codec_id = av_guess_codec(oc->oformat, NULL, oc->filename, NULL, AVMEDIA_TYPE_VIDEO); - - if(codec_id != CV_CODEC(CODEC_ID_NONE)){ - c->codec_id = codec_id; - } - - //if(codec_tag) c->codec_tag=codec_tag; - codec = avcodec_find_encoder(c->codec_id); + c->codec_id = codec->id; c->codec_type = AVMEDIA_TYPE_VIDEO; // Set per-codec defaults @@ -1792,13 +1936,13 @@ static AVStream *icv_add_video_stream_FFMPEG(AVFormatContext *oc, } } if (best == NULL) - return NULL; + return false; c->time_base.den= best->num; c->time_base.num= best->den; } c->gop_size = 12; /* emit one intra frame every twelve frames at most */ - c->pix_fmt = (AVPixelFormat) pixel_format; + c->pix_fmt = pixel_format; if (c->codec_id == CV_CODEC(CODEC_ID_MPEG2VIDEO)) { c->max_b_frames = 2; @@ -1845,14 +1989,14 @@ static AVStream *icv_add_video_stream_FFMPEG(AVFormatContext *oc, st->time_base = c->time_base; #endif - return st; + return true; } static const int OPENCV_NO_FRAMES_WRITTEN_CODE = 1000; static int icv_av_write_frame_FFMPEG( AVFormatContext * oc, AVStream * video_st, uint8_t *, uint32_t, - AVFrame * picture ) + AVFrame * picture, int frame_idx) { AVCodecContext* c = video_st->codec; int ret = OPENCV_NO_FRAMES_WRITTEN_CODE; @@ -1877,7 +2021,13 @@ static int icv_av_write_frame_FFMPEG( AVFormatContext * oc, AVStream * video_st, { /* encode the image */ #if USE_AV_SEND_FRAME_API - ret = avcodec_send_frame(c, picture); + if (picture == NULL && frame_idx == 0) { + ret = 0; + } else { + ret = avcodec_send_frame(c, picture); + if (ret < 0) + CV_LOG_ERROR(NULL, "Error sending frame to encoder (avcodec_send_frame)"); + } while (ret >= 0) { AVPacket* pkt = av_packet_alloc(); @@ -1896,6 +2046,7 @@ static int icv_av_write_frame_FFMPEG( AVFormatContext * oc, AVStream * video_st, break; } #else + CV_UNUSED(frame_idx); AVPacket pkt; av_init_packet(&pkt); int got_output = 0; @@ -1956,7 +2107,7 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int // 2. (dataend - SIMD_SIZE) and (dataend + SIMD_SIZE) is from the same 4k page const int CV_STEP_ALIGNMENT = 32; const size_t CV_SIMD_SIZE = 32; - const size_t CV_PAGE_MASK = ~(4096 - 1); + const size_t CV_PAGE_MASK = ~(size_t)(4096 - 1); const unsigned char* dataend = data + ((size_t)height * step); if (step % CV_STEP_ALIGNMENT != 0 || (((size_t)dataend - CV_SIMD_SIZE) & CV_PAGE_MASK) != (((size_t)dataend + CV_SIMD_SIZE) & CV_PAGE_MASK)) @@ -1984,7 +2135,12 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int step = aligned_step; } - if ( c->pix_fmt != input_pix_fmt ) { + AVPixelFormat sw_pix_fmt = c->pix_fmt; +#if USE_AV_HW_CODECS + if (c->hw_frames_ctx) + sw_pix_fmt = ((AVHWFramesContext*)c->hw_frames_ctx->data)->sw_format; +#endif + if ( sw_pix_fmt != input_pix_fmt ) { assert( input_picture ); // let input_picture point to the raw data buffer of 'image' _opencv_ffmpeg_av_image_fill_arrays(input_picture, (uint8_t *) data, @@ -1998,7 +2154,7 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int (AVPixelFormat)input_pix_fmt, c->width, c->height, - c->pix_fmt, + sw_pix_fmt, SWS_BICUBIC, NULL, NULL, NULL); if( !img_convert_ctx ) @@ -2017,13 +2173,58 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int picture->linesize[0] = step; } - picture->pts = frame_idx; - bool ret = icv_av_write_frame_FFMPEG( oc, video_st, outbuf, outbuf_size, picture) >= 0; + bool ret; +#if USE_AV_HW_CODECS + if (video_st->codec->hw_device_ctx) { + // copy data to HW frame + AVFrame* hw_frame = av_frame_alloc(); + if (!hw_frame) { + CV_LOG_ERROR(NULL, "Error allocating AVFrame (av_frame_alloc)"); + return false; + } + if (av_hwframe_get_buffer(video_st->codec->hw_frames_ctx, hw_frame, 0) < 0) { + CV_LOG_ERROR(NULL, "Error obtaining HW frame (av_hwframe_get_buffer)"); + av_frame_free(&hw_frame); + return false; + } + if (av_hwframe_transfer_data(hw_frame, picture, 0) < 0) { + CV_LOG_ERROR(NULL, "Error copying data from CPU to GPU (av_hwframe_transfer_data)"); + av_frame_free(&hw_frame); + return false; + } + hw_frame->pts = frame_idx; + int ret_write = icv_av_write_frame_FFMPEG(oc, video_st, outbuf, outbuf_size, hw_frame, frame_idx); + ret = ret_write >= 0 ? true : false; + av_frame_free(&hw_frame); + } else +#endif + { + picture->pts = frame_idx; + int ret_write = icv_av_write_frame_FFMPEG(oc, video_st, outbuf, outbuf_size, picture, frame_idx); + ret = ret_write >= 0 ? true : false; + } + frame_idx++; return ret; } +double CvVideoWriter_FFMPEG::getProperty(int propId) const +{ + CV_UNUSED(propId); +#if USE_AV_HW_CODECS + if (propId == VIDEOWRITER_PROP_HW_ACCELERATION) + { + return static_cast(va_type); + } + else if (propId == VIDEOWRITER_PROP_HW_DEVICE) + { + return static_cast(hw_device); + } +#endif + return 0; +} + /// close video output stream and free associated memory void CvVideoWriter_FFMPEG::close() { @@ -2045,7 +2246,7 @@ void CvVideoWriter_FFMPEG::close() { for(;;) { - int ret = icv_av_write_frame_FFMPEG( oc, video_st, outbuf, outbuf_size, NULL); + int ret = icv_av_write_frame_FFMPEG( oc, video_st, outbuf, outbuf_size, NULL, frame_idx); if( ret == OPENCV_NO_FRAMES_WRITTEN_CODE || ret < 0 ) break; } @@ -2135,15 +2336,48 @@ static inline void cv_ff_codec_tag_dump(const AVCodecTag *const *tags) /// Create a video writer object that uses FFMPEG bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, - double fps, int width, int height, bool is_color ) + double fps, int width, int height, const VideoWriterParameters& params) { InternalFFMpegRegister::init(); CV_CODEC_ID codec_id = CV_CODEC(CODEC_ID_NONE); - int err, codec_pix_fmt; + AVPixelFormat codec_pix_fmt; double bitrate_scale = 1; close(); + const bool is_color = params.get(VIDEOWRITER_PROP_IS_COLOR, true); + if (params.has(VIDEOWRITER_PROP_HW_ACCELERATION)) + { + va_type = params.get(VIDEOWRITER_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_NONE); +#if !USE_AV_HW_CODECS + if (va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: FFmpeg backend is build without acceleration support. Can't handle VIDEOWRITER_PROP_HW_ACCELERATION parameter. Bailout"); + return false; + } +#endif + } + if (params.has(VIDEOWRITER_PROP_HW_DEVICE)) + { + hw_device = params.get(VIDEOWRITER_PROP_HW_DEVICE, -1); + if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: Invalid usage of VIDEOWRITER_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); + return false; + } + if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: Invalid usage of VIDEOWRITER_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); + return false; + } + } + + if (params.warnUnusedParameters()) + { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in VideoWriter, see logger INFO channel for details"); + return false; + } + // check arguments if( !filename ) return false; @@ -2196,7 +2430,18 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, #endif codec_bmp_tags, // fallback for avformat < 54.1 NULL }; - if( (codec_id = av_codec_get_id(fallback_tags, fourcc)) == CV_CODEC(CODEC_ID_NONE) ) + if (codec_id == CV_CODEC(CODEC_ID_NONE)) { + codec_id = av_codec_get_id(fallback_tags, fourcc); + } + if (codec_id == CV_CODEC(CODEC_ID_NONE)) { + char *p = (char *) &fourcc; + char name[] = {(char)tolower(p[0]), (char)tolower(p[1]), (char)tolower(p[2]), (char)tolower(p[3]), 0}; + const AVCodecDescriptor *desc = avcodec_descriptor_get_by_name(name); + if (desc) + codec_id = desc->id; + } + + if (codec_id == CV_CODEC(CODEC_ID_NONE)) { fflush(stdout); fprintf(stderr, "OpenCV: FFMPEG: tag 0x%08x/'%c%c%c%c' is not found (format '%s / %s')'\n", @@ -2344,45 +2589,133 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, double bitrate = std::min(bitrate_scale*fps*width*height, (double)INT_MAX/2); - // TODO -- safe to ignore output audio stream? - video_st = icv_add_video_stream_FFMPEG(oc, codec_id, - width, height, (int)(bitrate + 0.5), - fps, codec_pix_fmt); + if (codec_id == AV_CODEC_ID_NONE) { + codec_id = av_guess_codec(oc->oformat, NULL, oc->filename, NULL, AVMEDIA_TYPE_VIDEO); + } + + // Add video stream to output file + video_st = avformat_new_stream(oc, 0); + if (!video_st) { + CV_WARN("Could not allocate stream"); + return false; + } + + AVDictionary *dict = NULL; +#if !defined(NO_GETENV) && (LIBAVUTIL_VERSION_MAJOR >= 53) + char* options = getenv("OPENCV_FFMPEG_WRITER_OPTIONS"); + if (options) { + av_dict_parse_string(&dict, options, ";", "|", 0); + } +#endif + + AVCodecContext *c = video_st->codec; + + // find and open encoder, try HW acceleration types specified in 'hw_acceleration' list (in order) + int err = -1; + AVCodec* codec = NULL; +#if USE_AV_HW_CODECS + AVBufferRef* hw_device_ctx = NULL; + HWAccelIterator accel_iter(va_type, true/*isEncoder*/, dict); + while (accel_iter.good()) + { +#else + do { +#endif +#if USE_AV_HW_CODECS + accel_iter.parse_next(); + AVHWDeviceType hw_type = accel_iter.hw_type(); + codec = NULL; + AVPixelFormat hw_format = AV_PIX_FMT_NONE; + if (hw_device_ctx) + av_buffer_unref(&hw_device_ctx); + if (hw_type != AV_HWDEVICE_TYPE_NONE) + { + codec = hw_find_codec(codec_id, hw_type, av_codec_is_encoder, accel_iter.disabled_codecs().c_str(), &hw_format); + if (!codec) + continue; + + hw_device_ctx = hw_create_device(hw_type, hw_device, accel_iter.device_subname()); + if (!hw_device_ctx) + continue; + } + else if (hw_type == AV_HWDEVICE_TYPE_NONE) +#endif + { + codec = avcodec_find_encoder(codec_id); + if (!codec) { + CV_LOG_ERROR(NULL, "Could not find encoder for codec_id=" << (int)codec_id << ", error: " + << icvFFMPEGErrStr(AVERROR_ENCODER_NOT_FOUND)); + } + } + if (!codec) + continue; +#if USE_AV_HW_CODECS + AVPixelFormat format = (hw_format != AV_PIX_FMT_NONE) ? hw_format : codec_pix_fmt; +#else + AVPixelFormat format = codec_pix_fmt; +#endif + + if (!icv_configure_video_stream_FFMPEG(oc, video_st, codec, + width, height, (int) (bitrate + 0.5), + fps, format)) { + continue; + } #if 0 #if FF_API_DUMP_FORMAT - dump_format(oc, 0, filename, 1); + dump_format(oc, 0, filename, 1); #else - av_dump_format(oc, 0, filename, 1); + av_dump_format(oc, 0, filename, 1); #endif #endif - /* now that all the parameters are set, we can open the audio and - video codecs and allocate the necessary encode buffers */ - if (!video_st){ - return false; - } + c->codec_tag = fourcc; - AVCodecContext* c = video_st->codec; +#if USE_AV_HW_CODECS + if (hw_device_ctx) { + c->hw_device_ctx = av_buffer_ref(hw_device_ctx); + if (hw_format != AV_PIX_FMT_NONE) { + c->hw_frames_ctx = hw_create_frames(NULL, hw_device_ctx, width, height, hw_format); + if (!c->hw_frames_ctx) + continue; + } + } +#endif - c->codec_tag = fourcc; - /* find the video encoder */ - AVCodec* codec = avcodec_find_encoder(c->codec_id); - if (!codec) { - fprintf(stderr, "Could not find encoder for codec id %d: %s\n", c->codec_id, - icvFFMPEGErrStr(AVERROR_ENCODER_NOT_FOUND)); - return false; - } + int64_t lbit_rate = (int64_t) c->bit_rate; + lbit_rate += (int64_t)(bitrate / 2); + lbit_rate = std::min(lbit_rate, (int64_t) INT_MAX); + c->bit_rate_tolerance = (int) lbit_rate; + c->bit_rate = (int) lbit_rate; - int64_t lbit_rate = (int64_t)c->bit_rate; - lbit_rate += (bitrate / 2); - lbit_rate = std::min(lbit_rate, (int64_t)INT_MAX); - c->bit_rate_tolerance = (int)lbit_rate; - c->bit_rate = (int)lbit_rate; + /* open the codec */ + err = avcodec_open2(c, codec, NULL); + if (err >= 0) { +#if USE_AV_HW_CODECS + va_type = hw_type_to_va_type(hw_type); + if (hw_type != AV_HWDEVICE_TYPE_NONE && hw_device < 0) + hw_device = 0; +#endif + break; + } else { + CV_LOG_ERROR(NULL, "Could not open codec " << codec->name << ", error: " << icvFFMPEGErrStr(err)); + } +#if USE_AV_HW_CODECS + } // while (accel_iter.good()) +#else + } while (0); +#endif - /* open the codec */ - if ((err= avcodec_open2(c, codec, NULL)) < 0) { - fprintf(stderr, "Could not open codec '%s': %s\n", codec->name, icvFFMPEGErrStr(err)); +#if USE_AV_HW_CODECS + if (hw_device_ctx) + av_buffer_unref(&hw_device_ctx); +#endif + + if (dict != NULL) + av_dict_free(&dict); + + if (err < 0) { + CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: Failed to initialize VideoWriter"); return false; } @@ -2400,10 +2733,16 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc, } bool need_color_convert; - need_color_convert = (c->pix_fmt != input_pix_fmt); + AVPixelFormat sw_pix_fmt = c->pix_fmt; +#if USE_AV_HW_CODECS + if (c->hw_frames_ctx) + sw_pix_fmt = ((AVHWFramesContext*)c->hw_frames_ctx->data)->sw_format; +#endif + + need_color_convert = (sw_pix_fmt != input_pix_fmt); /* allocate the encoded raw picture */ - picture = icv_alloc_picture_FFMPEG(c->pix_fmt, c->width, c->height, need_color_convert); + picture = icv_alloc_picture_FFMPEG(sw_pix_fmt, c->width, c->height, need_color_convert); if (!picture) { return false; } @@ -2493,20 +2832,28 @@ int cvRetrieveFrame_FFMPEG(CvCapture_FFMPEG* capture, unsigned char** data, int* return capture->retrieveFrame(0, data, step, width, height, cn); } -CvVideoWriter_FFMPEG* cvCreateVideoWriter_FFMPEG( const char* filename, int fourcc, double fps, - int width, int height, int isColor ) +static CvVideoWriter_FFMPEG* cvCreateVideoWriterWithParams_FFMPEG( const char* filename, int fourcc, double fps, + int width, int height, const VideoWriterParameters& params ) { CvVideoWriter_FFMPEG* writer = (CvVideoWriter_FFMPEG*)malloc(sizeof(*writer)); if (!writer) return 0; writer->init(); - if( writer->open( filename, fourcc, fps, width, height, isColor != 0 )) + if( writer->open( filename, fourcc, fps, width, height, params )) return writer; writer->close(); free(writer); return 0; } +CvVideoWriter_FFMPEG* cvCreateVideoWriter_FFMPEG( const char* filename, int fourcc, double fps, + int width, int height, int isColor ) +{ + VideoWriterParameters params; + params.add(VIDEOWRITER_PROP_IS_COLOR, isColor); + return cvCreateVideoWriterWithParams_FFMPEG(filename, fourcc, fps, width, height, params); +} + void cvReleaseVideoWriter_FFMPEG( CvVideoWriter_FFMPEG** writer ) { if( writer && *writer ) diff --git a/modules/videoio/src/cap_gstreamer.cpp b/modules/videoio/src/cap_gstreamer.cpp index 47c9f2b..60ecf66 100644 --- a/modules/videoio/src/cap_gstreamer.cpp +++ b/modules/videoio/src/cap_gstreamer.cpp @@ -281,6 +281,22 @@ bool is_gst_element_exists(const std::string& name) return (bool)testfac; } +static void find_hw_element(const GValue *item, gpointer va_type) +{ + GstElement *element = GST_ELEMENT(g_value_get_object(item)); + const gchar *name = g_type_name(G_OBJECT_TYPE(element)); + if (name) { + std::string name_lower = toLowerCase(name); + if (name_lower.find("vaapi") != std::string::npos) { + *(int*)va_type = VIDEO_ACCELERATION_VAAPI; + } else if (name_lower.find("mfx") != std::string::npos || name_lower.find("msdk") != std::string::npos) { + *(int*)va_type = VIDEO_ACCELERATION_MFX; + } else if (name_lower.find("d3d11") != std::string::npos) { + *(int*)va_type = VIDEO_ACCELERATION_D3D11; + } + } +} + //================================================================================================== class GStreamerCapture CV_FINAL : public IVideoCapture @@ -300,6 +316,8 @@ private: bool isPosFramesEmulated; gint64 emulatedFrameNumber; + VideoAccelerationType va_type; + int hw_device; public: GStreamerCapture(); virtual ~GStreamerCapture() CV_OVERRIDE; @@ -309,8 +327,8 @@ public: virtual bool setProperty(int propId, double value) CV_OVERRIDE; virtual bool isOpened() const CV_OVERRIDE { return (bool)pipeline; } virtual int getCaptureDomain() CV_OVERRIDE { return cv::CAP_GSTREAMER; } - bool open(int id); - bool open(const String &filename_); + bool open(int id, const cv::VideoCaptureParameters& params); + bool open(const String &filename_, const cv::VideoCaptureParameters& params); static void newPad(GstElement * /*elem*/, GstPad *pad, gpointer data); protected: @@ -327,6 +345,8 @@ GStreamerCapture::GStreamerCapture() : isPosFramesSupported(false), isPosFramesEmulated(false), emulatedFrameNumber(-1) + , va_type(VIDEO_ACCELERATION_NONE) + , hw_device(-1) { } @@ -754,7 +774,7 @@ void GStreamerCapture::newPad(GstElement *, GstPad *pad, gpointer data) * is really slow if we need to restart the pipeline over and over again. * */ -bool GStreamerCapture::open(int id) +bool GStreamerCapture::open(int id, const cv::VideoCaptureParameters& params) { gst_initializer::init(); @@ -764,13 +784,37 @@ bool GStreamerCapture::open(int id) desc << "v4l2src device=/dev/video" << id << " ! " << COLOR_ELEM << " ! appsink drop=true"; - return open(desc.str()); + return open(desc.str(), params); } -bool GStreamerCapture::open(const String &filename_) +bool GStreamerCapture::open(const String &filename_, const cv::VideoCaptureParameters& params) { gst_initializer::init(); + if (params.has(CAP_PROP_HW_ACCELERATION)) + { + va_type = params.get(CAP_PROP_HW_ACCELERATION); + } + if (params.has(CAP_PROP_HW_DEVICE)) + { + hw_device = params.get(CAP_PROP_HW_DEVICE); + if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); + return false; + } + if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of CAP_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); + return false; + } + if (hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: CAP_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout"); + return false; + } + } + const gchar* filename = filename_.c_str(); bool file = false; @@ -1046,6 +1090,35 @@ bool GStreamerCapture::open(const String &filename_) GST_DEBUG_BIN_TO_DOT_FILE(GST_BIN(pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "pipeline"); } + std::vector unused_params = params.getUnused(); + for (int key : unused_params) { + if (!setProperty(key, params.get(key))) { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: can't set property " << key); + return false; + } + } + + if (pipeline) + { + VideoAccelerationType actual_va_type = VIDEO_ACCELERATION_NONE; + GstIterator *iter = gst_bin_iterate_recurse(GST_BIN (pipeline.get())); + gst_iterator_foreach(iter, find_hw_element, (gpointer)&actual_va_type); + gst_iterator_free(iter); + if (va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + if (va_type != actual_va_type) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Can't select requested video acceleration through CAP_PROP_HW_ACCELERATION: " + << va_type << " (actual is " << actual_va_type << "). Bailout"); + return false; + } + } + else + { + va_type = actual_va_type; + } + } + return true; } @@ -1129,6 +1202,10 @@ double GStreamerCapture::getProperty(int propId) const } } break; + case CAP_PROP_HW_ACCELERATION: + return static_cast(va_type); + case CAP_PROP_HW_DEVICE: + return static_cast(hw_device); case CV_CAP_GSTREAMER_QUEUE_LENGTH: if(!sink) { @@ -1276,6 +1353,10 @@ bool GStreamerCapture::setProperty(int propId, double value) case CV_CAP_PROP_GAIN: case CV_CAP_PROP_CONVERT_RGB: break; + case cv::CAP_PROP_HW_ACCELERATION: + return false; // open-only + case cv::CAP_PROP_HW_DEVICE: + return false; // open-only case CV_CAP_GSTREAMER_QUEUE_LENGTH: { if(!sink) @@ -1297,18 +1378,18 @@ bool GStreamerCapture::setProperty(int propId, double value) } -Ptr createGStreamerCapture_file(const String& filename) +Ptr createGStreamerCapture_file(const String& filename, const cv::VideoCaptureParameters& params) { Ptr cap = makePtr(); - if (cap && cap->open(filename)) + if (cap && cap->open(filename, params)) return cap; return Ptr(); } -Ptr createGStreamerCapture_cam(int index) +Ptr createGStreamerCapture_cam(int index, const cv::VideoCaptureParameters& params) { Ptr cap = makePtr(); - if (cap && cap->open(index)) + if (cap && cap->open(index, params)) return cap; return Ptr(); } @@ -1325,6 +1406,7 @@ public: CvVideoWriter_GStreamer() : ipl_depth(CV_8U) , input_pix_fmt(0), num_frames(0), framerate(0) + , va_type(VIDEO_ACCELERATION_NONE), hw_device(0) { } virtual ~CvVideoWriter_GStreamer() CV_OVERRIDE @@ -1346,11 +1428,14 @@ public: int getCaptureDomain() const CV_OVERRIDE { return cv::CAP_GSTREAMER; } bool open(const std::string &filename, int fourcc, - double fps, const Size &frameSize, bool isColor, int depth ); + double fps, const Size &frameSize, const VideoWriterParameters& params ); void close(); bool writeFrame( const IplImage* image ) CV_OVERRIDE; int getIplDepth() const { return ipl_depth; } + + virtual double getProperty(int) const CV_OVERRIDE; + protected: const char* filenameToMimetype(const char* filename); GSafePtr pipeline; @@ -1360,6 +1445,9 @@ protected: int num_frames; double framerate; + VideoAccelerationType va_type; + int hw_device; + void close_(); }; @@ -1423,6 +1511,8 @@ void CvVideoWriter_GStreamer::close() close_(); source.release(); pipeline.release(); + va_type = VIDEO_ACCELERATION_NONE; + hw_device = -1; } /*! @@ -1480,8 +1570,7 @@ const char* CvVideoWriter_GStreamer::filenameToMimetype(const char *filename) * \param fourcc desired codec fourcc * \param fps desired framerate * \param frameSize the size of the expected frames - * \param is_color color or grayscale - * \param depth the depth of the expected frames + * \param params other parameters * \return success * * We support 2 modes of operation. Either the user enters a filename and a fourcc @@ -1495,13 +1584,46 @@ const char* CvVideoWriter_GStreamer::filenameToMimetype(const char *filename) */ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc, double fps, const cv::Size &frameSize, - bool is_color, int depth ) + const VideoWriterParameters& params ) { // check arguments CV_Assert(!filename.empty()); CV_Assert(fps > 0); CV_Assert(frameSize.width > 0 && frameSize.height > 0); + const bool is_color = params.get(VIDEOWRITER_PROP_IS_COLOR, true); + const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U); + + if (params.has(VIDEOWRITER_PROP_HW_ACCELERATION)) + { + va_type = params.get(VIDEOWRITER_PROP_HW_ACCELERATION); + } + if (params.has(VIDEOWRITER_PROP_HW_DEVICE)) + { + hw_device = params.get(VIDEOWRITER_PROP_HW_DEVICE); + if (va_type == VIDEO_ACCELERATION_NONE && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of VIDEOWRITER_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); + return false; + } + if (va_type == VIDEO_ACCELERATION_ANY && hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Invalid usage of VIDEOWRITER_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); + return false; + } + if (hw_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: VIDEOWRITER_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout"); + return false; + } + } + + if (params.warnUnusedParameters()) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: unsupported parameters in VideoWriter, see logger INFO channel for details"); + return false; + } + // init gstreamer gst_initializer::init(); @@ -1732,6 +1854,28 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc, handleMessage(pipeline); + if (pipeline) + { + VideoAccelerationType actual_va_type = VIDEO_ACCELERATION_NONE; + GstIterator *iter = gst_bin_iterate_recurse(GST_BIN (pipeline.get())); + gst_iterator_foreach(iter, find_hw_element, (gpointer)&actual_va_type); + gst_iterator_free(iter); + if (va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + if (va_type != actual_va_type) + { + CV_LOG_ERROR(NULL, "VIDEOIO/GStreamer: Can't select requested VideoWriter acceleration through VIDEOWRITER_PROP_HW_ACCELERATION: " + << va_type << " (actual is " << actual_va_type << "). Bailout"); + close(); + return false; + } + } + else + { + va_type = actual_va_type; + } + } + return true; } @@ -1812,15 +1956,27 @@ bool CvVideoWriter_GStreamer::writeFrame( const IplImage * image ) return true; } + +double CvVideoWriter_GStreamer::getProperty(int propId) const +{ + if (propId == VIDEOWRITER_PROP_HW_ACCELERATION) + { + return static_cast(va_type); + } + else if (propId == VIDEOWRITER_PROP_HW_DEVICE) + { + return static_cast(hw_device); + } + return 0; +} + Ptr create_GStreamer_writer(const std::string& filename, int fourcc, double fps, const cv::Size& frameSize, const VideoWriterParameters& params) { CvVideoWriter_GStreamer* wrt = new CvVideoWriter_GStreamer; - const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true); - const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U); try { - if (wrt->open(filename, fourcc, fps, frameSize, isColor, depth)) + if (wrt->open(filename, fourcc, fps, frameSize, params)) return makePtr(wrt); delete wrt; } @@ -1923,7 +2079,7 @@ void handleMessage(GstElement * pipeline) #if defined(BUILD_PLUGIN) #define CAPTURE_ABI_VERSION 1 -#define CAPTURE_API_VERSION 0 +#define CAPTURE_API_VERSION 1 #include "plugin_capture_api.hpp" #define WRITER_ABI_VERSION 1 #define WRITER_API_VERSION 1 @@ -1932,7 +2088,11 @@ void handleMessage(GstElement * pipeline) namespace cv { static -CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_OUT CvPluginCapture* handle) +CvResult CV_API_CALL cv_capture_open_with_params( + const char* filename, int camera_index, + int* params, unsigned n_params, + CV_OUT CvPluginCapture* handle +) { if (!handle) return CV_ERROR_FAIL; @@ -1942,12 +2102,13 @@ CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_ GStreamerCapture *cap = 0; try { + cv::VideoCaptureParameters parameters(params, n_params); cap = new GStreamerCapture(); bool res; if (filename) - res = cap->open(std::string(filename)); + res = cap->open(std::string(filename), parameters); else - res = cap->open(camera_index); + res = cap->open(camera_index, parameters); if (res) { *handle = (CvPluginCapture)cap; @@ -1968,6 +2129,12 @@ CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_ } static +CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_OUT CvPluginCapture* handle) +{ + return cv_capture_open_with_params(filename, camera_index, NULL, 0, handle); +} + +static CvResult CV_API_CALL cv_capture_release(CvPluginCapture handle) { if (!handle) @@ -2083,31 +2250,9 @@ CvResult CV_API_CALL cv_writer_open_with_params( try { CvSize sz = { width, height }; - bool isColor = true; - int depth = CV_8U; - if (params) - { - for (unsigned i = 0; i < n_params; ++i) - { - const int prop = params[i*2]; - const int value = params[i*2 + 1]; - switch (prop) - { - case VIDEOWRITER_PROP_IS_COLOR: - isColor = value != 0; - break; - case VIDEOWRITER_PROP_DEPTH: - depth = value; - break; - default: - // TODO emit message about non-recognized propert - // FUTURE: there should be mandatory and optional properties - return CV_ERROR_FAIL; - } - } - } + VideoWriterParameters parameters(params, n_params); wrt = new CvVideoWriter_GStreamer(); - if (wrt && wrt->open(filename, fourcc, fps, sz, isColor, depth)) + if (wrt && wrt->open(filename, fourcc, fps, sz, parameters)) { *handle = (CvPluginWriter)wrt; return CV_ERROR_OK; @@ -2145,11 +2290,25 @@ CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle) } static -CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter /*handle*/, int /*prop*/, CV_OUT double* /*val*/) +CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter handle, int prop, CV_OUT double* val) { - return CV_ERROR_FAIL; + if (!handle) + return CV_ERROR_FAIL; + if (!val) + return CV_ERROR_FAIL; + try + { + CvVideoWriter_GStreamer* instance = (CvVideoWriter_GStreamer*)handle; + *val = instance->getProperty(prop); + return CV_ERROR_OK; + } + catch (...) + { + return CV_ERROR_FAIL; + } } + static CvResult CV_API_CALL cv_writer_set_prop(CvPluginWriter /*handle*/, int /*prop*/, double /*val*/) { @@ -2197,6 +2356,9 @@ static const OpenCV_VideoIO_Capture_Plugin_API capture_api = /* 5*/cv_capture_set_prop, /* 6*/cv_capture_grab, /* 7*/cv_capture_retrieve, + }, + { + /* 8*/cv_capture_open_with_params, } }; diff --git a/modules/videoio/src/cap_interface.hpp b/modules/videoio/src/cap_interface.hpp index 80ad9e2..eef515b 100644 --- a/modules/videoio/src/cap_interface.hpp +++ b/modules/videoio/src/cap_interface.hpp @@ -29,6 +29,7 @@ struct CvVideoWriter virtual ~CvVideoWriter() {} virtual bool writeFrame(const IplImage*) { return false; } virtual int getCaptureDomain() const { return cv::CAP_ANY; } // Return the type of the capture object: CAP_FFMPEG, etc... + virtual double getProperty(int) const { return 0; } }; //=================================================== @@ -185,7 +186,7 @@ public: { found = true; CV_LOG_INFO(NULL, "VIDEOIO: unused parameter: [" << param.key << "]=" << - cv::format("%lld / 0x%16llx", (long long)param.value, (long long)param.value)); + cv::format("%lld / 0x%016llx", (long long)param.value, (long long)param.value)); } } return found; @@ -312,8 +313,12 @@ public: { cvReleaseVideoWriter(&writer); } - double getProperty(int) const CV_OVERRIDE + double getProperty(int propId) const CV_OVERRIDE { + if (writer) + { + return writer->getProperty(propId); + } return 0.; } bool setProperty(int, double) CV_OVERRIDE @@ -337,13 +342,13 @@ public: //================================================================================================== -Ptr cvCreateFileCapture_FFMPEG_proxy(const std::string &filename, const cv::VideoCaptureParameters& params); +Ptr cvCreateFileCapture_FFMPEG_proxy(const std::string &filename, const VideoCaptureParameters& params); Ptr cvCreateVideoWriter_FFMPEG_proxy(const std::string& filename, int fourcc, double fps, const Size& frameSize, const VideoWriterParameters& params); -Ptr createGStreamerCapture_file(const std::string& filename); -Ptr createGStreamerCapture_cam(int index); +Ptr createGStreamerCapture_file(const std::string& filename, const cv::VideoCaptureParameters& params); +Ptr createGStreamerCapture_cam(int index, const cv::VideoCaptureParameters& params); Ptr create_GStreamer_writer(const std::string& filename, int fourcc, double fps, const Size& frameSize, const VideoWriterParameters& params); @@ -361,8 +366,8 @@ Ptr create_AVFoundation_writer(const std::string& filename, int fo Ptr create_WRT_capture(int device); -Ptr cvCreateCapture_MSMF(int index); -Ptr cvCreateCapture_MSMF(const std::string& filename); +Ptr cvCreateCapture_MSMF(int index, const VideoCaptureParameters& params); +Ptr cvCreateCapture_MSMF(const std::string& filename, const VideoCaptureParameters& params); Ptr cvCreateVideoWriter_MSMF(const std::string& filename, int fourcc, double fps, const Size& frameSize, const VideoWriterParameters& params); @@ -411,6 +416,21 @@ bool VideoCapture_V4L_waitAny( CV_OUT std::vector& ready, int64 timeoutNs); +static inline +std::ostream& operator<<(std::ostream& out, const VideoAccelerationType& va_type) +{ + switch (va_type) + { + case VIDEO_ACCELERATION_NONE: out << "NONE"; return out; + case VIDEO_ACCELERATION_ANY: out << "ANY"; return out; + case VIDEO_ACCELERATION_D3D11: out << "D3D11"; return out; + case VIDEO_ACCELERATION_VAAPI: out << "VAAPI"; return out; + case VIDEO_ACCELERATION_MFX: out << "MFX"; return out; + } + out << cv::format("UNKNOWN(0x%ux)", static_cast(va_type)); + return out; +} + } // cv:: #endif // CAP_INTERFACE_HPP diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index a15c295..1cb537d 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #ifdef HAVE_MSMF_DXVA #include @@ -45,6 +46,7 @@ #pragma comment(lib, "mfuuid") #pragma comment(lib, "Strmiids") #pragma comment(lib, "Mfreadwrite") +#pragma comment(lib, "dxgi") #ifdef HAVE_MSMF_DXVA #pragma comment(lib, "d3d11") // MFCreateDXGIDeviceManager() is available since Win8 only. @@ -82,6 +84,8 @@ struct IMFAttributes; #define CV_CAP_MODE_GRAY CV_FOURCC_MACRO('G','R','E','Y') #define CV_CAP_MODE_YUYV CV_FOURCC_MACRO('Y', 'U', 'Y', 'V') +using namespace cv; + namespace { @@ -576,8 +580,9 @@ public: } MSMFCapture_Mode; CvCapture_MSMF(); virtual ~CvCapture_MSMF(); - virtual bool open(int); - virtual bool open(const cv::String&); + bool configureHW(const cv::VideoCaptureParameters& params); + virtual bool open(int, const cv::VideoCaptureParameters* params); + virtual bool open(const cv::String&, const cv::VideoCaptureParameters* params); virtual void close(); virtual double getProperty(int) const CV_OVERRIDE; virtual bool setProperty(int, double) CV_OVERRIDE; @@ -597,10 +602,14 @@ protected: _ComPtr getDefaultSourceConfig(UINT32 num = 10); bool initStream(DWORD streamID, const MediaType& mt); + bool openFinalize_(const VideoCaptureParameters* params); + Media_Foundation& MF; cv::String filename; int camid; MSMFCapture_Mode captureMode; + VideoAccelerationType va_type; + int hwDeviceIndex; #ifdef HAVE_MSMF_DXVA _ComPtr D3DDev; _ComPtr D3DMgr; @@ -624,6 +633,8 @@ CvCapture_MSMF::CvCapture_MSMF(): filename(""), camid(-1), captureMode(MODE_SW), + va_type(VIDEO_ACCELERATION_NONE), + hwDeviceIndex(-1), #ifdef HAVE_MSMF_DXVA D3DDev(NULL), D3DMgr(NULL), @@ -635,7 +646,6 @@ CvCapture_MSMF::CvCapture_MSMF(): sampleTime(0), isOpen(false) { - configureHW(true); } CvCapture_MSMF::~CvCapture_MSMF() @@ -732,10 +742,19 @@ bool CvCapture_MSMF::configureHW(bool enable) close(); if (enable) { + _ComPtr pAdapter; + if (hwDeviceIndex >= 0) { + _ComPtr pDXGIFactory; + if (FAILED(CreateDXGIFactory(__uuidof(IDXGIFactory2), (void**)& pDXGIFactory)) || + FAILED(pDXGIFactory->EnumAdapters(hwDeviceIndex, &pAdapter))) { + return false; + } + } D3D_FEATURE_LEVEL levels[] = { D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, D3D_FEATURE_LEVEL_10_0, D3D_FEATURE_LEVEL_9_3, D3D_FEATURE_LEVEL_9_2, D3D_FEATURE_LEVEL_9_1 }; - if (SUCCEEDED(D3D11CreateDevice(NULL, D3D_DRIVER_TYPE_HARDWARE, NULL, D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + D3D_DRIVER_TYPE driverType = pAdapter ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE; + if (SUCCEEDED(D3D11CreateDevice(pAdapter.Get(), driverType, NULL, D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT, levels, sizeof(levels) / sizeof(*levels), D3D11_SDK_VERSION, &D3DDev, NULL, NULL))) { // NOTE: Getting ready for multi-threaded operation @@ -750,7 +769,23 @@ bool CvCapture_MSMF::configureHW(bool enable) if (SUCCEEDED(D3DMgr->ResetDevice(D3DDev.Get(), mgrRToken))) { captureMode = MODE_HW; - return reopen ? (prevcam >= 0 ? open(prevcam) : open(prevfile.c_str())) : true; + if (hwDeviceIndex < 0) + hwDeviceIndex = 0; + // Log adapter description + _ComPtr dxgiDevice; + if (SUCCEEDED(D3DDev->QueryInterface(__uuidof(IDXGIDevice), reinterpret_cast(&dxgiDevice)))) { + _ComPtr adapter; + if (SUCCEEDED(dxgiDevice->GetAdapter(&adapter))) { + DXGI_ADAPTER_DESC desc; + if (SUCCEEDED(adapter->GetDesc(&desc))) { + std::wstring name(desc.Description); + std::wstring_convert> conv; + CV_LOG_INFO(NULL, "MSMF: Using D3D11 video acceleration on GPU device: " << conv.to_bytes(name)); + } + } + } + // Reopen if needed + return reopen ? (prevcam >= 0 ? open(prevcam, NULL) : open(prevfile.c_str(), NULL)) : true; } D3DMgr.Release(); } @@ -766,13 +801,26 @@ bool CvCapture_MSMF::configureHW(bool enable) if (D3DDev) D3DDev.Release(); captureMode = MODE_SW; - return reopen ? (prevcam >= 0 ? open(prevcam) : open(prevfile.c_str())) : true; + return reopen ? (prevcam >= 0 ? open(prevcam, NULL) : open(prevfile.c_str(), NULL)) : true; } #else return !enable; #endif } +bool CvCapture_MSMF::configureHW(const VideoCaptureParameters& params) +{ + va_type = params.get(CAP_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_ANY); + hwDeviceIndex = params.get(CAP_PROP_HW_DEVICE, -1); +#ifndef HAVE_MSMF_DXVA + if (va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + CV_LOG_INFO(NULL, "VIDEOIO/MSMF: MSMF backend is build without DXVA acceleration support. Can't handle CAP_PROP_HW_ACCELERATION parameter: " << va_type); + } +#endif + return configureHW(va_type == VIDEO_ACCELERATION_D3D11 || va_type == VIDEO_ACCELERATION_ANY); +} + bool CvCapture_MSMF::configureOutput(MediaType newType, cv::uint32_t outFormat) { FormatStorage formats; @@ -820,11 +868,17 @@ bool CvCapture_MSMF::configureOutput(MediaType newType, cv::uint32_t outFormat) return initStream(dwStreamIndex, newFormat); } -bool CvCapture_MSMF::open(int index) +bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) { close(); if (index < 0) return false; + + if (params) + { + configureHW(*params); + } + DeviceList devices; UINT32 count = devices.read(); if (count == 0 || static_cast(index) > count) @@ -850,15 +904,27 @@ bool CvCapture_MSMF::open(int index) { frameStep = captureFormat.getFrameStep(); } + + if (isOpen && !openFinalize_(params)) + { + close(); + return false; + } + return isOpen; } -bool CvCapture_MSMF::open(const cv::String& _filename) +bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCaptureParameters* params) { close(); if (_filename.empty()) return false; + if (params) + { + configureHW(*params); + } + // Set source reader parameters _ComPtr attr = getDefaultSourceConfig(); cv::AutoBuffer unicodeFileName(_filename.length() + 1); @@ -884,9 +950,48 @@ bool CvCapture_MSMF::open(const cv::String& _filename) } } + if (isOpen && !openFinalize_(params)) + { + close(); + return false; + } + return isOpen; } +bool CvCapture_MSMF::openFinalize_(const VideoCaptureParameters* params) +{ + if (params) + { + std::vector unused_params = params->getUnused(); + for (int key : unused_params) + { + if (!setProperty(key, params->get(key))) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: can't set property " << key); + return false; + } + } + } + + VideoAccelerationType actual_va_type = (captureMode == MODE_HW) ? VIDEO_ACCELERATION_D3D11 : VIDEO_ACCELERATION_NONE; + if (va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + if (va_type != actual_va_type) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: Can't select requested video acceleration through CAP_PROP_HW_ACCELERATION: " + << va_type << " (actual is " << actual_va_type << "). Bailout"); + return false; + } + } + else + { + va_type = actual_va_type; + } + + return true; +} + bool CvCapture_MSMF::grabFrame() { CV_TRACE_FUNCTION(); @@ -1151,7 +1256,11 @@ double CvCapture_MSMF::getProperty( int property_id ) const switch (property_id) { case CV_CAP_PROP_MODE: - return captureMode; + return captureMode; + case cv::CAP_PROP_HW_DEVICE: + return hwDeviceIndex; + case cv::CAP_PROP_HW_ACCELERATION: + return static_cast(va_type); case CV_CAP_PROP_CONVERT_RGB: return convertFormat ? 1 : 0; case CV_CAP_PROP_SAR_NUM: @@ -1415,24 +1524,24 @@ bool CvCapture_MSMF::setProperty( int property_id, double value ) return false; } -cv::Ptr cv::cvCreateCapture_MSMF( int index ) +cv::Ptr cv::cvCreateCapture_MSMF( int index, const cv::VideoCaptureParameters& params) { cv::Ptr capture = cv::makePtr(); if (capture) { - capture->open(index); + capture->open(index, ¶ms); if (capture->isOpened()) return capture; } return cv::Ptr(); } -cv::Ptr cv::cvCreateCapture_MSMF (const cv::String& filename) +cv::Ptr cv::cvCreateCapture_MSMF (const cv::String& filename, const cv::VideoCaptureParameters& params) { cv::Ptr capture = cv::makePtr(); if (capture) { - capture->open(filename); + capture->open(filename, ¶ms); if (capture->isOpened()) return capture; } @@ -1451,17 +1560,20 @@ public: CvVideoWriter_MSMF(); virtual ~CvVideoWriter_MSMF(); virtual bool open(const cv::String& filename, int fourcc, - double fps, cv::Size frameSize, bool isColor); + double fps, cv::Size frameSize, const cv::VideoWriterParameters& params); virtual void close(); virtual void write(cv::InputArray); - virtual double getProperty(int) const { return 0; } + virtual double getProperty(int) const override; virtual bool setProperty(int, double) { return false; } virtual bool isOpened() const { return initiated; } int getCaptureDomain() const CV_OVERRIDE { return cv::CAP_MSMF; } private: Media_Foundation& MF; + VideoAccelerationType va_type; + int va_device; + UINT32 videoWidth; UINT32 videoHeight; double fps; @@ -1483,6 +1595,8 @@ private: CvVideoWriter_MSMF::CvVideoWriter_MSMF(): MF(Media_Foundation::getInstance()), + va_type(VIDEO_ACCELERATION_NONE), + va_device(-1), videoWidth(0), videoHeight(0), fps(0), @@ -1556,10 +1670,40 @@ const GUID CvVideoWriter_MSMF::FourCC2GUID(int fourcc) } bool CvVideoWriter_MSMF::open( const cv::String& filename, int fourcc, - double _fps, cv::Size _frameSize, bool /*isColor*/ ) + double _fps, cv::Size _frameSize, const cv::VideoWriterParameters& params) { if (initiated) close(); + + if (params.has(VIDEOWRITER_PROP_HW_ACCELERATION)) + { + va_type = params.get(VIDEOWRITER_PROP_HW_ACCELERATION); + if (va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: MSMF backend doesn't support writer acceleration support. Can't handle VIDEOWRITER_PROP_HW_ACCELERATION parameter. Bailout"); + return false; + } + } + if (params.has(VIDEOWRITER_PROP_HW_DEVICE)) + { + va_device = params.get(VIDEOWRITER_PROP_HW_DEVICE); + if (va_type == VIDEO_ACCELERATION_NONE && va_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: Invalid usage of VIDEOWRITER_PROP_HW_DEVICE without requested H/W acceleration. Bailout"); + return false; + } + if (va_type == VIDEO_ACCELERATION_ANY && va_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: Invalid usage of VIDEOWRITER_PROP_HW_DEVICE with 'ANY' H/W acceleration. Bailout"); + return false; + } + if (va_device != -1) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: VIDEOWRITER_PROP_HW_DEVICE is not supported. Specify -1 (auto) value. Bailout"); + return false; + } + } + videoWidth = _frameSize.width; videoHeight = _frameSize.height; fps = _fps; @@ -1608,6 +1752,23 @@ bool CvVideoWriter_MSMF::open( const cv::String& filename, int fourcc, initiated = true; rtStart = 0; MFFrameRateToAverageTimePerFrame((UINT32)(fps * 1000), 1000, &rtDuration); + + VideoAccelerationType actual_va_type = VIDEO_ACCELERATION_NONE; + if (va_type != VIDEO_ACCELERATION_NONE && va_type != VIDEO_ACCELERATION_ANY) + { + if (va_type != actual_va_type) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: Can't select requested video acceleration through VIDEOWRITER_PROP_HW_ACCELERATION: " + << va_type << " (actual is " << actual_va_type << "). Bailout"); + close(); + return false; + } + } + else + { + va_type = actual_va_type; + } + return true; } } @@ -1663,6 +1824,20 @@ void CvVideoWriter_MSMF::write(cv::InputArray img) } } + +double CvVideoWriter_MSMF::getProperty(int propId) const +{ + if (propId == VIDEOWRITER_PROP_HW_ACCELERATION) + { + return static_cast(va_type); + } + else if (propId == VIDEOWRITER_PROP_HW_DEVICE) + { + return static_cast(va_device); + } + return 0; +} + cv::Ptr cv::cvCreateVideoWriter_MSMF( const std::string& filename, int fourcc, double fps, const cv::Size& frameSize, const VideoWriterParameters& params) @@ -1670,8 +1845,7 @@ cv::Ptr cv::cvCreateVideoWriter_MSMF( const std::string& filen cv::Ptr writer = cv::makePtr(); if (writer) { - const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true); - writer->open(filename, fourcc, fps, frameSize, isColor); + writer->open(filename, fourcc, fps, frameSize, params); if (writer->isOpened()) return writer; } @@ -1680,9 +1854,20 @@ cv::Ptr cv::cvCreateVideoWriter_MSMF( const std::string& filen #if defined(BUILD_PLUGIN) +#define NEW_PLUGIN + +#ifndef NEW_PLUGIN #define ABI_VERSION 0 #define API_VERSION 0 #include "plugin_api.hpp" +#else +#define CAPTURE_ABI_VERSION 1 +#define CAPTURE_API_VERSION 1 +#include "plugin_capture_api.hpp" +#define WRITER_ABI_VERSION 1 +#define WRITER_API_VERSION 1 +#include "plugin_writer_api.hpp" +#endif namespace cv { @@ -1690,7 +1875,11 @@ typedef CvCapture_MSMF CaptureT; typedef CvVideoWriter_MSMF WriterT; static -CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_OUT CvPluginCapture* handle) +CvResult CV_API_CALL cv_capture_open_with_params( + const char* filename, int camera_index, + int* params, unsigned n_params, + CV_OUT CvPluginCapture* handle +) { if (!handle) return CV_ERROR_FAIL; @@ -1700,12 +1889,13 @@ CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_ CaptureT* cap = 0; try { + cv::VideoCaptureParameters parameters(params, n_params); cap = new CaptureT(); bool res; if (filename) - res = cap->open(std::string(filename)); + res = cap->open(std::string(filename), ¶meters); else - res = cap->open(camera_index); + res = cap->open(camera_index, ¶meters); if (res) { *handle = (CvPluginCapture)cap; @@ -1726,6 +1916,12 @@ CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_ } static +CvResult CV_API_CALL cv_capture_open(const char* filename, int camera_index, CV_OUT CvPluginCapture* handle) +{ + return cv_capture_open_with_params(filename, camera_index, NULL, 0, handle); +} + +static CvResult CV_API_CALL cv_capture_release(CvPluginCapture handle) { if (!handle) @@ -1806,7 +2002,7 @@ CvResult CV_API_CALL cv_capture_grab(CvPluginCapture handle) } static -CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx, cv_videoio_retrieve_cb_t callback, void* userdata) +CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx, cv_videoio_capture_retrieve_cb_t callback, void* userdata) { if (!handle) return CV_ERROR_FAIL; @@ -1815,7 +2011,11 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx, CaptureT* instance = (CaptureT*)handle; Mat img; if (instance->retrieveFrame(stream_idx, img)) +#ifndef NEW_PLUGIN return callback(stream_idx, img.data, (int)img.step, img.cols, img.rows, img.channels(), userdata); +#else + return callback(stream_idx, img.data, (int)img.step, img.cols, img.rows, img.type(), userdata); +#endif return CV_ERROR_FAIL; } catch (const std::exception& e) @@ -1831,14 +2031,18 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx, } static -CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor, CV_OUT CvPluginWriter* handle) +CvResult CV_API_CALL cv_writer_open_with_params( + const char* filename, int fourcc, double fps, int width, int height, + int* params, unsigned n_params, + CV_OUT CvPluginWriter* handle) { WriterT* wrt = 0; try { + VideoWriterParameters parameters(params, n_params); wrt = new WriterT(); Size sz(width, height); - if (wrt && wrt->open(filename, fourcc, fps, sz, isColor != 0)) + if (wrt && wrt->open(filename, fourcc, fps, sz, parameters)) { *handle = (CvPluginWriter)wrt; return CV_ERROR_OK; @@ -1858,6 +2062,14 @@ CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps } static +CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor, + CV_OUT CvPluginWriter* handle) +{ + int params[2] = { VIDEOWRITER_PROP_IS_COLOR, isColor }; + return cv_writer_open_with_params(filename, fourcc, fps, width, height, params, 1, handle); +} + +static CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle) { if (!handle) @@ -1868,9 +2080,22 @@ CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle) } static -CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter /*handle*/, int /*prop*/, CV_OUT double* /*val*/) +CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter handle, int prop, CV_OUT double* val) { - return CV_ERROR_FAIL; + if (!handle) + return CV_ERROR_FAIL; + if (!val) + return CV_ERROR_FAIL; + try + { + WriterT* instance = (WriterT*)handle; + *val = instance->getProperty(prop); + return CV_ERROR_OK; + } + catch (...) + { + return CV_ERROR_FAIL; + } } static @@ -1905,6 +2130,10 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char* } } +} // namespace + +#ifndef NEW_PLUGIN + static const OpenCV_VideoIO_Plugin_API_preview plugin_api = { { @@ -1913,28 +2142,85 @@ static const OpenCV_VideoIO_Plugin_API_preview plugin_api = "Microsoft Media Foundation OpenCV Video I/O plugin" }, { - /* 1*/CAP_MSMF, - /* 2*/cv_capture_open, - /* 3*/cv_capture_release, - /* 4*/cv_capture_get_prop, - /* 5*/cv_capture_set_prop, - /* 6*/cv_capture_grab, - /* 7*/cv_capture_retrieve, - /* 8*/cv_writer_open, - /* 9*/cv_writer_release, - /* 10*/cv_writer_get_prop, - /* 11*/cv_writer_set_prop, - /* 12*/cv_writer_write + /* 1*/cv::CAP_MSMF, + /* 2*/cv::cv_capture_open, + /* 3*/cv::cv_capture_release, + /* 4*/cv::cv_capture_get_prop, + /* 5*/cv::cv_capture_set_prop, + /* 6*/cv::cv_capture_grab, + /* 7*/cv::cv_capture_retrieve, + /* 8*/cv::cv_writer_open, + /* 9*/cv::cv_writer_release, + /* 10*/cv::cv_writer_get_prop, + /* 11*/cv::cv_writer_set_prop, + /* 12*/cv::cv_writer_write } }; -} // namespace - const OpenCV_VideoIO_Plugin_API_preview* opencv_videoio_plugin_init_v0(int requested_abi_version, int requested_api_version, void* /*reserved=NULL*/) CV_NOEXCEPT { if (requested_abi_version == ABI_VERSION && requested_api_version <= API_VERSION) - return &cv::plugin_api; + return &plugin_api; + return NULL; +} + +#else // NEW_PLUGIN + +static const OpenCV_VideoIO_Capture_Plugin_API capture_plugin_api = +{ + { + sizeof(OpenCV_VideoIO_Capture_Plugin_API), CAPTURE_ABI_VERSION, CAPTURE_API_VERSION, + CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS, + "Microsoft Media Foundation OpenCV Video I/O plugin" + }, + { + /* 1*/cv::CAP_MSMF, + /* 2*/cv::cv_capture_open, + /* 3*/cv::cv_capture_release, + /* 4*/cv::cv_capture_get_prop, + /* 5*/cv::cv_capture_set_prop, + /* 6*/cv::cv_capture_grab, + /* 7*/cv::cv_capture_retrieve, + }, + { + /* 8*/cv::cv_capture_open_with_params, + } +}; + +const OpenCV_VideoIO_Capture_Plugin_API* opencv_videoio_capture_plugin_init_v1(int requested_abi_version, int requested_api_version, void* /*reserved=NULL*/) CV_NOEXCEPT +{ + if (requested_abi_version == CAPTURE_ABI_VERSION && requested_api_version <= CAPTURE_API_VERSION) + return &capture_plugin_api; return NULL; } +static const OpenCV_VideoIO_Writer_Plugin_API writer_plugin_api = +{ + { + sizeof(OpenCV_VideoIO_Writer_Plugin_API), WRITER_ABI_VERSION, WRITER_API_VERSION, + CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS, + "Microsoft Media Foundation OpenCV Video I/O plugin" + }, + { + /* 1*/cv::CAP_MSMF, + /* 2*/cv::cv_writer_open, + /* 3*/cv::cv_writer_release, + /* 4*/cv::cv_writer_get_prop, + /* 5*/cv::cv_writer_set_prop, + /* 6*/cv::cv_writer_write + }, + { + /* 7*/cv::cv_writer_open_with_params + } +}; + +const OpenCV_VideoIO_Writer_Plugin_API* opencv_videoio_writer_plugin_init_v1(int requested_abi_version, int requested_api_version, void* /*reserved=NULL*/) CV_NOEXCEPT +{ + if (requested_abi_version == WRITER_ABI_VERSION && requested_api_version <= WRITER_API_VERSION) + return &writer_plugin_api; + return NULL; +} + +#endif // NEW_PLUGIN + #endif // BUILD_PLUGIN diff --git a/modules/videoio/src/precomp.hpp b/modules/videoio/src/precomp.hpp index eeea218..65f5da1 100644 --- a/modules/videoio/src/precomp.hpp +++ b/modules/videoio/src/precomp.hpp @@ -44,6 +44,7 @@ #if defined(__OPENCV_BUILD) && defined(BUILD_PLUGIN) #undef __OPENCV_BUILD // allow public API only +#define OPENCV_HAVE_CVCONFIG_H 1 // but we still have access to cvconfig.h (TODO remove this) #include #include #endif diff --git a/modules/videoio/test/test_precomp.hpp b/modules/videoio/test/test_precomp.hpp index 0b43cd0..5bc2ccd 100644 --- a/modules/videoio/test/test_precomp.hpp +++ b/modules/videoio/test/test_precomp.hpp @@ -15,11 +15,35 @@ namespace cv { -inline std::ostream &operator<<(std::ostream &out, const VideoCaptureAPIs& api) +static inline +std::ostream& operator<<(std::ostream& out, const VideoCaptureAPIs& api) { out << cv::videoio_registry::getBackendName(api); return out; } +static inline +std::ostream& operator<<(std::ostream& out, const VideoAccelerationType& va_type) +{ + struct { + VideoAccelerationType va_type; + const char* str; + } va_types[] = { + {VIDEO_ACCELERATION_ANY, "ANY"}, + {VIDEO_ACCELERATION_NONE, "NONE"}, + {VIDEO_ACCELERATION_D3D11, "D3D11"}, + {VIDEO_ACCELERATION_VAAPI, "VAAPI"}, + {VIDEO_ACCELERATION_MFX, "MFX"}, + }; + for (const auto& va : va_types) { + if (va_type == va.va_type) { + out << va.str; + return out; + } + } + out << cv::format("UNKNOWN(0x%ux)", static_cast(va_type)); + return out; +} + static inline void PrintTo(const cv::VideoCaptureAPIs& api, std::ostream* os) { *os << cv::videoio_registry::getBackendName(api); diff --git a/modules/videoio/test/test_video_io.cpp b/modules/videoio/test/test_video_io.cpp index ff2a444..a6a06c2 100644 --- a/modules/videoio/test/test_video_io.cpp +++ b/modules/videoio/test/test_video_io.cpp @@ -431,11 +431,11 @@ static Ext_Fourcc_PSNR synthetic_params[] = { {"mkv", "MPEG", 30.f, CAP_FFMPEG}, {"mkv", "MJPG", 30.f, CAP_FFMPEG}, - {"avi", "MPEG", 30.f, CAP_GSTREAMER}, + {"avi", "MPEG", 28.f, CAP_GSTREAMER}, {"avi", "MJPG", 30.f, CAP_GSTREAMER}, {"avi", "H264", 30.f, CAP_GSTREAMER}, - {"mkv", "MPEG", 30.f, CAP_GSTREAMER}, + {"mkv", "MPEG", 28.f, CAP_GSTREAMER}, {"mkv", "MJPG", 30.f, CAP_GSTREAMER}, {"mkv", "H264", 30.f, CAP_GSTREAMER}, @@ -649,4 +649,332 @@ TEST_P(safe_capture, frames_independency) static VideoCaptureAPIs safe_apis[] = {CAP_FFMPEG, CAP_GSTREAMER, CAP_MSMF,CAP_AVFOUNDATION}; INSTANTIATE_TEST_CASE_P(videoio, safe_capture, testing::ValuesIn(safe_apis)); +//================================================================================================== +// TEST_P(videocapture_acceleration, ...) + +struct VideoCaptureAccelerationInput +{ + const char* filename; + double psnr_threshold; +}; + +static inline +std::ostream& operator<<(std::ostream& out, const VideoCaptureAccelerationInput& p) +{ + out << p.filename; + return out; +} + +typedef testing::TestWithParam> videocapture_acceleration; + +TEST_P(videocapture_acceleration, read) +{ + auto param = GetParam(); + std::string filename = get<0>(param).filename; + double psnr_threshold = get<0>(param).psnr_threshold; + VideoCaptureAPIs backend = get<1>(param); + VideoAccelerationType va_type = get<2>(param); + bool use_umat = get<3>(param); + int device_idx = -1; + const int frameNum = 15; + + std::string filepath = cvtest::findDataFile("video/" + filename); + + if (backend == CAP_MSMF && ( + filename == "sample_322x242_15frames.yuv420p.mjpeg.mp4" || + filename == "sample_322x242_15frames.yuv420p.libx265.mp4" || + filename == "sample_322x242_15frames.yuv420p.libaom-av1.mp4" || + filename == "sample_322x242_15frames.yuv420p.mpeg2video.mp4" + )) + throw SkipTestException("Format/codec is not supported"); + + + std::string backend_name = cv::videoio_registry::getBackendName(backend); + if (!videoio_registry::hasBackend(backend)) + throw SkipTestException(cv::String("Backend is not available/disabled: ") + backend_name); + + + // HW reader + VideoCapture hw_reader(filepath, backend, { + CAP_PROP_HW_ACCELERATION, static_cast(va_type), + CAP_PROP_HW_DEVICE, device_idx + }); + if (!hw_reader.isOpened()) + { + if (va_type == VIDEO_ACCELERATION_ANY || va_type == VIDEO_ACCELERATION_NONE) + { + // ANY HW acceleration should have fallback to SW codecs + VideoCapture sw_reader(filepath, backend, { + CAP_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_NONE + }); + if (!sw_reader.isOpened()) + throw SkipTestException(backend_name + " VideoCapture on " + filename + " not supported, skipping"); + + ASSERT_TRUE(hw_reader.isOpened()) << "ANY HW acceleration should have fallback to SW codecs"; + } + else + { + throw SkipTestException(backend_name + " VideoCapture on " + filename + " not supported with HW acceleration, skipping"); + } + } + + VideoAccelerationType actual_va = static_cast(static_cast(hw_reader.get(CAP_PROP_HW_ACCELERATION))); + if (va_type != VIDEO_ACCELERATION_ANY && va_type != VIDEO_ACCELERATION_NONE) + { +#ifdef _WIN32 // FIXIT FFmpeg wrapper upgrade is required + if (actual_va == static_cast(0)) + throw SkipTestException(backend_name + " VideoCapture on " + filename + " not supported with HW acceleration (legacy FFmpeg wrapper), skipping"); +#endif + ASSERT_EQ((int)actual_va, (int)va_type) << "actual_va=" << actual_va << ", va_type=" << va_type; + } + std::cout << "VideoCapture " << backend_name << ":" << actual_va << std::endl << std::flush; + + double min_psnr_original = 1000; + for (int i = 0; i < frameNum; i++) + { + SCOPED_TRACE(cv::format("frame=%d", i)); + Mat frame; + if (use_umat) + { + UMat umat; + EXPECT_TRUE(hw_reader.read(umat)); + ASSERT_FALSE(umat.empty()); + umat.copyTo(frame); + } + else + { + EXPECT_TRUE(hw_reader.read(frame)); + } + ASSERT_FALSE(frame.empty()); + + if (cvtest::debugLevel > 0) + { + imwrite(cv::format("test_frame%03d.png", i), frame); + } + + Mat original(frame.size(), CV_8UC3, Scalar::all(0)); + generateFrame(i, frameNum, original); + double psnr = cvtest::PSNR(frame, original); + if (psnr < min_psnr_original) + min_psnr_original = psnr; + } + + std::ostringstream ss; ss << actual_va; + std::string actual_va_str = ss.str(); + std::cout << "VideoCapture with acceleration = " << cv::format("%-6s @ %-10s", actual_va_str.c_str(), backend_name.c_str()) + << " on " << filename + << " with PSNR-original = " << min_psnr_original + << std::endl << std::flush; + EXPECT_GE(min_psnr_original, psnr_threshold); +} + +static const VideoCaptureAccelerationInput hw_filename[] = { + { "sample_322x242_15frames.yuv420p.libxvid.mp4", 28.0 }, + { "sample_322x242_15frames.yuv420p.mjpeg.mp4", 20.0 }, + { "sample_322x242_15frames.yuv420p.mpeg2video.mp4", 24.0 }, // GSTREAMER on Ubuntu 18.04 + { "sample_322x242_15frames.yuv420p.libx264.mp4", 24.0 }, // GSTREAMER on Ubuntu 18.04 + { "sample_322x242_15frames.yuv420p.libx265.mp4", 30.0 }, + { "sample_322x242_15frames.yuv420p.libvpx-vp9.mp4", 30.0 }, + { "sample_322x242_15frames.yuv420p.libaom-av1.mp4", 30.0 } +}; + +static const VideoCaptureAPIs hw_backends[] = { + CAP_FFMPEG, + CAP_GSTREAMER, +#ifdef _WIN32 + CAP_MSMF, +#endif +}; + +static const VideoAccelerationType hw_types[] = { + VIDEO_ACCELERATION_NONE, + VIDEO_ACCELERATION_ANY, + VIDEO_ACCELERATION_MFX, +#ifdef _WIN32 + VIDEO_ACCELERATION_D3D11, +#else + VIDEO_ACCELERATION_VAAPI, +#endif +}; + +static bool hw_use_umat[] = { + false, + //true +}; + +INSTANTIATE_TEST_CASE_P(videoio, videocapture_acceleration, testing::Combine( + testing::ValuesIn(hw_filename), + testing::ValuesIn(hw_backends), + testing::ValuesIn(hw_types), + testing::ValuesIn(hw_use_umat) +)); + +////////////////////////////////////////// TEST_P(video_acceleration, write_read) + +typedef tuple VATestParams; + +typedef testing::TestWithParam videowriter_acceleration; + +TEST_P(videowriter_acceleration, write) +{ + auto param = GetParam(); + VideoCaptureAPIs backend = get<0>(param).api; + std::string codecid = get<0>(param).fourcc; + std::string extension = get<0>(param).ext; + double psnr_threshold = get<0>(param).PSNR; + VideoAccelerationType va_type = get<1>(param); + int device_idx = -1; + bool use_umat = get<2>(param); + std::string backend_name = cv::videoio_registry::getBackendName(backend); + if (!videoio_registry::hasBackend(backend)) + throw SkipTestException(cv::String("Backend is not available/disabled: ") + backend_name); + + const Size sz(640, 480); + const int frameNum = 15; + const double fps = 25; + + std::string filename = tempfile("videowriter_acceleration.") + extension; + + // Write video + VideoAccelerationType actual_va; + { + VideoWriter hw_writer( + filename, + backend, + VideoWriter::fourcc(codecid[0], codecid[1], codecid[2], codecid[3]), + fps, + sz, + { + VIDEOWRITER_PROP_HW_ACCELERATION, static_cast(va_type), + VIDEOWRITER_PROP_HW_DEVICE, device_idx + } + ); + + if (!hw_writer.isOpened()) { + if (va_type == VIDEO_ACCELERATION_ANY || va_type == VIDEO_ACCELERATION_NONE) + { + // ANY HW acceleration should have fallback to SW codecs + { + VideoWriter sw_writer( + filename, + backend, + VideoWriter::fourcc(codecid[0], codecid[1], codecid[2], codecid[3]), + fps, + sz, + { + VIDEOWRITER_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_NONE, + } + ); + if (!sw_writer.isOpened()) { + remove(filename.c_str()); + throw SkipTestException(backend_name + " VideoWriter on codec " + codecid + " not supported, skipping"); + } + } + remove(filename.c_str()); + ASSERT_TRUE(hw_writer.isOpened()) << "ANY HW acceleration should have fallback to SW codecs"; + } else { + throw SkipTestException(backend_name + " VideoWriter on " + filename + " not supported with HW acceleration, skipping"); + } + } + + actual_va = static_cast(static_cast(hw_writer.get(VIDEOWRITER_PROP_HW_ACCELERATION))); + if (va_type != VIDEO_ACCELERATION_ANY && va_type != VIDEO_ACCELERATION_NONE) + { +#ifdef _WIN32 // FIXIT FFmpeg wrapper upgrade is required + if (actual_va == static_cast(-1)) + throw SkipTestException(backend_name + " VideoWriter on " + filename + " not supported with HW acceleration (legacy FFmpeg wrapper), skipping"); +#endif + ASSERT_EQ((int)actual_va, (int)va_type) << "actual_va=" << actual_va << ", va_type=" << va_type; + } + std::cout << "VideoWriter " << backend_name << ":" << actual_va << std::endl << std::flush; + + Mat frame(sz, CV_8UC3); + for (int i = 0; i < frameNum; ++i) { + generateFrame(i, frameNum, frame); + if (use_umat) { + UMat umat; + frame.copyTo(umat); + hw_writer.write(umat); + } + else { + hw_writer.write(frame); + } + } + } + + std::ifstream ofile(filename, std::ios::binary); + ofile.seekg(0, std::ios::end); + int64 fileSize = (int64)ofile.tellg(); + ASSERT_GT(fileSize, 0); + std::cout << "File size: " << fileSize << std::endl; + + // Read video and check PSNR on every frame + { + VideoCapture reader( + filename, + CAP_ANY /*backend*/, + { CAP_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_NONE } + ); + ASSERT_TRUE(reader.isOpened()); + double min_psnr = 1000; + Mat reference(sz, CV_8UC3); + for (int i = 0; i < frameNum; ++i) { + Mat actual; + if (use_umat) { + UMat umat; + EXPECT_TRUE(reader.read(umat)); + umat.copyTo(actual); + } + else { + EXPECT_TRUE(reader.read(actual)); + } + EXPECT_FALSE(actual.empty()); + generateFrame(i, frameNum, reference); + EXPECT_EQ(reference.size(), actual.size()); + EXPECT_EQ(reference.depth(), actual.depth()); + EXPECT_EQ(reference.channels(), actual.channels()); + double psnr = cvtest::PSNR(actual, reference); + EXPECT_GE(psnr, psnr_threshold) << " frame " << i; + if (psnr < min_psnr) + min_psnr = psnr; + } + Mat actual; + EXPECT_FALSE(reader.read(actual)); + { + std::ostringstream ss; ss << actual_va; + std::string actual_va_str = ss.str(); + std::cout << "VideoWriter with acceleration = " << cv::format("%-6s @ %-10s", actual_va_str.c_str(), backend_name.c_str()) + << " on codec=" << codecid << " (." << extension << ")" + << ", bitrate = " << fileSize / (frameNum / fps) + << ", with PSNR-original = " << min_psnr + << std::endl << std::flush; + } + remove(filename.c_str()); + } +} + +static Ext_Fourcc_PSNR hw_codecs[] = { + {"mp4", "MPEG", 29.f, CAP_FFMPEG}, + {"mp4", "H264", 29.f, CAP_FFMPEG}, + {"mp4", "HEVC", 29.f, CAP_FFMPEG}, + {"avi", "MJPG", 29.f, CAP_FFMPEG}, + {"avi", "XVID", 29.f, CAP_FFMPEG}, + //{"webm", "VP8", 29.f, CAP_FFMPEG}, + //{"webm", "VP9", 29.f, CAP_FFMPEG}, + + {"mkv", "MPEG", 29.f, CAP_GSTREAMER}, + {"mkv", "H264", 29.f, CAP_GSTREAMER}, + +#ifdef _WIN32 + {"mp4", "MPEG", 29.f, CAP_MSMF}, + {"mp4", "H264", 29.f, CAP_MSMF}, +#endif +}; + +INSTANTIATE_TEST_CASE_P(videoio, videowriter_acceleration, testing::Combine( + testing::ValuesIn(hw_codecs), + testing::ValuesIn(hw_types), + testing::ValuesIn(hw_use_umat) +)); + } // namespace diff --git a/samples/tapi/video_acceleration.cpp b/samples/tapi/video_acceleration.cpp new file mode 100644 index 0000000..2c99771 --- /dev/null +++ b/samples/tapi/video_acceleration.cpp @@ -0,0 +1,211 @@ +#include +#include +#include "opencv2/core.hpp" +#include "opencv2/core/ocl.hpp" +#include "opencv2/core/utility.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/videoio.hpp" +#include "opencv2/highgui.hpp" + +using namespace cv; +using namespace std; + +const char* keys = +"{ i input | | input video file }" +"{ o output | | output video file, or specify 'null' to measure decoding without rendering to screen}" +"{ backend | any | VideoCapture and VideoWriter backend, valid values: 'any', 'ffmpeg', 'msmf', 'gstreamer' }" +"{ accel | any | GPU Video Acceleration, valid values: 'none', 'any', 'd3d11', 'vaapi', 'mfx' }" +"{ device | -1 | Video Acceleration device (GPU) index (-1 means default device) }" +"{ out_w | | output width (resize by calling cv::resize) }" +"{ out_h | | output height (resize by calling cv::resize) }" +"{ bitwise_not| false | apply simple image processing - bitwise_not pixels by calling cv::bitwise_not }" +"{ opencl | true | use OpenCL (inside VideoCapture/VideoWriter and for image processing) }" +"{ codec | H264 | codec id (four characters string) of output file encoder }" +"{ h help | | print help message }"; + +struct { + cv::VideoCaptureAPIs backend; + const char* str; +} backend_strings[] = { + { cv::CAP_ANY, "any" }, + { cv::CAP_FFMPEG, "ffmpeg" }, + { cv::CAP_MSMF, "msmf" }, + { cv::CAP_GSTREAMER, "gstreamer" }, +}; + +struct { + VideoAccelerationType acceleration; + const char* str; +} acceleration_strings[] = { + { VIDEO_ACCELERATION_NONE, "none" }, + { VIDEO_ACCELERATION_ANY, "any" }, + { VIDEO_ACCELERATION_D3D11, "d3d11" }, + { VIDEO_ACCELERATION_VAAPI, "vaapi" }, + { VIDEO_ACCELERATION_MFX, "mfx" }, +}; + +class FPSCounter { +public: + FPSCounter(double _interval) : interval(_interval) { + } + + ~FPSCounter() { + NewFrame(true); + } + + void NewFrame(bool last_frame = false) { + num_frames++; + auto now = std::chrono::high_resolution_clock::now(); + if (!last_time.time_since_epoch().count()) { + last_time = now; + } + + double sec = std::chrono::duration_cast>(now - last_time).count(); + if (sec >= interval || last_frame) { + printf("FPS(last %.2f sec) = %.2f\n", sec, num_frames / sec); + fflush(stdout); + num_frames = 0; + last_time = now; + } + } + +private: + double interval = 1; + std::chrono::time_point last_time; + int num_frames = 0; +}; + +int main(int argc, char** argv) +{ + cv::CommandLineParser cmd(argc, argv, keys); + if (cmd.has("help")) + { + cout << "Usage : video_acceleration [options]" << endl; + cout << "Available options:" << endl; + cmd.printMessage(); + return EXIT_SUCCESS; + } + + string infile = cmd.get("i"); + string outfile = cmd.get("o"); + string codec = cmd.get("codec"); + int device = cmd.get("device"); + int out_w = cmd.get("out_w"); + int out_h = cmd.get("out_h"); + bool use_opencl = cmd.get("opencl"); + bool bitwise_not = cmd.get("bitwise_not"); + + cv::VideoCaptureAPIs backend = cv::CAP_ANY; + string backend_str = cmd.get("backend"); + for (size_t i = 0; i < sizeof(backend_strings)/sizeof(backend_strings[0]); i++) { + if (backend_str == backend_strings[i].str) { + backend = backend_strings[i].backend; + break; + } + } + + VideoAccelerationType accel = VIDEO_ACCELERATION_ANY; + string accel_str = cmd.get("accel"); + for (size_t i = 0; i < sizeof(acceleration_strings) / sizeof(acceleration_strings[0]); i++) { + if (accel_str == acceleration_strings[i].str) { + accel = acceleration_strings[i].acceleration; + break; + } + } + + ocl::setUseOpenCL(use_opencl); + + VideoCapture capture(infile, backend, { + CAP_PROP_HW_ACCELERATION, (int)accel, + CAP_PROP_HW_DEVICE, device + }); + if (!capture.isOpened()) { + cerr << "Failed to open VideoCapture" << endl; + return 1; + } + cout << "VideoCapture backend = " << capture.getBackendName() << endl; + VideoAccelerationType actual_accel = static_cast(static_cast(capture.get(CAP_PROP_HW_ACCELERATION))); + for (size_t i = 0; i < sizeof(acceleration_strings) / sizeof(acceleration_strings[0]); i++) { + if (actual_accel == acceleration_strings[i].acceleration) { + cout << "VideoCapture acceleration = " << acceleration_strings[i].str << endl; + cout << "VideoCapture acceleration device = " << (int)capture.get(CAP_PROP_HW_DEVICE) << endl; + break; + } + } + + VideoWriter writer; + if (!outfile.empty() && outfile != "null") { + const char* codec_str = codec.c_str(); + int fourcc = VideoWriter::fourcc(codec_str[0], codec_str[1], codec_str[2], codec_str[3]); + double fps = capture.get(CAP_PROP_FPS); + Size frameSize = { out_w, out_h }; + if (!out_w || !out_h) { + frameSize = { (int)capture.get(CAP_PROP_FRAME_WIDTH), (int)capture.get(CAP_PROP_FRAME_HEIGHT) }; + } + writer = VideoWriter(outfile, backend, fourcc, fps, frameSize, { + VIDEOWRITER_PROP_HW_ACCELERATION, (int)accel, + VIDEOWRITER_PROP_HW_DEVICE, device + }); + if (!writer.isOpened()) { + cerr << "Failed to open VideoWriter" << endl; + return 1; + } + cout << "VideoWriter backend = " << writer.getBackendName() << endl; + actual_accel = static_cast(static_cast(writer.get(CAP_PROP_HW_ACCELERATION))); + for (size_t i = 0; i < sizeof(acceleration_strings) / sizeof(acceleration_strings[0]); i++) { + if (actual_accel == acceleration_strings[i].acceleration) { + cout << "VideoWriter acceleration = " << acceleration_strings[i].str << endl; + cout << "VideoWriter acceleration device = " << (int)writer.get(CAP_PROP_HW_DEVICE) << endl; + break; + } + } + } + + cout << "\nStarting frame loop. Press ESC to exit\n"; + + FPSCounter fps_counter(0.5); // print FPS every 0.5 seconds + + UMat frame, frame2, frame3; + + for (;;) + { + capture.read(frame); + if (frame.empty()) { + cout << "End of stream" << endl; + break; + } + + if (out_w && out_h) { + cv::resize(frame, frame2, cv::Size(out_w, out_h)); + //cv::cvtColor(frame, outframe, COLOR_BGRA2RGBA); + } + else { + frame2 = frame; + } + + if (bitwise_not) { + cv::bitwise_not(frame2, frame3); + } + else { + frame3 = frame2; + } + + if (writer.isOpened()) { + writer.write(frame3); + } + + if (outfile.empty()) { + imshow("output", frame3); + char key = (char) waitKey(1); + if (key == 27) + break; + else if (key == 'm') { + ocl::setUseOpenCL(!cv::ocl::useOpenCL()); + cout << "Switched to " << (ocl::useOpenCL() ? "OpenCL enabled" : "CPU") << " mode\n"; + } + } + fps_counter.NewFrame(); + } + + return EXIT_SUCCESS; +}