From cb08c15616e69168de0e3ff9ca63ad464dcf3cae Mon Sep 17 00:00:00 2001 From: MaximMilashchenko <67949029+MaximMilashchenko@users.noreply.github.com> Date: Sat, 4 Dec 2021 13:37:10 +0300 Subject: [PATCH] Merge pull request #21145 from MaximMilashchenko:AudioUpdate Audio MSMF: added the ability to set sample per second * Audio MSMF: added the ability to set sample per second * changed the valid sampling rate check * fixed docs * add test * fixed warning * fixed error * fixed error --- modules/videoio/include/opencv2/videoio.hpp | 2 +- modules/videoio/src/cap_msmf.cpp | 72 +++++++++++++++++++++++++---- modules/videoio/test/test_audio.cpp | 11 +++++ 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 8d97c31..93ea8cd 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -195,7 +195,7 @@ enum VideoCaptureProperties { CAP_PROP_AUDIO_POS = 59, //!< (read-only) Audio position is measured in samples. Accurate audio sample timestamp of previous grabbed fragment. See CAP_PROP_AUDIO_SAMPLES_PER_SECOND and CAP_PROP_AUDIO_SHIFT_NSEC. CAP_PROP_AUDIO_SHIFT_NSEC = 60, //!< (read only) Contains the time difference between the start of the audio stream and the video stream in nanoseconds. Positive value means that audio is started after the first video frame. Negative value means that audio is started before the first video frame. CAP_PROP_AUDIO_DATA_DEPTH = 61, //!< (open, read) Alternative definition to bits-per-sample, but with clear handling of 32F / 32S - CAP_PROP_AUDIO_SAMPLES_PER_SECOND = 62, //!< (read-only) determined from file/codec input. If not specified, then selected audio sample rate is 44100 + CAP_PROP_AUDIO_SAMPLES_PER_SECOND = 62, //!< (open, read) determined from file/codec input. If not specified, then selected audio sample rate is 44100 CAP_PROP_AUDIO_BASE_INDEX = 63, //!< (read-only) Index of the first audio channel for .retrieve() calls. That audio channel number continues enumeration after video channels. CAP_PROP_AUDIO_TOTAL_CHANNELS = 64, //!< (read-only) Number of audio channels in the selected audio stream (mono, stereo, etc) CAP_PROP_AUDIO_TOTAL_STREAMS = 65, //!< (read-only) Number of audio streams. diff --git a/modules/videoio/src/cap_msmf.cpp b/modules/videoio/src/cap_msmf.cpp index 39f191e..68171ea 100644 --- a/modules/videoio/src/cap_msmf.cpp +++ b/modules/videoio/src/cap_msmf.cpp @@ -727,6 +727,7 @@ protected: bool configureHW(bool enable); bool configureStreams(const cv::VideoCaptureParameters&); bool setAudioProperties(const cv::VideoCaptureParameters&); + bool checkAudioProperties(); template bool readComplexPropery(long prop, long& val) const; @@ -766,6 +767,7 @@ protected: unsigned int audioBaseIndex; int outputVideoFormat; int outputAudioFormat; + UINT32 audioSamplesPerSecond; bool convertFormat; MFTIME duration; LONGLONG frameStep; @@ -818,6 +820,7 @@ CvCapture_MSMF::CvCapture_MSMF(): audioBaseIndex(1), outputVideoFormat(CV_CAP_MODE_BGR), outputAudioFormat(CV_16S), + audioSamplesPerSecond(0), convertFormat(true), duration(0), frameStep(0), @@ -1047,7 +1050,7 @@ bool CvCapture_MSMF::configureAudioOutput(MediaType newType) MediaType newFormat = bestMatch.second; newFormat.majorType = MFMediaType_Audio; - newFormat.nSamplesPerSec = 44100; + newFormat.nSamplesPerSec = (audioSamplesPerSecond == 0) ? 44100 : audioSamplesPerSecond; switch (outputAudioFormat) { case CV_8S: @@ -1147,7 +1150,8 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) if (params) { configureHW(*params); - configureStreams(*params); + if (!(configureStreams(*params) && setAudioProperties(*params))) + return false; } if (videoStream != -1 && audioStream != -1 || videoStream == -1 && audioStream == -1) { @@ -1189,6 +1193,12 @@ bool CvCapture_MSMF::open(int index, const cv::VideoCaptureParameters* params) close(); return false; } + if (isOpen) + { + if (audioStream != -1) + if (!checkAudioProperties()) + return false; + } return isOpen; } @@ -1202,8 +1212,8 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar if (params) { configureHW(*params); - configureStreams(*params); - setAudioProperties(*params); + if (!(configureStreams(*params) && setAudioProperties(*params))) + return false; } // Set source reader parameters _ComPtr attr = getDefaultSourceConfig(); @@ -1235,12 +1245,19 @@ bool CvCapture_MSMF::open(const cv::String& _filename, const cv::VideoCapturePar return false; } if (isOpen) - if (audioStream != -1 && videoStream != -1) + { + if (audioStream != -1) { - isOpen = grabFrame(); - if (isOpen) - grabIsDone = true; + if (!checkAudioProperties()) + return false; + if (videoStream != -1) + { + isOpen = grabFrame(); + if (isOpen) + grabIsDone = true; + } } + } return isOpen; } @@ -1318,14 +1335,49 @@ bool CvCapture_MSMF::setAudioProperties(const cv::VideoCaptureParameters& params outputAudioFormat = value; } } + if (params.has(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)) + { + int value = static_cast(params.get(CAP_PROP_AUDIO_SAMPLES_PER_SECOND)); + if (value < 0) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter can't be negative: " << value); + return false; + } + else + { + audioSamplesPerSecond = value; + } + } if (params.has(CAP_PROP_AUDIO_SYNCHRONIZE)) { - int value = static_cast(params.get(CAP_PROP_AUDIO_SYNCHRONIZE)); + int value = static_cast(params.get(CAP_PROP_AUDIO_SYNCHRONIZE)); syncLastFrame = (value != 0) ? true : false; } return true; } - +bool CvCapture_MSMF::checkAudioProperties() +{ + if (audioSamplesPerSecond != 0) + { + _ComPtr type; + UINT32 actualAudioSamplesPerSecond = 0; + HRESULT hr = videoFileSource->GetCurrentMediaType(dwAudioStreamIndex, &type); + if (SUCCEEDED(hr)) + { + type->GetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND , &actualAudioSamplesPerSecond); + if (actualAudioSamplesPerSecond != audioSamplesPerSecond) + { + CV_LOG_ERROR(NULL, "VIDEOIO/MSMF: CAP_PROP_AUDIO_SAMPLES_PER_SECOND parameter value is invalid/unsupported: " << audioSamplesPerSecond + << ". Current value of CAP_PROP_AUDIO_SAMPLES_PER_SECOND: " << actualAudioSamplesPerSecond); + close(); + return false; + } + return true; + } + return false; + } + return true; +} bool CvCapture_MSMF::grabVideoFrame() { DWORD streamIndex, flags; diff --git a/modules/videoio/test/test_audio.cpp b/modules/videoio/test/test_audio.cpp index 0b637ae..7c66b83 100644 --- a/modules/videoio/test/test_audio.cpp +++ b/modules/videoio/test/test_audio.cpp @@ -281,4 +281,15 @@ TEST(AudioOpenCheck, bad_arg_invalid_audio_stream) ASSERT_FALSE(cap.isOpened()); } +TEST(AudioOpenCheck, bad_arg_invalid_audio_sample_per_second) +{ + std::string fileName = "audio/test_audio.mp4"; + std::vector params { CAP_PROP_AUDIO_STREAM, 0, + CAP_PROP_VIDEO_STREAM, -1, + CAP_PROP_AUDIO_SAMPLES_PER_SECOND, (int)1e9 }; + VideoCapture cap; + cap.open(findDataFile(fileName), cv::CAP_MSMF, params); + ASSERT_FALSE(cap.isOpened()); +} + }} //namespace -- 2.7.4