From: Stefan Dragnev Date: Thu, 10 Nov 2022 17:08:59 +0000 (+0100) Subject: add support for CAP_PROP_ORIENTATION_AUTO to AVFoundation backend X-Git-Tag: accepted/tizen/unified/20230127.161057~1^2~68^2 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a462f49b993aeeaa73b3d1176b3c122901e702ad;p=platform%2Fupstream%2Fopencv.git add support for CAP_PROP_ORIENTATION_AUTO to AVFoundation backend * extract rotateFrame as free function, rename to applyMetadataRotation * LegacyCapture::get() always return 0, if cap is null --- diff --git a/modules/videoio/include/opencv2/videoio.hpp b/modules/videoio/include/opencv2/videoio.hpp index 91aa083ef0..59f1fcb9b0 100644 --- a/modules/videoio/include/opencv2/videoio.hpp +++ b/modules/videoio/include/opencv2/videoio.hpp @@ -182,8 +182,8 @@ enum VideoCaptureProperties { CAP_PROP_WB_TEMPERATURE=45, //!< white-balance color temperature CAP_PROP_CODEC_PIXEL_FORMAT =46, //!< (read-only) codec's pixel format. 4-character code - see VideoWriter::fourcc . Subset of [AV_PIX_FMT_*](https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/raw.c) or -1 if unknown CAP_PROP_BITRATE =47, //!< (read-only) Video bitrate in kbits/s - CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only) - CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499) + CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg and AVFoundation back-ends only) + CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata (applicable for FFmpeg and AVFoundation back-ends only) (https://github.com/opencv/opencv/issues/15499) CAP_PROP_HW_ACCELERATION=50, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in cv::VideoCapture constructor / .open() method. Default value is backend-specific. CAP_PROP_HW_DEVICE =51, //!< (**open-only**) Hardware device index (select GPU if multiple available). Device enumeration is acceleration type specific. CAP_PROP_HW_ACCELERATION_USE_OPENCL=52, //!< (**open-only**) If non-zero, create new OpenCL context and bind it to current thread. The OpenCL context created with Video Acceleration context attached it (if not attached yet) for optimized GPU data copy between HW accelerated decoder and cv::UMat. diff --git a/modules/videoio/src/cap.cpp b/modules/videoio/src/cap.cpp index fa958e2c8f..691fb9ab38 100644 --- a/modules/videoio/src/cap.cpp +++ b/modules/videoio/src/cap.cpp @@ -698,4 +698,29 @@ int VideoWriter::fourcc(char c1, char c2, char c3, char c4) return (c1 & 255) + ((c2 & 255) << 8) + ((c3 & 255) << 16) + ((c4 & 255) << 24); } + +void applyMetadataRotation(const IVideoCapture& cap, OutputArray mat) +{ + bool rotation_auto = 0 != cap.getProperty(CAP_PROP_ORIENTATION_AUTO); + int rotation_angle = static_cast(cap.getProperty(CAP_PROP_ORIENTATION_META)); + + if(!rotation_auto || rotation_angle%360 == 0) + { + return; + } + + cv::RotateFlags flag; + if(rotation_angle == 90 || rotation_angle == -270) { // Rotate clockwise 90 degrees + flag = cv::ROTATE_90_CLOCKWISE; + } else if(rotation_angle == 270 || rotation_angle == -90) { // Rotate clockwise 270 degrees + flag = cv::ROTATE_90_COUNTERCLOCKWISE; + } else if(rotation_angle == 180 || rotation_angle == -180) { // Rotate clockwise 180 degrees + flag = cv::ROTATE_180; + } else { // Unsupported rotation + return; + } + + cv::rotate(mat, mat, flag); +} + } // namespace cv diff --git a/modules/videoio/src/cap_avfoundation.mm b/modules/videoio/src/cap_avfoundation.mm index 17c5879014..ef13dd6d14 100644 --- a/modules/videoio/src/cap_avfoundation.mm +++ b/modules/videoio/src/cap_avfoundation.mm @@ -162,6 +162,7 @@ private: bool setupReadingAt(CMTime position); IplImage* retrieveFramePixelBuffer(); + int getPreferredOrientationDegrees() const; CMTime mFrameTimestamp; size_t mFrameNum; @@ -1098,6 +1099,13 @@ IplImage* CvCaptureFile::retrieveFramePixelBuffer() { return mOutImage; } +int CvCaptureFile::getPreferredOrientationDegrees() const { + if (mAssetTrack == nil) return 0; + + CGAffineTransform transform = mAssetTrack.preferredTransform; + double radians = atan2(transform.b, transform.a); + return static_cast(round(radians * 180 / M_PI)); +} IplImage* CvCaptureFile::retrieveFrame(int) { return retrieveFramePixelBuffer(); @@ -1129,6 +1137,8 @@ double CvCaptureFile::getProperty(int property_id) const{ return mFormat; case CV_CAP_PROP_FOURCC: return mMode; + case cv::CAP_PROP_ORIENTATION_META: + return getPreferredOrientationDegrees(); default: break; } diff --git a/modules/videoio/src/cap_avfoundation_mac.mm b/modules/videoio/src/cap_avfoundation_mac.mm index f33574104f..bdd4a934f8 100644 --- a/modules/videoio/src/cap_avfoundation_mac.mm +++ b/modules/videoio/src/cap_avfoundation_mac.mm @@ -169,6 +169,7 @@ private: bool setupReadingAt(CMTime position); IplImage* retrieveFramePixelBuffer(); + int getPreferredOrientationDegrees() const; CMTime mFrameTimestamp; size_t mFrameNum; @@ -1064,6 +1065,13 @@ IplImage* CvCaptureFile::retrieveFramePixelBuffer() { return mOutImage; } +int CvCaptureFile::getPreferredOrientationDegrees() const { + if (mAssetTrack == nil) return 0; + + CGAffineTransform transform = mAssetTrack.preferredTransform; + double radians = atan2(transform.b, transform.a); + return static_cast(round(radians * 180 / M_PI)); +} IplImage* CvCaptureFile::retrieveFrame(int) { return retrieveFramePixelBuffer(); @@ -1095,6 +1103,8 @@ double CvCaptureFile::getProperty(int property_id) const{ return mFormat; case CV_CAP_PROP_FOURCC: return mMode; + case cv::CAP_PROP_ORIENTATION_META: + return getPreferredOrientationDegrees(); default: break; } diff --git a/modules/videoio/src/cap_ffmpeg.cpp b/modules/videoio/src/cap_ffmpeg.cpp index 7030d0e653..ed2e4336db 100644 --- a/modules/videoio/src/cap_ffmpeg.cpp +++ b/modules/videoio/src/cap_ffmpeg.cpp @@ -112,7 +112,7 @@ public: } cv::Mat tmp(height, width, CV_MAKETYPE(CV_8U, cn), data, step); - this->rotateFrame(tmp); + applyMetadataRotation(*this, tmp); tmp.copyTo(frame); return true; @@ -137,30 +137,6 @@ public: protected: CvCapture_FFMPEG* ffmpegCapture; - - void rotateFrame(cv::Mat &mat) const - { - bool rotation_auto = 0 != getProperty(CAP_PROP_ORIENTATION_AUTO); - int rotation_angle = static_cast(getProperty(CAP_PROP_ORIENTATION_META)); - - if(!rotation_auto || rotation_angle%360 == 0) - { - return; - } - - cv::RotateFlags flag; - if(rotation_angle == 90 || rotation_angle == -270) { // Rotate clockwise 90 degrees - flag = cv::ROTATE_90_CLOCKWISE; - } else if(rotation_angle == 270 || rotation_angle == -90) { // Rotate clockwise 270 degrees - flag = cv::ROTATE_90_COUNTERCLOCKWISE; - } else if(rotation_angle == 180 || rotation_angle == -180) { // Rotate clockwise 180 degrees - flag = cv::ROTATE_180; - } else { // Unsupported rotation - return; - } - - cv::rotate(mat, mat, flag); - } }; } // namespace diff --git a/modules/videoio/src/cap_interface.hpp b/modules/videoio/src/cap_interface.hpp index 3b3d5398fc..52639f3605 100644 --- a/modules/videoio/src/cap_interface.hpp +++ b/modules/videoio/src/cap_interface.hpp @@ -221,6 +221,8 @@ public: virtual int getCaptureDomain() { return CAP_ANY; } // Return the type of the capture object: CAP_DSHOW, etc... }; +void applyMetadataRotation(const IVideoCapture& cap, OutputArray mat); + class IVideoWriter { public: @@ -249,21 +251,58 @@ class LegacyCapture : public IVideoCapture { private: CvCapture * cap; + bool autorotate; LegacyCapture(const LegacyCapture &); LegacyCapture& operator=(const LegacyCapture &); + + bool shouldSwapWidthHeight() const + { + if (!autorotate) + return false; + int rotation = static_cast(cap->getProperty(cv::CAP_PROP_ORIENTATION_META)); + return std::abs(rotation % 180) == 90; + } + public: - LegacyCapture(CvCapture * cap_) : cap(cap_) {} + LegacyCapture(CvCapture * cap_) : cap(cap_), autorotate(true) {} ~LegacyCapture() { cvReleaseCapture(&cap); } double getProperty(int propId) const CV_OVERRIDE { - return cap ? cap->getProperty(propId) : 0; + if (!cap) + return 0; + + switch(propId) + { + case cv::CAP_PROP_ORIENTATION_AUTO: + return static_cast(autorotate); + + case cv::CAP_PROP_FRAME_WIDTH: + return shouldSwapWidthHeight() ? cap->getProperty(cv::CAP_PROP_FRAME_HEIGHT) : cap->getProperty(cv::CAP_PROP_FRAME_WIDTH); + + case cv::CAP_PROP_FRAME_HEIGHT: + return shouldSwapWidthHeight() ? cap->getProperty(cv::CAP_PROP_FRAME_WIDTH) : cap->getProperty(cv::CAP_PROP_FRAME_HEIGHT); + + default: + return cap->getProperty(propId); + } } bool setProperty(int propId, double value) CV_OVERRIDE { - return cvSetCaptureProperty(cap, propId, value) != 0; + if (!cap) + return false; + + switch(propId) + { + case cv::CAP_PROP_ORIENTATION_AUTO: + autorotate = (value != 0); + return true; + + default: + return cvSetCaptureProperty(cap, propId, value) != 0; + } } bool grabFrame() CV_OVERRIDE { @@ -286,6 +325,7 @@ public: Mat temp = cv::cvarrToMat(_img); flip(temp, image, 0); } + applyMetadataRotation(*this, image); return true; } bool isOpened() const CV_OVERRIDE diff --git a/modules/videoio/test/test_ffmpeg.cpp b/modules/videoio/test/test_ffmpeg.cpp index 3ae27de1b4..cab49dcabb 100644 --- a/modules/videoio/test/test_ffmpeg.cpp +++ b/modules/videoio/test/test_ffmpeg.cpp @@ -475,68 +475,6 @@ const ffmpeg_get_fourcc_param_t ffmpeg_get_fourcc_param[] = INSTANTIATE_TEST_CASE_P(videoio, ffmpeg_get_fourcc, testing::ValuesIn(ffmpeg_get_fourcc_param)); -// related issue: https://github.com/opencv/opencv/issues/15499 -TEST(videoio, mp4_orientation_meta_auto) -{ - if (!videoio_registry::hasBackend(CAP_FFMPEG)) - throw SkipTestException("FFmpeg backend was not found"); - - string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/big_buck_bunny_rotated.mp4"; - - VideoCapture cap; - EXPECT_NO_THROW(cap.open(video_file, CAP_FFMPEG)); - ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << CAP_FFMPEG << std::endl; - - // related issue: https://github.com/opencv/opencv/issues/22088 - EXPECT_EQ(90, cap.get(CAP_PROP_ORIENTATION_META)); - - cap.set(CAP_PROP_ORIENTATION_AUTO, true); - if (cap.get(CAP_PROP_ORIENTATION_AUTO) == 0) - throw SkipTestException("FFmpeg frame rotation metadata is not supported"); - - Size actual; - EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH), - (int)cap.get(CAP_PROP_FRAME_HEIGHT))); - EXPECT_EQ(384, actual.width); - EXPECT_EQ(672, actual.height); - - Mat frame; - - cap >> frame; - - ASSERT_EQ(384, frame.cols); - ASSERT_EQ(672, frame.rows); -} - -// related issue: https://github.com/opencv/opencv/issues/15499 -TEST(videoio, mp4_orientation_no_rotation) -{ - if (!videoio_registry::hasBackend(CAP_FFMPEG)) - throw SkipTestException("FFmpeg backend was not found"); - - string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/big_buck_bunny_rotated.mp4"; - - VideoCapture cap; - EXPECT_NO_THROW(cap.open(video_file, CAP_FFMPEG)); - cap.set(CAP_PROP_ORIENTATION_AUTO, 0); - ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << CAP_FFMPEG << std::endl; - ASSERT_FALSE(cap.get(CAP_PROP_ORIENTATION_AUTO)); - - Size actual; - EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH), - (int)cap.get(CAP_PROP_FRAME_HEIGHT))); - EXPECT_EQ(672, actual.width); - EXPECT_EQ(384, actual.height); - - Mat frame; - - cap >> frame; - - ASSERT_EQ(672, frame.cols); - ASSERT_EQ(384, frame.rows); -} - - static void ffmpeg_check_read_raw(VideoCapture& cap) { ASSERT_TRUE(cap.isOpened()) << "Can't open the video"; diff --git a/modules/videoio/test/test_orientation.cpp b/modules/videoio/test/test_orientation.cpp new file mode 100644 index 0000000000..96530e2a03 --- /dev/null +++ b/modules/videoio/test/test_orientation.cpp @@ -0,0 +1,76 @@ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +#include "test_precomp.hpp" + +using namespace std; + +namespace opencv_test { namespace { + +typedef TestWithParam VideoCaptureAPITests; + +// related issue: https://github.com/opencv/opencv/issues/15499 +TEST_P(VideoCaptureAPITests, mp4_orientation_meta_auto) +{ + cv::VideoCaptureAPIs api = GetParam(); + if (!videoio_registry::hasBackend(api)) + throw SkipTestException("backend " + std::to_string(int(api)) + " was not found"); + + string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/rotated_metadata.mp4"; + + VideoCapture cap; + EXPECT_NO_THROW(cap.open(video_file, api)); + ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << api << std::endl; + + // related issue: https://github.com/opencv/opencv/issues/22088 + EXPECT_EQ(90, cap.get(CAP_PROP_ORIENTATION_META)); + + EXPECT_TRUE(cap.set(CAP_PROP_ORIENTATION_AUTO, true)); + + Size actual; + EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH), + (int)cap.get(CAP_PROP_FRAME_HEIGHT))); + EXPECT_EQ(270, actual.width); + EXPECT_EQ(480, actual.height); + + Mat frame; + + cap >> frame; + + ASSERT_EQ(270, frame.cols); + ASSERT_EQ(480, frame.rows); +} + +// related issue: https://github.com/opencv/opencv/issues/15499 +TEST_P(VideoCaptureAPITests, mp4_orientation_no_rotation) +{ + cv::VideoCaptureAPIs api = GetParam(); + if (!videoio_registry::hasBackend(api)) + throw SkipTestException("backend " + std::to_string(int(api)) + " was not found"); + + string video_file = string(cvtest::TS::ptr()->get_data_path()) + "video/rotated_metadata.mp4"; + + VideoCapture cap; + EXPECT_NO_THROW(cap.open(video_file, api)); + cap.set(CAP_PROP_ORIENTATION_AUTO, 0); + ASSERT_TRUE(cap.isOpened()) << "Can't open the video: " << video_file << " with backend " << api << std::endl; + ASSERT_FALSE(cap.get(CAP_PROP_ORIENTATION_AUTO)); + + Size actual; + EXPECT_NO_THROW(actual = Size((int)cap.get(CAP_PROP_FRAME_WIDTH), + (int)cap.get(CAP_PROP_FRAME_HEIGHT))); + EXPECT_EQ(480, actual.width); + EXPECT_EQ(270, actual.height); + + Mat frame; + + cap >> frame; + + ASSERT_EQ(480, frame.cols); + ASSERT_EQ(270, frame.rows); +} + +INSTANTIATE_TEST_CASE_P(videoio, VideoCaptureAPITests, testing::Values(CAP_FFMPEG, CAP_AVFOUNDATION)); + +}} // namespace