From: Timur Pocheptsov Date: Wed, 4 Mar 2015 15:34:32 +0000 (+0100) Subject: AVCaptureDeviceFormat - avoid duplicates (OS X/iOS) X-Git-Tag: v5.5.90+alpha1~29^2~4 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f839f9e3eb2cd89f2b458349a08f9e5444744f1b;p=platform%2Fupstream%2Fqtmultimedia.git AVCaptureDeviceFormat - avoid duplicates (OS X/iOS) Excluding video range (iOS) is not the right way to avoid "duplicates" - with other devices there can be also duplicates (formats with the same resolutions), but completely different pixel formats. Since we do not know what they will be in advance, we take the media subtype from the initial preset for a capture device and use it as a filter. Update viewfinder and image encoder settings controls. Change-Id: If20aea24b19b43574d5c3e9bf2ba85f50fc08916 Reviewed-by: Yoann Lopes --- diff --git a/src/plugins/avfoundation/camera/avfcamerasession.h b/src/plugins/avfoundation/camera/avfcamerasession.h index 4f418cb..9fd0b18 100644 --- a/src/plugins/avfoundation/camera/avfcamerasession.h +++ b/src/plugins/avfoundation/camera/avfcamerasession.h @@ -80,6 +80,7 @@ public: void addProbe(AVFMediaVideoProbeControl *probe); void removeProbe(AVFMediaVideoProbeControl *probe); + FourCharCode defaultCodec(); public Q_SLOTS: void setState(QCamera::State state); @@ -119,6 +120,7 @@ private: QSet m_videoProbes; QMutex m_videoProbesMutex; + FourCharCode m_defaultCodec; }; QT_END_NAMESPACE diff --git a/src/plugins/avfoundation/camera/avfcamerasession.mm b/src/plugins/avfoundation/camera/avfcamerasession.mm index 98fbb98..af30dd3 100644 --- a/src/plugins/avfoundation/camera/avfcamerasession.mm +++ b/src/plugins/avfoundation/camera/avfcamerasession.mm @@ -41,6 +41,7 @@ #include "avfmediavideoprobecontrol.h" #include "avfcameraviewfindersettingscontrol.h" #include "avfimageencodercontrol.h" +#include "avfcamerautility.h" #include #include @@ -143,6 +144,7 @@ AVFCameraSession::AVFCameraSession(AVFCameraService *service, QObject *parent) , m_active(false) , m_videoInput(nil) , m_audioInput(nil) + , m_defaultCodec(0) { m_captureSession = [[AVCaptureSession alloc] init]; m_observer = [[AVFCameraSessionObserver alloc] initWithCameraSession:this]; @@ -277,6 +279,8 @@ void AVFCameraSession::setState(QCamera::State newState) Q_EMIT readyToConfigureConnections(); [m_captureSession commitConfiguration]; [m_captureSession startRunning]; + m_defaultCodec = 0; + defaultCodec(); applyImageEncoderSettings(); applyViewfinderSettings(); } @@ -407,6 +411,25 @@ void AVFCameraSession::removeProbe(AVFMediaVideoProbeControl *probe) m_videoProbesMutex.unlock(); } +FourCharCode AVFCameraSession::defaultCodec() +{ + if (!m_defaultCodec) { +#if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0) + if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) { + if (AVCaptureDevice *device = videoCaptureDevice()) { + AVCaptureDeviceFormat *format = device.activeFormat; + if (!format || !format.formatDescription) + return m_defaultCodec; + m_defaultCodec = CMVideoFormatDescriptionGetCodecType(format.formatDescription); + } + } +#else + // TODO: extract media subtype. +#endif + } + return m_defaultCodec; +} + void AVFCameraSession::onCameraFrameFetched(const QVideoFrame &frame) { m_videoProbesMutex.lock(); diff --git a/src/plugins/avfoundation/camera/avfcamerautility.h b/src/plugins/avfoundation/camera/avfcamerautility.h index 1c13736..03a6146 100644 --- a/src/plugins/avfoundation/camera/avfcamerautility.h +++ b/src/plugins/avfoundation/camera/avfcamerautility.h @@ -97,13 +97,17 @@ AVFRational qt_float_to_rational(qreal par, int limit); #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0) -bool qt_is_video_range_subtype(AVCaptureDeviceFormat *format); +QVector qt_unique_device_formats(AVCaptureDevice *captureDevice, + FourCharCode preferredFormat); QSize qt_device_format_resolution(AVCaptureDeviceFormat *format); QSize qt_device_format_high_resolution(AVCaptureDeviceFormat *format); QSize qt_device_format_pixel_aspect_ratio(AVCaptureDeviceFormat *format); QVector qt_device_format_framerates(AVCaptureDeviceFormat *format); -AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &res); -AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice, Float64 fps); +AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &res, + FourCharCode preferredFormat); +AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice, + FourCharCode preferredFormat, + Float64 fps); AVFrameRateRange *qt_find_supported_framerate_range(AVCaptureDeviceFormat *format, Float64 fps); #endif diff --git a/src/plugins/avfoundation/camera/avfcamerautility.mm b/src/plugins/avfoundation/camera/avfcamerautility.mm index 554c25c..f8d5647 100644 --- a/src/plugins/avfoundation/camera/avfcamerautility.mm +++ b/src/plugins/avfoundation/camera/avfcamerautility.mm @@ -37,6 +37,7 @@ #include #include +#include #include #include @@ -107,19 +108,6 @@ AVFRational qt_float_to_rational(qreal par, int limit) #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0) -bool qt_is_video_range_subtype(AVCaptureDeviceFormat *format) -{ - Q_ASSERT(format); -#ifdef Q_OS_IOS - // Use only 420f on iOS, not 420v. - const FourCharCode subType = CMFormatDescriptionGetMediaSubType(format.formatDescription); - return subType == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; -#else - Q_UNUSED(format) -#endif - return false; -} - namespace { inline bool qt_area_sane(const QSize &size) @@ -128,40 +116,25 @@ inline bool qt_area_sane(const QSize &size) && std::numeric_limits::max() / size.width() >= size.height(); } -inline bool avf_format_compare(AVCaptureDeviceFormat *f1, AVCaptureDeviceFormat *f2) +struct ResolutionPredicate : std::binary_function { - Q_ASSERT(f1); - Q_ASSERT(f2); - const QSize r1(qt_device_format_resolution(f1)); - const QSize r2(qt_device_format_resolution(f2)); - return r1.width() > r2.width() && r1.height() > r2.height(); -} + bool operator() (AVCaptureDeviceFormat *f1, AVCaptureDeviceFormat *f2)const + { + Q_ASSERT(f1 && f2); + const QSize r1(qt_device_format_resolution(f1)); + const QSize r2(qt_device_format_resolution(f2)); + return r1.width() < r2.width() || (r2.width() == r1.width() && r1.height() < r2.height()); + } +}; -QVector qt_sort_device_formats(AVCaptureDevice *captureDevice) +struct FormatHasNoFPSRange : std::unary_function { - // Select only formats with framerate ranges + sort them by resoluions, - Q_ASSERT(captureDevice); - - QVectorsorted; - - NSArray *formats = captureDevice.formats; - if (!formats || !formats.count) - return sorted; - - sorted.reserve(formats.count); - for (AVCaptureDeviceFormat *format in formats) { - if (qt_is_video_range_subtype(format)) - continue; - if (format.videoSupportedFrameRateRanges && format.videoSupportedFrameRateRanges.count) { - const QSize resolution(qt_device_format_resolution(format)); - if (!resolution.isNull() && resolution.isValid()) - sorted << format; - } + bool operator() (AVCaptureDeviceFormat *format) + { + Q_ASSERT(format); + return !format.videoSupportedFrameRateRanges || !format.videoSupportedFrameRateRanges.count; } - - std::sort(sorted.begin(), sorted.end(), avf_format_compare); - return sorted; -} +}; Float64 qt_find_min_framerate_distance(AVCaptureDeviceFormat *format, Float64 fps) { @@ -180,6 +153,50 @@ Float64 qt_find_min_framerate_distance(AVCaptureDeviceFormat *format, Float64 fp } // Unnamed namespace. +QVector qt_unique_device_formats(AVCaptureDevice *captureDevice, FourCharCode filter) +{ + // 'filter' is the format we prefer if we have duplicates. + Q_ASSERT(captureDevice); + + QVector formats; + + if (!captureDevice.formats || !captureDevice.formats.count) + return formats; + + formats.reserve(captureDevice.formats.count); + for (AVCaptureDeviceFormat *format in captureDevice.formats) { + const QSize resolution(qt_device_format_resolution(format)); + if (resolution.isNull() || !resolution.isValid()) + continue; + formats << format; + } + + if (!formats.size()) + return formats; + + std::sort(formats.begin(), formats.end(), ResolutionPredicate()); + + QSize size(qt_device_format_resolution(formats[0])); + FourCharCode codec = CMVideoFormatDescriptionGetCodecType(formats[0].formatDescription); + int last = 0; + for (int i = 1; i < formats.size(); ++i) { + const QSize nextSize(qt_device_format_resolution(formats[i])); + if (nextSize == size) { + if (codec == filter) + continue; + formats[last] = formats[i]; + } else { + ++last; + formats[last] = formats[i]; + size = nextSize; + } + codec = CMVideoFormatDescriptionGetCodecType(formats[i].formatDescription); + } + formats.resize(last + 1); + + return formats; +} + QSize qt_device_format_resolution(AVCaptureDeviceFormat *format) { Q_ASSERT(format); @@ -246,7 +263,9 @@ QSize qt_device_format_pixel_aspect_ratio(AVCaptureDeviceFormat *format) return QSize(asRatio.first, asRatio.second); } -AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &request) +AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, + const QSize &request, + FourCharCode filter) { Q_ASSERT(captureDevice); Q_ASSERT(!request.isNull() && request.isValid()); @@ -254,9 +273,10 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev if (!captureDevice.formats || !captureDevice.formats.count) return 0; - for (AVCaptureDeviceFormat *format in captureDevice.formats) { - if (qt_is_video_range_subtype(format)) - continue; + QVector formats(qt_unique_device_formats(captureDevice, filter)); + + for (int i = 0; i < formats.size(); ++i) { + AVCaptureDeviceFormat *format = formats[i]; if (qt_device_format_resolution(format) == request) return format; // iOS only (still images). @@ -269,31 +289,30 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev typedef QPair FormatPair; - QVector formats; - formats.reserve(captureDevice.formats.count); + QVector pairs; // default|HR sizes + pairs.reserve(formats.size()); - for (AVCaptureDeviceFormat *format in captureDevice.formats) { - if (qt_is_video_range_subtype(format)) - continue; + for (int i = 0; i < formats.size(); ++i) { + AVCaptureDeviceFormat *format = formats[i]; const QSize res(qt_device_format_resolution(format)); if (!res.isNull() && res.isValid() && qt_area_sane(res)) - formats << FormatPair(res, format); + pairs << FormatPair(res, format); const QSize highRes(qt_device_format_high_resolution(format)); if (!highRes.isNull() && highRes.isValid() && qt_area_sane(highRes)) - formats << FormatPair(highRes, format); + pairs << FormatPair(highRes, format); } - if (!formats.size()) + if (!pairs.size()) return 0; - AVCaptureDeviceFormat *best = formats[0].second; - QSize next(formats[0].first); + AVCaptureDeviceFormat *best = pairs[0].second; + QSize next(pairs[0].first); int wDiff = qAbs(request.width() - next.width()); int hDiff = qAbs(request.height() - next.height()); const int area = request.width() * request.height(); int areaDiff = qAbs(area - next.width() * next.height()); - for (int i = 1; i < formats.size(); ++i) { - next = formats[i].first; + for (int i = 1; i < pairs.size(); ++i) { + next = pairs[i].first; const int newWDiff = qAbs(next.width() - request.width()); const int newHDiff = qAbs(next.height() - request.height()); const int newAreaDiff = qAbs(area - next.width() * next.height()); @@ -302,7 +321,7 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev || ((newWDiff <= wDiff || newHDiff <= hDiff) && newAreaDiff <= areaDiff)) { wDiff = newWDiff; hDiff = newHDiff; - best = formats[i].second; + best = pairs[i].second; areaDiff = newAreaDiff; } } @@ -310,15 +329,21 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev return best; } -AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice, Float64 fps) +AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice, + FourCharCode filter, + Float64 fps) { Q_ASSERT(captureDevice); Q_ASSERT(fps > 0.); const qreal epsilon = 0.1; - // Sort formats by their resolution. - const QVector sorted(qt_sort_device_formats(captureDevice)); + QVectorsorted(qt_unique_device_formats(captureDevice, filter)); + // Sort formats by their resolution in decreasing order: + std::sort(sorted.begin(), sorted.end(), std::not2(ResolutionPredicate())); + // We can use only formats with framerate ranges: + sorted.erase(std::remove_if(sorted.begin(), sorted.end(), FormatHasNoFPSRange()), sorted.end()); + if (!sorted.size()) return nil; diff --git a/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm b/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm index c5aa573..250aae9 100644 --- a/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm +++ b/src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm @@ -279,9 +279,11 @@ QList AVFCameraViewfinderSettingsControl2::supportedV return supportedSettings; } - for (AVCaptureDeviceFormat *format in m_captureDevice.formats) { - if (qt_is_video_range_subtype(format)) - continue; + const QVector formats(qt_unique_device_formats(m_captureDevice, + m_session->defaultCodec())); + for (int i = 0; i < formats.size(); ++i) { + AVCaptureDeviceFormat *format = formats[i]; + const QSize res(qt_device_format_resolution(format)); if (res.isNull() || !res.isValid()) continue; @@ -435,12 +437,14 @@ AVCaptureDeviceFormat *AVFCameraViewfinderSettingsControl2::findBestFormatMatch( #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0) if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) { Q_ASSERT(m_captureDevice); + Q_ASSERT(m_session); const QSize &resolution = settings.resolution(); if (!resolution.isNull() && resolution.isValid()) { // Either the exact match (including high resolution for images on iOS) // or a format with a resolution close to the requested one. - return qt_find_best_resolution_match(m_captureDevice, resolution); + return qt_find_best_resolution_match(m_captureDevice, resolution, + m_session->defaultCodec()); } // No resolution requested, what about framerates? @@ -453,7 +457,8 @@ AVCaptureDeviceFormat *AVFCameraViewfinderSettingsControl2::findBestFormatMatch( const qreal minFPS(settings.minimumFrameRate()); const qreal maxFPS(settings.maximumFrameRate()); if (minFPS || maxFPS) - return qt_find_best_framerate_match(m_captureDevice, maxFPS ? maxFPS : minFPS); + return qt_find_best_framerate_match(m_captureDevice, maxFPS ? maxFPS : minFPS, + m_session->defaultCodec()); // Ignore PAR for the moment (PAR without resolution can // pick a format with really bad resolution). // No need to test pixel format, just return settings. diff --git a/src/plugins/avfoundation/camera/avfimageencodercontrol.mm b/src/plugins/avfoundation/camera/avfimageencodercontrol.mm index ea25665..36050c3 100644 --- a/src/plugins/avfoundation/camera/avfimageencodercontrol.mm +++ b/src/plugins/avfoundation/camera/avfimageencodercontrol.mm @@ -48,20 +48,6 @@ QT_BEGIN_NAMESPACE -QSize qt_image_high_resolution(AVCaptureDeviceFormat *format) -{ - Q_ASSERT(format); - QSize res; -#if defined(Q_OS_IOS) && QT_IOS_PLATFORM_SDK_EQUAL_OR_ABOVE(__IPHONE_8_0) - if (QSysInfo::MacintoshVersion >= QSysInfo::MV_IOS_8_0) { - const CMVideoDimensions hrDim(format.highResolutionStillImageDimensions); - res.setWidth(hrDim.width); - res.setHeight(hrDim.height); - } -#endif - return res; -} - AVFImageEncoderControl::AVFImageEncoderControl(AVFCameraService *service) : m_service(service) { @@ -94,9 +80,11 @@ QList AVFImageEncoderControl::supportedResolutions(const QImageEncoderSet #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0) if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) { AVCaptureDevice *captureDevice = m_service->session()->videoCaptureDevice(); - for (AVCaptureDeviceFormat *format in captureDevice.formats) { - if (qt_is_video_range_subtype(format)) - continue; + const QVector formats(qt_unique_device_formats(captureDevice, + m_service->session()->defaultCodec())); + + for (int i = 0; i < formats.size(); ++i) { + AVCaptureDeviceFormat *format = formats[i]; const QSize res(qt_device_format_resolution(format)); if (!res.isNull() && res.isValid()) @@ -108,7 +96,7 @@ QList AVFImageEncoderControl::supportedResolutions(const QImageEncoderSet // its source AVCaptureDevice instance’s activeFormat.formatDescription. However, // if you set this property to YES, the receiver emits still images at the capture // device’s highResolutionStillImageDimensions value. - const QSize hrRes(qt_image_high_resolution(format)); + const QSize hrRes(qt_device_format_high_resolution(format)); if (!hrRes.isNull() && hrRes.isValid()) resolutions << res; } @@ -152,7 +140,7 @@ QImageEncoderSettings AVFImageEncoderControl::imageSettings() const AVCaptureStillImageOutput *stillImageOutput = m_service->imageCaptureControl()->stillImageOutput(); if (stillImageOutput.highResolutionStillImageOutputEnabled) - res = qt_image_high_resolution(captureDevice.activeFormat); + res = qt_device_format_high_resolution(captureDevice.activeFormat); } #endif if (res.isNull() || !res.isValid()) { @@ -179,7 +167,6 @@ void AVFImageEncoderControl::setImageSettings(const QImageEncoderSettings &setti return; m_settings = settings; - applySettings(); } @@ -223,7 +210,8 @@ void AVFImageEncoderControl::applySettings() #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0) if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) { AVCaptureDevice *captureDevice = m_service->session()->videoCaptureDevice(); - AVCaptureDeviceFormat *match = qt_find_best_resolution_match(captureDevice, res); + AVCaptureDeviceFormat *match = qt_find_best_resolution_match(captureDevice, res, + m_service->session()->defaultCodec()); if (!match) { qDebugCamera() << Q_FUNC_INFO << "unsupported resolution:" << res; @@ -242,7 +230,7 @@ void AVFImageEncoderControl::applySettings() #if defined(Q_OS_IOS) && QT_IOS_PLATFORM_SDK_EQUAL_OR_ABOVE(__IPHONE_8_0) if (QSysInfo::MacintoshVersion >= QSysInfo::MV_IOS_8_0) { AVCaptureStillImageOutput *imageOutput = m_service->imageCaptureControl()->stillImageOutput(); - if (res == qt_image_high_resolution(captureDevice.activeFormat)) + if (res == qt_device_format_high_resolution(captureDevice.activeFormat)) imageOutput.highResolutionStillImageOutputEnabled = YES; else imageOutput.highResolutionStillImageOutputEnabled = NO;