AVCaptureDeviceFormat - avoid duplicates (OS X/iOS)
authorTimur Pocheptsov <Timur.Pocheptsov@digia.com>
Wed, 4 Mar 2015 15:34:32 +0000 (16:34 +0100)
committerTimur Pocheptsov <Timur.Pocheptsov@digia.com>
Tue, 10 Mar 2015 16:35:59 +0000 (16:35 +0000)
Excluding video range (iOS) is not the right way to avoid "duplicates" - with
other devices there can be also duplicates (formats with the same resolutions),
but completely different pixel formats. Since we do not know what they will be in advance,
we take the media subtype from the initial preset for a capture device and use it
as a filter. Update viewfinder and image encoder settings controls.

Change-Id: If20aea24b19b43574d5c3e9bf2ba85f50fc08916
Reviewed-by: Yoann Lopes <yoann.lopes@theqtcompany.com>
src/plugins/avfoundation/camera/avfcamerasession.h
src/plugins/avfoundation/camera/avfcamerasession.mm
src/plugins/avfoundation/camera/avfcamerautility.h
src/plugins/avfoundation/camera/avfcamerautility.mm
src/plugins/avfoundation/camera/avfcameraviewfindersettingscontrol.mm
src/plugins/avfoundation/camera/avfimageencodercontrol.mm

index 4f418cb..9fd0b18 100644 (file)
@@ -80,6 +80,7 @@ public:
 
     void addProbe(AVFMediaVideoProbeControl *probe);
     void removeProbe(AVFMediaVideoProbeControl *probe);
+    FourCharCode defaultCodec();
 
 public Q_SLOTS:
     void setState(QCamera::State state);
@@ -119,6 +120,7 @@ private:
     QSet<AVFMediaVideoProbeControl *> m_videoProbes;
     QMutex m_videoProbesMutex;
 
+    FourCharCode m_defaultCodec;
 };
 
 QT_END_NAMESPACE
index 98fbb98..af30dd3 100644 (file)
@@ -41,6 +41,7 @@
 #include "avfmediavideoprobecontrol.h"
 #include "avfcameraviewfindersettingscontrol.h"
 #include "avfimageencodercontrol.h"
+#include "avfcamerautility.h"
 
 #include <CoreFoundation/CoreFoundation.h>
 #include <Foundation/Foundation.h>
@@ -143,6 +144,7 @@ AVFCameraSession::AVFCameraSession(AVFCameraService *service, QObject *parent)
    , m_active(false)
    , m_videoInput(nil)
    , m_audioInput(nil)
+   , m_defaultCodec(0)
 {
     m_captureSession = [[AVCaptureSession alloc] init];
     m_observer = [[AVFCameraSessionObserver alloc] initWithCameraSession:this];
@@ -277,6 +279,8 @@ void AVFCameraSession::setState(QCamera::State newState)
         Q_EMIT readyToConfigureConnections();
         [m_captureSession commitConfiguration];
         [m_captureSession startRunning];
+        m_defaultCodec = 0;
+        defaultCodec();
         applyImageEncoderSettings();
         applyViewfinderSettings();
     }
@@ -407,6 +411,25 @@ void AVFCameraSession::removeProbe(AVFMediaVideoProbeControl *probe)
     m_videoProbesMutex.unlock();
 }
 
+FourCharCode AVFCameraSession::defaultCodec()
+{
+    if (!m_defaultCodec) {
+#if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
+        if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) {
+            if (AVCaptureDevice *device = videoCaptureDevice()) {
+                AVCaptureDeviceFormat *format = device.activeFormat;
+                if (!format || !format.formatDescription)
+                    return m_defaultCodec;
+                m_defaultCodec = CMVideoFormatDescriptionGetCodecType(format.formatDescription);
+            }
+        }
+#else
+    // TODO: extract media subtype.
+#endif
+    }
+    return m_defaultCodec;
+}
+
 void AVFCameraSession::onCameraFrameFetched(const QVideoFrame &frame)
 {
     m_videoProbesMutex.lock();
index 1c13736..03a6146 100644 (file)
@@ -97,13 +97,17 @@ AVFRational qt_float_to_rational(qreal par, int limit);
 
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
 
-bool qt_is_video_range_subtype(AVCaptureDeviceFormat *format);
+QVector<AVCaptureDeviceFormat *> qt_unique_device_formats(AVCaptureDevice *captureDevice,
+                                                          FourCharCode preferredFormat);
 QSize qt_device_format_resolution(AVCaptureDeviceFormat *format);
 QSize qt_device_format_high_resolution(AVCaptureDeviceFormat *format);
 QSize qt_device_format_pixel_aspect_ratio(AVCaptureDeviceFormat *format);
 QVector<AVFPSRange> qt_device_format_framerates(AVCaptureDeviceFormat *format);
-AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &res);
-AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice, Float64 fps);
+AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &res,
+                                                     FourCharCode preferredFormat);
+AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice,
+                                                    FourCharCode preferredFormat,
+                                                    Float64 fps);
 AVFrameRateRange *qt_find_supported_framerate_range(AVCaptureDeviceFormat *format, Float64 fps);
 
 #endif
index 554c25c..f8d5647 100644 (file)
@@ -37,6 +37,7 @@
 #include <QtCore/qvector.h>
 #include <QtCore/qpair.h>
 
+#include <functional>
 #include <algorithm>
 #include <limits>
 
@@ -107,19 +108,6 @@ AVFRational qt_float_to_rational(qreal par, int limit)
 
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
 
-bool qt_is_video_range_subtype(AVCaptureDeviceFormat *format)
-{
-    Q_ASSERT(format);
-#ifdef Q_OS_IOS
-    // Use only 420f on iOS, not 420v.
-    const FourCharCode subType = CMFormatDescriptionGetMediaSubType(format.formatDescription);
-    return subType == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
-#else
-    Q_UNUSED(format)
-#endif
-    return false;
-}
-
 namespace {
 
 inline bool qt_area_sane(const QSize &size)
@@ -128,40 +116,25 @@ inline bool qt_area_sane(const QSize &size)
            && std::numeric_limits<int>::max() / size.width() >= size.height();
 }
 
-inline bool avf_format_compare(AVCaptureDeviceFormat *f1, AVCaptureDeviceFormat *f2)
+struct ResolutionPredicate : std::binary_function<AVCaptureDeviceFormat *, AVCaptureDeviceFormat *, bool>
 {
-    Q_ASSERT(f1);
-    Q_ASSERT(f2);
-    const QSize r1(qt_device_format_resolution(f1));
-    const QSize r2(qt_device_format_resolution(f2));
-    return r1.width() > r2.width() && r1.height() > r2.height();
-}
+    bool operator() (AVCaptureDeviceFormat *f1, AVCaptureDeviceFormat *f2)const
+    {
+        Q_ASSERT(f1 && f2);
+        const QSize r1(qt_device_format_resolution(f1));
+        const QSize r2(qt_device_format_resolution(f2));
+        return r1.width() < r2.width() || (r2.width() == r1.width() && r1.height() < r2.height());
+    }
+};
 
-QVector<AVCaptureDeviceFormat *> qt_sort_device_formats(AVCaptureDevice *captureDevice)
+struct FormatHasNoFPSRange : std::unary_function<AVCaptureDeviceFormat *, bool>
 {
-    // Select only formats with framerate ranges + sort them by resoluions,
-    Q_ASSERT(captureDevice);
-
-    QVector<AVCaptureDeviceFormat *>sorted;
-
-    NSArray *formats = captureDevice.formats;
-    if (!formats || !formats.count)
-        return sorted;
-
-    sorted.reserve(formats.count);
-    for (AVCaptureDeviceFormat *format in formats) {
-        if (qt_is_video_range_subtype(format))
-            continue;
-        if (format.videoSupportedFrameRateRanges && format.videoSupportedFrameRateRanges.count) {
-            const QSize resolution(qt_device_format_resolution(format));
-            if (!resolution.isNull() && resolution.isValid())
-                sorted << format;
-        }
+    bool operator() (AVCaptureDeviceFormat *format)
+    {
+        Q_ASSERT(format);
+        return !format.videoSupportedFrameRateRanges || !format.videoSupportedFrameRateRanges.count;
     }
-
-    std::sort(sorted.begin(), sorted.end(), avf_format_compare);
-    return sorted;
-}
+};
 
 Float64 qt_find_min_framerate_distance(AVCaptureDeviceFormat *format, Float64 fps)
 {
@@ -180,6 +153,50 @@ Float64 qt_find_min_framerate_distance(AVCaptureDeviceFormat *format, Float64 fp
 
 } // Unnamed namespace.
 
+QVector<AVCaptureDeviceFormat *> qt_unique_device_formats(AVCaptureDevice *captureDevice, FourCharCode filter)
+{
+    // 'filter' is the format we prefer if we have duplicates.
+    Q_ASSERT(captureDevice);
+
+    QVector<AVCaptureDeviceFormat *> formats;
+
+    if (!captureDevice.formats || !captureDevice.formats.count)
+        return formats;
+
+    formats.reserve(captureDevice.formats.count);
+    for (AVCaptureDeviceFormat *format in captureDevice.formats) {
+        const QSize resolution(qt_device_format_resolution(format));
+        if (resolution.isNull() || !resolution.isValid())
+            continue;
+        formats << format;
+    }
+
+    if (!formats.size())
+        return formats;
+
+    std::sort(formats.begin(), formats.end(), ResolutionPredicate());
+
+    QSize size(qt_device_format_resolution(formats[0]));
+    FourCharCode codec = CMVideoFormatDescriptionGetCodecType(formats[0].formatDescription);
+    int last = 0;
+    for (int i = 1; i < formats.size(); ++i) {
+        const QSize nextSize(qt_device_format_resolution(formats[i]));
+        if (nextSize == size) {
+            if (codec == filter)
+                continue;
+            formats[last] = formats[i];
+        } else {
+            ++last;
+            formats[last] = formats[i];
+            size = nextSize;
+        }
+        codec = CMVideoFormatDescriptionGetCodecType(formats[i].formatDescription);
+    }
+    formats.resize(last + 1);
+
+    return formats;
+}
+
 QSize qt_device_format_resolution(AVCaptureDeviceFormat *format)
 {
     Q_ASSERT(format);
@@ -246,7 +263,9 @@ QSize qt_device_format_pixel_aspect_ratio(AVCaptureDeviceFormat *format)
     return QSize(asRatio.first, asRatio.second);
 }
 
-AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, const QSize &request)
+AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice,
+                                                     const QSize &request,
+                                                     FourCharCode filter)
 {
     Q_ASSERT(captureDevice);
     Q_ASSERT(!request.isNull() && request.isValid());
@@ -254,9 +273,10 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev
     if (!captureDevice.formats || !captureDevice.formats.count)
         return 0;
 
-    for (AVCaptureDeviceFormat *format in captureDevice.formats) {
-        if (qt_is_video_range_subtype(format))
-            continue;
+    QVector<AVCaptureDeviceFormat *> formats(qt_unique_device_formats(captureDevice, filter));
+
+    for (int i = 0; i < formats.size(); ++i) {
+        AVCaptureDeviceFormat *format = formats[i];
         if (qt_device_format_resolution(format) == request)
             return format;
         // iOS only (still images).
@@ -269,31 +289,30 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev
 
     typedef QPair<QSize, AVCaptureDeviceFormat *> FormatPair;
 
-    QVector<FormatPair> formats;
-    formats.reserve(captureDevice.formats.count);
+    QVector<FormatPair> pairs; // default|HR sizes
+    pairs.reserve(formats.size());
 
-    for (AVCaptureDeviceFormat *format in captureDevice.formats) {
-        if (qt_is_video_range_subtype(format))
-            continue;
+    for (int i = 0; i < formats.size(); ++i) {
+        AVCaptureDeviceFormat *format = formats[i];
         const QSize res(qt_device_format_resolution(format));
         if (!res.isNull() && res.isValid() && qt_area_sane(res))
-            formats << FormatPair(res, format);
+            pairs << FormatPair(res, format);
         const QSize highRes(qt_device_format_high_resolution(format));
         if (!highRes.isNull() && highRes.isValid() && qt_area_sane(highRes))
-            formats << FormatPair(highRes, format);
+            pairs << FormatPair(highRes, format);
     }
 
-    if (!formats.size())
+    if (!pairs.size())
         return 0;
 
-    AVCaptureDeviceFormat *best = formats[0].second;
-    QSize next(formats[0].first);
+    AVCaptureDeviceFormat *best = pairs[0].second;
+    QSize next(pairs[0].first);
     int wDiff = qAbs(request.width() - next.width());
     int hDiff = qAbs(request.height() - next.height());
     const int area = request.width() * request.height();
     int areaDiff = qAbs(area - next.width() * next.height());
-    for (int i = 1; i < formats.size(); ++i) {
-        next = formats[i].first;
+    for (int i = 1; i < pairs.size(); ++i) {
+        next = pairs[i].first;
         const int newWDiff = qAbs(next.width() - request.width());
         const int newHDiff = qAbs(next.height() - request.height());
         const int newAreaDiff = qAbs(area - next.width() * next.height());
@@ -302,7 +321,7 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev
             || ((newWDiff <= wDiff || newHDiff <= hDiff) && newAreaDiff <= areaDiff)) {
             wDiff = newWDiff;
             hDiff = newHDiff;
-            best = formats[i].second;
+            best = pairs[i].second;
             areaDiff = newAreaDiff;
         }
     }
@@ -310,15 +329,21 @@ AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDev
     return best;
 }
 
-AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice, Float64 fps)
+AVCaptureDeviceFormat *qt_find_best_framerate_match(AVCaptureDevice *captureDevice,
+                                                    FourCharCode filter,
+                                                    Float64 fps)
 {
     Q_ASSERT(captureDevice);
     Q_ASSERT(fps > 0.);
 
     const qreal epsilon = 0.1;
 
-    // Sort formats by their resolution.
-    const QVector<AVCaptureDeviceFormat *> sorted(qt_sort_device_formats(captureDevice));
+    QVector<AVCaptureDeviceFormat *>sorted(qt_unique_device_formats(captureDevice, filter));
+    // Sort formats by their resolution in decreasing order:
+    std::sort(sorted.begin(), sorted.end(), std::not2(ResolutionPredicate()));
+    // We can use only formats with framerate ranges:
+    sorted.erase(std::remove_if(sorted.begin(), sorted.end(), FormatHasNoFPSRange()), sorted.end());
+
     if (!sorted.size())
         return nil;
 
index c5aa573..250aae9 100644 (file)
@@ -279,9 +279,11 @@ QList<QCameraViewfinderSettings> AVFCameraViewfinderSettingsControl2::supportedV
             return supportedSettings;
         }
 
-        for (AVCaptureDeviceFormat *format in m_captureDevice.formats) {
-            if (qt_is_video_range_subtype(format))
-                continue;
+        const QVector<AVCaptureDeviceFormat *> formats(qt_unique_device_formats(m_captureDevice,
+                                                       m_session->defaultCodec()));
+        for (int i = 0; i < formats.size(); ++i) {
+            AVCaptureDeviceFormat *format = formats[i];
+
             const QSize res(qt_device_format_resolution(format));
             if (res.isNull() || !res.isValid())
                 continue;
@@ -435,12 +437,14 @@ AVCaptureDeviceFormat *AVFCameraViewfinderSettingsControl2::findBestFormatMatch(
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
     if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) {
         Q_ASSERT(m_captureDevice);
+        Q_ASSERT(m_session);
 
         const QSize &resolution = settings.resolution();
         if (!resolution.isNull() && resolution.isValid()) {
             // Either the exact match (including high resolution for images on iOS)
             // or a format with a resolution close to the requested one.
-            return qt_find_best_resolution_match(m_captureDevice, resolution);
+            return qt_find_best_resolution_match(m_captureDevice, resolution,
+                                                 m_session->defaultCodec());
         }
 
         // No resolution requested, what about framerates?
@@ -453,7 +457,8 @@ AVCaptureDeviceFormat *AVFCameraViewfinderSettingsControl2::findBestFormatMatch(
         const qreal minFPS(settings.minimumFrameRate());
         const qreal maxFPS(settings.maximumFrameRate());
         if (minFPS || maxFPS)
-            return qt_find_best_framerate_match(m_captureDevice, maxFPS ? maxFPS : minFPS);
+            return qt_find_best_framerate_match(m_captureDevice, maxFPS ? maxFPS : minFPS,
+                                                m_session->defaultCodec());
         // Ignore PAR for the moment (PAR without resolution can
         // pick a format with really bad resolution).
         // No need to test pixel format, just return settings.
index ea25665..36050c3 100644 (file)
 
 QT_BEGIN_NAMESPACE
 
-QSize qt_image_high_resolution(AVCaptureDeviceFormat *format)
-{
-    Q_ASSERT(format);
-    QSize res;
-#if defined(Q_OS_IOS) && QT_IOS_PLATFORM_SDK_EQUAL_OR_ABOVE(__IPHONE_8_0)
-    if (QSysInfo::MacintoshVersion >= QSysInfo::MV_IOS_8_0) {
-        const CMVideoDimensions hrDim(format.highResolutionStillImageDimensions);
-        res.setWidth(hrDim.width);
-        res.setHeight(hrDim.height);
-    }
-#endif
-    return res;
-}
-
 AVFImageEncoderControl::AVFImageEncoderControl(AVFCameraService *service)
     : m_service(service)
 {
@@ -94,9 +80,11 @@ QList<QSize> AVFImageEncoderControl::supportedResolutions(const QImageEncoderSet
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
     if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) {
         AVCaptureDevice *captureDevice = m_service->session()->videoCaptureDevice();
-        for (AVCaptureDeviceFormat *format in captureDevice.formats) {
-            if (qt_is_video_range_subtype(format))
-                continue;
+        const QVector<AVCaptureDeviceFormat *> formats(qt_unique_device_formats(captureDevice,
+                                                       m_service->session()->defaultCodec()));
+
+        for (int i = 0; i < formats.size(); ++i) {
+            AVCaptureDeviceFormat *format = formats[i];
 
             const QSize res(qt_device_format_resolution(format));
             if (!res.isNull() && res.isValid())
@@ -108,7 +96,7 @@ QList<QSize> AVFImageEncoderControl::supportedResolutions(const QImageEncoderSet
                 // its source AVCaptureDevice instance’s activeFormat.formatDescription. However,
                 // if you set this property to YES, the receiver emits still images at the capture
                 // device’s highResolutionStillImageDimensions value.
-                const QSize hrRes(qt_image_high_resolution(format));
+                const QSize hrRes(qt_device_format_high_resolution(format));
                 if (!hrRes.isNull() && hrRes.isValid())
                     resolutions << res;
             }
@@ -152,7 +140,7 @@ QImageEncoderSettings AVFImageEncoderControl::imageSettings() const
 
             AVCaptureStillImageOutput *stillImageOutput = m_service->imageCaptureControl()->stillImageOutput();
             if (stillImageOutput.highResolutionStillImageOutputEnabled)
-                res = qt_image_high_resolution(captureDevice.activeFormat);
+                res = qt_device_format_high_resolution(captureDevice.activeFormat);
         }
 #endif
         if (res.isNull() || !res.isValid()) {
@@ -179,7 +167,6 @@ void AVFImageEncoderControl::setImageSettings(const QImageEncoderSettings &setti
         return;
 
     m_settings = settings;
-
     applySettings();
 }
 
@@ -223,7 +210,8 @@ void AVFImageEncoderControl::applySettings()
 #if QT_MAC_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_10_7, __IPHONE_7_0)
     if (QSysInfo::MacintoshVersion >= qt_OS_limit(QSysInfo::MV_10_7, QSysInfo::MV_IOS_7_0)) {
         AVCaptureDevice *captureDevice = m_service->session()->videoCaptureDevice();
-        AVCaptureDeviceFormat *match = qt_find_best_resolution_match(captureDevice, res);
+        AVCaptureDeviceFormat *match = qt_find_best_resolution_match(captureDevice, res,
+                                                                     m_service->session()->defaultCodec());
 
         if (!match) {
             qDebugCamera() << Q_FUNC_INFO << "unsupported resolution:" << res;
@@ -242,7 +230,7 @@ void AVFImageEncoderControl::applySettings()
 #if defined(Q_OS_IOS) && QT_IOS_PLATFORM_SDK_EQUAL_OR_ABOVE(__IPHONE_8_0)
         if (QSysInfo::MacintoshVersion >= QSysInfo::MV_IOS_8_0) {
             AVCaptureStillImageOutput *imageOutput = m_service->imageCaptureControl()->stillImageOutput();
-            if (res == qt_image_high_resolution(captureDevice.activeFormat))
+            if (res == qt_device_format_high_resolution(captureDevice.activeFormat))
                 imageOutput.highResolutionStillImageOutputEnabled = YES;
             else
                 imageOutput.highResolutionStillImageOutputEnabled = NO;