vtenc: Disable HW acceleration for interlaced ProRes

Due to a bug in the VT API, attempting to encode interlaced content
with ProRes results in an error, halting the pipeline instead of
gracefully falling back to software encoding.
Should be removed in the future if Apple ever fixes this issue.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3878>

diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c
index 3138f25..218e218 100644 (file)
--- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c
+++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c
@@ -1208,11 +1208,22 @@ gst_vtenc_create_session (GstVTEnc * self)
   const GstVTEncoderDetails *codec_details =
       GST_VTENC_CLASS_GET_CODEC_DETAILS (G_OBJECT_GET_CLASS (self));
 
+  /* Apple's M1 hardware encoding fails when provided with an interlaced ProRes source.
+   * It's most likely a bug in VideoToolbox, as no such limitation has been officially mentioned anywhere.
+   * For now let's disable HW encoding entirely when such case occurs. */
+  gboolean enable_hw = !(GST_VIDEO_INFO_IS_INTERLACED (&self->video_info)
+      && codec_details->format_id == GST_kCMVideoCodecType_Some_AppleProRes);
+
+  if (!enable_hw)
+    GST_WARNING_OBJECT (self,
+        "Interlaced content detected, disabling HW-accelerated encoding due to https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1429");
+
   encoder_spec =
       CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
       &kCFTypeDictionaryValueCallBacks);
   gst_vtutil_dict_set_boolean (encoder_spec,
-      kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, true);
+      kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
+      enable_hw);
   if (codec_details->require_hardware)
     gst_vtutil_dict_set_boolean (encoder_spec,
         kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,