/* leave some headroom for new GstVideoCodecFrameFlags flags */
VTDEC_FRAME_FLAG_SKIP = (1 << 10),
VTDEC_FRAME_FLAG_DROP = (1 << 11),
+ VTDEC_FRAME_FLAG_ERROR = (1 << 12),
};
static void gst_vtdec_finalize (GObject * object);
GST_STATIC_CAPS ("video/x-h264, stream-format=avc, alignment=au,"
" width=(int)[1, MAX], height=(int)[1, MAX];"
"video/mpeg, mpegversion=2, systemstream=false, parsed=true;"
- "image/jpeg")
+ "image/jpeg;"
+ "video/x-prores, variant = { (string)standard, (string)hq, (string)lt,"
+ " (string)proxy, (string)4444, (string)4444xq };")
);
/* define EnableHardwareAcceleratedVideoDecoder in < 10.9 */
CFSTR ("RequireHardwareAcceleratedVideoDecoder");
#endif
-#if defined(APPLEMEDIA_MOLTENVK)
-#define VIDEO_SRC_CAPS \
- GST_VIDEO_CAPS_MAKE("NV12") ";" \
- GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
- "NV12") ", " \
- "texture-target = (string) rectangle ; " \
- GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,\
- "NV12")
-#else
-#define VIDEO_SRC_CAPS \
- GST_VIDEO_CAPS_MAKE("NV12") ";" \
+#define VIDEO_SRC_CAPS_FORMATS "{ NV12, AYUV64 }"
+
+#define VIDEO_SRC_CAPS_NATIVE \
+ GST_VIDEO_CAPS_MAKE(VIDEO_SRC_CAPS_FORMATS) ";" \
GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
- "NV12") ", " \
+ VIDEO_SRC_CAPS_FORMATS) ", " \
"texture-target = (string) rectangle "
+
+#if defined(APPLEMEDIA_MOLTENVK)
+#define VIDEO_SRC_CAPS VIDEO_SRC_CAPS_NATIVE "; " \
+ GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, \
+ VIDEO_SRC_CAPS_FORMATS)
+#else
+#define VIDEO_SRC_CAPS VIDEO_SRC_CAPS_NATIVE
#endif
G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER);
}
static void
-setup_texture_cache (GstVtdec * vtdec)
+setup_texture_cache (GstVtdec * vtdec, GstVideoFormat format)
{
GstVideoCodecState *output_state;
+ GST_INFO_OBJECT (vtdec, "setting up texture cache");
output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
- gst_video_texture_cache_set_format (vtdec->texture_cache,
- GST_VIDEO_FORMAT_NV12, output_state->caps);
+ gst_video_texture_cache_set_format (vtdec->texture_cache, format,
+ output_state->caps);
gst_video_codec_state_unref (output_state);
}
+/*
+ * Unconditionally output a high bit-depth + alpha format when decoding Apple
+ * ProRes video if downstream supports it.
+ * TODO: read src_pix_fmt to get the preferred output format
+ * https://wiki.multimedia.cx/index.php/Apple_ProRes#Frame_header
+ */
+static GstVideoFormat
+get_preferred_video_format (GstStructure * s, gboolean prores)
+{
+ const GValue *list = gst_structure_get_value (s, "format");
+ guint i, size = gst_value_list_get_size (list);
+ for (i = 0; i < size; i++) {
+ const GValue *value = gst_value_list_get_value (list, i);
+ const char *fmt = g_value_get_string (value);
+ GstVideoFormat vfmt = gst_video_format_from_string (fmt);
+ switch (vfmt) {
+ case GST_VIDEO_FORMAT_NV12:
+ if (!prores)
+ return vfmt;
+ break;
+ case GST_VIDEO_FORMAT_AYUV64:
+ if (prores)
+ return vfmt;
+ break;
+ default:
+ break;
+ }
+ }
+ return GST_VIDEO_FORMAT_UNKNOWN;
+}
+
static gboolean
gst_vtdec_negotiate (GstVideoDecoder * decoder)
{
GstVideoCodecState *output_state = NULL;
GstCaps *peercaps = NULL, *caps = NULL, *templcaps = NULL, *prevcaps = NULL;
- GstVideoFormat format;
- GstStructure *structure;
- const gchar *s;
+ GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN;
GstVtdec *vtdec;
OSStatus err = noErr;
GstCapsFeatures *features = NULL;
gst_caps_unref (peercaps);
caps = gst_caps_truncate (gst_caps_make_writable (caps));
- structure = gst_caps_get_structure (caps, 0);
- s = gst_structure_get_string (structure, "format");
- format = gst_video_format_from_string (s);
+
+ /* Try to use whatever video format downstream prefers */
+ {
+ GstStructure *s = gst_caps_get_structure (caps, 0);
+
+ if (gst_structure_has_field_typed (s, "format", GST_TYPE_LIST)) {
+ GstStructure *is = gst_caps_get_structure (vtdec->input_state->caps, 0);
+ const char *name = gst_structure_get_name (is);
+ format = get_preferred_video_format (s,
+ g_strcmp0 (name, "video/x-prores") == 0);
+ }
+
+ if (format == GST_VIDEO_FORMAT_UNKNOWN) {
+ const char *fmt;
+ gst_structure_fixate_field (s, "format");
+ fmt = gst_structure_get_string (s, "format");
+ if (fmt)
+ format = gst_video_format_from_string (fmt);
+ else
+ /* If all fails, just use NV12 */
+ format = GST_VIDEO_FORMAT_NV12;
+ }
+ }
+
features = gst_caps_get_features (caps, 0);
if (features)
features = gst_caps_features_copy (features);
if (!vtdec->texture_cache) {
vtdec->texture_cache =
gst_video_texture_cache_gl_new (vtdec->ctxh->context);
- setup_texture_cache (vtdec);
+ setup_texture_cache (vtdec, format);
}
}
#if defined(APPLEMEDIA_MOLTENVK)
if (!vtdec->texture_cache) {
vtdec->texture_cache =
gst_video_texture_cache_vulkan_new (vtdec->device);
- setup_texture_cache (vtdec);
+ setup_texture_cache (vtdec, format);
}
}
#endif
cm_format = kCMVideoCodecType_MPEG2Video;
} else if (!strcmp (caps_name, "image/jpeg")) {
cm_format = kCMVideoCodecType_JPEG;
+ } else if (!strcmp (caps_name, "video/x-prores")) {
+ const char *variant = gst_structure_get_string (structure, "variant");
+
+ if (variant)
+ cm_format = gst_vtutil_codec_type_from_prores_variant (variant);
+
+ if (cm_format == GST_kCMVideoCodecType_Some_AppleProRes) {
+ GST_ERROR_OBJECT (vtdec, "Invalid ProRes variant %s", variant);
+ return FALSE;
+ }
}
if (cm_format == kCMVideoCodecType_H264 && state->codec_data == NULL) {
case GST_VIDEO_FORMAT_NV12:
cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
break;
+ case GST_VIDEO_FORMAT_AYUV64:
+/* This is fine for now because Apple only ships LE devices */
+#if G_BYTE_ORDER != G_LITTLE_ENDIAN
+#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
+#endif
+ cv_format = kCVPixelFormatType_4444AYpCbCr16;
+ break;
default:
g_warn_if_reached ();
break;
* example) or we're draining/flushing
*/
if (frame) {
- if (flush || frame->flags & VTDEC_FRAME_FLAG_SKIP)
+ if (frame->flags & VTDEC_FRAME_FLAG_ERROR) {
gst_video_decoder_release_frame (decoder, frame);
- else if (frame->flags & VTDEC_FRAME_FLAG_DROP)
+ ret = GST_FLOW_ERROR;
+ } else if (flush || frame->flags & VTDEC_FRAME_FLAG_SKIP) {
+ gst_video_decoder_release_frame (decoder, frame);
+ } else if (frame->flags & VTDEC_FRAME_FLAG_DROP) {
gst_video_decoder_drop_frame (decoder, frame);
- else
+ } else {
ret = gst_video_decoder_finish_frame (decoder, frame);
+ }
}
if (!frame || ret != GST_FLOW_OK)
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ NV12, I420 }"));
#else
static GstStaticCaps sink_caps =
-GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ UYVY, NV12, I420 }"));
+GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE ("{ AYUV64, UYVY, NV12, I420 }"));
#endif
static void
"height", GST_TYPE_INT_RANGE, min_height, max_height,
"framerate", GST_TYPE_FRACTION_RANGE,
min_fps_n, min_fps_d, max_fps_n, max_fps_d, NULL);
- if (codec_details->format_id == kCMVideoCodecType_H264) {
- gst_structure_set (gst_caps_get_structure (src_caps, 0),
- "stream-format", G_TYPE_STRING, "avc",
- "alignment", G_TYPE_STRING, "au", NULL);
+ switch (codec_details->format_id) {
+ case kCMVideoCodecType_H264:
+ gst_structure_set (gst_caps_get_structure (src_caps, 0),
+ "stream-format", G_TYPE_STRING, "avc",
+ "alignment", G_TYPE_STRING, "au", NULL);
+ break;
+ case GST_kCMVideoCodecType_Some_AppleProRes:
+ if (g_strcmp0 (codec_details->mimetype, "video/x-prores") == 0) {
+ G_GNUC_BEGIN_IGNORE_DEPRECATIONS;
+ GValueArray *arr = g_value_array_new (6);
+ GValue val = G_VALUE_INIT;
+
+ g_value_init (&val, G_TYPE_STRING);
+ g_value_set_string (&val, "standard");
+ arr = g_value_array_append (arr, &val);
+ g_value_set_string (&val, "4444xq");
+ arr = g_value_array_append (arr, &val);
+ g_value_set_string (&val, "4444");
+ arr = g_value_array_append (arr, &val);
+ g_value_set_string (&val, "hq");
+ arr = g_value_array_append (arr, &val);
+ g_value_set_string (&val, "lt");
+ arr = g_value_array_append (arr, &val);
+ g_value_set_string (&val, "proxy");
+ arr = g_value_array_append (arr, &val);
+ gst_structure_set_list (gst_caps_get_structure (src_caps, 0),
+ "variant", arr);
+ g_value_array_free (arr);
+ g_value_unset (&val);
+ G_GNUC_END_IGNORE_DEPRECATIONS;
+ break;
+ }
+ /* fall through */
+ default:
+ g_assert_not_reached ();
}
src_template = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS,
src_caps);
} else if (!strcmp (profile, "main")) {
profile = "Main";
} else {
- g_assert_not_reached ();
+ GST_ERROR_OBJECT (self, "invalid profile: %s", profile);
+ return ret;
}
if (strlen (level) == 1) {
}
static gboolean
-gst_vtenc_negotiate_profile_and_level (GstVideoEncoder * enc)
+gst_vtenc_negotiate_profile_and_level (GstVTEnc * self, GstStructure * s)
+{
+ const gchar *profile = gst_structure_get_string (s, "profile");
+ const gchar *level = gst_structure_get_string (s, "level");
+
+ if (self->profile_level)
+ CFRelease (self->profile_level);
+ self->profile_level = gst_vtenc_profile_level_key (self, profile, level);
+ if (self->profile_level == NULL) {
+ GST_ERROR_OBJECT (self, "unsupported h264 profile '%s' or level '%s'",
+ profile, level);
+ return FALSE;
+ }
+
+ return TRUE;
+}
+
+static gboolean
+gst_vtenc_negotiate_prores_variant (GstVTEnc * self, GstStructure * s)
+{
+ const char *variant = gst_structure_get_string (s, "variant");
+ CMVideoCodecType codec_type =
+ gst_vtutil_codec_type_from_prores_variant (variant);
+
+ if (codec_type == GST_kCMVideoCodecType_Some_AppleProRes) {
+ GST_ERROR_OBJECT (self, "unsupported prores variant: %s", variant);
+ return FALSE;
+ }
+
+ self->specific_format_id = codec_type;
+ return TRUE;
+}
+
+static gboolean
+gst_vtenc_negotiate_specific_format_details (GstVideoEncoder * enc)
{
GstVTEnc *self = GST_VTENC_CAST (enc);
GstCaps *allowed_caps = NULL;
gboolean ret = TRUE;
- const gchar *profile = NULL;
- const gchar *level = NULL;
allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SRC_PAD (enc));
if (allowed_caps) {
allowed_caps = gst_caps_make_writable (allowed_caps);
allowed_caps = gst_caps_fixate (allowed_caps);
s = gst_caps_get_structure (allowed_caps, 0);
-
- profile = gst_structure_get_string (s, "profile");
- level = gst_structure_get_string (s, "level");
- }
-
- if (self->profile_level)
- CFRelease (self->profile_level);
- self->profile_level = gst_vtenc_profile_level_key (self, profile, level);
- if (self->profile_level == NULL) {
- GST_ERROR_OBJECT (enc, "invalid profile and level");
- goto fail;
+ switch (self->details->format_id) {
+ case kCMVideoCodecType_H264:
+ self->specific_format_id = kCMVideoCodecType_H264;
+ if (!gst_vtenc_negotiate_profile_and_level (self, s))
+ goto fail;
+ break;
+ case GST_kCMVideoCodecType_Some_AppleProRes:
+ if (g_strcmp0 (self->details->mimetype, "video/x-prores") != 0) {
+ GST_ERROR_OBJECT (self, "format_id == %i mimetype must be Apple "
+ "ProRes", GST_kCMVideoCodecType_Some_AppleProRes);
+ goto fail;
+ }
+ if (!gst_vtenc_negotiate_prores_variant (self, s))
+ goto fail;
+ break;
+ default:
+ g_assert_not_reached ();
+ }
}
out:
gst_vtenc_destroy_session (self, &self->session);
GST_OBJECT_UNLOCK (self);
- gst_vtenc_negotiate_profile_and_level (enc);
+ gst_vtenc_negotiate_specific_format_details (enc);
session = gst_vtenc_create_session (self);
GST_OBJECT_LOCK (self);
"framerate", GST_TYPE_FRACTION,
self->negotiated_fps_n, self->negotiated_fps_d, NULL);
- if (self->details->format_id == kCMVideoCodecType_H264) {
- CMFormatDescriptionRef fmt;
- CFDictionaryRef atoms;
- CFStringRef avccKey;
- CFDataRef avcc;
- guint8 *codec_data;
- gsize codec_data_size;
- GstBuffer *codec_data_buf;
- guint8 sps[3];
-
- fmt = CMSampleBufferGetFormatDescription (sbuf);
- atoms = CMFormatDescriptionGetExtension (fmt,
- kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms);
- avccKey = CFStringCreateWithCString (NULL, "avcC", kCFStringEncodingUTF8);
- avcc = CFDictionaryGetValue (atoms, avccKey);
- CFRelease (avccKey);
- codec_data_size = CFDataGetLength (avcc);
- codec_data = g_malloc (codec_data_size);
- CFDataGetBytes (avcc, CFRangeMake (0, codec_data_size), codec_data);
- codec_data_buf = gst_buffer_new_wrapped (codec_data, codec_data_size);
-
- gst_structure_set (s, "codec_data", GST_TYPE_BUFFER, codec_data_buf, NULL);
-
- sps[0] = codec_data[1];
- sps[1] = codec_data[2] & ~0xDF;
- sps[2] = codec_data[3];
-
- gst_codec_utils_h264_caps_set_level_and_profile (caps, sps, 3);
-
- gst_buffer_unref (codec_data_buf);
+ switch (self->details->format_id) {
+ case kCMVideoCodecType_H264:
+ {
+ CMFormatDescriptionRef fmt;
+ CFDictionaryRef atoms;
+ CFStringRef avccKey;
+ CFDataRef avcc;
+ guint8 *codec_data;
+ gsize codec_data_size;
+ GstBuffer *codec_data_buf;
+ guint8 sps[3];
+
+ fmt = CMSampleBufferGetFormatDescription (sbuf);
+ atoms = CMFormatDescriptionGetExtension (fmt,
+ kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms);
+ avccKey = CFStringCreateWithCString (NULL, "avcC", kCFStringEncodingUTF8);
+ avcc = CFDictionaryGetValue (atoms, avccKey);
+ CFRelease (avccKey);
+ codec_data_size = CFDataGetLength (avcc);
+ codec_data = g_malloc (codec_data_size);
+ CFDataGetBytes (avcc, CFRangeMake (0, codec_data_size), codec_data);
+ codec_data_buf = gst_buffer_new_wrapped (codec_data, codec_data_size);
+
+ gst_structure_set (s, "codec_data", GST_TYPE_BUFFER, codec_data_buf,
+ NULL);
+
+ sps[0] = codec_data[1];
+ sps[1] = codec_data[2] & ~0xDF;
+ sps[2] = codec_data[3];
+
+ gst_codec_utils_h264_caps_set_level_and_profile (caps, sps, 3);
+
+ gst_buffer_unref (codec_data_buf);
+ }
+ break;
+ case GST_kCMVideoCodecType_Some_AppleProRes:
+ gst_structure_set (s, "variant", G_TYPE_STRING,
+ gst_vtutil_codec_type_to_prores_variant (self->specific_format_id),
+ NULL);
+ break;
+ default:
+ g_assert_not_reached ();
}
state =
gst_vtenc_create_session (GstVTEnc * self)
{
VTCompressionSessionRef session = NULL;
- CFMutableDictionaryRef encoder_spec = NULL, pb_attrs;
+ CFMutableDictionaryRef encoder_spec = NULL, pb_attrs = NULL;
OSStatus status;
#if !HAVE_IOS
TRUE);
#endif
- pb_attrs = CFDictionaryCreateMutable (NULL, 0, &kCFTypeDictionaryKeyCallBacks,
- &kCFTypeDictionaryValueCallBacks);
- gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferWidthKey,
- self->negotiated_width);
- gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferHeightKey,
- self->negotiated_height);
+ if (self->profile_level) {
+ pb_attrs = CFDictionaryCreateMutable (NULL, 0,
+ &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);
+ gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferWidthKey,
+ self->negotiated_width);
+ gst_vtutil_dict_set_i32 (pb_attrs, kCVPixelBufferHeightKey,
+ self->negotiated_height);
+ }
+
+ /* This was set in gst_vtenc_negotiate_specific_format_details() */
+ g_assert_cmpint (self->specific_format_id, !=, 0);
status = VTCompressionSessionCreate (NULL,
self->negotiated_width, self->negotiated_height,
- self->details->format_id, encoder_spec, pb_attrs, NULL,
+ self->specific_format_id, encoder_spec, pb_attrs, NULL,
gst_vtenc_enqueue_buffer, self, &session);
GST_INFO_OBJECT (self, "VTCompressionSessionCreate for %d x %d => %d",
self->negotiated_width, self->negotiated_height, (int) status);
goto beach;
}
- gst_vtenc_session_configure_expected_framerate (self, session,
- (gdouble) self->negotiated_fps_n / (gdouble) self->negotiated_fps_d);
+ if (self->profile_level) {
+ gst_vtenc_session_configure_expected_framerate (self, session,
+ (gdouble) self->negotiated_fps_n / (gdouble) self->negotiated_fps_d);
- status = VTSessionSetProperty (session,
- kVTCompressionPropertyKey_ProfileLevel, self->profile_level);
- GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_ProfileLevel => %d",
- (int) status);
+ /*
+ * https://developer.apple.com/documentation/videotoolbox/vtcompressionsession/compression_properties/profile_and_level_constants
+ */
+ status = VTSessionSetProperty (session,
+ kVTCompressionPropertyKey_ProfileLevel, self->profile_level);
+ GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_ProfileLevel => %d",
+ (int) status);
- status = VTSessionSetProperty (session,
- kVTCompressionPropertyKey_AllowTemporalCompression, kCFBooleanTrue);
- GST_DEBUG_OBJECT (self,
- "kVTCompressionPropertyKey_AllowTemporalCompression => %d", (int) status);
+ status = VTSessionSetProperty (session,
+ kVTCompressionPropertyKey_AllowTemporalCompression, kCFBooleanTrue);
+ GST_DEBUG_OBJECT (self,
+ "kVTCompressionPropertyKey_AllowTemporalCompression => %d",
+ (int) status);
+
+ gst_vtenc_session_configure_max_keyframe_interval (self, session,
+ self->max_keyframe_interval);
+ gst_vtenc_session_configure_max_keyframe_interval_duration (self, session,
+ self->max_keyframe_interval_duration / ((gdouble) GST_SECOND));
- gst_vtenc_session_configure_max_keyframe_interval (self, session,
- self->max_keyframe_interval);
- gst_vtenc_session_configure_max_keyframe_interval_duration (self, session,
- self->max_keyframe_interval_duration / ((gdouble) GST_SECOND));
+ gst_vtenc_session_configure_bitrate (self, session,
+ gst_vtenc_get_bitrate (self));
+ }
- gst_vtenc_session_configure_bitrate (self, session,
- gst_vtenc_get_bitrate (self));
gst_vtenc_session_configure_realtime (self, session,
gst_vtenc_get_realtime (self));
gst_vtenc_session_configure_allow_frame_reordering (self, session,
beach:
if (encoder_spec)
CFRelease (encoder_spec);
- CFRelease (pb_attrs);
+ if (pb_attrs)
+ CFRelease (pb_attrs);
return session;
}
}
switch (GST_VIDEO_INFO_FORMAT (&self->video_info)) {
+ case GST_VIDEO_FORMAT_AYUV64:
+/* This is fine for now because Apple only ships LE devices */
+#if G_BYTE_ORDER != G_LITTLE_ENDIAN
+#error "AYUV64 is NE but kCVPixelFormatType_4444AYpCbCr16 is LE"
+#endif
+ pixel_format_type = kCVPixelFormatType_4444AYpCbCr16;
+ break;
case GST_VIDEO_FORMAT_I420:
pixel_format_type = kCVPixelFormatType_420YpCbCr8Planar;
break;
plane_bytes_per_row, gst_pixel_buffer_release_cb, vframe, NULL,
&pbuf);
if (cv_ret != kCVReturnSuccess) {
- GST_ERROR_OBJECT (self, "CVPixelBufferCreateWithPlanarBytes failed: %i", cv_ret);
+ GST_ERROR_OBJECT (self, "CVPixelBufferCreateWithPlanarBytes failed: %i",
+ cv_ret);
gst_vtenc_frame_free (vframe);
goto cv_error;
}
#ifndef HAVE_IOS
{"H.264 (HW only)", "h264_hw", "video/x-h264", kCMVideoCodecType_H264, TRUE},
#endif
+ {"Apple ProRes", "prores", "video/x-prores",
+ GST_kCMVideoCodecType_Some_AppleProRes, FALSE},
};
void
projects / platform / upstream / gstreamer.git / commitdiff