From 8bf2acba1d2a81ded284a3f6092972a2c20b9cc5 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 20 Oct 2014 10:34:27 +1100 Subject: [PATCH] videoaggregator: operate on caps rather than video info Otherwise the CapsFeatures will be lost along with the possibility of multiple output types and formats. https://bugzilla.gnome.org/show_bug.cgi?id=738129 --- ext/gl/gstglmixer.c | 96 +++++++++++++++++++++++++++++++++ ext/gl/gstglmixer.h | 1 + ext/gl/gstglvideomixer.c | 26 +++++---- gst-libs/gst/video/gstvideoaggregator.c | 38 ++++++++----- gst-libs/gst/video/gstvideoaggregator.h | 10 ++-- gst/compositor/compositor.c | 18 ++++--- 6 files changed, 155 insertions(+), 34 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 0a299db..37aa8f0 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -36,6 +36,10 @@ #include "gstglmixer.h" +#if GST_GL_HAVE_PLATFORM_EGL +#include +#endif + #define gst_gl_mixer_parent_class parent_class G_DEFINE_ABSTRACT_TYPE (GstGLMixer, gst_gl_mixer, GST_TYPE_VIDEO_AGGREGATOR); static gboolean gst_gl_mixer_do_bufferpool (GstGLMixer * mix, @@ -559,6 +563,98 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) return TRUE; } +static GstCaps * +gst_gl_mixer_set_caps_features (const GstCaps * caps, + const gchar * feature_name) +{ + GstCaps *tmp = gst_caps_copy (caps); + guint n = gst_caps_get_size (tmp); + guint i = 0; + + for (i = 0; i < n; i++) { + GstCapsFeatures *features = gst_caps_get_features (tmp, i); + if (features) { + guint n_f = gst_caps_features_get_size (features); + guint j = 0; + for (j = 0; j < n_f; j++) { + gst_caps_features_remove_id (features, + gst_caps_features_get_nth_id (features, j)); + } + } + + gst_caps_features_add (features, feature_name); + gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL); + } + + return tmp; +} + +/* copies the given caps */ +static GstCaps * +gst_gl_mixer_caps_remove_format_info (GstCaps * caps) +{ + GstStructure *st; + GstCapsFeatures *f; + gint i, n; + GstCaps *res; + + res = gst_caps_new_empty (); + + n = gst_caps_get_size (caps); + for (i = 0; i < n; i++) { + st = gst_caps_get_structure (caps, i); + f = gst_caps_get_features (caps, i); + + /* If this is already expressed by the existing caps + * skip this structure */ + if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) + continue; + + st = gst_structure_copy (st); + /* Only remove format info for the cases when we can actually convert */ + if (!gst_caps_features_is_any (f) + && gst_caps_features_is_equal (f, + GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) + gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", + "width", "height", NULL); + + gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); + } + + return res; +} + +GstCaps * +gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) +{ + GstCaps *result = NULL; + GstCaps *glcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); +#if GST_GL_HAVE_PLATFORM_EGL + GstCaps *eglcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); +#endif + GstCaps *uploadcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); + GstCaps *raw_caps = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); + + result = gst_caps_new_empty (); + + result = gst_caps_merge (result, glcaps); +#if GST_GL_HAVE_PLATFORM_EGL + result = gst_caps_merge (result, eglcaps); +#endif + result = gst_caps_merge (result, uploadcaps); + result = gst_caps_merge (result, raw_caps); + + result = gst_caps_merge (result, gst_gl_mixer_caps_remove_format_info (caps)); + + GST_DEBUG_OBJECT (mix, "returning %" GST_PTR_FORMAT, result); + + return result; +} + static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query) { diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 5ea246b..f25c30a 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -92,6 +92,7 @@ struct _GstGLMixerFrameData GType gst_gl_mixer_get_type(void); gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf); +GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps); G_END_DECLS #endif /* __GST_GL_MIXER_H__ */ diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 16843be..39e17ae 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -65,7 +65,7 @@ static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static gboolean _update_info (GstVideoAggregator * vagg, GstVideoInfo * info); +static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); static void gst_gl_video_mixer_reset (GstGLMixer * mixer); static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); @@ -335,7 +335,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_video_mixer_process_textures; - vagg_class->update_info = _update_info; + vagg_class->update_caps = _update_caps; agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; } @@ -380,12 +380,17 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id, } } -static gboolean -_update_info (GstVideoAggregator * vagg, GstVideoInfo * info) +static GstCaps * +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) { GList *l; gint best_width = -1, best_height = -1; - gboolean ret = FALSE; + GstVideoInfo info; + GstCaps *ret = NULL; + int i; + + caps = gst_caps_make_writable (caps); + gst_video_info_from_caps (&info, caps); GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -417,10 +422,13 @@ _update_info (GstVideoAggregator * vagg, GstVideoInfo * info) } GST_OBJECT_UNLOCK (vagg); - if (best_width > 0 && best_height > 0) { - info->width = best_width; - info->height = best_height; - ret = TRUE; + ret = gst_gl_mixer_update_caps (GST_GL_MIXER (vagg), caps); + + for (i = 0; i < gst_caps_get_size (ret); i++) { + GstStructure *s = gst_caps_get_structure (ret, i); + + gst_structure_set (s, "width", G_TYPE_INT, best_width, "height", G_TYPE_INT, + best_height, NULL); } return ret; diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 8f46b7b..83bf08b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -559,7 +559,7 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) } if (best_width > 0 && best_height > 0 && best_fps > 0) { - GstCaps *caps, *peercaps; + GstCaps *caps, *peercaps, *info_caps; GstStructure *s; GstVideoInfo info; @@ -578,25 +578,40 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) info.par_n = GST_VIDEO_INFO_PAR_N (&vagg->info); info.par_d = GST_VIDEO_INFO_PAR_D (&vagg->info); - if (vagg_klass->update_info) { - if (!vagg_klass->update_info (vagg, &info)) { + info_caps = gst_video_info_to_caps (&info); + + if (vagg_klass->update_caps) { + if (!(caps = vagg_klass->update_caps (vagg, info_caps))) { + gst_caps_unref (info_caps); ret = FALSE; goto done; } + gst_caps_unref (info_caps); + } else { + caps = info_caps; } - caps = gst_video_info_to_caps (&info); - peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); if (peercaps) { GstCaps *tmp; + int i; s = gst_caps_get_structure (caps, 0); - gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, "height", - GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE, - 0, 1, G_MAXINT, 1, NULL); + gst_structure_get (s, "width", G_TYPE_INT, &best_width, "height", + G_TYPE_INT, &best_height, NULL); + + for (i = 0; i < gst_caps_get_size (caps); i++) { + s = gst_caps_get_structure (caps, i); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", + GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + } tmp = gst_caps_intersect (caps, peercaps); + GST_DEBUG_OBJECT (vagg, "intersecting %" GST_PTR_FORMAT + " with peer caps %" GST_PTR_FORMAT " result %" GST_PTR_FORMAT, caps, + peercaps, tmp); + gst_caps_unref (caps); gst_caps_unref (peercaps); caps = tmp; @@ -608,8 +623,8 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) caps = gst_caps_truncate (caps); s = gst_caps_get_structure (caps, 0); - gst_structure_fixate_field_nearest_int (s, "width", info.width); - gst_structure_fixate_field_nearest_int (s, "height", info.height); + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, best_fps_d); @@ -618,9 +633,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) gst_structure_get_fraction (s, "framerate", &info.fps_n, &info.fps_d); } - gst_caps_unref (caps); - caps = gst_video_info_to_caps (&info); - if (gst_videoaggregator_src_setcaps (vagg, caps)) { if (vagg_klass->negotiated_caps) ret = diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index b9fd531..2bdd619 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -73,9 +73,9 @@ struct _GstVideoAggregator * @disable_frame_conversion: Optional. * Allows subclasses to disable the frame colorspace * conversion feature - * @update_info: Optional. - * Lets subclasses update the src #GstVideoInfo representing - * the src pad caps before usage. + * @update_caps: Optional. + * Lets subclasses update the #GstCaps representing + * the src pad caps before usage. Return %NULL to indicate failure. * @aggregate_frames: Lets subclasses aggregate frames that are ready. Subclasses * should iterate the GstElement.sinkpads and use the already * mapped #GstVideoFrame from GstVideoAggregatorPad.aggregated_frame @@ -96,8 +96,8 @@ struct _GstVideoAggregatorClass /*< public >*/ gboolean disable_frame_conversion; - gboolean (*update_info) (GstVideoAggregator * videoaggregator, - GstVideoInfo * info); + GstCaps * (*update_caps) (GstVideoAggregator * videoaggregator, + GstCaps * caps); GstFlowReturn (*aggregate_frames) (GstVideoAggregator * videoaggregator, GstBuffer * outbuffer); GstFlowReturn (*get_output_buffer) (GstVideoAggregator * videoaggregator, diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 805b674..8b68950 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -437,12 +437,15 @@ set_functions (GstCompositor * self, GstVideoInfo * info) return ret; } -static gboolean -_update_info (GstVideoAggregator * vagg, GstVideoInfo * info) +static GstCaps * +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) { GList *l; gint best_width = -1, best_height = -1; - gboolean ret = FALSE; + GstVideoInfo info; + GstCaps *ret = NULL; + + gst_video_info_from_caps (&info, caps); GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -468,9 +471,10 @@ _update_info (GstVideoAggregator * vagg, GstVideoInfo * info) GST_OBJECT_UNLOCK (vagg); if (best_width > 0 && best_height > 0) { - gst_video_info_set_format (info, GST_VIDEO_INFO_FORMAT (info), - best_width, best_height); - ret = set_functions (GST_COMPOSITOR (vagg), info); + info.width = best_width; + info.height = best_height; + if (set_functions (GST_COMPOSITOR (vagg), &info)) + ret = gst_video_info_to_caps (&info); } return ret; @@ -559,7 +563,7 @@ gst_compositor_class_init (GstCompositorClass * klass) gobject_class->set_property = gst_compositor_set_property; agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD; - videoaggregator_class->update_info = _update_info; + videoaggregator_class->update_caps = _update_caps; videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames; g_object_class_install_property (gobject_class, PROP_BACKGROUND, -- 2.7.4