From 25aa8a4042b66ac6f61f66ce88b19ab469a88d01 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 19 Jun 2012 15:54:24 +1000 Subject: [PATCH] [512/906] GstGLMixer: update for 1.0 based on the videomixer2 element in -good --- gst-libs/gst/gl/gstglmixer.c | 1333 +++++++++++++++++++++++++++------------ gst-libs/gst/gl/gstglmixer.h | 33 +- gst-libs/gst/gl/gstglmixerpad.h | 14 +- 3 files changed, 953 insertions(+), 427 deletions(-) diff --git a/gst-libs/gst/gl/gstglmixer.c b/gst-libs/gst/gl/gstglmixer.c index 4f57428..f4e26c2 100644 --- a/gst-libs/gst/gl/gstglmixer.c +++ b/gst-libs/gst/gl/gstglmixer.c @@ -25,7 +25,6 @@ #include #include -//#include #include #ifdef HAVE_STDLIB_H @@ -40,12 +39,12 @@ #define GST_CAT_DEFAULT gst_gl_mixer_debug GST_DEBUG_CATEGORY (gst_gl_mixer_debug); -#define GST_GL_MIXER_GET_STATE_LOCK(mix) \ - (GST_GL_MIXER(mix)->state_lock) -#define GST_GL_MIXER_STATE_LOCK(mix) \ - (g_mutex_lock(GST_GL_MIXER_GET_STATE_LOCK (mix))) -#define GST_GL_MIXER_STATE_UNLOCK(mix) \ - (g_mutex_unlock(GST_GL_MIXER_GET_STATE_LOCK (mix))) +#define GST_GL_MIXER_GET_LOCK(mix) \ + (GST_GL_MIXER(mix)->lock) +#define GST_GL_MIXER_LOCK(mix) \ + (g_mutex_lock(&GST_GL_MIXER_GET_LOCK (mix))) +#define GST_GL_MIXER_UNLOCK(mix) \ + (g_mutex_unlock(&GST_GL_MIXER_GET_LOCK (mix))) static void gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass); static void gst_gl_mixer_pad_init (GstGLMixerPad * mixerpad); @@ -55,9 +54,12 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); -static gboolean gst_gl_mixer_src_event (GstPad * pad, GstEvent * event); -static gboolean gst_gl_mixer_sink_event (GstPad * pad, GstEvent * event); - +static gboolean gst_gl_mixer_src_event (GstPad * pad, GstObject * object, + GstEvent * event); +static gboolean gst_gl_mixer_sink_event (GstCollectPads * pads, + GstCollectData * cdata, GstEvent * event, GstGLMixer * mix); +static gboolean gst_gl_mixer_src_setcaps (GstPad * pad, GstGLMixer * mix, + GstCaps * caps); enum { @@ -79,8 +81,6 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - //GstGLMixerPad *pad = GST_GL_MIXER_PAD (object); - switch (prop_id) { default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -92,8 +92,6 @@ static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - //GstGLMixerPad *pad = GST_GL_MIXER_PAD (object); - switch (prop_id) { default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -101,81 +99,274 @@ gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, } } -static void -gst_gl_mixer_set_master_geometry (GstGLMixer * mix) +static gboolean +gst_gl_mixer_update_src_caps (GstGLMixer * mix) { - GSList *walk = mix->sinkpads; - gint fps_n = 0; - gint fps_d = 0; - GstGLMixerPad *master = NULL; + GSList *l; + gint best_width = -1, best_height = -1; + gdouble best_fps = -1, cur_fps; + gint best_fps_n = -1, best_fps_d = -1; + gboolean ret = TRUE; + + GST_GL_MIXER_LOCK (mix); + + for (l = mix->sinkpads; l; l = l->next) { + GstGLMixerPad *mpad = l->data; + gint this_width, this_height; + gint fps_n, fps_d; + gint width, height; + + fps_n = GST_VIDEO_INFO_FPS_N (&mpad->info); + fps_d = GST_VIDEO_INFO_FPS_D (&mpad->info); + width = GST_VIDEO_INFO_WIDTH (&mpad->info); + height = GST_VIDEO_INFO_HEIGHT (&mpad->info); + + if (fps_n == 0 || fps_d == 0 || width == 0 || height == 0) + continue; + + this_width = width; + this_height = height; + + if (best_width < this_width) + best_width = this_width; + if (best_height < this_height) + best_height = this_height; + + if (fps_d == 0) + cur_fps = 0.0; + else + gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); + + if (best_fps < cur_fps) { + best_fps = cur_fps; + best_fps_n = fps_n; + best_fps_d = fps_d; + } + } - while (walk) { - GstGLMixerPad *mixpad = GST_GL_MIXER_PAD (walk->data); + if (best_fps_n <= 0 && best_fps_d <= 0) { + best_fps_n = 25; + best_fps_d = 1; + best_fps = 25.0; + } - walk = g_slist_next (walk); + if (best_width > 0 && best_height > 0 && best_fps > 0) { + GstCaps *caps, *peercaps; + GstStructure *s; + GstVideoInfo info; - /* If mix framerate < mixpad framerate, using fractions */ - GST_DEBUG_OBJECT (mix, "comparing framerate %d/%d to mixpad's %d/%d", - fps_n, fps_d, mixpad->fps_n, mixpad->fps_d); - if ((!fps_n && !fps_d) || - ((gint64) fps_n * mixpad->fps_d < (gint64) mixpad->fps_n * fps_d)) { - fps_n = mixpad->fps_n; - fps_d = mixpad->fps_d; - GST_DEBUG_OBJECT (mix, "becomes the master pad"); - master = mixpad; + if (GST_VIDEO_INFO_FPS_N (&mix->info) != best_fps_n || + GST_VIDEO_INFO_FPS_D (&mix->info) != best_fps_d) { + if (mix->segment.position != -1) { + mix->ts_offset = mix->segment.position - mix->segment.start; + mix->nframes = 0; + } + } + gst_video_info_set_format (&info, GST_VIDEO_INFO_FORMAT (&mix->info), + best_width, best_height); + info.fps_n = best_fps_n; + info.fps_d = best_fps_d; + info.par_n = GST_VIDEO_INFO_PAR_N (&mix->info); + info.par_d = GST_VIDEO_INFO_PAR_D (&mix->info); + + caps = gst_video_info_to_caps (&info); + + peercaps = gst_pad_peer_query_caps (mix->srcpad, NULL); + if (peercaps) { + GstCaps *tmp; + + s = gst_caps_get_structure (caps, 0); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, "height", + GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE, + 0, 1, G_MAXINT, 1, NULL); + + tmp = gst_caps_intersect (caps, peercaps); + gst_caps_unref (caps); + gst_caps_unref (peercaps); + caps = tmp; + if (gst_caps_is_empty (caps)) { + GST_DEBUG_OBJECT (mix, "empty caps"); + ret = FALSE; + GST_GL_MIXER_UNLOCK (mix); + goto done; + } + + caps = gst_caps_truncate (caps); + s = gst_caps_get_structure (caps, 0); + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); + gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d); + + gst_structure_get_int (s, "width", &info.width); + gst_structure_get_int (s, "height", &info.height); + gst_structure_get_fraction (s, "fraction", &info.fps_n, &info.fps_d); } - } - /* set results */ - if (mix->master != master || mix->fps_n != fps_n || mix->fps_d != fps_d) { - mix->setcaps = TRUE; - mix->sendseg = TRUE; - mix->master = master; - mix->fps_n = fps_n; - mix->fps_d = fps_d; + caps = gst_video_info_to_caps (&info); + + GST_GL_MIXER_UNLOCK (mix); + ret = gst_gl_mixer_src_setcaps (mix->srcpad, mix, caps); + gst_caps_unref (caps); + } else { + GST_GL_MIXER_UNLOCK (mix); } + +done: + return ret; } static gboolean -gst_gl_mixer_pad_sink_setcaps (GstPad * pad, GstCaps * vscaps) +gst_gl_mixer_pad_sink_setcaps (GstPad * pad, GstObject * parent, GstCaps * caps) { - GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent (pad)); - GstGLMixerPad *mixpad = GST_GL_MIXER_PAD (pad); - GstStructure *structure = gst_caps_get_structure (vscaps, 0); - gint width = 0; - gint height = 0; + GstGLMixer *mix; + GstGLMixerPad *mixpad; + GstVideoInfo info; gboolean ret = FALSE; - const GValue *framerate = gst_structure_get_value (structure, "framerate"); - GST_INFO_OBJECT (mix, "Setting caps %" GST_PTR_FORMAT, vscaps); + GST_INFO_OBJECT (pad, "Setting caps %" GST_PTR_FORMAT, caps); + + mix = GST_GL_MIXER (parent); + mixpad = GST_GL_MIXER_PAD (pad); - if (!gst_structure_get_int (structure, "width", &width) || - !gst_structure_get_int (structure, "height", &height) || !framerate) + if (!gst_video_info_from_caps (&info, caps)) { + GST_ERROR_OBJECT (pad, "Failed to parse caps"); goto beach; + } - GST_GL_MIXER_STATE_LOCK (mix); - mixpad->fps_n = gst_value_get_fraction_numerator (framerate); - mixpad->fps_d = gst_value_get_fraction_denominator (framerate); + GST_GL_MIXER_LOCK (mix); + if (GST_VIDEO_INFO_FORMAT (&mix->info) != GST_VIDEO_FORMAT_UNKNOWN) { + if (GST_VIDEO_INFO_FORMAT (&mix->info) != GST_VIDEO_INFO_FORMAT (&info) || + GST_VIDEO_INFO_PAR_N (&mix->info) != GST_VIDEO_INFO_PAR_N (&info) || + GST_VIDEO_INFO_PAR_D (&mix->info) != GST_VIDEO_INFO_PAR_D (&info)) { + GST_ERROR_OBJECT (pad, "Caps not compatible with other pads' caps"); + GST_GL_MIXER_UNLOCK (mix); + goto beach; + } + } - mixpad->width = width; - mixpad->height = height; + mix->info = info; + mixpad->info = info; - gst_gl_mixer_set_master_geometry (mix); - GST_GL_MIXER_STATE_UNLOCK (mix); + GST_GL_MIXER_UNLOCK (mix); - ret = TRUE; + ret = gst_gl_mixer_update_src_caps (mix); beach: - gst_object_unref (mix); + return ret; +} +static GstCaps * +gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstObject * parent, + GstCaps * filter) +{ + GstGLMixer *mix; + GstCaps *srccaps; + GstStructure *s; + gint i, n; + + mix = GST_GL_MIXER (parent); + + srccaps = gst_pad_get_current_caps (GST_PAD (mix->srcpad)); + if (srccaps == NULL) + srccaps = gst_pad_get_pad_template_caps (GST_PAD (mix->srcpad)); + + srccaps = gst_caps_make_writable (srccaps); + + n = gst_caps_get_size (srccaps); + for (i = 0; i < n; i++) { + s = gst_caps_get_structure (srccaps, i); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + if (!gst_structure_has_field (s, "pixel-aspect-ratio")) + gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, + NULL); + } + + GST_DEBUG_OBJECT (pad, "Returning %" GST_PTR_FORMAT, srccaps); + + return srccaps; +} + +static gboolean +gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstObject * parent, + GstCaps * caps) +{ + gboolean ret; + GstGLMixer *mix; + GstCaps *accepted_caps; + gint i, n; + GstStructure *s; + + mix = GST_GL_MIXER (parent); + GST_DEBUG_OBJECT (pad, "%" GST_PTR_FORMAT, caps); + + accepted_caps = gst_pad_get_current_caps (GST_PAD (mix->srcpad)); + if (accepted_caps == NULL) + accepted_caps = gst_pad_get_pad_template_caps (GST_PAD (mix->srcpad)); + + accepted_caps = gst_caps_make_writable (accepted_caps); + GST_LOG_OBJECT (pad, "src caps %" GST_PTR_FORMAT, accepted_caps); + + n = gst_caps_get_size (accepted_caps); + for (i = 0; i < n; i++) { + s = gst_caps_get_structure (accepted_caps, i); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + if (!gst_structure_has_field (s, "pixel-aspect-ratio")) + gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, + NULL); + } + + ret = gst_caps_can_intersect (caps, accepted_caps); + GST_INFO_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT, (ret ? "" : "not "), + caps); + GST_INFO_OBJECT (pad, "acceptable caps are %" GST_PTR_FORMAT, accepted_caps); + gst_caps_unref (accepted_caps); + + return ret; +} + +static gboolean +gst_gl_mixer_pad_sink_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + gboolean ret = FALSE; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_gl_mixer_pad_sink_getcaps (pad, parent, filter); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + ret = TRUE; + break; + } + case GST_QUERY_ACCEPT_CAPS: + { + GstCaps *caps; + + gst_query_parse_accept_caps (query, &caps); + ret = gst_gl_mixer_pad_sink_acceptcaps (pad, parent, caps); + gst_query_set_accept_caps_result (query, ret); + ret = TRUE; + break; + } + default: + ret = gst_pad_query_default (pad, parent, query); + break; + } return ret; } static void gst_gl_mixer_pad_init (GstGLMixerPad * mixerpad) { - gst_pad_set_setcaps_function (GST_PAD (mixerpad), - gst_gl_mixer_pad_sink_setcaps); + gst_pad_set_query_function (GST_PAD (mixerpad), gst_gl_mixer_pad_sink_query); mixerpad->display = NULL; } @@ -206,14 +397,16 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%d", static void gst_gl_mixer_finalize (GObject * object); -static GstCaps *gst_gl_mixer_getcaps (GstPad * pad); -static gboolean gst_gl_mixer_setcaps (GstPad * pad, GstCaps * caps); -static gboolean gst_gl_mixer_query (GstPad * pad, GstQuery * query); +static gboolean gst_gl_mixer_src_query (GstPad * pad, GstObject * object, + GstQuery * query); +static GstFlowReturn gst_gl_mixer_sink_clip (GstCollectPads * pads, + GstCollectData * data, GstBuffer * buf, GstBuffer ** outbuf, + GstGLMixer * mix); static GstFlowReturn gst_gl_mixer_collected (GstCollectPads * pads, GstGLMixer * mix); static GstPad *gst_gl_mixer_request_new_pad (GstElement * element, - GstPadTemplate * templ, const gchar * name); + GstPadTemplate * templ, const gchar * name, const GstCaps * caps); static void gst_gl_mixer_release_pad (GstElement * element, GstPad * pad); static void gst_gl_mixer_set_property (GObject * object, guint prop_id, @@ -223,40 +416,39 @@ static void gst_gl_mixer_get_property (GObject * object, guint prop_id, static GstStateChangeReturn gst_gl_mixer_change_state (GstElement * element, GstStateChange transition); +static gboolean gst_gl_mixer_query_caps (GstPad * pad, GstObject * parent, + GstQuery * query); +static gboolean gst_gl_mixer_query_duration (GstGLMixer * mix, + GstQuery * query); +static gboolean gst_gl_mixer_query_latency (GstGLMixer * mix, GstQuery * query); + +static gint64 gst_gl_mixer_do_qos (GstGLMixer * mix, GstClockTime timestamp); +static void gst_gl_mixer_update_qos (GstGLMixer * mix, gdouble proportion, + GstClockTimeDiff diff, GstClockTime timestamp); +static void gst_gl_mixer_reset_qos (GstGLMixer * mix); +static void gst_gl_mixer_read_qos (GstGLMixer * mix, gdouble * proportion, + GstClockTime * time); + static void gst_gl_mixer_child_proxy_init (gpointer g_iface, gpointer iface_data); -static void _do_init (GType object_type); #define gst_gl_mixer_parent_class parent_class -G_DEFINE_TYPE_WITH_CODE (GstGLMixer, gst_gl_mixer, GST_TYPE_ELEMENT, _do_init); +G_DEFINE_TYPE_WITH_CODE (GstGLMixer, gst_gl_mixer, GST_TYPE_ELEMENT, + G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, gst_gl_mixer_child_proxy_init); + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "opengl mixer")); static void gst_gl_mixer_finalize (GObject * object); -static void -_do_init (GType object_type) -{ - static const GInterfaceInfo child_proxy_info = { - (GInterfaceInitFunc) gst_gl_mixer_child_proxy_init, - NULL, - NULL - }; - - GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "opengl mixer"); - g_type_add_interface_static (object_type, GST_TYPE_CHILD_PROXY, - &child_proxy_info); - GST_INFO ("GstChildProxy interface registered"); -} - -static GstObject * +static GObject * gst_gl_mixer_child_proxy_get_child_by_index (GstChildProxy * child_proxy, guint index) { GstGLMixer *mix = GST_GL_MIXER (child_proxy); - GstObject *obj; + GObject *obj; - GST_GL_MIXER_STATE_LOCK (mix); + GST_GL_MIXER_LOCK (mix); if ((obj = g_slist_nth_data (mix->sinkpads, index))) - gst_object_ref (obj); - GST_GL_MIXER_STATE_UNLOCK (mix); + g_object_ref (obj); + GST_GL_MIXER_UNLOCK (mix); return obj; } @@ -266,9 +458,9 @@ gst_gl_mixer_child_proxy_get_children_count (GstChildProxy * child_proxy) guint count = 0; GstGLMixer *mix = GST_GL_MIXER (child_proxy); - GST_GL_MIXER_STATE_LOCK (mix); + GST_GL_MIXER_LOCK (mix); count = mix->numpads; - GST_GL_MIXER_STATE_UNLOCK (mix); + GST_GL_MIXER_UNLOCK (mix); GST_INFO_OBJECT (mix, "Children Count: %d", count); return count; } @@ -308,7 +500,7 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) element_class->change_state = GST_DEBUG_FUNCPTR (gst_gl_mixer_change_state); /* Register the pad class */ - (void) (GST_TYPE_GL_MIXER_PAD); + g_type_class_ref (GST_TYPE_GL_MIXER_PAD); klass->set_caps = NULL; } @@ -325,29 +517,29 @@ gst_gl_mixer_collect_free (GstGLMixerCollect * mixcol) static void gst_gl_mixer_reset (GstGLMixer * mix) { - GSList *walk; + GSList *l; - mix->width = 0; - mix->height = 0; - mix->fps_n = 0; - mix->fps_d = 0; - mix->setcaps = FALSE; - mix->sendseg = FALSE; - mix->segment_position = 0; - mix->segment_rate = 1.0; + gst_video_info_init (&mix->info); + mix->ts_offset = 0; + mix->nframes = 0; - mix->last_ts = 0; + gst_segment_init (&mix->segment, GST_FORMAT_TIME); + mix->segment.position = -1; /* clean up collect data */ - walk = mix->collect->data; - while (walk) { - GstGLMixerCollect *data = (GstGLMixerCollect *) walk->data; + for (l = mix->sinkpads; l; l = l->next) { + GstGLMixerPad *p = l->data; + GstGLMixerCollect *mixcol = p->mixcol; - gst_gl_mixer_collect_free (data); - walk = g_slist_next (walk); + gst_buffer_replace (&mixcol->buffer, NULL); + mixcol->start_time = -1; + mixcol->end_time = -1; + + gst_video_info_init (&p->info); } - mix->next_sinkpad = 0; + mix->newseg_pending = TRUE; + mix->flush_stop_pending = FALSE; } static void @@ -358,12 +550,8 @@ gst_gl_mixer_init (GstGLMixer * mix) mix->srcpad = gst_pad_new_from_template (gst_element_class_get_pad_template (klass, "src"), "src"); - gst_pad_set_getcaps_function (GST_PAD (mix->srcpad), - GST_DEBUG_FUNCPTR (gst_gl_mixer_getcaps)); - gst_pad_set_setcaps_function (GST_PAD (mix->srcpad), - GST_DEBUG_FUNCPTR (gst_gl_mixer_setcaps)); gst_pad_set_query_function (GST_PAD (mix->srcpad), - GST_DEBUG_FUNCPTR (gst_gl_mixer_query)); + GST_DEBUG_FUNCPTR (gst_gl_mixer_src_query)); gst_pad_set_event_function (GST_PAD (mix->srcpad), GST_DEBUG_FUNCPTR (gst_gl_mixer_src_event)); gst_element_add_pad (GST_ELEMENT (mix), mix->srcpad); @@ -372,8 +560,12 @@ gst_gl_mixer_init (GstGLMixer * mix) gst_collect_pads_set_function (mix->collect, (GstCollectPadsFunction) GST_DEBUG_FUNCPTR (gst_gl_mixer_collected), mix); + gst_collect_pads_set_event_function (mix->collect, + (GstCollectPadsEventFunction) gst_gl_mixer_sink_event, mix); + gst_collect_pads_set_clip_function (mix->collect, + (GstCollectPadsClipFunction) gst_gl_mixer_sink_clip, mix); - mix->state_lock = g_mutex_new (); + g_mutex_init (&mix->lock); mix->array_buffers = 0; mix->display = NULL; @@ -390,7 +582,7 @@ gst_gl_mixer_finalize (GObject * object) GstGLMixer *mix = GST_GL_MIXER (object); gst_object_unref (mix->collect); - g_mutex_free (mix->state_lock); + g_mutex_clear (&mix->lock); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -398,6 +590,7 @@ gst_gl_mixer_finalize (GObject * object) static gboolean gst_gl_mixer_query_duration (GstGLMixer * mix, GstQuery * query) { + GValue item = { 0 }; gint64 max; gboolean res; GstFormat format; @@ -415,7 +608,6 @@ gst_gl_mixer_query_duration (GstGLMixer * mix, GstQuery * query) it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (mix)); while (!done) { GstIteratorResult ires; - gpointer item; ires = gst_iterator_next (it, &item); switch (ires) { @@ -424,11 +616,13 @@ gst_gl_mixer_query_duration (GstGLMixer * mix, GstQuery * query) break; case GST_ITERATOR_OK: { - GstPad *pad = GST_PAD_CAST (item); + GstPad *pad; gint64 duration; + pad = g_value_get_object (&item); + /* ask sink peer for duration */ - res &= gst_pad_query_peer_duration (pad, &format, &duration); + res &= gst_pad_peer_query_duration (pad, format, &duration); /* take max from all valid return values */ if (res) { /* valid unknown length, stop searching */ @@ -440,7 +634,7 @@ gst_gl_mixer_query_duration (GstGLMixer * mix, GstQuery * query) else if (duration > max) max = duration; } - gst_object_unref (pad); + g_value_reset (&item); break; } case GST_ITERATOR_RESYNC: @@ -454,6 +648,7 @@ gst_gl_mixer_query_duration (GstGLMixer * mix, GstQuery * query) break; } } + g_value_reset (&item); gst_iterator_free (it); if (res) { @@ -467,8 +662,42 @@ gst_gl_mixer_query_duration (GstGLMixer * mix, GstQuery * query) } static gboolean +gst_gl_mixer_query_caps (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstCaps *filter, *caps; + GstGLMixer *mix = GST_GL_MIXER (parent); + GstStructure *s; + gint n; + + gst_query_parse_caps (query, &filter); + + if (GST_VIDEO_INFO_FORMAT (&mix->info) != GST_VIDEO_FORMAT_UNKNOWN) { + caps = gst_pad_get_current_caps (mix->srcpad); + } else { + caps = gst_pad_get_pad_template_caps (mix->srcpad); + } + + caps = gst_caps_make_writable (caps); + + n = gst_caps_get_size (caps) - 1; + for (; n >= 0; n--) { + s = gst_caps_get_structure (caps, n); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL); + if (GST_VIDEO_INFO_FPS_D (&mix->info) != 0) { + gst_structure_set (s, + "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + } + } + gst_query_set_caps_result (query, caps); + + return TRUE; +} + +static gboolean gst_gl_mixer_query_latency (GstGLMixer * mix, GstQuery * query) { + GValue item = { 0 }; GstClockTime min, max; gboolean live; gboolean res; @@ -485,7 +714,6 @@ gst_gl_mixer_query_latency (GstGLMixer * mix, GstQuery * query) it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (mix)); while (!done) { GstIteratorResult ires; - gpointer item; ires = gst_iterator_next (it, &item); switch (ires) { @@ -494,14 +722,12 @@ gst_gl_mixer_query_latency (GstGLMixer * mix, GstQuery * query) break; case GST_ITERATOR_OK: { - GstPad *pad = GST_PAD_CAST (item); - + GstPad *pad; GstQuery *peerquery; - GstClockTime min_cur, max_cur; - gboolean live_cur; + pad = g_value_get_object (&item); peerquery = gst_query_new_latency (); /* Ask peer for latency */ @@ -523,7 +749,7 @@ gst_gl_mixer_query_latency (GstGLMixer * mix, GstQuery * query) } gst_query_unref (peerquery); - gst_object_unref (pad); + g_value_reset (&item); break; } case GST_ITERATOR_RESYNC: @@ -539,6 +765,7 @@ gst_gl_mixer_query_latency (GstGLMixer * mix, GstQuery * query) break; } } + g_value_unset (&item); gst_iterator_free (it); if (res) { @@ -552,10 +779,51 @@ gst_gl_mixer_query_latency (GstGLMixer * mix, GstQuery * query) return res; } +static void +gst_gl_mixer_update_qos (GstGLMixer * mix, gdouble proportion, + GstClockTimeDiff diff, GstClockTime timestamp) +{ + GST_DEBUG_OBJECT (mix, + "Updating QoS: proportion %lf, diff %s%" GST_TIME_FORMAT ", timestamp %" + GST_TIME_FORMAT, proportion, (diff < 0) ? "-" : "", + GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp)); + + GST_OBJECT_LOCK (mix); + mix->proportion = proportion; + if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { + if (G_UNLIKELY (diff > 0)) + mix->earliest_time = + timestamp + 2 * diff + gst_util_uint64_scale_int (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&mix->info), GST_VIDEO_INFO_FPS_N (&mix->info)); + else + mix->earliest_time = timestamp + diff; + } else { + mix->earliest_time = GST_CLOCK_TIME_NONE; + } + GST_OBJECT_UNLOCK (mix); +} + +static void +gst_gl_mixer_reset_qos (GstGLMixer * mix) +{ + gst_gl_mixer_update_qos (mix, 0.5, 0, GST_CLOCK_TIME_NONE); + mix->qos_processed = mix->qos_dropped = 0; +} + +static void +gst_gl_mixer_read_qos (GstGLMixer * mix, gdouble * proportion, + GstClockTime * time) +{ + GST_OBJECT_LOCK (mix); + *proportion = mix->proportion; + *time = mix->earliest_time; + GST_OBJECT_UNLOCK (mix); +} + static gboolean -gst_gl_mixer_query (GstPad * pad, GstQuery * query) +gst_gl_mixer_src_query (GstPad * pad, GstObject * parent, GstQuery * query) { - GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent (pad)); + GstGLMixer *mix = GST_GL_MIXER (parent); gboolean res = FALSE; switch (GST_QUERY_TYPE (query)) { @@ -567,8 +835,9 @@ gst_gl_mixer_query (GstPad * pad, GstQuery * query) switch (format) { case GST_FORMAT_TIME: - /* FIXME, bring to stream time, might be tricky */ - gst_query_set_position (query, format, mix->last_ts); + gst_query_set_position (query, format, + gst_segment_to_stream_time (&mix->segment, GST_FORMAT_TIME, + mix->segment.position)); res = TRUE; break; default: @@ -582,13 +851,15 @@ gst_gl_mixer_query (GstPad * pad, GstQuery * query) case GST_QUERY_LATENCY: res = gst_gl_mixer_query_latency (mix, query); break; - + case GST_QUERY_CAPS: + res = gst_gl_mixer_query_caps (pad, parent, query); + break; case GST_QUERY_CUSTOM: { /* mix is a sink in terms of gl chain, so we are sharing the gldisplay that * comes from src pad with every display of the sink pads */ GSList *walk = mix->sinkpads; - GstStructure *structure = gst_query_get_structure (query); + GstStructure *structure = gst_query_writable_structure (query); res = g_strcmp0 (gst_element_get_name (mix), @@ -648,91 +919,83 @@ gst_gl_mixer_query (GstPad * pad, GstQuery * query) default: /* FIXME, needs a custom query handler because we have multiple * sinkpads, send to the master pad until then */ - res = gst_pad_query (GST_PAD_CAST (mix->master), query); + res = FALSE; + gst_query_unref (query); break; } - gst_object_unref (mix); return res; } -static GstCaps * -gst_gl_mixer_getcaps (GstPad * pad) -{ - GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent (pad)); - GstCaps *caps = gst_caps_copy (gst_pad_get_pad_template_caps (mix->srcpad)); - GstStructure *structure = gst_caps_get_structure (caps, 0); - - gst_structure_set (structure, "width", G_TYPE_INT, 8000, NULL); - gst_structure_set (structure, "height", G_TYPE_INT, /*G_MAXINT */ 6000, NULL); - if (mix->fps_d != 0) - gst_structure_set (structure, "framerate", GST_TYPE_FRACTION, mix->fps_n, - mix->fps_d, NULL); - - gst_object_unref (mix); - - return caps; -} - static gboolean -gst_gl_mixer_setcaps (GstPad * pad, GstCaps * caps) +gst_gl_mixer_src_setcaps (GstPad * pad, GstGLMixer * mix, GstCaps * caps) { - GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent_element (pad)); GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); - GstStructure *structure = gst_caps_get_structure (caps, 0); - gint width = 0; - gint height = 0; + GstVideoInfo info; + gboolean ret; GST_INFO_OBJECT (mix, "set src caps: %" GST_PTR_FORMAT, caps); - if (!gst_structure_get_int (structure, "width", &width) || - !gst_structure_get_int (structure, "height", &height)) { - gst_object_unref (mix); - return FALSE; + if (!gst_video_info_from_caps (&info, caps)) { + ret = TRUE; + goto done; } - GST_GL_MIXER_STATE_LOCK (mix); + GST_GL_MIXER_LOCK (mix); - mix->width = width; - mix->height = height; + if (GST_VIDEO_INFO_FPS_N (&mix->info) != GST_VIDEO_INFO_FPS_N (&info) || + GST_VIDEO_INFO_FPS_D (&mix->info) != GST_VIDEO_INFO_FPS_D (&info)) { + if (mix->segment.position != -1) { + mix->ts_offset = mix->segment.position - mix->segment.start; + mix->nframes = 0; + } + gst_gl_mixer_reset_qos (mix); + } - GST_GL_MIXER_STATE_UNLOCK (mix); + mix->info = info; - if (!gst_gl_display_gen_fbo (mix->display, mix->width, mix->height, - &mix->fbo, &mix->depthbuffer)) { + GST_GL_MIXER_UNLOCK (mix); + + if (!gst_gl_display_gen_fbo (mix->display, GST_VIDEO_INFO_WIDTH (&mix->info), + GST_VIDEO_INFO_HEIGHT (&mix->info), &mix->fbo, &mix->depthbuffer)) { GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, GST_GL_DISPLAY_ERR_MSG (mix->display), (NULL)); - gst_object_unref (mix); - return FALSE; + ret = FALSE; + goto done; } if (mixer_class->set_caps) mixer_class->set_caps (mix, caps); - gst_object_unref (mix); + ret = TRUE; +done: - return TRUE; + return ret; } static GstPad * gst_gl_mixer_request_new_pad (GstElement * element, - GstPadTemplate * templ, const gchar * req_name) + GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps) { - GstGLMixer *mix = GST_GL_MIXER (element); - GstGLMixerPad *mixpad = NULL; + GstGLMixer *mix; + GstGLMixerPad *mixpad; GstElementClass *klass = GST_ELEMENT_GET_CLASS (element); + mix = GST_GL_MIXER (element); + if (templ == gst_element_class_get_pad_template (klass, "sink_%d")) { gint serial = 0; gchar *name = NULL; GstGLMixerCollect *mixcol = NULL; - if (req_name == NULL || strlen (req_name) < 6) { + GST_GL_MIXER_LOCK (mix); + if (req_name == NULL || strlen (req_name) < 6 + || !g_str_has_prefix (req_name, "sink_")) { /* no name given when requesting the pad, use next available int */ serial = mix->next_sinkpad++; } else { /* parse serial number from requested padname */ - serial = atoi (&req_name[5]); + serial = g_ascii_strtoull (&req_name[5], NULL, 10); if (serial >= mix->next_sinkpad) mix->next_sinkpad = serial + 1; } @@ -742,36 +1005,32 @@ gst_gl_mixer_request_new_pad (GstElement * element, templ->direction, "template", templ, NULL); g_free (name); - GST_GL_MIXER_STATE_LOCK (mix); - mixcol = (GstGLMixerCollect *) - gst_collect_pads_add_pad (mix->collect, GST_PAD (mixpad), - sizeof (GstGLMixerCollect)); - - /* FIXME: hacked way to override/extend the event function of - * GstCollectPads; because it sets its own event function giving the - * element no access to events */ - mix->collect_event = - (GstPadEventFunction) GST_PAD_EVENTFUNC (GST_PAD (mixpad)); - gst_pad_set_event_function (GST_PAD (mixpad), - GST_DEBUG_FUNCPTR (gst_gl_mixer_sink_event)); + gst_collect_pads_add_pad_full (mix->collect, GST_PAD (mixpad), + sizeof (GstGLMixerCollect), + (GstCollectDataDestroyNotify) gst_gl_mixer_collect_free, TRUE); /* Keep track of each other */ mixcol->mixpad = mixpad; mixpad->mixcol = mixcol; + mixcol->start_time = -1; + mixcol->end_time = -1; + /* Keep an internal list of mixpads for zordering */ mix->sinkpads = g_slist_append (mix->sinkpads, mixpad); mix->numpads++; - GST_GL_MIXER_STATE_UNLOCK (mix); + GST_GL_MIXER_UNLOCK (mix); } else { - g_warning ("glmixer: this is not our template!"); return NULL; } + GST_DEBUG_OBJECT (element, "Adding pad %s", GST_PAD_NAME (mixpad)); + /* add the pad to the element */ gst_element_add_pad (element, GST_PAD (mixpad)); - gst_child_proxy_child_added (GST_OBJECT (mix), GST_OBJECT (mixpad)); + gst_child_proxy_child_added (G_OBJECT (mix), G_OBJECT (mixpad), + GST_OBJECT_NAME (mixpad)); return GST_PAD (mixpad); } @@ -779,10 +1038,13 @@ gst_gl_mixer_request_new_pad (GstElement * element, static void gst_gl_mixer_release_pad (GstElement * element, GstPad * pad) { - GstGLMixer *mix = GST_GL_MIXER (element); - GstGLMixerPad *mixpad = NULL; + GstGLMixer *mix; + GstGLMixerPad *mixpad; + gboolean update_caps; + + mix = GST_GL_MIXER (element); - GST_GL_MIXER_STATE_LOCK (mix); + GST_GL_MIXER_LOCK (mix); if (G_UNLIKELY (g_slist_find (mix->sinkpads, pad) == NULL)) { g_warning ("Unknown pad %s", GST_PAD_NAME (pad)); goto error; @@ -791,114 +1053,187 @@ gst_gl_mixer_release_pad (GstElement * element, GstPad * pad) mixpad = GST_GL_MIXER_PAD (pad); mix->sinkpads = g_slist_remove (mix->sinkpads, pad); - gst_gl_mixer_collect_free (mixpad->mixcol); - gst_collect_pads_remove_pad (mix->collect, pad); - gst_child_proxy_child_removed (GST_OBJECT (mix), GST_OBJECT (mixpad)); - /* determine possibly new geometry and master */ - gst_gl_mixer_set_master_geometry (mix); + gst_child_proxy_child_removed (G_OBJECT (mix), G_OBJECT (mixpad), + GST_OBJECT_NAME (mixpad)); mix->numpads--; - GST_GL_MIXER_STATE_UNLOCK (mix); + + update_caps = GST_VIDEO_INFO_FORMAT (&mix->info) != GST_VIDEO_FORMAT_UNKNOWN; + GST_GL_MIXER_UNLOCK (mix); + + gst_collect_pads_remove_pad (mix->collect, pad); + + if (update_caps) + gst_gl_mixer_update_src_caps (mix); gst_element_remove_pad (element, pad); return; error: - GST_GL_MIXER_STATE_UNLOCK (mix); + GST_GL_MIXER_UNLOCK (mix); } /* try to get a buffer on all pads. As long as the queued value is * negative, we skip buffers */ -static gboolean -gst_gl_mixer_fill_queues (GstGLMixer * mix) +static gint +gst_gl_mixer_fill_queues (GstGLMixer * mix, + GstClockTime output_start_time, GstClockTime output_end_time) { - GSList *walk = NULL; + GSList *l; gboolean eos = TRUE; + gboolean need_more_data = FALSE; - g_return_val_if_fail (GST_IS_GL_MIXER (mix), FALSE); + for (l = mix->sinkpads; l; l = l->next) { + GstGLMixerPad *pad = l->data; + GstGLMixerCollect *mixcol = pad->mixcol; + GstSegment *segment = &pad->mixcol->collect.segment; + GstBuffer *buf; + + buf = gst_collect_pads_peek (mix->collect, &mixcol->collect); + if (buf) { + GstClockTime start_time, end_time; + + start_time = GST_BUFFER_TIMESTAMP (buf); + if (start_time == -1) { + gst_buffer_unref (buf); + GST_ERROR_OBJECT (pad, "Need timestamped buffers!"); + return -2; + } - /* try to make sure we have a buffer from each usable pad first */ - walk = mix->collect->data; - while (walk) { - GstCollectData *data = (GstCollectData *) walk->data; - GstGLMixerCollect *mixcol = (GstGLMixerCollect *) data; - GstGLMixerPad *mixpad = mixcol->mixpad; + /* FIXME: Make all this work with negative rates */ + + if ((mixcol->buffer && start_time < GST_BUFFER_TIMESTAMP (mixcol->buffer)) + || (mixcol->queued + && start_time < GST_BUFFER_TIMESTAMP (mixcol->queued))) { + GST_WARNING_OBJECT (pad, "Buffer from the past, dropping"); + gst_buffer_unref (buf); + buf = gst_collect_pads_pop (mix->collect, &mixcol->collect); + gst_buffer_unref (buf); + need_more_data = TRUE; + continue; + } - walk = g_slist_next (walk); + if (mixcol->queued) { + end_time = start_time - GST_BUFFER_TIMESTAMP (mixcol->queued); + start_time = GST_BUFFER_TIMESTAMP (mixcol->queued); + gst_buffer_unref (buf); + buf = gst_buffer_ref (mixcol->queued); + } else { + end_time = GST_BUFFER_DURATION (buf); - if (mixcol->buffer == NULL) { - GstBuffer *buf = NULL; + if (end_time == -1) { + mixcol->queued = buf; + need_more_data = TRUE; + continue; + } + } - GST_LOG_OBJECT (mix, "we need a new buffer"); + g_assert (start_time != -1 && end_time != -1); + end_time += start_time; /* convert from duration to position */ - buf = gst_collect_pads_peek (mix->collect, data); + if (mixcol->end_time != -1 && mixcol->end_time > end_time) { + GST_WARNING_OBJECT (pad, "Buffer from the past, dropping"); + if (buf == mixcol->queued) { + gst_buffer_unref (buf); + gst_buffer_replace (&mixcol->queued, NULL); + } else { + gst_buffer_unref (buf); + buf = gst_collect_pads_pop (mix->collect, &mixcol->collect); + gst_buffer_unref (buf); + } - if (buf) { - guint64 duration; + need_more_data = TRUE; + continue; + } - GST_LOG_OBJECT (mix, "we have a buffer !"); + /* Check if it's inside the segment */ + if (start_time >= segment->stop || end_time < segment->start) { + GST_DEBUG_OBJECT (pad, "Buffer outside the segment"); - mixcol->buffer = buf; + if (buf == mixcol->queued) { + gst_buffer_unref (buf); + gst_buffer_replace (&mixcol->queued, NULL); + } else { + gst_buffer_unref (buf); + buf = gst_collect_pads_pop (mix->collect, &mixcol->collect); + gst_buffer_unref (buf); + } - duration = GST_BUFFER_DURATION (mixcol->buffer); - /* no duration on the buffer, use the framerate */ - if (!GST_CLOCK_TIME_IS_VALID (duration)) { - if (mixpad->fps_n == 0) { - duration = GST_CLOCK_TIME_NONE; - } else { - duration = - gst_util_uint64_scale_int (GST_SECOND, mixpad->fps_d, - mixpad->fps_n); - } + need_more_data = TRUE; + continue; + } + + /* Clip to segment and convert to running time */ + start_time = MAX (start_time, segment->start); + if (segment->stop != -1) + end_time = MIN (end_time, segment->stop); + start_time = + gst_segment_to_running_time (segment, GST_FORMAT_TIME, start_time); + end_time = + gst_segment_to_running_time (segment, GST_FORMAT_TIME, end_time); + g_assert (start_time != -1 && end_time != -1); + + /* Convert to the output segment rate */ + if (ABS (mix->segment.rate) != 1.0) { + start_time *= ABS (mix->segment.rate); + end_time *= ABS (mix->segment.rate); + } + + if (end_time >= output_start_time && start_time < output_end_time) { + GST_DEBUG_OBJECT (pad, + "Taking new buffer with start time %" GST_TIME_FORMAT, + GST_TIME_ARGS (start_time)); + gst_buffer_replace (&mixcol->buffer, buf); + mixcol->start_time = start_time; + mixcol->end_time = end_time; + + if (buf == mixcol->queued) { + gst_buffer_unref (buf); + gst_buffer_replace (&mixcol->queued, NULL); + } else { + gst_buffer_unref (buf); + buf = gst_collect_pads_pop (mix->collect, &mixcol->collect); + gst_buffer_unref (buf); } - if (GST_CLOCK_TIME_IS_VALID (duration)) - mixpad->queued += duration; - else if (!mixpad->queued) - mixpad->queued = GST_CLOCK_TIME_NONE; + eos = FALSE; + } else if (start_time >= output_end_time) { + GST_DEBUG_OBJECT (pad, "Keeping buffer until %" GST_TIME_FORMAT, + GST_TIME_ARGS (start_time)); + gst_buffer_unref (buf); + eos = FALSE; } else { - GST_LOG_OBJECT (mix, "pop returned a NULL buffer"); - } - } - if (mix->sendseg && (mixpad == mix->master)) { - GstEvent *event; - gint64 stop, start; - GstSegment *segment = &data->segment; - - /* FIXME, use rate/applied_rate as set on all sinkpads. - * - currently we just set rate as received from last seek-event - * We could potentially figure out the duration as well using - * the current segment positions and the stated stop positions. - * Also we just start from stream time 0 which is rather - * weird. For non-synchronized mixing, the time should be - * the min of the stream times of all received segments, - * rationale being that the duration is at least going to - * be as long as the earliest stream we start mixing. This - * would also be correct for synchronized mixing but then - * the later streams would be delayed until the stream times - * match. - */ - GST_INFO_OBJECT (mix, "_sending play segment"); - - start = segment->accum; - - /* get the duration of the segment if we can and add it to the accumulated - * time on the segment. */ - if (segment->stop != -1 && segment->start != -1) - stop = start + (segment->stop - segment->start); - else - stop = -1; - - event = gst_event_new_new_segment_full (FALSE, segment->rate, 1.0, - segment->format, start, stop, start + mix->segment_position); - gst_pad_push_event (mix->srcpad, event); - mix->sendseg = FALSE; - } + GST_DEBUG_OBJECT (pad, "Too old buffer -- dropping"); + if (buf == mixcol->queued) { + gst_buffer_unref (buf); + gst_buffer_replace (&mixcol->queued, NULL); + } else { + gst_buffer_unref (buf); + buf = gst_collect_pads_pop (mix->collect, &mixcol->collect); + gst_buffer_unref (buf); + } - if (mixcol->buffer != NULL && GST_CLOCK_TIME_IS_VALID (mixpad->queued)) { - /* got a buffer somewhere so we're not eos */ - eos = FALSE; + need_more_data = TRUE; + continue; + } + } else { + if (mixcol->end_time != -1) { + if (mixcol->end_time < output_start_time) { + gst_buffer_replace (&mixcol->buffer, NULL); + mixcol->start_time = mixcol->end_time = -1; + if (!GST_COLLECT_PADS_STATE_IS_SET (mixcol, + GST_COLLECT_PADS_STATE_EOS)) + need_more_data = TRUE; + } else { + eos = FALSE; + } + } } } - return eos; + if (need_more_data) + return 0; + if (eos) + return -1; + + return 1; } static void @@ -928,25 +1263,25 @@ gst_gl_mixer_process_buffers (GstGLMixer * mix, GstBuffer * outbuf) /* sync object properties on stream time */ if (GST_CLOCK_TIME_IS_VALID (stream_time)) - gst_object_sync_values (G_OBJECT (pad), stream_time); + gst_object_sync_values (GST_OBJECT (pad), stream_time); /* put buffer into array */ mix->array_buffers->pdata[array_index] = mixcol->buffer; - if (pad == mix->master) { - gint64 running_time; + /*if (pad == mix->master) { + gint64 running_time; - running_time = - gst_segment_to_running_time (seg, GST_FORMAT_TIME, timestamp); + running_time = + gst_segment_to_running_time (seg, GST_FORMAT_TIME, timestamp); */ - /* outgoing buffers need the running_time */ - GST_BUFFER_TIMESTAMP (outbuf) = running_time; - GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (mixcol->buffer); + /* outgoing buffers need the running_time */ + /*GST_BUFFER_TIMESTAMP (outbuf) = running_time; + GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (mixcol->buffer); - mix->last_ts = running_time; - if (GST_BUFFER_DURATION_IS_VALID (outbuf)) - mix->last_ts += GST_BUFFER_DURATION (outbuf); - } + mix->last_ts = running_time; + if (GST_BUFFER_DURATION_IS_VALID (outbuf)) + mix->last_ts += GST_BUFFER_DURATION (outbuf); + } */ } ++array_index; } @@ -954,113 +1289,169 @@ gst_gl_mixer_process_buffers (GstGLMixer * mix, GstBuffer * outbuf) mix_class->process_buffers (mix, mix->array_buffers, outbuf); } -/* remove buffers from the queue that were expired in the - * interval of the master, we also prepare the queued value - * in the pad so that we can skip and fill buffers later on */ -static void -gst_gl_mixer_update_queues (GstGLMixer * mix) +/* Perform qos calculations before processing the next frame. Returns TRUE if + * the frame should be processed, FALSE if the frame can be dropped entirely */ +static gint64 +gst_gl_mixer_do_qos (GstGLMixer * mix, GstClockTime timestamp) { - GSList *walk; - gint64 interval; + GstClockTime qostime, earliest_time; + gdouble proportion; + gint64 jitter; + + /* no timestamp, can't do QoS => process frame */ + if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (timestamp))) { + GST_LOG_OBJECT (mix, "invalid timestamp, can't do QoS, process frame"); + return -1; + } - interval = mix->master->queued; - if (interval <= 0) { - if (mix->fps_n == 0) { - interval = G_MAXINT64; - } else { - interval = GST_SECOND * mix->fps_d / mix->fps_n; - } - GST_LOG_OBJECT (mix, "set interval to %" G_GINT64_FORMAT " nanoseconds", - interval); + /* get latest QoS observation values */ + gst_gl_mixer_read_qos (mix, &proportion, &earliest_time); + + /* skip qos if we have no observation (yet) => process frame */ + if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (earliest_time))) { + GST_LOG_OBJECT (mix, "no observation yet, process frame"); + return -1; } - walk = mix->sinkpads; - while (walk) { - GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); - GstGLMixerCollect *mixcol = pad->mixcol; + /* qos is done on running time */ + qostime = + gst_segment_to_running_time (&mix->segment, GST_FORMAT_TIME, timestamp); - walk = g_slist_next (walk); + /* see how our next timestamp relates to the latest qos timestamp */ + GST_LOG_OBJECT (mix, "qostime %" GST_TIME_FORMAT ", earliest %" + GST_TIME_FORMAT, GST_TIME_ARGS (qostime), GST_TIME_ARGS (earliest_time)); - if (mixcol->buffer != NULL) { - pad->queued -= interval; - GST_LOG_OBJECT (pad, "queued now %" G_GINT64_FORMAT, pad->queued); - if (pad->queued <= 0) { - GstBuffer *buffer = - gst_collect_pads_pop (mix->collect, &mixcol->collect); - GST_LOG_OBJECT (pad, "unreffing buffer"); - if (buffer) - gst_buffer_unref (buffer); - else - GST_WARNING_OBJECT (pad, - "Buffer was removed by GstCollectPads in the meantime"); - gst_buffer_unref (mixcol->buffer); - mixcol->buffer = NULL; - } - } + jitter = GST_CLOCK_DIFF (qostime, earliest_time); + if (qostime != GST_CLOCK_TIME_NONE && jitter > 0) { + GST_DEBUG_OBJECT (mix, "we are late, drop frame"); + return jitter; } + + GST_LOG_OBJECT (mix, "process frame"); + return jitter; } static GstFlowReturn gst_gl_mixer_collected (GstCollectPads * pads, GstGLMixer * mix) { - GstFlowReturn ret = GST_FLOW_OK; + GstFlowReturn ret; + GstClockTime output_start_time, output_end_time; GstBuffer *outbuf = NULL; - GstGLBuffer *gl_outbuf = NULL; - gboolean eos = FALSE; - gint width = 0; - gint height = 0; + gint res; + gint64 jitter; g_return_val_if_fail (GST_IS_GL_MIXER (mix), GST_FLOW_ERROR); - if (mix->width == 0) { - GstCaps *newcaps = gst_caps_make_writable - (gst_pad_get_negotiated_caps (GST_PAD (mix->master))); + /* If we're not negotiated yet... */ + if (GST_VIDEO_INFO_FORMAT (&mix->info) == GST_VIDEO_FORMAT_UNKNOWN) + return GST_FLOW_NOT_NEGOTIATED; - gst_pad_set_caps (mix->srcpad, newcaps); + if (g_atomic_int_compare_and_exchange (&mix->flush_stop_pending, TRUE, FALSE)) { + GST_DEBUG_OBJECT (mix, "pending flush stop"); + gst_pad_push_event (mix->srcpad, gst_event_new_flush_stop (TRUE)); } - GST_LOG_OBJECT (mix, "all pads are collected"); - GST_GL_MIXER_STATE_LOCK (mix); + GST_GL_MIXER_LOCK (mix); + + if (mix->newseg_pending) { + GST_DEBUG_OBJECT (mix, "Sending NEWSEGMENT event"); + if (!gst_pad_push_event (mix->srcpad, + gst_event_new_segment (&mix->segment))) { + ret = GST_FLOW_ERROR; + goto error; + } + mix->newseg_pending = FALSE; + } - eos = gst_gl_mixer_fill_queues (mix); + if (mix->segment.position == -1) + output_start_time = mix->segment.start; + else + output_start_time = mix->segment.position; - if (eos) { - /* Push EOS downstream */ - GST_LOG_OBJECT (mix, "all our sinkpads are EOS, pushing downstream"); + if (output_start_time >= mix->segment.stop) { + GST_DEBUG_OBJECT (mix, "Segment done"); gst_pad_push_event (mix->srcpad, gst_event_new_eos ()); - ret = GST_FLOW_WRONG_STATE; + ret = GST_FLOW_EOS; goto error; } - /* Calculating out buffer size from input size */ - ret = gst_gl_buffer_parse_caps (GST_PAD_CAPS (mix->srcpad), &width, &height); + output_end_time = + mix->ts_offset + gst_util_uint64_scale (mix->nframes + 1, + GST_SECOND * GST_VIDEO_INFO_FPS_D (&mix->info), + GST_VIDEO_INFO_FPS_N (&mix->info)); + if (mix->segment.stop != -1) + output_end_time = MIN (output_end_time, mix->segment.stop); - if (!ret) + res = gst_gl_mixer_fill_queues (mix, output_start_time, output_end_time); + + if (res == 0) { + GST_DEBUG_OBJECT (mix, "Need more data for decisions"); + ret = GST_FLOW_OK; + goto error; + } else if (res == -1) { + GST_DEBUG_OBJECT (mix, "All sinkpads are EOS -- forwarding"); + gst_pad_push_event (mix->srcpad, gst_event_new_eos ()); + ret = GST_FLOW_EOS; + goto error; + } else if (res == -2) { + GST_ERROR_OBJECT (mix, "Error collecting buffers"); + ret = GST_FLOW_ERROR; goto error; + } + + jitter = gst_gl_mixer_do_qos (mix, output_start_time); + if (jitter <= 0) { + /* FIXME: Use GstGLMeta */ + /* Calculating out buffer size from input size */ + /*ret = gst_gl_buffer_parse_caps (GST_PAD_CAPS (mix->srcpad), &width, &height); - gl_outbuf = gst_gl_buffer_new (mix->display, mix->width, mix->height); + gl_outbuf = gst_gl_buffer_new (mix->display, mix->width, mix->height); - outbuf = GST_BUFFER (gl_outbuf); - gst_buffer_set_caps (outbuf, GST_PAD_CAPS (mix->srcpad)); + outbuf = GST_BUFFER (gl_outbuf); + gst_buffer_set_caps (outbuf, GST_PAD_CAPS (mix->srcpad)); */ - gst_gl_mixer_process_buffers (mix, outbuf); + gst_gl_mixer_process_buffers (mix, outbuf); + mix->qos_processed++; + } else { + GstMessage *msg; + + mix->qos_dropped++; + + /* TODO: live */ + msg = + gst_message_new_qos (GST_OBJECT_CAST (mix), FALSE, + gst_segment_to_running_time (&mix->segment, GST_FORMAT_TIME, + output_start_time), gst_segment_to_stream_time (&mix->segment, + GST_FORMAT_TIME, output_start_time), output_start_time, + output_end_time - output_start_time); + gst_message_set_qos_values (msg, jitter, mix->proportion, 1000000); + gst_message_set_qos_stats (msg, GST_FORMAT_BUFFERS, mix->qos_processed, + mix->qos_dropped); + gst_element_post_message (GST_ELEMENT_CAST (mix), msg); + + ret = GST_FLOW_OK; + } - gst_gl_mixer_update_queues (mix); - GST_GL_MIXER_STATE_UNLOCK (mix); + mix->segment.position = output_end_time; + mix->nframes++; - ret = gst_pad_push (mix->srcpad, outbuf); + GST_GL_MIXER_UNLOCK (mix); + if (outbuf) { + GST_LOG_OBJECT (mix, + "Pushing buffer with ts %" GST_TIME_FORMAT " and duration %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (outbuf))); + ret = gst_pad_push (mix->srcpad, outbuf); + } -beach: +done: return ret; /* ERRORS */ error: { - if (outbuf) - gst_buffer_unref (outbuf); - - GST_GL_MIXER_STATE_UNLOCK (mix); - goto beach; + GST_GL_MIXER_UNLOCK (mix); + goto done; } } @@ -1088,7 +1479,7 @@ forward_event_func (GstPad * pad, GValue * ret, GstEvent * event) * sinkpads. */ static gboolean -forward_event (GstGLMixer * mix, GstEvent * event) +gst_gl_mixer_push_sink_event (GstGLMixer * mix, GstEvent * event) { GstIterator *it; GValue vret = { 0 }; @@ -1107,105 +1498,243 @@ forward_event (GstGLMixer * mix, GstEvent * event) return g_value_get_boolean (&vret); } +static GstFlowReturn +gst_gl_mixer_sink_clip (GstCollectPads * pads, + GstCollectData * data, GstBuffer * buf, GstBuffer ** outbuf, + GstGLMixer * mix) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (data->pad); + GstGLMixerCollect *mixcol = pad->mixcol; + GstClockTime start_time, end_time; + + start_time = GST_BUFFER_TIMESTAMP (buf); + if (start_time == -1) { + GST_ERROR_OBJECT (pad, "Timestamped buffers required!"); + gst_buffer_unref (buf); + return GST_FLOW_ERROR; + } + + end_time = GST_BUFFER_DURATION (buf); + if (end_time == -1) + end_time = + gst_util_uint64_scale_int (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&pad->info), GST_VIDEO_INFO_FPS_N (&pad->info)); + if (end_time == -1) { + *outbuf = buf; + return GST_FLOW_OK; + } + + start_time = MAX (start_time, mixcol->collect.segment.start); + start_time = + gst_segment_to_running_time (&mixcol->collect.segment, + GST_FORMAT_TIME, start_time); + + end_time += GST_BUFFER_TIMESTAMP (buf); + if (mixcol->collect.segment.stop != -1) + end_time = MIN (end_time, mixcol->collect.segment.stop); + end_time = + gst_segment_to_running_time (&mixcol->collect.segment, + GST_FORMAT_TIME, end_time); + + /* Convert to the output segment rate */ + if (ABS (mix->segment.rate) != 1.0) { + start_time *= ABS (mix->segment.rate); + end_time *= ABS (mix->segment.rate); + } + + if (mixcol->buffer != NULL && end_time < mixcol->end_time) { + gst_buffer_unref (buf); + *outbuf = NULL; + return GST_FLOW_OK; + } + + *outbuf = buf; + return GST_FLOW_OK; +} + static gboolean -gst_gl_mixer_src_event (GstPad * pad, GstEvent * event) +gst_gl_mixer_src_event (GstPad * pad, GstObject * parent, GstEvent * event) { - GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent (pad)); + GstGLMixer *mix = GST_GL_MIXER (parent); gboolean result; switch (GST_EVENT_TYPE (event)) { case GST_EVENT_QOS: - /* QoS might be tricky */ - result = FALSE; + { + GstQOSType type; + GstClockTimeDiff diff; + GstClockTime timestamp; + gdouble proportion; + + gst_event_parse_qos (event, &type, &proportion, &diff, ×tamp); + + gst_gl_mixer_update_qos (mix, proportion, diff, timestamp); + + result = gst_gl_mixer_push_sink_event (mix, event); break; + } case GST_EVENT_SEEK: { + gdouble rate; + GstFormat fmt; GstSeekFlags flags; - GstSeekType curtype; - gint64 cur; + GstSeekType start_type, stop_type; + gint64 start, stop; + GSList *l; + gdouble abs_rate; /* parse the seek parameters */ - gst_event_parse_seek (event, NULL, NULL, &flags, &curtype, - &cur, NULL, NULL); + gst_event_parse_seek (event, &rate, &fmt, &flags, &start_type, + &start, &stop_type, &stop); + + if (rate <= 0.0) { + GST_ERROR_OBJECT (mix, "Negative rates not supported yet"); + result = FALSE; + gst_event_unref (event); + break; + } + + GST_DEBUG_OBJECT (mix, "Handling SEEK event"); /* check if we are flushing */ if (flags & GST_SEEK_FLAG_FLUSH) { - /* make sure we accept nothing anymore and return WRONG_STATE */ - gst_collect_pads_set_flushing (mix->collect, TRUE); - /* flushing seek, start flush downstream, the flush will be done * when all pads received a FLUSH_STOP. */ gst_pad_push_event (mix->srcpad, gst_event_new_flush_start ()); + + /* make sure we accept nothing anymore and return WRONG_STATE */ + gst_collect_pads_set_flushing (mix->collect, TRUE); } /* now wait for the collected to be finished and mark a new * segment */ - GST_OBJECT_LOCK (mix->collect); - if (curtype == GST_SEEK_TYPE_SET) - mix->segment_position = cur; - else - mix->segment_position = 0; - mix->sendseg = TRUE; - GST_OBJECT_UNLOCK (mix->collect); - - result = forward_event (mix, event); + GST_COLLECT_PADS_STREAM_LOCK (mix->collect); + + abs_rate = ABS (rate); + + GST_GL_MIXER_LOCK (mix); + for (l = mix->sinkpads; l; l = l->next) { + GstGLMixerPad *p = l->data; + + if (flags & GST_SEEK_FLAG_FLUSH) { + gst_buffer_replace (&p->mixcol->buffer, NULL); + p->mixcol->start_time = p->mixcol->end_time = -1; + continue; + } + + /* Convert to the output segment rate */ + if (ABS (mix->segment.rate) != abs_rate) { + if (ABS (mix->segment.rate) != 1.0 && p->mixcol->buffer) { + p->mixcol->start_time /= ABS (mix->segment.rate); + p->mixcol->end_time /= ABS (mix->segment.rate); + } + if (abs_rate != 1.0 && p->mixcol->buffer) { + p->mixcol->start_time *= abs_rate; + p->mixcol->end_time *= abs_rate; + } + } + } + GST_GL_MIXER_UNLOCK (mix); + + gst_segment_do_seek (&mix->segment, rate, fmt, flags, start_type, start, + stop_type, stop, NULL); + mix->segment.position = -1; + mix->ts_offset = 0; + mix->nframes = 0; + mix->newseg_pending = TRUE; + + if (flags & GST_SEEK_FLAG_FLUSH) { + gst_collect_pads_set_flushing (mix->collect, FALSE); + + /* we can't send FLUSH_STOP here since upstream could start pushing data + * after we unlock mix->collect. + * We set flush_stop_pending to TRUE instead and send FLUSH_STOP after + * forwarding the seek upstream or from gst_gl_mixer_collected, + * whichever happens first. + */ + mix->flush_stop_pending = TRUE; + } + + GST_COLLECT_PADS_STREAM_UNLOCK (mix->collect); + + gst_gl_mixer_reset_qos (mix); + + result = gst_gl_mixer_push_sink_event (mix, event); + + if (g_atomic_int_compare_and_exchange (&mix->flush_stop_pending, TRUE, + FALSE)) { + GST_DEBUG_OBJECT (mix, "pending flush stop"); + gst_pad_push_event (mix->srcpad, gst_event_new_flush_stop (TRUE)); + } + break; } case GST_EVENT_NAVIGATION: /* navigation is rather pointless. */ result = FALSE; + gst_event_unref (event); break; default: /* just forward the rest for now */ - result = forward_event (mix, event); + result = gst_gl_mixer_push_sink_event (mix, event); break; } - gst_object_unref (mix); return result; } static gboolean -gst_gl_mixer_sink_event (GstPad * pad, GstEvent * event) +gst_gl_mixer_sink_event (GstCollectPads * pads, GstCollectData * cdata, + GstEvent * event, GstGLMixer * mix) { - GstGLMixerPad *vpad = GST_GL_MIXER_PAD (pad); - GstGLMixer *videomixer = GST_GL_MIXER (gst_pad_get_parent (pad)); - gboolean ret; + GstGLMixerPad *pad = GST_GL_MIXER_PAD (cdata->pad); + gboolean ret = TRUE; GST_DEBUG_OBJECT (pad, "Got %s event on pad %s:%s", GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad)); switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_FLUSH_STOP: - /* mark a pending new segment. This event is synchronized - * with the streaming thread so we can safely update the - * variable without races. It's somewhat weird because we - * assume the collectpads forwarded the FLUSH_STOP past us - * and downstream (using our source pad, the bastard!). - */ - videomixer->sendseg = TRUE; - - /* Reset pad state after FLUSH_STOP */ - if (vpad->mixcol->buffer) - gst_buffer_unref (vpad->mixcol->buffer); - vpad->mixcol->buffer = NULL; - vpad->queued = 0; + case GST_EVENT_CAPS: + { + GstCaps *caps; + + gst_event_parse_caps (event, &caps); + ret = + gst_gl_mixer_pad_sink_setcaps (GST_PAD (pad), GST_OBJECT (mix), caps); + gst_event_unref (event); + event = NULL; break; - case GST_EVENT_NEWSEGMENT: - videomixer->sendseg = TRUE; + } + case GST_EVENT_SEGMENT:{ + GstSegment seg; + gst_event_copy_segment (event, &seg); + + g_assert (seg.format == GST_FORMAT_TIME); + break; + } + case GST_EVENT_FLUSH_STOP: + mix->newseg_pending = TRUE; + mix->flush_stop_pending = FALSE; + gst_gl_mixer_reset_qos (mix); + gst_buffer_replace (&pad->mixcol->buffer, NULL); + pad->mixcol->start_time = -1; + pad->mixcol->end_time = -1; + + gst_segment_init (&mix->segment, GST_FORMAT_TIME); + mix->segment.position = -1; + mix->ts_offset = 0; + mix->nframes = 0; break; default: break; } - /* now GstCollectPads can take care of the rest, e.g. EOS */ - ret = videomixer->collect_event (pad, event); + if (event != NULL) + return gst_collect_pads_event_default (pads, cdata, event, FALSE); - gst_object_unref (videomixer); return ret; } - static void gst_gl_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) @@ -1261,8 +1790,8 @@ gst_gl_mixer_change_state (GstElement * element, GstStateChange transition) return FALSE; } - structure = gst_structure_new (gst_element_get_name (mix), NULL); - query = gst_query_new_application (GST_QUERY_CUSTOM, structure); + structure = gst_structure_new_empty (gst_element_get_name (mix)); + query = gst_query_new_custom (GST_QUERY_CUSTOM, structure); /* retrieve the gldisplay that is owned by gl elements after the gl mixer */ isPerformed = gst_element_query (parent, query); diff --git a/gst-libs/gst/gl/gstglmixer.h b/gst-libs/gst/gl/gstglmixer.h index 9cdbc8b..47f1402 100644 --- a/gst-libs/gst/gl/gstglmixer.h +++ b/gst-libs/gst/gl/gstglmixer.h @@ -56,36 +56,31 @@ struct _GstGLMixer GstPad *srcpad; /* Lock to prevent the state to change while blending */ - GMutex *state_lock; + GMutex lock; /* Sink pads using Collect Pads from core's base library */ GstCollectPads *collect; + /* sinkpads, a GSList of GstGLMixerPads */ GSList *sinkpads; - - GPtrArray *array_buffers; - gint numpads; + /* Next available sinkpad index */ + gint next_sinkpad; - GstClockTime last_ts; - - /* the master pad */ - GstGLMixerPad *master; + GPtrArray *array_buffers; - gint width; - gint height; - gboolean setcaps; - gboolean sendseg; + GstVideoInfo info; - gint fps_n; - gint fps_d; + gboolean newseg_pending; + gboolean flush_stop_pending; - /* Next available sinkpad index */ - gint next_sinkpad; + GstSegment segment; + GstClockTime ts_offset; + guint64 nframes; /* sink event handling */ - GstPadEventFunction collect_event; - guint64 segment_position; - gdouble segment_rate; + gdouble proportion; + GstClockTime earliest_time; + guint64 qos_processed, qos_dropped; GstGLDisplay *display; GLuint fbo; diff --git a/gst-libs/gst/gl/gstglmixerpad.h b/gst-libs/gst/gl/gstglmixerpad.h index cfd4df7..d3ec5f2 100644 --- a/gst-libs/gst/gl/gstglmixerpad.h +++ b/gst-libs/gst/gl/gstglmixerpad.h @@ -46,7 +46,11 @@ struct _GstGLMixerCollect { GstCollectData collect; /* we extend the CollectData */ + GstBuffer *queued; + GstBuffer *buffer; /* the queued buffer for this pad */ + GstClockTime start_time; + GstClockTime end_time; GstGLMixerPad *mixpad; }; @@ -56,12 +60,8 @@ struct _GstGLMixerPad { GstPad parent; /* subclass the pad */ - gint64 queued; - - gint width; - gint height; - gint fps_n; - gint fps_d; + /* */ + GstVideoInfo info; GstGLDisplay *display; @@ -73,5 +73,7 @@ struct _GstGLMixerPadClass GstPadClass parent_class; }; +GType gst_gl_mixer_pad_get_type (void); + G_END_DECLS #endif /* __GST_GL_MIXER_PAD_H__ */ -- 2.7.4