1 /* Generic video aggregator plugin
2 * Copyright (C) 2004, 2008 Wim Taymans <wim@fluendo.com>
3 * Copyright (C) 2010 Sebastian Dröge <sebastian.droege@collabora.co.uk>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 * SECTION:gstvideoaggregator
23 * @short_description: Base class for video aggregators
25 * VideoAggregator can accept AYUV, ARGB and BGRA video streams. For each of the requested
26 * sink pads it will compare the incoming geometry and framerate to define the
27 * output parameters. Indeed output video frames will have the geometry of the
28 * biggest incoming video stream and the framerate of the fastest incoming one.
30 * VideoAggregator will do colorspace conversion.
32 * Zorder for each input stream can be configured on the
33 * #GstVideoAggregatorPad.
43 #include "gstvideoaggregator.h"
44 #include "gstvideoaggregatorpad.h"
45 #include "videoconvert.h"
47 GST_DEBUG_CATEGORY_STATIC (gst_videoaggregator_debug);
48 #define GST_CAT_DEFAULT gst_videoaggregator_debug
50 /* Needed prototypes */
51 static void gst_videoaggregator_reset_qos (GstVideoAggregator * vagg);
53 /****************************************
54 * GstVideoAggregatorPad implementation *
55 ****************************************/
57 #define DEFAULT_PAD_ZORDER 0
64 G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad,
65 GST_TYPE_AGGREGATOR_PAD);
68 gst_videoaggregator_pad_get_property (GObject * object, guint prop_id,
69 GValue * value, GParamSpec * pspec)
71 GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (object);
75 g_value_set_uint (value, pad->zorder);
78 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
84 pad_zorder_compare (const GstVideoAggregatorPad * pad1,
85 const GstVideoAggregatorPad * pad2)
87 return pad1->zorder - pad2->zorder;
91 gst_videoaggregator_pad_set_property (GObject * object, guint prop_id,
92 const GValue * value, GParamSpec * pspec)
94 GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (object);
95 GstVideoAggregator *vagg =
96 GST_VIDEO_AGGREGATOR (gst_pad_get_parent (GST_PAD (pad)));
100 GST_OBJECT_LOCK (vagg);
101 pad->zorder = g_value_get_uint (value);
102 GST_ELEMENT (vagg)->sinkpads = g_list_sort (GST_ELEMENT (vagg)->sinkpads,
103 (GCompareFunc) pad_zorder_compare);
104 GST_OBJECT_UNLOCK (vagg);
107 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
111 gst_object_unref (vagg);
115 _flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator)
117 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (aggregator);
118 GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (aggpad);
120 gst_videoaggregator_reset_qos (vagg);
121 gst_buffer_replace (&pad->buffer, NULL);
122 pad->start_time = -1;
129 gst_videoaggregator_pad_finalize (GObject * o)
131 GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (o);
133 if (vaggpad->convert)
134 videoconvert_convert_free (vaggpad->convert);
135 vaggpad->convert = NULL;
137 G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->dispose (o);
141 gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass)
143 GObjectClass *gobject_class = (GObjectClass *) klass;
144 GstAggregatorPadClass *aggpadclass = (GstAggregatorPadClass *) klass;
146 gobject_class->set_property = gst_videoaggregator_pad_set_property;
147 gobject_class->get_property = gst_videoaggregator_pad_get_property;
148 gobject_class->finalize = gst_videoaggregator_pad_finalize;
150 g_object_class_install_property (gobject_class, PROP_PAD_ZORDER,
151 g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture",
152 0, 10000, DEFAULT_PAD_ZORDER,
153 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
155 aggpadclass->flush = GST_DEBUG_FUNCPTR (_flush_pad);
159 gst_videoaggregator_pad_init (GstVideoAggregatorPad * vaggpad)
161 vaggpad->zorder = DEFAULT_PAD_ZORDER;
162 vaggpad->convert = NULL;
163 vaggpad->need_conversion_update = FALSE;
164 vaggpad->aggregated_frame = NULL;
165 vaggpad->converted_buffer = NULL;
168 /*********************************
169 * GstChildProxy implementation *
170 *********************************/
172 gst_videoaggregator_child_proxy_get_child_by_index (GstChildProxy * child_proxy,
176 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (child_proxy);
178 GST_OBJECT_LOCK (vagg);
179 if ((obj = g_list_nth_data (GST_ELEMENT (vagg)->sinkpads, index)))
181 GST_OBJECT_UNLOCK (vagg);
187 gst_videoaggregator_child_proxy_get_children_count (GstChildProxy * child_proxy)
191 GST_OBJECT_LOCK (child_proxy);
192 count = GST_ELEMENT (child_proxy)->numsinkpads;
193 GST_OBJECT_UNLOCK (child_proxy);
195 GST_INFO_OBJECT (child_proxy, "Children Count: %d", count);
201 gst_videoaggregator_child_proxy_init (gpointer g_iface, gpointer iface_data)
203 GstChildProxyInterface *iface = g_iface;
205 GST_INFO ("intializing child proxy interface");
206 iface->get_child_by_index =
207 gst_videoaggregator_child_proxy_get_child_by_index;
208 iface->get_children_count =
209 gst_videoaggregator_child_proxy_get_children_count;
212 /**************************************
213 * GstVideoAggregator implementation *
214 **************************************/
216 #define GST_VIDEO_AGGREGATOR_GET_LOCK(vagg) (&GST_VIDEO_AGGREGATOR(vagg)->priv->lock)
218 #define GST_VIDEO_AGGREGATOR_LOCK(vagg) G_STMT_START { \
219 GST_LOG_OBJECT (vagg, "Taking EVENT lock from thread %p", \
221 g_mutex_lock(GST_VIDEO_AGGREGATOR_GET_LOCK(vagg)); \
222 GST_LOG_OBJECT (vagg, "Took EVENT lock from thread %p", \
226 #define GST_VIDEO_AGGREGATOR_UNLOCK(vagg) G_STMT_START { \
227 GST_LOG_OBJECT (vagg, "Releasing EVENT lock from thread %p", \
229 g_mutex_unlock(GST_VIDEO_AGGREGATOR_GET_LOCK(vagg)); \
230 GST_LOG_OBJECT (vagg, "Took EVENT lock from thread %p", \
235 #define GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg) (&GST_VIDEO_AGGREGATOR(vagg)->priv->setcaps_lock)
236 #define GST_VIDEO_AGGREGATOR_SETCAPS_LOCK(vagg) G_STMT_START { \
237 GST_LOG_OBJECT (vagg, "Taking SETCAPS lock from thread %p", \
239 g_mutex_lock(GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg)); \
240 GST_LOG_OBJECT (vagg, "Took SETCAPS lock from thread %p", \
244 #define GST_VIDEO_AGGREGATOR_SETCAPS_UNLOCK(vagg) G_STMT_START { \
245 GST_LOG_OBJECT (vagg, "Releasing SETCAPS lock from thread %p", \
247 g_mutex_unlock(GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg)); \
248 GST_LOG_OBJECT (vagg, "Took SETCAPS lock from thread %p", \
252 struct _GstVideoAggregatorPrivate
254 /* Lock to prevent the state to change while aggregating */
257 /* Lock to prevent two src setcaps from happening at the same time */
260 /* Current downstream segment */
261 GstClockTime ts_offset;
266 GstClockTime earliest_time;
267 guint64 qos_processed, qos_dropped;
270 GstCaps *current_caps;
274 G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVideoAggregator, gst_videoaggregator,
275 GST_TYPE_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
276 gst_videoaggregator_child_proxy_init));
279 _find_best_video_format (GstVideoAggregator * vagg, GstCaps * downstream_caps,
280 GstVideoInfo * best_info, GstVideoFormat * best_format,
281 gboolean * at_least_one_alpha)
284 GstCaps *possible_caps;
285 GstVideoAggregatorPad *pad;
286 gboolean need_alpha = FALSE;
287 gint best_format_number = 0;
288 GHashTable *formats_table = g_hash_table_new (g_direct_hash, g_direct_equal);
290 GST_OBJECT_LOCK (vagg);
291 for (tmp = GST_ELEMENT (vagg)->sinkpads; tmp; tmp = tmp->next) {
297 if (!pad->info.finfo)
300 if (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)
301 *at_least_one_alpha = TRUE;
303 /* If we want alpha, disregard all the other formats */
304 if (need_alpha && !(pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA))
307 /* This can happen if we release a pad and another pad hasn't been negotiated_caps yet */
308 if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN)
311 possible_caps = gst_video_info_to_caps (&pad->info);
313 s = gst_caps_get_structure (possible_caps, 0);
314 gst_structure_remove_fields (s, "width", "height", "framerate",
315 "pixel-aspect-ratio", "interlace-mode", NULL);
317 /* Can downstream accept this format ? */
318 if (!gst_caps_can_intersect (downstream_caps, possible_caps)) {
319 gst_caps_unref (possible_caps);
323 gst_caps_unref (possible_caps);
326 GPOINTER_TO_INT (g_hash_table_lookup (formats_table,
327 GINT_TO_POINTER (GST_VIDEO_INFO_FORMAT (&pad->info))));
330 g_hash_table_replace (formats_table,
331 GINT_TO_POINTER (GST_VIDEO_INFO_FORMAT (&pad->info)),
332 GINT_TO_POINTER (format_number));
334 /* If that pad is the first with alpha, set it as the new best format */
335 if (!need_alpha && (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) {
337 *best_format = GST_VIDEO_INFO_FORMAT (&pad->info);
338 *best_info = pad->info;
339 best_format_number = format_number;
340 } else if (format_number > best_format_number) {
341 *best_format = GST_VIDEO_INFO_FORMAT (&pad->info);
342 *best_info = pad->info;
343 best_format_number = format_number;
346 GST_OBJECT_UNLOCK (vagg);
348 g_hash_table_unref (formats_table);
352 gst_videoaggregator_update_converters (GstVideoAggregator * vagg)
355 GstVideoAggregatorPad *pad;
356 GstVideoFormat best_format;
357 GstVideoInfo best_info;
358 gboolean at_least_one_alpha = FALSE;
359 GstCaps *downstream_caps;
360 gchar *best_colorimetry;
361 const gchar *best_chroma;
362 GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg);
363 GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass;
364 GstAggregator *agg = GST_AGGREGATOR (vagg);
366 best_format = GST_VIDEO_FORMAT_UNKNOWN;
367 gst_video_info_init (&best_info);
369 downstream_caps = gst_pad_get_allowed_caps (agg->srcpad);
371 if (!downstream_caps || gst_caps_is_empty (downstream_caps)) {
372 GST_INFO_OBJECT (vagg, "No downstream caps found %"
373 GST_PTR_FORMAT, downstream_caps);
378 if (vagg_klass->disable_frame_conversion == FALSE)
379 _find_best_video_format (vagg, downstream_caps, &best_info, &best_format,
380 &at_least_one_alpha);
382 if (best_format == GST_VIDEO_FORMAT_UNKNOWN) {
383 downstream_caps = gst_caps_fixate (downstream_caps);
384 gst_video_info_from_caps (&best_info, downstream_caps);
385 best_format = GST_VIDEO_INFO_FORMAT (&best_info);
388 gst_caps_unref (downstream_caps);
390 if (at_least_one_alpha
391 && !(best_info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) {
392 GST_ELEMENT_ERROR (vagg, CORE, NEGOTIATION,
393 ("At least one of the input pads contains alpha, but downstream can't support alpha."),
394 ("Either convert your inputs to not contain alpha or add a videoconvert after the aggregator"));
398 best_colorimetry = gst_video_colorimetry_to_string (&(best_info.colorimetry));
399 best_chroma = gst_video_chroma_to_string (best_info.chroma_site);
400 vagg->info = best_info;
402 GST_DEBUG_OBJECT (vagg,
403 "The output format will now be : %d with colorimetry : %s and chroma : %s",
404 best_format, best_colorimetry, best_chroma);
406 /* Then browse the sinks once more, setting or unsetting conversion if needed */
407 GST_OBJECT_LOCK (vagg);
408 for (tmp = GST_ELEMENT (vagg)->sinkpads; tmp; tmp = tmp->next) {
414 if (!pad->info.finfo)
417 if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN)
421 videoconvert_convert_free (pad->convert);
425 colorimetry = gst_video_colorimetry_to_string (&(pad->info.colorimetry));
426 chroma = gst_video_chroma_to_string (pad->info.chroma_site);
428 if (best_format != GST_VIDEO_INFO_FORMAT (&pad->info) ||
429 g_strcmp0 (colorimetry, best_colorimetry) ||
430 g_strcmp0 (chroma, best_chroma)) {
431 GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d",
432 GST_VIDEO_INFO_FORMAT (&pad->info),
433 GST_VIDEO_INFO_FORMAT (&best_info));
434 pad->convert = videoconvert_convert_new (&pad->info, &best_info);
435 pad->need_conversion_update = TRUE;
437 g_free (colorimetry);
438 g_free (best_colorimetry);
439 GST_WARNING ("No path found for conversion");
440 GST_OBJECT_UNLOCK (vagg);
444 GST_DEBUG_OBJECT (pad, "This pad will not need conversion");
446 g_free (colorimetry);
448 GST_OBJECT_UNLOCK (vagg);
450 g_free (best_colorimetry);
455 gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps)
457 GstAggregator *agg = GST_AGGREGATOR (vagg);
458 gboolean ret = FALSE;
461 GstPad *pad = GST_AGGREGATOR (vagg)->srcpad;
463 GST_INFO_OBJECT (pad, "set src caps: %" GST_PTR_FORMAT, caps);
465 if (!gst_video_info_from_caps (&info, caps))
470 GST_VIDEO_AGGREGATOR_LOCK (vagg);
472 if (GST_VIDEO_INFO_FPS_N (&vagg->info) != GST_VIDEO_INFO_FPS_N (&info) ||
473 GST_VIDEO_INFO_FPS_D (&vagg->info) != GST_VIDEO_INFO_FPS_D (&info)) {
474 if (agg->segment.position != -1) {
475 vagg->priv->ts_offset = agg->segment.position - agg->segment.start;
476 vagg->priv->nframes = 0;
478 gst_videoaggregator_reset_qos (vagg);
483 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
485 if (vagg->priv->current_caps == NULL ||
486 gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) {
487 gst_caps_replace (&vagg->priv->current_caps, caps);
488 vagg->priv->send_caps = TRUE;
496 gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg)
499 gint best_width = -1, best_height = -1;
500 gdouble best_fps = -1, cur_fps;
501 gint best_fps_n = -1, best_fps_d = -1;
503 GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg);
504 GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass;
505 GstAggregator *agg = GST_AGGREGATOR (vagg);
507 GST_VIDEO_AGGREGATOR_SETCAPS_LOCK (vagg);
508 GST_VIDEO_AGGREGATOR_LOCK (vagg);
509 GST_OBJECT_LOCK (vagg);
510 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
511 GstVideoAggregatorPad *mpad = l->data;
512 gint this_width, this_height;
516 fps_n = GST_VIDEO_INFO_FPS_N (&mpad->info);
517 fps_d = GST_VIDEO_INFO_FPS_D (&mpad->info);
518 width = GST_VIDEO_INFO_WIDTH (&mpad->info);
519 height = GST_VIDEO_INFO_HEIGHT (&mpad->info);
521 if (width == 0 || height == 0)
525 this_height = height;
527 if (best_width < this_width)
528 best_width = this_width;
529 if (best_height < this_height)
530 best_height = this_height;
535 gst_util_fraction_to_double (fps_n, fps_d, &cur_fps);
537 if (best_fps < cur_fps) {
543 GST_OBJECT_UNLOCK (vagg);
545 if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) {
551 if (best_width > 0 && best_height > 0 && best_fps > 0) {
552 GstCaps *caps, *peercaps;
556 if (GST_VIDEO_INFO_FPS_N (&vagg->info) != best_fps_n ||
557 GST_VIDEO_INFO_FPS_D (&vagg->info) != best_fps_d) {
558 if (agg->segment.position != -1) {
559 vagg->priv->ts_offset = agg->segment.position - agg->segment.start;
560 vagg->priv->nframes = 0;
563 gst_video_info_init (&info);
564 gst_video_info_set_format (&info, GST_VIDEO_INFO_FORMAT (&vagg->info),
565 best_width, best_height);
566 info.fps_n = best_fps_n;
567 info.fps_d = best_fps_d;
568 info.par_n = GST_VIDEO_INFO_PAR_N (&vagg->info);
569 info.par_d = GST_VIDEO_INFO_PAR_D (&vagg->info);
571 if (vagg_klass->update_info) {
572 if (!vagg_klass->update_info (vagg, &info)) {
574 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
579 caps = gst_video_info_to_caps (&info);
581 peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL);
585 s = gst_caps_get_structure (caps, 0);
586 gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, "height",
587 GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE,
588 0, 1, G_MAXINT, 1, NULL);
590 tmp = gst_caps_intersect (caps, peercaps);
591 gst_caps_unref (caps);
592 gst_caps_unref (peercaps);
594 if (gst_caps_is_empty (caps)) {
595 GST_DEBUG_OBJECT (vagg, "empty caps");
597 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
598 GST_OBJECT_UNLOCK (vagg);
602 caps = gst_caps_truncate (caps);
603 s = gst_caps_get_structure (caps, 0);
604 gst_structure_fixate_field_nearest_int (s, "width", info.width);
605 gst_structure_fixate_field_nearest_int (s, "height", info.height);
606 gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n,
609 gst_structure_get_int (s, "width", &info.width);
610 gst_structure_get_int (s, "height", &info.height);
611 gst_structure_get_fraction (s, "framerate", &info.fps_n, &info.fps_d);
614 gst_caps_unref (caps);
615 caps = gst_video_info_to_caps (&info);
617 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
618 GST_OBJECT_UNLOCK (vagg);
620 if (gst_videoaggregator_src_setcaps (vagg, caps)) {
621 if (vagg_klass->negotiated_caps)
623 GST_VIDEO_AGGREGATOR_GET_CLASS (vagg)->negotiated_caps (vagg, caps);
625 gst_caps_unref (caps);
627 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
631 GST_VIDEO_AGGREGATOR_SETCAPS_UNLOCK (vagg);
637 gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent,
640 GstVideoAggregator *vagg;
641 GstVideoAggregatorPad *vaggpad;
643 gboolean ret = FALSE;
645 GST_INFO_OBJECT (pad, "Setting caps %" GST_PTR_FORMAT, caps);
647 vagg = GST_VIDEO_AGGREGATOR (parent);
648 vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad);
650 if (!gst_video_info_from_caps (&info, caps)) {
651 GST_DEBUG_OBJECT (pad, "Failed to parse caps");
655 GST_VIDEO_AGGREGATOR_LOCK (vagg);
656 if (GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN) {
657 if (GST_VIDEO_INFO_PAR_N (&vagg->info) != GST_VIDEO_INFO_PAR_N (&info)
658 || GST_VIDEO_INFO_PAR_D (&vagg->info) != GST_VIDEO_INFO_PAR_D (&info) ||
659 GST_VIDEO_INFO_INTERLACE_MODE (&vagg->info) !=
660 GST_VIDEO_INFO_INTERLACE_MODE (&info)) {
661 GST_ERROR_OBJECT (pad,
662 "got input caps %" GST_PTR_FORMAT ", but " "current caps are %"
663 GST_PTR_FORMAT, caps, vagg->priv->current_caps);
664 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
669 vaggpad->info = info;
671 ret = gst_videoaggregator_update_converters (vagg);
672 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
675 ret = gst_videoaggregator_update_src_caps (vagg);
682 gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg,
686 GstCaps *template_caps;
687 GstCaps *filtered_caps;
688 GstCaps *returned_caps;
690 gboolean had_current_caps = TRUE;
692 GstAggregator *agg = GST_AGGREGATOR (vagg);
694 template_caps = gst_pad_get_pad_template_caps (GST_PAD (agg->srcpad));
696 srccaps = gst_pad_get_current_caps (GST_PAD (agg->srcpad));
697 if (srccaps == NULL) {
698 had_current_caps = FALSE;
699 srccaps = template_caps;
702 srccaps = gst_caps_make_writable (srccaps);
704 n = gst_caps_get_size (srccaps);
705 for (i = 0; i < n; i++) {
706 s = gst_caps_get_structure (srccaps, i);
707 gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
708 "height", GST_TYPE_INT_RANGE, 1, G_MAXINT,
709 "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL);
710 if (!gst_structure_has_field (s, "pixel-aspect-ratio"))
711 gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
714 gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format",
718 filtered_caps = srccaps;
720 filtered_caps = gst_caps_intersect (srccaps, filter);
721 returned_caps = gst_caps_intersect (filtered_caps, template_caps);
723 gst_caps_unref (srccaps);
725 gst_caps_unref (filtered_caps);
726 if (had_current_caps)
727 gst_caps_unref (template_caps);
729 return returned_caps;
733 gst_videoaggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion,
734 GstClockTimeDiff diff, GstClockTime timestamp)
736 GST_DEBUG_OBJECT (vagg,
737 "Updating QoS: proportion %lf, diff %s%" GST_TIME_FORMAT ", timestamp %"
738 GST_TIME_FORMAT, proportion, (diff < 0) ? "-" : "",
739 GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp));
741 GST_OBJECT_LOCK (vagg);
742 vagg->priv->proportion = proportion;
743 if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) {
744 if (G_UNLIKELY (diff > 0))
745 vagg->priv->earliest_time =
746 timestamp + 2 * diff + gst_util_uint64_scale_int_round (GST_SECOND,
747 GST_VIDEO_INFO_FPS_D (&vagg->info),
748 GST_VIDEO_INFO_FPS_N (&vagg->info));
750 vagg->priv->earliest_time = timestamp + diff;
752 vagg->priv->earliest_time = GST_CLOCK_TIME_NONE;
754 GST_OBJECT_UNLOCK (vagg);
758 gst_videoaggregator_reset_qos (GstVideoAggregator * vagg)
760 gst_videoaggregator_update_qos (vagg, 0.5, 0, GST_CLOCK_TIME_NONE);
761 vagg->priv->qos_processed = vagg->priv->qos_dropped = 0;
765 gst_videoaggregator_read_qos (GstVideoAggregator * vagg, gdouble * proportion,
768 GST_OBJECT_LOCK (vagg);
769 *proportion = vagg->priv->proportion;
770 *time = vagg->priv->earliest_time;
771 GST_OBJECT_UNLOCK (vagg);
775 gst_videoaggregator_reset (GstVideoAggregator * vagg)
777 GstAggregator *agg = GST_AGGREGATOR (vagg);
780 gst_video_info_init (&vagg->info);
781 vagg->priv->ts_offset = 0;
782 vagg->priv->nframes = 0;
784 gst_segment_init (&agg->segment, GST_FORMAT_TIME);
785 agg->segment.position = -1;
787 gst_videoaggregator_reset_qos (vagg);
789 GST_OBJECT_LOCK (vagg);
790 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
791 GstVideoAggregatorPad *p = l->data;
793 gst_buffer_replace (&p->buffer, NULL);
797 gst_video_info_init (&p->info);
799 GST_OBJECT_UNLOCK (vagg);
802 #define GST_FLOW_NEEDS_DATA GST_FLOW_CUSTOM_ERROR
804 gst_videoaggregator_fill_queues (GstVideoAggregator * vagg,
805 GstClockTime output_start_time, GstClockTime output_end_time)
807 GstAggregator *agg = GST_AGGREGATOR (vagg);
810 gboolean need_more_data = FALSE;
812 GST_OBJECT_LOCK (vagg);
813 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
814 GstVideoAggregatorPad *pad = l->data;
816 GstAggregatorPad *bpad;
821 bpad = GST_AGGREGATOR_PAD (pad);
822 segment = &bpad->segment;
824 buf = gst_aggregator_pad_get_buffer (bpad);
826 GstClockTime start_time, end_time;
828 start_time = GST_BUFFER_TIMESTAMP (buf);
829 if (start_time == -1) {
830 gst_buffer_unref (buf);
831 GST_DEBUG_OBJECT (pad, "Need timestamped buffers!");
832 GST_OBJECT_UNLOCK (vagg);
833 return GST_FLOW_ERROR;
838 /* FIXME: Make all this work with negative rates */
840 if ((pad->buffer && start_time < GST_BUFFER_TIMESTAMP (pad->buffer))
841 || (pad->queued && start_time < GST_BUFFER_TIMESTAMP (pad->queued))) {
842 GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping");
843 gst_buffer_unref (buf);
844 buf = gst_aggregator_pad_steal_buffer (bpad);
845 gst_buffer_unref (buf);
846 need_more_data = TRUE;
851 end_time = start_time - GST_BUFFER_TIMESTAMP (pad->queued);
852 start_time = GST_BUFFER_TIMESTAMP (pad->queued);
853 gst_buffer_unref (buf);
854 buf = gst_buffer_ref (pad->queued);
855 vinfo = &pad->queued_vinfo;
857 end_time = GST_BUFFER_DURATION (buf);
859 if (end_time == -1) {
861 buf = gst_aggregator_pad_steal_buffer (bpad);
862 gst_buffer_unref (buf);
863 pad->queued_vinfo = pad->info;
864 GST_DEBUG ("end time is -1 and nothing queued");
865 need_more_data = TRUE;
870 g_assert (start_time != -1 && end_time != -1);
871 end_time += start_time; /* convert from duration to position */
873 /* Check if it's inside the segment */
874 if (start_time >= segment->stop || end_time < segment->start) {
875 GST_DEBUG_OBJECT (pad,
876 "Buffer outside the segment : segment: [%" GST_TIME_FORMAT " -- %"
877 GST_TIME_FORMAT "]" " Buffer [%" GST_TIME_FORMAT " -- %"
878 GST_TIME_FORMAT "]", GST_TIME_ARGS (segment->stop),
879 GST_TIME_ARGS (segment->start), GST_TIME_ARGS (start_time),
880 GST_TIME_ARGS (end_time));
882 if (buf == pad->queued) {
883 gst_buffer_unref (buf);
884 gst_buffer_replace (&pad->queued, NULL);
886 gst_buffer_unref (buf);
887 buf = gst_aggregator_pad_steal_buffer (bpad);
888 gst_buffer_unref (buf);
891 need_more_data = TRUE;
895 /* Clip to segment and convert to running time */
896 start_time = MAX (start_time, segment->start);
897 if (segment->stop != -1)
898 end_time = MIN (end_time, segment->stop);
900 gst_segment_to_running_time (segment, GST_FORMAT_TIME, start_time);
902 gst_segment_to_running_time (segment, GST_FORMAT_TIME, end_time);
903 g_assert (start_time != -1 && end_time != -1);
905 /* Convert to the output segment rate */
906 if (ABS (agg->segment.rate) != 1.0) {
907 start_time *= ABS (agg->segment.rate);
908 end_time *= ABS (agg->segment.rate);
911 if (pad->end_time != -1 && pad->end_time > end_time) {
912 GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping");
913 if (buf == pad->queued) {
914 gst_buffer_unref (buf);
915 gst_buffer_replace (&pad->queued, NULL);
917 gst_buffer_unref (buf);
918 buf = gst_aggregator_pad_steal_buffer (bpad);
919 gst_buffer_unref (buf);
922 need_more_data = TRUE;
926 if (end_time >= output_start_time && start_time < output_end_time) {
927 GST_DEBUG_OBJECT (pad,
928 "Taking new buffer with start time %" GST_TIME_FORMAT,
929 GST_TIME_ARGS (start_time));
930 gst_buffer_replace (&pad->buffer, buf);
931 pad->buffer_vinfo = *vinfo;
932 pad->start_time = start_time;
933 pad->end_time = end_time;
935 if (buf == pad->queued) {
936 gst_buffer_unref (buf);
937 gst_buffer_replace (&pad->queued, NULL);
939 gst_buffer_unref (buf);
940 buf = gst_aggregator_pad_steal_buffer (bpad);
941 gst_buffer_unref (buf);
944 } else if (start_time >= output_end_time) {
945 GST_DEBUG_OBJECT (pad, "Keeping buffer until %" GST_TIME_FORMAT,
946 GST_TIME_ARGS (start_time));
949 GST_DEBUG_OBJECT (pad, "Too old buffer -- dropping");
950 if (buf == pad->queued) {
951 gst_buffer_unref (buf);
952 gst_buffer_replace (&pad->queued, NULL);
954 gst_buffer_unref (buf);
955 buf = gst_aggregator_pad_steal_buffer (bpad);
956 gst_buffer_unref (buf);
959 need_more_data = TRUE;
963 if (pad->end_time != -1) {
964 if (pad->end_time <= output_start_time) {
965 gst_buffer_replace (&pad->buffer, NULL);
966 pad->start_time = pad->end_time = -1;
968 GST_DEBUG ("I just need more data");
969 need_more_data = TRUE;
977 GST_OBJECT_UNLOCK (vagg);
980 return GST_FLOW_NEEDS_DATA;
988 prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad)
990 GstAggregatorPad *bpad = GST_AGGREGATOR_PAD (pad);
992 static GstAllocationParams params = { 0, 15, 0, 0, };
994 if (pad->buffer != NULL) {
996 GstClockTime timestamp;
999 GstVideoFrame *converted_frame = g_slice_new0 (GstVideoFrame);
1000 GstBuffer *converted_buf = NULL;
1001 GstVideoFrame *frame = g_slice_new0 (GstVideoFrame);
1003 seg = &bpad->segment;
1005 timestamp = GST_BUFFER_TIMESTAMP (pad->buffer);
1007 stream_time = gst_segment_to_stream_time (seg, GST_FORMAT_TIME, timestamp);
1009 /* sync object properties on stream time */
1010 if (GST_CLOCK_TIME_IS_VALID (stream_time))
1011 gst_object_sync_values (GST_OBJECT (pad), stream_time);
1014 if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer,
1016 GST_WARNING_OBJECT (vagg, "Could not map input buffer");
1020 gint converted_size;
1022 /* We wait until here to set the conversion infos, in case vagg->info changed */
1023 if (pad->need_conversion_update) {
1024 pad->conversion_info = vagg->info;
1025 gst_video_info_set_format (&(pad->conversion_info),
1026 GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width,
1028 pad->need_conversion_update = FALSE;
1031 converted_size = pad->conversion_info.size;
1032 outsize = GST_VIDEO_INFO_SIZE (&vagg->info);
1033 converted_size = converted_size > outsize ? converted_size : outsize;
1034 converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms);
1036 if (!gst_video_frame_map (converted_frame, &(pad->conversion_info),
1037 converted_buf, GST_MAP_READWRITE)) {
1038 GST_WARNING_OBJECT (vagg, "Could not map converted frame");
1043 videoconvert_convert_convert (pad->convert, converted_frame, frame);
1044 pad->converted_buffer = converted_buf;
1045 gst_video_frame_unmap (frame);
1047 converted_frame = frame;
1048 converted_buf = pad->buffer;
1051 pad->aggregated_frame = converted_frame;
1058 clean_pad (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad)
1060 if (pad->aggregated_frame) {
1061 gst_video_frame_unmap (pad->aggregated_frame);
1062 g_slice_free (GstVideoFrame, pad->aggregated_frame);
1063 pad->aggregated_frame = NULL;
1066 if (pad->converted_buffer) {
1067 gst_buffer_unref (pad->converted_buffer);
1068 pad->converted_buffer = NULL;
1074 static GstFlowReturn
1075 gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg,
1076 GstClockTime output_start_time, GstClockTime output_end_time,
1077 GstBuffer ** outbuf)
1079 GstFlowReturn ret = GST_FLOW_OK;
1080 GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg);
1081 GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass;
1083 g_assert (vagg_klass->aggregate_frames != NULL);
1084 g_assert (vagg_klass->get_output_buffer != NULL);
1086 if ((ret = vagg_klass->get_output_buffer (vagg, outbuf)) != GST_FLOW_OK) {
1087 GST_WARNING_OBJECT (vagg, "Could not get an output buffer, reason: %s",
1088 gst_flow_get_name (ret));
1092 GST_BUFFER_TIMESTAMP (*outbuf) = output_start_time;
1093 GST_BUFFER_DURATION (*outbuf) = output_end_time - output_start_time;
1095 if (vagg_klass->disable_frame_conversion == FALSE) {
1096 /* Here we convert all the frames the subclass will have to aggregate */
1097 gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg),
1098 (GstAggregatorPadForeachFunc) prepare_frames, NULL);
1101 ret = vagg_klass->aggregate_frames (vagg, *outbuf);
1103 gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg),
1104 (GstAggregatorPadForeachFunc) clean_pad, NULL);
1109 /* Perform qos calculations before processing the next frame. Returns TRUE if
1110 * the frame should be processed, FALSE if the frame can be dropped entirely */
1112 gst_videoaggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp)
1114 GstAggregator *agg = GST_AGGREGATOR (vagg);
1115 GstClockTime qostime, earliest_time;
1119 /* no timestamp, can't do QoS => process frame */
1120 if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (timestamp))) {
1121 GST_LOG_OBJECT (vagg, "invalid timestamp, can't do QoS, process frame");
1125 /* get latest QoS observation values */
1126 gst_videoaggregator_read_qos (vagg, &proportion, &earliest_time);
1128 /* skip qos if we have no observation (yet) => process frame */
1129 if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (earliest_time))) {
1130 GST_LOG_OBJECT (vagg, "no observation yet, process frame");
1134 /* qos is done on running time */
1136 gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, timestamp);
1138 /* see how our next timestamp relates to the latest qos timestamp */
1139 GST_LOG_OBJECT (vagg, "qostime %" GST_TIME_FORMAT ", earliest %"
1140 GST_TIME_FORMAT, GST_TIME_ARGS (qostime), GST_TIME_ARGS (earliest_time));
1142 jitter = GST_CLOCK_DIFF (qostime, earliest_time);
1143 if (qostime != GST_CLOCK_TIME_NONE && jitter > 0) {
1144 GST_DEBUG_OBJECT (vagg, "we are late, drop frame");
1148 GST_LOG_OBJECT (vagg, "process frame");
1152 static GstFlowReturn
1153 gst_videoaggregator_aggregate (GstAggregator * agg)
1156 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1157 GstClockTime output_start_time, output_end_time;
1158 GstBuffer *outbuf = NULL;
1162 /* If we're not negotiated_caps yet... */
1163 if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN) {
1164 GST_INFO_OBJECT (agg, "Not negotiated yet!");
1165 return GST_FLOW_NOT_NEGOTIATED;
1168 if (gst_pad_check_reconfigure (agg->srcpad))
1169 gst_videoaggregator_update_src_caps (vagg);
1171 if (vagg->priv->send_caps) {
1172 gst_aggregator_set_src_caps (agg, vagg->priv->current_caps);
1173 vagg->priv->send_caps = FALSE;
1176 GST_VIDEO_AGGREGATOR_LOCK (vagg);
1178 if (agg->segment.position == -1)
1179 output_start_time = agg->segment.start;
1181 output_start_time = agg->segment.position;
1184 vagg->priv->ts_offset + gst_util_uint64_scale_round (vagg->priv->nframes +
1185 1, GST_SECOND * GST_VIDEO_INFO_FPS_D (&vagg->info),
1186 GST_VIDEO_INFO_FPS_N (&vagg->info)) + agg->segment.start;
1188 if (agg->segment.stop != -1)
1189 output_end_time = MIN (output_end_time, agg->segment.stop);
1192 gst_videoaggregator_fill_queues (vagg, output_start_time,
1195 if (res == GST_FLOW_NEEDS_DATA) {
1196 GST_DEBUG_OBJECT (vagg, "Need more data for decisions");
1199 } else if (res == GST_FLOW_EOS) {
1200 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
1201 GST_DEBUG_OBJECT (vagg, "All sinkpads are EOS -- forwarding");
1204 } else if (res == GST_FLOW_ERROR) {
1205 GST_WARNING_OBJECT (vagg, "Error collecting buffers");
1206 ret = GST_FLOW_ERROR;
1210 jitter = gst_videoaggregator_do_qos (vagg, output_start_time);
1212 ret = gst_videoaggregator_do_aggregate (vagg, output_start_time,
1213 output_end_time, &outbuf);
1214 vagg->priv->qos_processed++;
1218 vagg->priv->qos_dropped++;
1222 gst_message_new_qos (GST_OBJECT_CAST (vagg), FALSE,
1223 gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME,
1224 output_start_time), gst_segment_to_stream_time (&agg->segment,
1225 GST_FORMAT_TIME, output_start_time), output_start_time,
1226 output_end_time - output_start_time);
1227 gst_message_set_qos_values (msg, jitter, vagg->priv->proportion, 1000000);
1228 gst_message_set_qos_stats (msg, GST_FORMAT_BUFFERS,
1229 vagg->priv->qos_processed, vagg->priv->qos_dropped);
1230 gst_element_post_message (GST_ELEMENT_CAST (vagg), msg);
1235 agg->segment.position = output_end_time;
1236 vagg->priv->nframes++;
1238 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
1240 GST_DEBUG_OBJECT (vagg,
1241 "Pushing buffer with ts %" GST_TIME_FORMAT " and duration %"
1242 GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)),
1243 GST_TIME_ARGS (GST_BUFFER_DURATION (outbuf)));
1245 ret = gst_aggregator_finish_buffer (agg, outbuf);
1250 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
1256 /* FIXME, the duration query should reflect how long you will produce
1257 * data, that is the amount of stream time until you will emit EOS.
1259 * For synchronized aggregating this is always the max of all the durations
1260 * of upstream since we emit EOS when all of them finished.
1262 * We don't do synchronized aggregating so this really depends on where the
1263 * streams where punched in and what their relative offsets are against
1264 * each other which we can get from the first timestamps we see.
1266 * When we add a new stream (or remove a stream) the duration might
1267 * also become invalid again and we need to post a new DURATION
1268 * message to notify this fact to the parent.
1269 * For now we take the max of all the upstream elements so the simple
1270 * cases work at least somewhat.
1273 gst_videoaggregator_query_duration (GstVideoAggregator * vagg, GstQuery * query)
1275 GValue item = { 0 };
1283 gst_query_parse_duration (query, &format, NULL);
1289 /* Take maximum of all durations */
1290 it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (vagg));
1292 switch (gst_iterator_next (it, &item)) {
1293 case GST_ITERATOR_DONE:
1296 case GST_ITERATOR_OK:
1301 pad = g_value_get_object (&item);
1303 /* ask sink peer for duration */
1304 res &= gst_pad_peer_query_duration (pad, format, &duration);
1305 /* take max from all valid return values */
1307 /* valid unknown length, stop searching */
1308 if (duration == -1) {
1312 /* else see if bigger than current max */
1313 else if (duration > max)
1316 g_value_reset (&item);
1319 case GST_ITERATOR_RESYNC:
1322 gst_iterator_resync (it);
1330 g_value_unset (&item);
1331 gst_iterator_free (it);
1334 /* and store the max */
1335 GST_DEBUG_OBJECT (vagg, "Total duration in format %s: %"
1336 GST_TIME_FORMAT, gst_format_get_name (format), GST_TIME_ARGS (max));
1337 gst_query_set_duration (query, format, max);
1344 gst_videoaggregator_query_latency (GstVideoAggregator * vagg, GstQuery * query)
1346 GstClockTime min, max;
1351 GValue item = { 0 };
1357 max = GST_CLOCK_TIME_NONE;
1359 /* Take maximum of all latency values */
1360 it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (vagg));
1362 switch (gst_iterator_next (it, &item)) {
1363 case GST_ITERATOR_DONE:
1366 case GST_ITERATOR_OK:
1368 GstPad *pad = g_value_get_object (&item);
1369 GstQuery *peerquery;
1370 GstClockTime min_cur, max_cur;
1373 peerquery = gst_query_new_latency ();
1375 /* Ask peer for latency */
1376 res &= gst_pad_peer_query (pad, peerquery);
1378 /* take max from all valid return values */
1380 gst_query_parse_latency (peerquery, &live_cur, &min_cur, &max_cur);
1385 if (max_cur != GST_CLOCK_TIME_NONE &&
1386 ((max != GST_CLOCK_TIME_NONE && max_cur > max) ||
1387 (max == GST_CLOCK_TIME_NONE)))
1390 live = live || live_cur;
1393 gst_query_unref (peerquery);
1394 g_value_reset (&item);
1397 case GST_ITERATOR_RESYNC:
1400 max = GST_CLOCK_TIME_NONE;
1402 gst_iterator_resync (it);
1410 g_value_unset (&item);
1411 gst_iterator_free (it);
1414 /* store the results */
1415 GST_DEBUG_OBJECT (vagg, "Calculated total latency: live %s, min %"
1416 GST_TIME_FORMAT ", max %" GST_TIME_FORMAT,
1417 (live ? "yes" : "no"), GST_TIME_ARGS (min), GST_TIME_ARGS (max));
1418 gst_query_set_latency (query, live, min, max);
1425 gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query)
1427 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1428 gboolean res = FALSE;
1430 switch (GST_QUERY_TYPE (query)) {
1431 case GST_QUERY_POSITION:
1435 gst_query_parse_position (query, &format, NULL);
1438 case GST_FORMAT_TIME:
1439 gst_query_set_position (query, format,
1440 gst_segment_to_stream_time (&agg->segment, GST_FORMAT_TIME,
1441 agg->segment.position));
1449 case GST_QUERY_DURATION:
1450 res = gst_videoaggregator_query_duration (vagg, query);
1452 case GST_QUERY_LATENCY:
1453 res = gst_videoaggregator_query_latency (vagg, query);
1455 case GST_QUERY_CAPS:
1457 GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_query
1461 /* FIXME, needs a custom query handler because we have multiple
1470 gst_videoaggregator_src_event (GstAggregator * agg, GstEvent * event)
1472 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1474 switch (GST_EVENT_TYPE (event)) {
1478 GstClockTimeDiff diff;
1479 GstClockTime timestamp;
1482 gst_event_parse_qos (event, &type, &proportion, &diff, ×tamp);
1483 gst_videoaggregator_update_qos (vagg, proportion, diff, timestamp);
1486 case GST_EVENT_SEEK:
1488 GST_DEBUG_OBJECT (vagg, "Handling SEEK event");
1495 GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_event (agg,
1499 static GstFlowReturn
1500 gst_videoaggregator_sink_clip (GstAggregator * agg,
1501 GstAggregatorPad * bpad, GstBuffer * buf, GstBuffer ** outbuf)
1503 GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (bpad);
1504 GstClockTime start_time, end_time;
1506 start_time = GST_BUFFER_TIMESTAMP (buf);
1507 if (start_time == -1) {
1508 GST_DEBUG_OBJECT (pad, "Timestamped buffers required!");
1509 gst_buffer_unref (buf);
1510 return GST_FLOW_ERROR;
1513 end_time = GST_BUFFER_DURATION (buf);
1514 if (end_time == -1 && GST_VIDEO_INFO_FPS_N (&pad->info) != 0)
1516 gst_util_uint64_scale_int_round (GST_SECOND,
1517 GST_VIDEO_INFO_FPS_D (&pad->info), GST_VIDEO_INFO_FPS_N (&pad->info));
1518 if (end_time == -1) {
1523 start_time = MAX (start_time, bpad->segment.start);
1525 gst_segment_to_running_time (&bpad->segment, GST_FORMAT_TIME, start_time);
1527 end_time += GST_BUFFER_TIMESTAMP (buf);
1528 if (bpad->segment.stop != -1)
1529 end_time = MIN (end_time, bpad->segment.stop);
1531 gst_segment_to_running_time (&bpad->segment, GST_FORMAT_TIME, end_time);
1533 /* Convert to the output segment rate */
1534 if (ABS (agg->segment.rate) != 1.0) {
1535 start_time *= ABS (agg->segment.rate);
1536 end_time *= ABS (agg->segment.rate);
1539 if (bpad->buffer != NULL && end_time < pad->end_time) {
1540 gst_buffer_unref (buf);
1549 static GstFlowReturn
1550 gst_videoaggregator_flush (GstAggregator * agg)
1554 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1556 GST_INFO_OBJECT (agg, "Flushing");
1557 abs_rate = ABS (agg->segment.rate);
1558 GST_OBJECT_LOCK (vagg);
1559 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
1560 GstVideoAggregatorPad *p = l->data;
1562 /* Convert to the output segment rate */
1563 if (ABS (agg->segment.rate) != abs_rate) {
1564 if (ABS (agg->segment.rate) != 1.0 && p->buffer) {
1565 p->start_time /= ABS (agg->segment.rate);
1566 p->end_time /= ABS (agg->segment.rate);
1568 if (abs_rate != 1.0 && p->buffer) {
1569 p->start_time *= abs_rate;
1570 p->end_time *= abs_rate;
1574 GST_OBJECT_UNLOCK (vagg);
1576 agg->segment.position = -1;
1577 vagg->priv->ts_offset = 0;
1578 vagg->priv->nframes = 0;
1580 gst_videoaggregator_reset_qos (vagg);
1585 gst_videoaggregator_sink_event (GstAggregator * agg, GstAggregatorPad * bpad,
1588 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1589 GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (bpad);
1590 gboolean ret = TRUE;
1592 GST_DEBUG_OBJECT (pad, "Got %s event on pad %s:%s",
1593 GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad));
1595 switch (GST_EVENT_TYPE (event)) {
1596 case GST_EVENT_CAPS:
1600 gst_event_parse_caps (event, &caps);
1602 gst_videoaggregator_pad_sink_setcaps (GST_PAD (pad),
1603 GST_OBJECT (vagg), caps);
1604 gst_event_unref (event);
1608 case GST_EVENT_SEGMENT:{
1610 gst_event_copy_segment (event, &seg);
1612 g_assert (seg.format == GST_FORMAT_TIME);
1620 return GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->sink_event
1627 gst_videoaggregator_start (GstAggregator * agg)
1629 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1631 if (!GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->start (agg))
1634 vagg->priv->send_caps = TRUE;
1635 gst_segment_init (&agg->segment, GST_FORMAT_TIME);
1636 gst_caps_replace (&vagg->priv->current_caps, NULL);
1642 gst_videoaggregator_stop (GstAggregator * agg)
1644 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1646 if (!GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->stop (agg))
1649 gst_videoaggregator_reset (vagg);
1654 /* GstElement vmethods */
1656 gst_videoaggregator_request_new_pad (GstElement * element,
1657 GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps)
1659 GstVideoAggregator *vagg;
1660 GstVideoAggregatorPad *vaggpad;
1662 vagg = GST_VIDEO_AGGREGATOR (element);
1664 vaggpad = (GstVideoAggregatorPad *)
1665 GST_ELEMENT_CLASS (gst_videoaggregator_parent_class)->request_new_pad
1666 (element, templ, req_name, caps);
1668 if (vaggpad == NULL)
1671 GST_OBJECT_LOCK (vagg);
1672 vaggpad->zorder = GST_ELEMENT (vagg)->numsinkpads;
1673 vaggpad->start_time = -1;
1674 vaggpad->end_time = -1;
1675 element->sinkpads = g_list_sort (element->sinkpads,
1676 (GCompareFunc) pad_zorder_compare);
1677 GST_OBJECT_UNLOCK (vagg);
1679 gst_child_proxy_child_added (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad),
1680 GST_OBJECT_NAME (vaggpad));
1682 return GST_PAD (vaggpad);
1686 gst_videoaggregator_release_pad (GstElement * element, GstPad * pad)
1688 GstVideoAggregator *vagg = NULL;
1689 GstVideoAggregatorPad *vaggpad;
1690 gboolean update_caps;
1692 vagg = GST_VIDEO_AGGREGATOR (element);
1693 vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad);
1695 GST_VIDEO_AGGREGATOR_LOCK (vagg);
1696 gst_videoaggregator_update_converters (vagg);
1697 gst_buffer_replace (&vaggpad->buffer, NULL);
1698 update_caps = GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN;
1699 GST_VIDEO_AGGREGATOR_UNLOCK (vagg);
1701 gst_child_proxy_child_removed (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad),
1702 GST_OBJECT_NAME (vaggpad));
1704 GST_ELEMENT_CLASS (gst_videoaggregator_parent_class)->release_pad (GST_ELEMENT
1708 gst_videoaggregator_update_src_caps (vagg);
1713 static GstFlowReturn
1714 gst_videoaggregator_get_output_buffer (GstVideoAggregator * videoaggregator,
1715 GstBuffer ** outbuf)
1718 static GstAllocationParams params = { 0, 15, 0, 0, };
1720 outsize = GST_VIDEO_INFO_SIZE (&videoaggregator->info);
1721 *outbuf = gst_buffer_new_allocate (NULL, outsize, ¶ms);
1723 if (*outbuf == NULL) {
1724 GST_ERROR_OBJECT (videoaggregator,
1725 "Could not instantiate buffer of size: %d", outsize);
1732 gst_videoaggregator_pad_sink_acceptcaps (GstPad * pad,
1733 GstVideoAggregator * vagg, GstCaps * caps)
1736 GstCaps *modified_caps;
1737 GstCaps *accepted_caps;
1738 GstCaps *template_caps;
1739 gboolean had_current_caps = TRUE;
1742 GstAggregator *agg = GST_AGGREGATOR (vagg);
1744 GST_DEBUG_OBJECT (pad, "%" GST_PTR_FORMAT, caps);
1746 accepted_caps = gst_pad_get_current_caps (GST_PAD (agg->srcpad));
1748 template_caps = gst_pad_get_pad_template_caps (GST_PAD (agg->srcpad));
1750 if (accepted_caps == NULL) {
1751 accepted_caps = template_caps;
1752 had_current_caps = FALSE;
1755 accepted_caps = gst_caps_make_writable (accepted_caps);
1757 GST_LOG_OBJECT (pad, "src caps %" GST_PTR_FORMAT, accepted_caps);
1759 n = gst_caps_get_size (accepted_caps);
1760 for (i = 0; i < n; i++) {
1761 s = gst_caps_get_structure (accepted_caps, i);
1762 gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
1763 "height", GST_TYPE_INT_RANGE, 1, G_MAXINT,
1764 "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL);
1765 if (!gst_structure_has_field (s, "pixel-aspect-ratio"))
1766 gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
1769 gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format",
1773 modified_caps = gst_caps_intersect (accepted_caps, template_caps);
1775 ret = gst_caps_can_intersect (caps, accepted_caps);
1776 GST_DEBUG_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT,
1777 (ret ? "" : "not "), caps);
1778 gst_caps_unref (accepted_caps);
1779 gst_caps_unref (modified_caps);
1780 if (had_current_caps)
1781 gst_caps_unref (template_caps);
1786 gst_videoaggregator_sink_query (GstAggregator * agg, GstAggregatorPad * bpad,
1789 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1790 GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (bpad);
1791 gboolean ret = FALSE;
1793 switch (GST_QUERY_TYPE (query)) {
1794 case GST_QUERY_CAPS:
1796 GstCaps *filter, *caps;
1798 gst_query_parse_caps (query, &filter);
1799 caps = gst_videoaggregator_pad_sink_getcaps (GST_PAD (pad), vagg, filter);
1800 gst_query_set_caps_result (query, caps);
1801 gst_caps_unref (caps);
1805 case GST_QUERY_ACCEPT_CAPS:
1809 gst_query_parse_accept_caps (query, &caps);
1810 ret = gst_videoaggregator_pad_sink_acceptcaps (GST_PAD (pad), vagg, caps);
1811 gst_query_set_accept_caps_result (query, ret);
1817 GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->sink_query
1824 /* GObject vmethods */
1826 gst_videoaggregator_finalize (GObject * o)
1828 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (o);
1830 g_mutex_clear (&vagg->priv->lock);
1831 g_mutex_clear (&vagg->priv->setcaps_lock);
1833 G_OBJECT_CLASS (gst_videoaggregator_parent_class)->finalize (o);
1837 gst_videoaggregator_dispose (GObject * o)
1839 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (o);
1841 gst_caps_replace (&vagg->priv->current_caps, NULL);
1845 gst_videoaggregator_get_property (GObject * object,
1846 guint prop_id, GValue * value, GParamSpec * pspec)
1850 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1856 gst_videoaggregator_set_property (GObject * object,
1857 guint prop_id, const GValue * value, GParamSpec * pspec)
1861 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1866 /* GObject boilerplate */
1868 gst_videoaggregator_class_init (GstVideoAggregatorClass * klass)
1870 GObjectClass *gobject_class = (GObjectClass *) klass;
1871 GstElementClass *gstelement_class = (GstElementClass *) klass;
1872 GstAggregatorClass *agg_class = (GstAggregatorClass *) klass;
1874 GST_DEBUG_CATEGORY_INIT (gst_videoaggregator_debug, "videoaggregator", 0,
1875 "base video aggregator");
1877 g_type_class_add_private (klass, sizeof (GstVideoAggregatorPrivate));
1879 gobject_class->finalize = gst_videoaggregator_finalize;
1880 gobject_class->dispose = gst_videoaggregator_dispose;
1882 gobject_class->get_property = gst_videoaggregator_get_property;
1883 gobject_class->set_property = gst_videoaggregator_set_property;
1885 gstelement_class->request_new_pad =
1886 GST_DEBUG_FUNCPTR (gst_videoaggregator_request_new_pad);
1887 gstelement_class->release_pad =
1888 GST_DEBUG_FUNCPTR (gst_videoaggregator_release_pad);
1890 gst_element_class_set_static_metadata (gstelement_class,
1891 "Video aggregator base class", "Filter/Editor/Video",
1892 "Aggregate multiple video streams",
1893 "Wim Taymans <wim@fluendo.com>, "
1894 "Sebastian Dröge <sebastian.droege@collabora.co.uk>, "
1895 "Mathieu Duponchelle <mathieu.duponchelle@opencreed.com>, "
1896 "Thibault Saunier <tsaunier@gnome.org>");
1898 agg_class->sinkpads_type = GST_TYPE_VIDEO_AGGREGATOR_PAD;
1899 agg_class->start = gst_videoaggregator_start;
1900 agg_class->stop = gst_videoaggregator_stop;
1901 agg_class->sink_query = gst_videoaggregator_sink_query;
1902 agg_class->sink_event = gst_videoaggregator_sink_event;
1903 agg_class->flush = gst_videoaggregator_flush;
1904 agg_class->clip = gst_videoaggregator_sink_clip;
1905 agg_class->aggregate = gst_videoaggregator_aggregate;
1906 agg_class->src_event = gst_videoaggregator_src_event;
1907 agg_class->src_query = gst_videoaggregator_src_query;
1909 klass->get_output_buffer = gst_videoaggregator_get_output_buffer;
1911 /* Register the pad class */
1912 g_type_class_ref (GST_TYPE_VIDEO_AGGREGATOR_PAD);
1916 gst_videoaggregator_init (GstVideoAggregator * vagg)
1919 G_TYPE_INSTANCE_GET_PRIVATE (vagg, GST_TYPE_VIDEO_AGGREGATOR,
1920 GstVideoAggregatorPrivate);
1922 vagg->priv->current_caps = NULL;
1924 g_mutex_init (&vagg->priv->lock);
1925 g_mutex_init (&vagg->priv->setcaps_lock);
1926 /* initialize variables */
1927 gst_videoaggregator_reset (vagg);