1 /* Video compositor plugin
2 * Copyright (C) 2004, 2008 Wim Taymans <wim@fluendo.com>
3 * Copyright (C) 2010 Sebastian Dröge <sebastian.droege@collabora.co.uk>
4 * Copyright (C) 2014 Mathieu Duponchelle <mathieu.duponchelle@opencreed.com>
5 * Copyright (C) 2014 Thibault Saunier <tsaunier@gnome.org>
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
20 * Boston, MA 02110-1301, USA.
24 * SECTION:element-compositor
27 * Compositor can accept AYUV, ARGB and BGRA video streams. For each of the requested
28 * sink pads it will compare the incoming geometry and framerate to define the
29 * output parameters. Indeed output video frames will have the geometry of the
30 * biggest incoming video stream and the framerate of the fastest incoming one.
32 * Compositor will do colorspace conversion.
34 * Individual parameters for each input stream can be configured on the
37 * * "xpos": The x-coordinate position of the top-left corner of the picture (#gint)
38 * * "ypos": The y-coordinate position of the top-left corner of the picture (#gint)
39 * * "width": The width of the picture; the input will be scaled if necessary (#gint)
40 * * "height": The height of the picture; the input will be scaled if necessary (#gint)
41 * * "alpha": The transparency of the picture; between 0.0 and 1.0. The blending
42 * is a simple copy when fully-transparent (0.0) and fully-opaque (1.0). (#gdouble)
43 * * "zorder": The z-order position of the picture in the composition (#guint)
48 * videotestsrc pattern=1 ! \
49 * video/x-raw,format=AYUV,framerate=\(fraction\)10/1,width=100,height=100 ! \
50 * videobox border-alpha=0 top=-70 bottom=-70 right=-220 ! \
51 * compositor name=comp sink_0::alpha=0.7 sink_1::alpha=0.5 ! \
52 * videoconvert ! xvimagesink \
54 * video/x-raw,format=AYUV,framerate=\(fraction\)5/1,width=320,height=240 ! comp.
55 * ]| A pipeline to demonstrate compositor used together with videobox.
56 * This should show a 320x240 pixels video test source with some transparency
57 * showing the background checker pattern. Another video test source with just
58 * the snow pattern of 100x100 pixels is overlaid on top of the first one on
59 * the left vertically centered with a small transparency showing the first
60 * video test source behind and the checker pattern under it. Note that the
61 * framerate of the output video is 10 frames per second.
63 * gst-launch-1.0 videotestsrc pattern=1 ! \
64 * video/x-raw, framerate=\(fraction\)10/1, width=100, height=100 ! \
65 * compositor name=comp ! videoconvert ! ximagesink \
67 * video/x-raw, framerate=\(fraction\)5/1, width=320, height=240 ! comp.
68 * ]| A pipeline to demostrate bgra comping. (This does not demonstrate alpha blending).
70 * gst-launch-1.0 videotestsrc pattern=1 ! \
71 * video/x-raw,format =I420, framerate=\(fraction\)10/1, width=100, height=100 ! \
72 * compositor name=comp ! videoconvert ! ximagesink \
74 * video/x-raw,format=I420, framerate=\(fraction\)5/1, width=320, height=240 ! comp.
75 * ]| A pipeline to test I420
77 * gst-launch-1.0 compositor name=comp sink_1::alpha=0.5 sink_1::xpos=50 sink_1::ypos=50 ! \
78 * videoconvert ! ximagesink \
79 * videotestsrc pattern=snow timestamp-offset=3000000000 ! \
80 * "video/x-raw,format=AYUV,width=640,height=480,framerate=(fraction)30/1" ! \
81 * timeoverlay ! queue2 ! comp. \
82 * videotestsrc pattern=smpte ! \
83 * "video/x-raw,format=AYUV,width=800,height=600,framerate=(fraction)10/1" ! \
84 * timeoverlay ! queue2 ! comp.
85 * ]| A pipeline to demonstrate synchronized compositing (the second stream starts after 3 seconds)
95 #include "compositor.h"
98 #define orc_memset memset
100 #include <orc/orcfunctions.h>
103 GST_DEBUG_CATEGORY_STATIC (gst_compositor_debug);
104 #define GST_CAT_DEFAULT gst_compositor_debug
106 #define FORMATS " { AYUV, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, "\
107 " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, "\
110 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
113 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS))
116 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
119 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS))
122 static void gst_compositor_child_proxy_init (gpointer g_iface,
123 gpointer iface_data);
125 #define GST_TYPE_COMPOSITOR_OPERATOR (gst_compositor_operator_get_type())
127 gst_compositor_operator_get_type (void)
129 static GType compositor_operator_type = 0;
131 static const GEnumValue compositor_operator[] = {
132 {COMPOSITOR_OPERATOR_SOURCE, "Source", "source"},
133 {COMPOSITOR_OPERATOR_OVER, "Over", "over"},
134 {COMPOSITOR_OPERATOR_ADD, "Add", "add"},
138 if (!compositor_operator_type) {
139 compositor_operator_type =
140 g_enum_register_static ("GstCompositorOperator", compositor_operator);
142 return compositor_operator_type;
145 #define DEFAULT_PAD_XPOS 0
146 #define DEFAULT_PAD_YPOS 0
147 #define DEFAULT_PAD_WIDTH 0
148 #define DEFAULT_PAD_HEIGHT 0
149 #define DEFAULT_PAD_ALPHA 1.0
150 #define DEFAULT_PAD_OPERATOR COMPOSITOR_OPERATOR_OVER
162 G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad,
163 GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD);
166 gst_compositor_pad_get_property (GObject * object, guint prop_id,
167 GValue * value, GParamSpec * pspec)
169 GstCompositorPad *pad = GST_COMPOSITOR_PAD (object);
173 g_value_set_int (value, pad->xpos);
176 g_value_set_int (value, pad->ypos);
179 g_value_set_int (value, pad->width);
181 case PROP_PAD_HEIGHT:
182 g_value_set_int (value, pad->height);
185 g_value_set_double (value, pad->alpha);
187 case PROP_PAD_OPERATOR:
188 g_value_set_enum (value, pad->op);
191 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
197 gst_compositor_pad_set_property (GObject * object, guint prop_id,
198 const GValue * value, GParamSpec * pspec)
200 GstCompositorPad *pad = GST_COMPOSITOR_PAD (object);
204 pad->xpos = g_value_get_int (value);
207 pad->ypos = g_value_get_int (value);
210 pad->width = g_value_get_int (value);
211 gst_video_aggregator_convert_pad_update_conversion_info
212 (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad));
214 case PROP_PAD_HEIGHT:
215 pad->height = g_value_get_int (value);
216 gst_video_aggregator_convert_pad_update_conversion_info
217 (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad));
220 pad->alpha = g_value_get_double (value);
222 case PROP_PAD_OPERATOR:
223 pad->op = g_value_get_enum (value);
224 gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad),
225 pad->op == COMPOSITOR_OPERATOR_ADD);
228 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
234 _mixer_pad_get_output_size (GstCompositorPad * comp_pad, gint out_par_n,
235 gint out_par_d, gint * width, gint * height)
237 GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad);
238 gint pad_width, pad_height;
241 /* FIXME: Anything better we can do here? */
242 if (!vagg_pad->info.finfo
243 || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) {
244 GST_DEBUG_OBJECT (comp_pad, "Have no caps yet");
252 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : comp_pad->width;
255 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height;
257 if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height,
258 GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
259 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) {
260 GST_WARNING_OBJECT (comp_pad, "Cannot calculate display aspect ratio");
261 *width = *height = 0;
264 GST_LOG_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width,
265 pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
266 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d);
268 /* Pick either height or width, whichever is an integer multiple of the
269 * display aspect ratio. However, prefer preserving the height to account
270 * for interlaced video. */
271 if (pad_height % dar_n == 0) {
272 pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
273 } else if (pad_width % dar_d == 0) {
274 pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n);
276 pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
280 *height = pad_height;
283 /* Test whether rectangle2 contains rectangle 1 (geometrically) */
285 is_rectangle_contained (GstVideoRectangle rect1, GstVideoRectangle rect2)
287 if ((rect2.x <= rect1.x) && (rect2.y <= rect1.y) &&
288 ((rect2.x + rect2.w) >= (rect1.x + rect1.w)) &&
289 ((rect2.y + rect2.h) >= (rect1.y + rect1.h)))
294 static GstVideoRectangle
295 clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width,
300 GstVideoRectangle clamped;
302 /* Clamp the x/y coordinates of this frame to the output boundaries to cover
303 * the case where (say, with negative xpos/ypos or w/h greater than the output
304 * size) the non-obscured portion of the frame could be outside the bounds of
305 * the video itself and hence not visible at all */
306 clamped.x = CLAMP (x, 0, outer_width);
307 clamped.y = CLAMP (y, 0, outer_height);
308 clamped.w = CLAMP (x2, 0, outer_width) - clamped.x;
309 clamped.h = CLAMP (y2, 0, outer_height) - clamped.y;
314 /* Call this with the lock taken */
316 _pad_obscures_rectangle (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad,
317 GstVideoRectangle rect, gboolean rect_transparent)
319 GstVideoRectangle pad_rect;
320 GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad);
322 /* No buffer to obscure the rectangle with */
323 if (!gst_video_aggregator_pad_has_current_buffer (pad))
326 /* Can't obscure if it's transparent and if the format has an alpha component
327 * we'd have to inspect every pixel to know if the frame is opaque, so assume
328 * it doesn't obscure. As a bonus, if the rectangle is fully transparent, we
329 * can also obscure it if we have alpha components on the pad */
330 if (!rect_transparent &&
331 (cpad->alpha != 1.0 || GST_VIDEO_INFO_HAS_ALPHA (&pad->info)))
334 pad_rect.x = cpad->xpos;
335 pad_rect.y = cpad->ypos;
336 /* Handle pixel and display aspect ratios to find the actual size */
337 _mixer_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (&vagg->info),
338 GST_VIDEO_INFO_PAR_D (&vagg->info), &(pad_rect.w), &(pad_rect.h));
340 if (!is_rectangle_contained (rect, pad_rect))
343 GST_DEBUG_OBJECT (pad, "Pad %s %ix%i@(%i,%i) obscures rect %ix%i@(%i,%i)",
344 GST_PAD_NAME (pad), pad_rect.w, pad_rect.h, pad_rect.x, pad_rect.y,
345 rect.w, rect.h, rect.x, rect.y);
351 gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad,
352 GstVideoAggregator * vagg, GstBuffer * buffer,
353 GstVideoFrame * prepared_frame)
355 GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad);
357 gboolean frame_obscured = FALSE;
359 /* The rectangle representing this frame, clamped to the video's boundaries.
360 * Due to the clamping, this is different from the frame width/height above. */
361 GstVideoRectangle frame_rect;
363 /* There's three types of width/height here:
364 * 1. GST_VIDEO_FRAME_WIDTH/HEIGHT:
365 * The frame width/height (same as pad->info.height/width;
366 * see gst_video_frame_map())
367 * 2. cpad->width/height:
368 * The optional pad property for scaling the frame (if zero, the video is
370 * 3. conversion_info.width/height:
371 * Equal to cpad->width/height if it's set, otherwise it's the pad
372 * width/height. See ->set_info()
375 _mixer_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (&vagg->info),
376 GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height);
378 if (cpad->alpha == 0.0) {
379 GST_DEBUG_OBJECT (pad, "Pad has alpha 0.0, not converting frame");
383 frame_rect = clamp_rectangle (cpad->xpos, cpad->ypos, width, height,
384 GST_VIDEO_INFO_WIDTH (&vagg->info), GST_VIDEO_INFO_HEIGHT (&vagg->info));
386 if (frame_rect.w == 0 || frame_rect.h == 0) {
387 GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height "
388 "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h);
392 GST_OBJECT_LOCK (vagg);
393 /* Check if this frame is obscured by a higher-zorder frame
394 * TODO: Also skip a frame if it's obscured by a combination of
395 * higher-zorder frames */
396 l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad)->next;
397 for (; l; l = l->next) {
398 if (_pad_obscures_rectangle (vagg, l->data, frame_rect, FALSE)) {
399 frame_obscured = TRUE;
403 GST_OBJECT_UNLOCK (vagg);
409 GST_VIDEO_AGGREGATOR_PAD_CLASS
410 (gst_compositor_pad_parent_class)->prepare_frame (pad, vagg, buffer,
419 gst_compositor_pad_create_conversion_info (GstVideoAggregatorConvertPad * pad,
420 GstVideoAggregator * vagg, GstVideoInfo * conversion_info)
422 GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad);
425 GST_VIDEO_AGGREGATOR_CONVERT_PAD_CLASS
426 (gst_compositor_pad_parent_class)->create_conversion_info (pad, vagg,
428 if (!conversion_info->finfo)
431 _mixer_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (&vagg->info),
432 GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height);
434 /* The only thing that can change here is the width
435 * and height, otherwise set_info would've been called */
436 if (GST_VIDEO_INFO_WIDTH (conversion_info) != width ||
437 GST_VIDEO_INFO_HEIGHT (conversion_info) != height) {
438 GstVideoInfo tmp_info;
440 /* Initialize with the wanted video format and our original width and
441 * height as we don't want to rescale. Then copy over the wanted
442 * colorimetry, and chroma-site and our current pixel-aspect-ratio
443 * and other relevant fields.
445 gst_video_info_set_format (&tmp_info,
446 GST_VIDEO_INFO_FORMAT (conversion_info), width, height);
447 tmp_info.chroma_site = conversion_info->chroma_site;
448 tmp_info.colorimetry = conversion_info->colorimetry;
449 tmp_info.par_n = conversion_info->par_n;
450 tmp_info.par_d = conversion_info->par_d;
451 tmp_info.fps_n = conversion_info->fps_n;
452 tmp_info.fps_d = conversion_info->fps_d;
453 tmp_info.flags = conversion_info->flags;
454 tmp_info.interlace_mode = conversion_info->interlace_mode;
456 *conversion_info = tmp_info;
461 gst_compositor_pad_class_init (GstCompositorPadClass * klass)
463 GObjectClass *gobject_class = (GObjectClass *) klass;
464 GstVideoAggregatorPadClass *vaggpadclass =
465 (GstVideoAggregatorPadClass *) klass;
466 GstVideoAggregatorConvertPadClass *vaggcpadclass =
467 (GstVideoAggregatorConvertPadClass *) klass;
469 gobject_class->set_property = gst_compositor_pad_set_property;
470 gobject_class->get_property = gst_compositor_pad_get_property;
472 g_object_class_install_property (gobject_class, PROP_PAD_XPOS,
473 g_param_spec_int ("xpos", "X Position", "X Position of the picture",
474 G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
475 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
476 g_object_class_install_property (gobject_class, PROP_PAD_YPOS,
477 g_param_spec_int ("ypos", "Y Position", "Y Position of the picture",
478 G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
479 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
480 g_object_class_install_property (gobject_class, PROP_PAD_WIDTH,
481 g_param_spec_int ("width", "Width", "Width of the picture",
482 G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH,
483 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
484 g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT,
485 g_param_spec_int ("height", "Height", "Height of the picture",
486 G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT,
487 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
488 g_object_class_install_property (gobject_class, PROP_PAD_ALPHA,
489 g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
491 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
492 g_object_class_install_property (gobject_class, PROP_PAD_OPERATOR,
493 g_param_spec_enum ("operator", "Operator",
494 "Blending operator to use for blending this pad over the previous ones",
495 GST_TYPE_COMPOSITOR_OPERATOR, DEFAULT_PAD_OPERATOR,
496 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
498 vaggpadclass->prepare_frame =
499 GST_DEBUG_FUNCPTR (gst_compositor_pad_prepare_frame);
501 vaggcpadclass->create_conversion_info =
502 GST_DEBUG_FUNCPTR (gst_compositor_pad_create_conversion_info);
506 gst_compositor_pad_init (GstCompositorPad * compo_pad)
508 compo_pad->xpos = DEFAULT_PAD_XPOS;
509 compo_pad->ypos = DEFAULT_PAD_YPOS;
510 compo_pad->alpha = DEFAULT_PAD_ALPHA;
511 compo_pad->op = DEFAULT_PAD_OPERATOR;
516 #define DEFAULT_BACKGROUND COMPOSITOR_BACKGROUND_CHECKER
523 #define GST_TYPE_COMPOSITOR_BACKGROUND (gst_compositor_background_get_type())
525 gst_compositor_background_get_type (void)
527 static GType compositor_background_type = 0;
529 static const GEnumValue compositor_background[] = {
530 {COMPOSITOR_BACKGROUND_CHECKER, "Checker pattern", "checker"},
531 {COMPOSITOR_BACKGROUND_BLACK, "Black", "black"},
532 {COMPOSITOR_BACKGROUND_WHITE, "White", "white"},
533 {COMPOSITOR_BACKGROUND_TRANSPARENT,
534 "Transparent Background to enable further compositing", "transparent"},
538 if (!compositor_background_type) {
539 compositor_background_type =
540 g_enum_register_static ("GstCompositorBackground",
541 compositor_background);
543 return compositor_background_type;
547 gst_compositor_get_property (GObject * object,
548 guint prop_id, GValue * value, GParamSpec * pspec)
550 GstCompositor *self = GST_COMPOSITOR (object);
553 case PROP_BACKGROUND:
554 g_value_set_enum (value, self->background);
557 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
563 gst_compositor_set_property (GObject * object,
564 guint prop_id, const GValue * value, GParamSpec * pspec)
566 GstCompositor *self = GST_COMPOSITOR (object);
569 case PROP_BACKGROUND:
570 self->background = g_value_get_enum (value);
573 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
578 #define gst_compositor_parent_class parent_class
579 G_DEFINE_TYPE_WITH_CODE (GstCompositor, gst_compositor,
580 GST_TYPE_VIDEO_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
581 gst_compositor_child_proxy_init));
584 set_functions (GstCompositor * self, GstVideoInfo * info)
586 gboolean ret = FALSE;
589 self->overlay = NULL;
590 self->fill_checker = NULL;
591 self->fill_color = NULL;
593 switch (GST_VIDEO_INFO_FORMAT (info)) {
594 case GST_VIDEO_FORMAT_AYUV:
595 self->blend = gst_compositor_blend_ayuv;
596 self->overlay = gst_compositor_overlay_ayuv;
597 self->fill_checker = gst_compositor_fill_checker_ayuv;
598 self->fill_color = gst_compositor_fill_color_ayuv;
601 case GST_VIDEO_FORMAT_ARGB:
602 self->blend = gst_compositor_blend_argb;
603 self->overlay = gst_compositor_overlay_argb;
604 self->fill_checker = gst_compositor_fill_checker_argb;
605 self->fill_color = gst_compositor_fill_color_argb;
608 case GST_VIDEO_FORMAT_BGRA:
609 self->blend = gst_compositor_blend_bgra;
610 self->overlay = gst_compositor_overlay_bgra;
611 self->fill_checker = gst_compositor_fill_checker_bgra;
612 self->fill_color = gst_compositor_fill_color_bgra;
615 case GST_VIDEO_FORMAT_ABGR:
616 self->blend = gst_compositor_blend_abgr;
617 self->overlay = gst_compositor_overlay_abgr;
618 self->fill_checker = gst_compositor_fill_checker_abgr;
619 self->fill_color = gst_compositor_fill_color_abgr;
622 case GST_VIDEO_FORMAT_RGBA:
623 self->blend = gst_compositor_blend_rgba;
624 self->overlay = gst_compositor_overlay_rgba;
625 self->fill_checker = gst_compositor_fill_checker_rgba;
626 self->fill_color = gst_compositor_fill_color_rgba;
629 case GST_VIDEO_FORMAT_Y444:
630 self->blend = gst_compositor_blend_y444;
631 self->overlay = self->blend;
632 self->fill_checker = gst_compositor_fill_checker_y444;
633 self->fill_color = gst_compositor_fill_color_y444;
636 case GST_VIDEO_FORMAT_Y42B:
637 self->blend = gst_compositor_blend_y42b;
638 self->overlay = self->blend;
639 self->fill_checker = gst_compositor_fill_checker_y42b;
640 self->fill_color = gst_compositor_fill_color_y42b;
643 case GST_VIDEO_FORMAT_YUY2:
644 self->blend = gst_compositor_blend_yuy2;
645 self->overlay = self->blend;
646 self->fill_checker = gst_compositor_fill_checker_yuy2;
647 self->fill_color = gst_compositor_fill_color_yuy2;
650 case GST_VIDEO_FORMAT_UYVY:
651 self->blend = gst_compositor_blend_uyvy;
652 self->overlay = self->blend;
653 self->fill_checker = gst_compositor_fill_checker_uyvy;
654 self->fill_color = gst_compositor_fill_color_uyvy;
657 case GST_VIDEO_FORMAT_YVYU:
658 self->blend = gst_compositor_blend_yvyu;
659 self->overlay = self->blend;
660 self->fill_checker = gst_compositor_fill_checker_yvyu;
661 self->fill_color = gst_compositor_fill_color_yvyu;
664 case GST_VIDEO_FORMAT_I420:
665 self->blend = gst_compositor_blend_i420;
666 self->overlay = self->blend;
667 self->fill_checker = gst_compositor_fill_checker_i420;
668 self->fill_color = gst_compositor_fill_color_i420;
671 case GST_VIDEO_FORMAT_YV12:
672 self->blend = gst_compositor_blend_yv12;
673 self->overlay = self->blend;
674 self->fill_checker = gst_compositor_fill_checker_yv12;
675 self->fill_color = gst_compositor_fill_color_yv12;
678 case GST_VIDEO_FORMAT_NV12:
679 self->blend = gst_compositor_blend_nv12;
680 self->overlay = self->blend;
681 self->fill_checker = gst_compositor_fill_checker_nv12;
682 self->fill_color = gst_compositor_fill_color_nv12;
685 case GST_VIDEO_FORMAT_NV21:
686 self->blend = gst_compositor_blend_nv21;
687 self->overlay = self->blend;
688 self->fill_checker = gst_compositor_fill_checker_nv21;
689 self->fill_color = gst_compositor_fill_color_nv21;
692 case GST_VIDEO_FORMAT_Y41B:
693 self->blend = gst_compositor_blend_y41b;
694 self->overlay = self->blend;
695 self->fill_checker = gst_compositor_fill_checker_y41b;
696 self->fill_color = gst_compositor_fill_color_y41b;
699 case GST_VIDEO_FORMAT_RGB:
700 self->blend = gst_compositor_blend_rgb;
701 self->overlay = self->blend;
702 self->fill_checker = gst_compositor_fill_checker_rgb;
703 self->fill_color = gst_compositor_fill_color_rgb;
706 case GST_VIDEO_FORMAT_BGR:
707 self->blend = gst_compositor_blend_bgr;
708 self->overlay = self->blend;
709 self->fill_checker = gst_compositor_fill_checker_bgr;
710 self->fill_color = gst_compositor_fill_color_bgr;
713 case GST_VIDEO_FORMAT_xRGB:
714 self->blend = gst_compositor_blend_xrgb;
715 self->overlay = self->blend;
716 self->fill_checker = gst_compositor_fill_checker_xrgb;
717 self->fill_color = gst_compositor_fill_color_xrgb;
720 case GST_VIDEO_FORMAT_xBGR:
721 self->blend = gst_compositor_blend_xbgr;
722 self->overlay = self->blend;
723 self->fill_checker = gst_compositor_fill_checker_xbgr;
724 self->fill_color = gst_compositor_fill_color_xbgr;
727 case GST_VIDEO_FORMAT_RGBx:
728 self->blend = gst_compositor_blend_rgbx;
729 self->overlay = self->blend;
730 self->fill_checker = gst_compositor_fill_checker_rgbx;
731 self->fill_color = gst_compositor_fill_color_rgbx;
734 case GST_VIDEO_FORMAT_BGRx:
735 self->blend = gst_compositor_blend_bgrx;
736 self->overlay = self->blend;
737 self->fill_checker = gst_compositor_fill_checker_bgrx;
738 self->fill_color = gst_compositor_fill_color_bgrx;
749 _fixate_caps (GstAggregator * agg, GstCaps * caps)
751 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
753 gint best_width = -1, best_height = -1;
754 gint best_fps_n = -1, best_fps_d = -1;
756 gdouble best_fps = 0.;
760 ret = gst_caps_make_writable (caps);
762 /* we need this to calculate how large to make the output frame */
763 s = gst_caps_get_structure (ret, 0);
764 if (gst_structure_has_field (s, "pixel-aspect-ratio")) {
765 gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1);
766 gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d);
771 GST_OBJECT_LOCK (vagg);
772 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
773 GstVideoAggregatorPad *vaggpad = l->data;
774 GstCompositorPad *compositor_pad = GST_COMPOSITOR_PAD (vaggpad);
775 gint this_width, this_height;
780 fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info);
781 fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info);
782 _mixer_pad_get_output_size (compositor_pad, par_n, par_d, &width, &height);
784 if (width == 0 || height == 0)
787 this_width = width + MAX (compositor_pad->xpos, 0);
788 this_height = height + MAX (compositor_pad->ypos, 0);
790 if (best_width < this_width)
791 best_width = this_width;
792 if (best_height < this_height)
793 best_height = this_height;
798 gst_util_fraction_to_double (fps_n, fps_d, &cur_fps);
800 if (best_fps < cur_fps) {
806 GST_OBJECT_UNLOCK (vagg);
808 if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) {
814 gst_structure_fixate_field_nearest_int (s, "width", best_width);
815 gst_structure_fixate_field_nearest_int (s, "height", best_height);
816 gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n,
818 ret = gst_caps_fixate (ret);
824 _negotiated_caps (GstAggregator * agg, GstCaps * caps)
828 GST_DEBUG_OBJECT (agg, "Negotiated caps %" GST_PTR_FORMAT, caps);
830 if (!gst_video_info_from_caps (&v_info, caps))
833 if (!set_functions (GST_COMPOSITOR (agg), &v_info)) {
834 GST_ERROR_OBJECT (agg, "Failed to setup vfuncs");
838 return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps);
842 _should_draw_background (GstVideoAggregator * vagg, gboolean bg_transparent)
844 GstVideoRectangle bg_rect;
845 gboolean draw = TRUE;
846 bg_rect.x = bg_rect.y = 0;
848 GST_OBJECT_LOCK (vagg);
849 bg_rect.w = GST_VIDEO_INFO_WIDTH (&vagg->info);
850 bg_rect.h = GST_VIDEO_INFO_HEIGHT (&vagg->info);
851 /* Check if the background is completely obscured by a pad
852 * TODO: Also skip if it's obscured by a combination of pads */
853 for (GList * l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
854 if (_pad_obscures_rectangle (vagg, l->data, bg_rect, bg_transparent)) {
859 GST_OBJECT_UNLOCK (vagg);
864 _draw_background (GstVideoAggregator * vagg, GstVideoFrame * outframe)
866 GstCompositor *comp = GST_COMPOSITOR (vagg);
867 BlendFunction composite = comp->blend;
869 /* If one of the frames to be composited completely obscures the background,
870 * don't bother drawing the background at all. We can also always use the
871 * 'blend' BlendFunction in that case because it only changes if we have to
872 * overlay on top of a transparent background. */
873 if (!_should_draw_background (vagg,
874 comp->background == COMPOSITOR_BACKGROUND_TRANSPARENT))
877 switch (comp->background) {
878 case COMPOSITOR_BACKGROUND_CHECKER:
879 comp->fill_checker (outframe);
881 case COMPOSITOR_BACKGROUND_BLACK:
882 comp->fill_color (outframe, 16, 128, 128);
884 case COMPOSITOR_BACKGROUND_WHITE:
885 comp->fill_color (outframe, 240, 128, 128);
887 case COMPOSITOR_BACKGROUND_TRANSPARENT:
889 guint i, plane, num_planes, height;
891 num_planes = GST_VIDEO_FRAME_N_PLANES (outframe);
892 for (plane = 0; plane < num_planes; ++plane) {
894 gsize rowsize, plane_stride;
896 pdata = GST_VIDEO_FRAME_PLANE_DATA (outframe, plane);
897 plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (outframe, plane);
898 rowsize = GST_VIDEO_FRAME_COMP_WIDTH (outframe, plane)
899 * GST_VIDEO_FRAME_COMP_PSTRIDE (outframe, plane);
900 height = GST_VIDEO_FRAME_COMP_HEIGHT (outframe, plane);
901 for (i = 0; i < height; ++i) {
902 memset (pdata, 0, rowsize);
903 pdata += plane_stride;
906 /* use overlay to keep background transparent */
907 composite = comp->overlay;
916 gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf)
919 BlendFunction composite;
920 GstVideoFrame out_frame, *outframe;
922 if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, GST_MAP_WRITE)) {
923 GST_WARNING_OBJECT (vagg, "Could not map output buffer");
924 return GST_FLOW_ERROR;
927 outframe = &out_frame;
928 composite = _draw_background (vagg, outframe);
930 GST_OBJECT_LOCK (vagg);
931 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
932 GstVideoAggregatorPad *pad = l->data;
933 GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad);
934 GstVideoFrame *prepared_frame =
935 gst_video_aggregator_pad_get_prepared_frame (pad);
936 GstCompositorBlendMode blend_mode = COMPOSITOR_BLEND_MODE_OVER;
938 switch (compo_pad->op) {
939 case COMPOSITOR_OPERATOR_SOURCE:
940 blend_mode = COMPOSITOR_BLEND_MODE_SOURCE;
942 case COMPOSITOR_OPERATOR_OVER:
943 blend_mode = COMPOSITOR_BLEND_MODE_OVER;
945 case COMPOSITOR_OPERATOR_ADD:
946 blend_mode = COMPOSITOR_BLEND_MODE_ADD;
949 g_assert_not_reached ();
953 if (prepared_frame != NULL) {
954 composite (prepared_frame,
956 compo_pad->ypos, compo_pad->alpha, outframe, blend_mode);
959 GST_OBJECT_UNLOCK (vagg);
961 gst_video_frame_unmap (outframe);
967 gst_compositor_request_new_pad (GstElement * element, GstPadTemplate * templ,
968 const gchar * req_name, const GstCaps * caps)
973 GST_ELEMENT_CLASS (parent_class)->request_new_pad (element,
974 templ, req_name, caps);
977 goto could_not_create;
979 gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad),
980 GST_OBJECT_NAME (newpad));
986 GST_DEBUG_OBJECT (element, "could not create/add pad");
992 gst_compositor_release_pad (GstElement * element, GstPad * pad)
994 GstCompositor *compositor;
996 compositor = GST_COMPOSITOR (element);
998 GST_DEBUG_OBJECT (compositor, "release pad %s:%s", GST_DEBUG_PAD_NAME (pad));
1000 gst_child_proxy_child_removed (GST_CHILD_PROXY (compositor), G_OBJECT (pad),
1001 GST_OBJECT_NAME (pad));
1003 GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad);
1007 _sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query)
1009 switch (GST_QUERY_TYPE (query)) {
1010 case GST_QUERY_ALLOCATION:{
1013 GstBufferPool *pool;
1015 GstStructure *structure;
1017 gst_query_parse_allocation (query, &caps, NULL);
1022 if (!gst_video_info_from_caps (&info, caps))
1025 size = GST_VIDEO_INFO_SIZE (&info);
1027 pool = gst_video_buffer_pool_new ();
1029 structure = gst_buffer_pool_get_config (pool);
1030 gst_buffer_pool_config_set_params (structure, caps, size, 0, 0);
1032 if (!gst_buffer_pool_set_config (pool, structure)) {
1033 gst_object_unref (pool);
1037 gst_query_add_allocation_pool (query, pool, size, 0, 0);
1038 gst_object_unref (pool);
1039 gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
1044 return GST_AGGREGATOR_CLASS (parent_class)->sink_query (agg, bpad, query);
1048 /* GObject boilerplate */
1050 gst_compositor_class_init (GstCompositorClass * klass)
1052 GObjectClass *gobject_class = (GObjectClass *) klass;
1053 GstElementClass *gstelement_class = (GstElementClass *) klass;
1054 GstVideoAggregatorClass *videoaggregator_class =
1055 (GstVideoAggregatorClass *) klass;
1056 GstAggregatorClass *agg_class = (GstAggregatorClass *) klass;
1058 gobject_class->get_property = gst_compositor_get_property;
1059 gobject_class->set_property = gst_compositor_set_property;
1061 gstelement_class->request_new_pad =
1062 GST_DEBUG_FUNCPTR (gst_compositor_request_new_pad);
1063 gstelement_class->release_pad =
1064 GST_DEBUG_FUNCPTR (gst_compositor_release_pad);
1065 agg_class->sink_query = _sink_query;
1066 agg_class->fixate_src_caps = _fixate_caps;
1067 agg_class->negotiated_src_caps = _negotiated_caps;
1068 videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames;
1070 g_object_class_install_property (gobject_class, PROP_BACKGROUND,
1071 g_param_spec_enum ("background", "Background", "Background type",
1072 GST_TYPE_COMPOSITOR_BACKGROUND,
1073 DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1075 gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
1076 &src_factory, GST_TYPE_AGGREGATOR_PAD);
1077 gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
1078 &sink_factory, GST_TYPE_COMPOSITOR_PAD);
1080 gst_element_class_set_static_metadata (gstelement_class, "Compositor",
1081 "Filter/Editor/Video/Compositor",
1082 "Composite multiple video streams", "Wim Taymans <wim@fluendo.com>, "
1083 "Sebastian Dröge <sebastian.droege@collabora.co.uk>");
1087 gst_compositor_init (GstCompositor * self)
1089 /* initialize variables */
1090 self->background = DEFAULT_BACKGROUND;
1093 /* GstChildProxy implementation */
1095 gst_compositor_child_proxy_get_child_by_index (GstChildProxy * child_proxy,
1098 GstCompositor *compositor = GST_COMPOSITOR (child_proxy);
1099 GObject *obj = NULL;
1101 GST_OBJECT_LOCK (compositor);
1102 obj = g_list_nth_data (GST_ELEMENT_CAST (compositor)->sinkpads, index);
1104 gst_object_ref (obj);
1105 GST_OBJECT_UNLOCK (compositor);
1111 gst_compositor_child_proxy_get_children_count (GstChildProxy * child_proxy)
1114 GstCompositor *compositor = GST_COMPOSITOR (child_proxy);
1116 GST_OBJECT_LOCK (compositor);
1117 count = GST_ELEMENT_CAST (compositor)->numsinkpads;
1118 GST_OBJECT_UNLOCK (compositor);
1119 GST_INFO_OBJECT (compositor, "Children Count: %d", count);
1125 gst_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data)
1127 GstChildProxyInterface *iface = g_iface;
1129 iface->get_child_by_index = gst_compositor_child_proxy_get_child_by_index;
1130 iface->get_children_count = gst_compositor_child_proxy_get_children_count;
1133 /* Element registration */
1135 plugin_init (GstPlugin * plugin)
1137 GST_DEBUG_CATEGORY_INIT (gst_compositor_debug, "compositor", 0, "compositor");
1139 gst_compositor_init_blend ();
1141 return gst_element_register (plugin, "compositor", GST_RANK_PRIMARY + 1,
1142 GST_TYPE_COMPOSITOR);
1145 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
1148 "Compositor", plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME,