1 /* Generic video mixer plugin
2 * Copyright (C) 2004 Wim Taymans <wim@fluendo.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
17 * Boston, MA 02111-1307, USA.
27 GST_DEBUG_CATEGORY_STATIC (gst_videomixer_debug);
28 #define GST_CAT_DEFAULT gst_videomixer_debug
30 #define GST_TYPE_VIDEO_MIXER_PAD (gst_videomixer_pad_get_type())
31 #define GST_VIDEO_MIXER_PAD(obj) \
32 (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_MIXER_PAD, GstVideoMixerPad))
33 #define GST_VIDEO_MIXER_PAD_CLASS(klass) \
34 (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VIDEO_MIXER_PAD, GstVideoMixerPadiClass))
35 #define GST_IS_VIDEO_MIXER_PAD(obj) \
36 (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VIDEO_MIXER_PAD))
37 #define GST_IS_VIDEO_MIXER_PAD_CLASS(obj) \
38 (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VIDEO_MIXER_PAD))
40 typedef struct _GstVideoMixerPad GstVideoMixerPad;
41 typedef struct _GstVideoMixerPadClass GstVideoMixerPadClass;
43 #define GST_TYPE_VIDEO_MIXER (gst_videomixer_get_type())
44 #define GST_VIDEO_MIXER(obj) \
45 (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_MIXER, GstVideoMixer))
46 #define GST_VIDEO_MIXER_CLASS(klass) \
47 (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VIDEO_MIXER, GstVideoMixerClass))
48 #define GST_IS_VIDEO_MIXER(obj) \
49 (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VIDEO_MIXER))
50 #define GST_IS_VIDEO_MIXER_CLASS(obj) \
51 (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VIDEO_MIXER))
53 static GType gst_videomixer_get_type (void);
55 typedef struct _GstVideoMixer GstVideoMixer;
56 typedef struct _GstVideoMixerClass GstVideoMixerClass;
58 static void gst_videomixer_pad_base_init (gpointer g_class);
59 static void gst_videomixer_pad_class_init (GstVideoMixerPadClass * klass);
60 static void gst_videomixer_pad_init (GstVideoMixerPad * mixerpad);
62 static void gst_videomixer_pad_get_property (GObject * object, guint prop_id,
63 GValue * value, GParamSpec * pspec);
64 static void gst_videomixer_pad_set_property (GObject * object, guint prop_id,
65 const GValue * value, GParamSpec * pspec);
67 static void gst_videomixer_sort_pads (GstVideoMixer * mix);
69 #define DEFAULT_PAD_ZORDER 0
70 #define DEFAULT_PAD_XPOS 0
71 #define DEFAULT_PAD_YPOS 0
72 #define DEFAULT_PAD_ALPHA 1.0
82 /* all information needed for one video stream */
83 struct _GstVideoMixerPad
85 GstRealPad parent; /* subclass the pad */
87 GstBuffer *buffer; /* the queued buffer for this pad */
92 guint in_width, in_height;
101 struct _GstVideoMixerPadClass
103 GstRealPadClass parent_class;
107 gst_videomixer_pad_get_type (void)
109 static GType videomixer_pad_type = 0;
111 if (!videomixer_pad_type) {
112 static const GTypeInfo videomixer_pad_info = {
113 sizeof (GstVideoMixerPadClass),
114 gst_videomixer_pad_base_init,
116 (GClassInitFunc) gst_videomixer_pad_class_init,
119 sizeof (GstVideoMixerPad),
121 (GInstanceInitFunc) gst_videomixer_pad_init,
124 videomixer_pad_type =
125 g_type_register_static (GST_TYPE_REAL_PAD,
126 "GstVideoMixerPad", &videomixer_pad_info, 0);
128 return videomixer_pad_type;
132 gst_videomixer_pad_base_init (gpointer g_class)
137 gst_videomixer_pad_class_init (GstVideoMixerPadClass * klass)
139 GObjectClass *gobject_class;
141 gobject_class = (GObjectClass *) klass;
143 gobject_class->set_property =
144 GST_DEBUG_FUNCPTR (gst_videomixer_pad_set_property);
145 gobject_class->get_property =
146 GST_DEBUG_FUNCPTR (gst_videomixer_pad_get_property);
148 g_object_class_install_property (G_OBJECT_CLASS (klass), ARG_PAD_ZORDER,
149 g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture",
150 0, 10000, DEFAULT_PAD_ZORDER, G_PARAM_READWRITE));
151 g_object_class_install_property (G_OBJECT_CLASS (klass), ARG_PAD_XPOS,
152 g_param_spec_int ("xpos", "X Position", "X Position of the picture",
153 G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, G_PARAM_READWRITE));
154 g_object_class_install_property (G_OBJECT_CLASS (klass), ARG_PAD_YPOS,
155 g_param_spec_int ("ypos", "Y Position", "Y Position of the picture",
156 G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, G_PARAM_READWRITE));
157 g_object_class_install_property (G_OBJECT_CLASS (klass), ARG_PAD_ALPHA,
158 g_param_spec_double ("alpha", "Alpha", "Alpha of the picture",
159 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE));
162 static const GstEventMask *
163 gst_videomixer_pad_get_sink_event_masks (GstPad * pad)
165 static const GstEventMask gst_videomixer_sink_event_masks[] = {
170 return gst_videomixer_sink_event_masks;
174 gst_videomixer_pad_get_property (GObject * object, guint prop_id,
175 GValue * value, GParamSpec * pspec)
177 GstVideoMixerPad *pad;
179 g_return_if_fail (GST_IS_VIDEO_MIXER_PAD (object));
181 pad = GST_VIDEO_MIXER_PAD (object);
185 g_value_set_uint (value, pad->zorder);
188 g_value_set_int (value, pad->xpos);
191 g_value_set_int (value, pad->ypos);
194 g_value_set_double (value, pad->alpha);
197 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
203 gst_videomixer_pad_set_property (GObject * object, guint prop_id,
204 const GValue * value, GParamSpec * pspec)
206 GstVideoMixerPad *pad;
209 g_return_if_fail (GST_IS_PAD (object));
211 pad = GST_VIDEO_MIXER_PAD (object);
212 mix = GST_VIDEO_MIXER (gst_pad_get_parent (GST_PAD (pad)));
216 pad->zorder = g_value_get_uint (value);
217 gst_videomixer_sort_pads (mix);
220 pad->xpos = g_value_get_int (value);
223 pad->ypos = g_value_get_int (value);
226 pad->alpha = g_value_get_double (value);
229 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
236 VIDEO_MIXER_BACKGROUND_CHECKER,
237 VIDEO_MIXER_BACKGROUND_BLACK,
238 VIDEO_MIXER_BACKGROUND_WHITE,
240 GstVideoMixerBackground;
242 struct _GstVideoMixer
249 /* sinkpads, a GSList of GstVideoMixerPads */
254 GstVideoMixerPad *master;
256 gint in_width, in_height;
257 gint out_width, out_height;
259 GstVideoMixerBackground background;
261 gdouble in_framerate;
264 struct _GstVideoMixerClass
266 GstElementClass parent_class;
269 static GstPadLinkReturn
270 gst_videomixer_pad_sinkconnect (GstPad * pad, const GstCaps * vscaps)
273 GstVideoMixerPad *mixpad;
274 GstStructure *structure;
276 mix = GST_VIDEO_MIXER (gst_pad_get_parent (pad));
277 mixpad = GST_VIDEO_MIXER_PAD (pad);
279 GST_DEBUG ("videomixer: sinkconnect triggered on %s", gst_pad_get_name (pad));
281 structure = gst_caps_get_structure (vscaps, 0);
283 gst_structure_get_int (structure, "width", &mixpad->in_width);
284 gst_structure_get_int (structure, "height", &mixpad->in_height);
285 gst_structure_get_double (structure, "framerate", &mixpad->in_framerate);
290 mix->in_width = MAX (mix->in_width, mixpad->in_width);
291 mix->in_height = MAX (mix->in_height, mixpad->in_height);
292 if (mix->in_framerate < mixpad->in_framerate) {
293 mix->in_framerate = mixpad->in_framerate;
294 mix->master = mixpad;
297 return GST_PAD_LINK_OK;
301 gst_videomixer_pad_link (GstPad * pad, GstPad * peer, gpointer data)
303 //GstVideoMixer *videomixer = GST_VIDEO_MIXER (data);
305 GST_DEBUG ("pad '%s' connected", gst_pad_get_name (pad));
309 gst_videomixer_pad_unlink (GstPad * pad, GstPad * peer, gpointer data)
311 //GstVideoMixer *videomixer = GST_VIDEO_MIXER (data);
313 GST_DEBUG ("pad '%s' unlinked", gst_pad_get_name (pad));
317 gst_videomixer_pad_init (GstVideoMixerPad * mixerpad)
319 g_signal_connect (mixerpad, "linked",
320 G_CALLBACK (gst_videomixer_pad_link), (gpointer) mixerpad);
321 g_signal_connect (mixerpad, "unlinked",
322 G_CALLBACK (gst_videomixer_pad_unlink), (gpointer) mixerpad);
324 /* setup some pad functions */
325 gst_pad_set_link_function (GST_PAD (mixerpad),
326 gst_videomixer_pad_sinkconnect);
327 gst_pad_set_event_mask_function (GST_PAD (mixerpad),
328 gst_videomixer_pad_get_sink_event_masks);
330 mixerpad->zorder = DEFAULT_PAD_ZORDER;
331 mixerpad->xpos = DEFAULT_PAD_XPOS;
332 mixerpad->ypos = DEFAULT_PAD_YPOS;
333 mixerpad->alpha = DEFAULT_PAD_ALPHA;
338 /* elementfactory information */
339 static GstElementDetails gst_videomixer_details =
340 GST_ELEMENT_DETAILS ("video mixer",
341 "Filter/Editor/Video",
342 "Mix multiple video streams",
343 "Wim Taymans <wim@fluendo.com>");
345 /* VideoMixer signals and args */
352 #define DEFAULT_BACKGROUND VIDEO_MIXER_BACKGROUND_CHECKER
359 #define GST_TYPE_VIDEO_MIXER_BACKGROUND (gst_video_mixer_background_get_type())
361 gst_video_mixer_background_get_type (void)
363 static GType video_mixer_background_type = 0;
364 static GEnumValue video_mixer_background[] = {
365 {VIDEO_MIXER_BACKGROUND_CHECKER, "0", "Checker pattern"},
366 {VIDEO_MIXER_BACKGROUND_BLACK, "1", "Black"},
367 {VIDEO_MIXER_BACKGROUND_WHITE, "2", "White"},
371 if (!video_mixer_background_type) {
372 video_mixer_background_type =
373 g_enum_register_static ("GstVideoMixerBackground",
374 video_mixer_background);
376 return video_mixer_background_type;
379 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
382 GST_STATIC_CAPS ("video/x-raw-yuv,"
383 "format = (fourcc) I420,"
384 "width = (int) [ 16, 4096 ],"
385 "height = (int) [ 16, 4096 ]," "framerate = (double) [ 0, max ]")
388 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%d",
391 GST_STATIC_CAPS ("video/x-raw-yuv,"
392 "format = (fourcc) AYUV,"
393 "width = (int) [ 16, 4096 ],"
394 "height = (int) [ 16, 4096 ]," "framerate = (double) [ 0, max ]")
397 static void gst_videomixer_base_init (gpointer g_class);
398 static void gst_videomixer_class_init (GstVideoMixerClass * klass);
399 static void gst_videomixer_init (GstVideoMixer * videomixer);
401 static GstCaps *gst_videomixer_getcaps (GstPad * pad);
403 static void gst_videomixer_loop (GstElement * element);
404 static gboolean gst_videomixer_handle_src_event (GstPad * pad,
406 static GstPad *gst_videomixer_request_new_pad (GstElement * element,
407 GstPadTemplate * templ, const gchar * name);
408 static void gst_videomixer_set_property (GObject * object, guint prop_id,
409 const GValue * value, GParamSpec * pspec);
410 static void gst_videomixer_get_property (GObject * object, guint prop_id,
411 GValue * value, GParamSpec * pspec);
412 static GstElementStateReturn gst_videomixer_change_state (GstElement * element);
414 static GstElementClass *parent_class = NULL;
416 /*static guint gst_videomixer_signals[LAST_SIGNAL] = { 0 }; */
419 gst_videomixer_get_type (void)
421 static GType videomixer_type = 0;
423 if (!videomixer_type) {
424 static const GTypeInfo videomixer_info = {
425 sizeof (GstVideoMixerClass),
426 gst_videomixer_base_init,
428 (GClassInitFunc) gst_videomixer_class_init,
431 sizeof (GstVideoMixer),
433 (GInstanceInitFunc) gst_videomixer_init,
437 g_type_register_static (GST_TYPE_ELEMENT, "GstVideoMixer",
438 &videomixer_info, 0);
440 return videomixer_type;
444 gst_videomixer_base_init (gpointer g_class)
446 GstElementClass *element_class = GST_ELEMENT_CLASS (g_class);
448 gst_element_class_add_pad_template (element_class,
449 gst_static_pad_template_get (&src_factory));
450 gst_element_class_add_pad_template (element_class,
451 gst_static_pad_template_get (&sink_factory));
453 gst_element_class_set_details (element_class, &gst_videomixer_details);
457 gst_videomixer_class_init (GstVideoMixerClass * klass)
459 GObjectClass *gobject_class;
460 GstElementClass *gstelement_class;
462 gobject_class = (GObjectClass *) klass;
463 gstelement_class = (GstElementClass *) klass;
465 parent_class = g_type_class_ref (GST_TYPE_ELEMENT);
467 g_object_class_install_property (G_OBJECT_CLASS (klass), ARG_BACKGROUND,
468 g_param_spec_enum ("background", "Background", "Background type",
469 GST_TYPE_VIDEO_MIXER_BACKGROUND,
470 DEFAULT_BACKGROUND, G_PARAM_READWRITE));
472 gstelement_class->request_new_pad = gst_videomixer_request_new_pad;
474 gstelement_class->change_state = gst_videomixer_change_state;
476 gstelement_class->get_property = gst_videomixer_get_property;
477 gstelement_class->set_property = gst_videomixer_set_property;
481 gst_videomixer_init (GstVideoMixer * mix)
483 GstElementClass *klass = GST_ELEMENT_GET_CLASS (mix);
486 gst_pad_new_from_template (gst_element_class_get_pad_template (klass,
488 gst_pad_set_getcaps_function (GST_PAD (mix->srcpad), gst_videomixer_getcaps);
489 gst_pad_set_event_function (mix->srcpad, gst_videomixer_handle_src_event);
490 gst_element_add_pad (GST_ELEMENT (mix), mix->srcpad);
492 GST_FLAG_SET (GST_ELEMENT (mix), GST_ELEMENT_EVENT_AWARE);
494 mix->sinkpads = NULL;
495 mix->background = DEFAULT_BACKGROUND;
501 gst_element_set_loop_function (GST_ELEMENT (mix), gst_videomixer_loop);
505 gst_videomixer_getcaps (GstPad * pad)
509 GstPadTemplate *src_pad_template;
510 GstStructure *structure;
513 mix = GST_VIDEO_MIXER (gst_pad_get_parent (pad));
514 src_pad_template = gst_static_pad_template_get (&src_factory);
515 caps = gst_caps_copy (gst_pad_template_get_caps (src_pad_template));
517 structure = gst_caps_get_structure (caps, 0);
519 if (mix->out_width != 0) {
520 gst_structure_set (structure, "width", G_TYPE_INT, mix->out_width, NULL);
522 if (mix->out_height != 0) {
523 gst_structure_set (structure, "height", G_TYPE_INT, mix->out_height, NULL);
525 if (mix->in_framerate != 0) {
526 gst_structure_set (structure,
527 "framerate", G_TYPE_DOUBLE, mix->in_framerate, NULL);
534 gst_videomixer_request_new_pad (GstElement * element,
535 GstPadTemplate * templ, const gchar * req_name)
539 GstElementClass *klass = GST_ELEMENT_GET_CLASS (element);
541 g_return_val_if_fail (templ != NULL, NULL);
543 if (templ->direction != GST_PAD_SINK) {
544 g_warning ("videomixer: request pad that is not a SINK pad\n");
548 g_return_val_if_fail (GST_IS_VIDEO_MIXER (element), NULL);
550 mix = GST_VIDEO_MIXER (element);
552 if (templ == gst_element_class_get_pad_template (klass, "sink_%d")) {
554 GstVideoMixerPad *mixpad;
556 /* create new pad with the name */
557 name = g_strdup_printf ("sink_%02d", mix->numpads);
559 gst_pad_custom_new_from_template (GST_TYPE_VIDEO_MIXER_PAD, templ,
563 mixpad = GST_VIDEO_MIXER_PAD (newpad);
565 mixpad->zorder = mix->numpads;
567 mix->sinkpads = g_slist_append (mix->sinkpads, newpad);
569 g_warning ("videomixer: this is not our template!\n");
573 /* dd the pad to the element */
574 gst_element_add_pad (element, newpad);
581 gst_videomixer_handle_src_event (GstPad * pad, GstEvent * event)
586 mix = GST_VIDEO_MIXER (gst_pad_get_parent (pad));
588 type = event ? GST_EVENT_TYPE (event) : GST_EVENT_UNKNOWN;
592 /* disable seeking for now */
598 return gst_pad_event_default (pad, event);
601 #define BLEND_NORMAL(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
602 Y = ((Y1*(255-alpha))+(Y2*alpha))>>8; \
603 U = ((U1*(255-alpha))+(U2*alpha))>>8; \
604 V = ((V1*(255-alpha))+(V2*alpha))>>8;
606 #define BLEND_ADD(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
607 Y = Y1+((Y2*alpha)>>8); \
608 U = U1+(((127*(255-alpha)+(U2*alpha)))>>8)-127; \
609 V = V1+(((127*(255-alpha)+(V2*alpha)))>>8)-127; \
611 gint mult = MAX (0, 288-Y); \
612 U = ((U*mult) + (127*(32-mult)))>>5; \
613 V = ((V*mult) + (127*(32-mult)))>>5; \
619 #define BLEND_SUBTRACT(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
620 Y = Y1-((Y2*alpha)>>8); \
621 U = U1+(((127*(255-alpha)+(U2*alpha)))>>8)-127; \
622 V = V1+(((127*(255-alpha)+(V2*alpha)))>>8)-127; \
624 gint mult = MIN (32, -Y); \
625 U = ((U*(32-mult)) + (127*mult))>>5; \
626 V = ((V*(32-mult)) + (127*mult))>>5; \
630 #define BLEND_DARKEN(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
632 Y = Y1; U = U1; V = V1; \
635 Y = ((Y1*(255-alpha))+(Y2*alpha))>>8; \
636 U = ((U1*(255-alpha))+(U2*alpha))>>8; \
637 V = ((V1*(255-alpha))+(V2*alpha))>>8; \
640 #define BLEND_LIGHTEN(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
642 Y = Y1; U = U1; V = V1; \
645 Y = ((Y1*(255-alpha))+(Y2*alpha))>>8; \
646 U = ((U1*(255-alpha))+(U2*alpha))>>8; \
647 V = ((V1*(255-alpha))+(V2*alpha))>>8; \
650 #define BLEND_MULTIPLY(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
651 Y = (Y1*(256*(255-alpha) +(Y2*alpha)))>>16; \
652 U = ((U1*(255-alpha)*256)+(alpha*(U1*Y2+128*(256-Y2))))>>16; \
653 V = ((V1*(255-alpha)*256)+(alpha*(V1*Y2+128*(256-Y2))))>>16;
655 #define BLEND_DIFFERENCE(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
656 Y = ABS((gint)Y1-(gint)Y2)+127; \
657 U = ABS((gint)U1-(gint)U2)+127; \
658 V = ABS((gint)V1-(gint)V2)+127; \
659 Y = ((Y*alpha)+(Y1*(255-alpha)))>>8; \
660 U = ((U*alpha)+(U1*(255-alpha)))>>8; \
661 V = ((V*alpha)+(V1*(255-alpha)))>>8; \
663 gint mult = MAX (0, 288-Y); \
664 U = ((U*mult) + (127*(32-mult)))>>5; \
665 V = ((V*mult) + (127*(32-mult)))>>5; \
668 gint mult = MIN (32, -Y); \
669 U = ((U*(32-mult)) + (127*mult))>>5; \
670 V = ((V*(32-mult)) + (127*mult))>>5; \
673 U = CLAMP(U, 0, 255); \
674 V = CLAMP(V, 0, 255);
676 #define BLEND_EXCLUSION(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
677 Y = ((gint)(Y1^0xff)*Y2+(gint)(Y2^0xff)*Y1)>>8; \
678 U = ((gint)(U1^0xff)*Y2+(gint)(Y2^0xff)*U1)>>8; \
679 V = ((gint)(V1^0xff)*Y2+(gint)(Y2^0xff)*V1)>>8; \
680 Y = ((Y*alpha)+(Y1*(255-alpha)))>>8; \
681 U = ((U*alpha)+(U1*(255-alpha)))>>8; \
682 V = ((V*alpha)+(V1*(255-alpha)))>>8; \
684 gint mult = MAX (0, 288-Y); \
685 U = ((U*mult) + (127*(32-mult)))>>5; \
686 V = ((V*mult) + (127*(32-mult)))>>5; \
689 gint mult = MIN (32, -Y); \
690 U = ((U*(32-mult)) + (127*mult))>>5; \
691 V = ((V*(32-mult)) + (127*mult))>>5; \
694 U = CLAMP(U, 0, 255); \
695 V = CLAMP(V, 0, 255);
697 #define BLEND_SOFTLIGHT(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
698 Y = (gint)Y1+(gint)Y2 - 127; \
699 U = (gint)U1+(gint)U2 - 127; \
700 V = (gint)V1+(gint)V2 - 127; \
701 Y = ((Y*alpha)+(Y1*(255-alpha)))>>8; \
702 U = ((U*alpha)+(U1*(255-alpha)))>>8; \
703 V = ((V*alpha)+(V1*(255-alpha)))>>8; \
705 gint mult = MAX (0, 288-Y); \
706 U = ((U*mult) + (127*(32-mult)))>>5; \
707 V = ((V*mult) + (127*(32-mult)))>>5; \
710 gint mult = MIN (32, -Y); \
711 U = ((U*(32-mult)) + (127*mult))>>5; \
712 V = ((V*(32-mult)) + (127*mult))>>5; \
716 #define BLEND_HARDLIGHT(Y1,U1,V1,Y2,U2,V2,alpha,Y,U,V) \
717 Y = (gint)Y1+(gint)Y2*2 - 255; \
718 U = (gint)U1+(gint)U2 - 127; \
719 V = (gint)V1+(gint)V2 - 127; \
720 Y = ((Y*alpha)+(Y1*(255-alpha)))>>8; \
721 U = ((U*alpha)+(U1*(255-alpha)))>>8; \
722 V = ((V*alpha)+(V1*(255-alpha)))>>8; \
724 gint mult = MAX (0, 288-Y); \
725 U = ((U*mult) + (127*(32-mult)))>>5; \
726 V = ((V*mult) + (127*(32-mult)))>>5; \
729 gint mult = MIN (32, -Y); \
730 U = ((U*(32-mult)) + (127*mult))>>5; \
731 V = ((V*(32-mult)) + (127*mult))>>5; \
735 #define BLEND_MODE BLEND_NORMAL
737 #define BLEND_MODE BLEND_NORMAL
738 #define BLEND_MODE BLEND_ADD
739 #define BLEND_MODE BLEND_SUBTRACT
740 #define BLEND_MODE BLEND_LIGHTEN
741 #define BLEND_MODE BLEND_DARKEN
742 #define BLEND_MODE BLEND_MULTIPLY
743 #define BLEND_MODE BLEND_DIFFERENCE
744 #define BLEND_MODE BLEND_EXCLUSION
745 #define BLEND_MODE BLEND_SOFTLIGHT
746 #define BLEND_MODE BLEND_HARDLIGHT
749 #define ROUND_UP_2(x) (((x) + 1) & ~1)
750 #define ROUND_UP_4(x) (((x) + 3) & ~3)
751 #define ROUND_UP_8(x) (((x) + 7) & ~7)
753 /* note that this function does packing conversion and blending at the
756 gst_videomixer_blend_ayuv_i420 (guint8 * src, gint xpos, gint ypos,
757 gint src_width, gint src_height, gdouble src_alpha,
758 guint8 * dest, gint dest_width, gint dest_height)
760 gint dest_size, dest_size2;
762 guint8 *destY1, *destY2, *destU, *destV;
767 gint src_add, destY_add, destC_add;
770 gint dest_stride, dest_stride2;
772 src_stride = ROUND_UP_2 (src_width) * 4;
774 dest_stride = ROUND_UP_4 (dest_width);
775 dest_size = dest_stride * ROUND_UP_2 (dest_height);
776 dest_stride2 = ROUND_UP_8 (dest_width) / 2;
777 dest_size2 = dest_stride2 * ROUND_UP_2 (dest_height) / 2;
779 b_alpha = (gint) (src_alpha * 255);
781 /* adjust src pointers for negative sizes */
788 src += -ypos * src_stride;
792 /* adjust width/height if the src is bigger than dest */
793 if (xpos + src_width > dest_width) {
794 src_width = dest_width - xpos;
796 if (ypos + src_height > dest_height) {
797 src_height = dest_height - ypos;
800 src_add = 2 * src_stride - (8 * ROUND_UP_2 (src_width) / 2);
801 destY_add = 2 * dest_stride - (2 * (ROUND_UP_2 (src_width) / 2));
802 destC_add = dest_stride2 - (ROUND_UP_2 (src_width) / 2);
804 destY1 = dest + xpos + (ypos * dest_stride);
805 destY2 = destY1 + dest_stride;
806 destU = dest + dest_size + (xpos / 2) + ((ypos / 2) * dest_stride2);
807 destV = destU + dest_size2;
810 src2 = src + src_stride;
812 /* we convert a square of 2x2 samples to generate 4 Luma and 2 chroma samples */
813 for (i = 0; i < ROUND_UP_2 (src_height) / 2; i++) {
814 for (j = 0; j < ROUND_UP_2 (src_width) / 2; j++) {
815 alpha = (src1[0] * b_alpha) >> 8;
816 BLEND_MODE (destY1[0], destU[0], destV[0], src1[1], src1[2], src1[3],
821 alpha = (src1[4] * b_alpha) >> 8;
822 BLEND_MODE (destY1[1], destU[0], destV[0], src1[5], src1[6], src1[7],
827 alpha = (src2[0] * b_alpha) >> 8;
828 BLEND_MODE (destY2[0], destU[0], destV[0], src2[1], src2[2], src2[3],
833 alpha = (src2[4] * b_alpha) >> 8;
834 BLEND_MODE (destY2[1], destU[0], destV[0], src2[5], src2[6], src2[7],
840 /* take the average of the 4 chroma samples to get the final value */
841 destU[0] = (accumU + 2) >> 2;
842 destV[0] = (accumV + 2) >> 2;
863 pad_zorder_compare (const GstVideoMixerPad * pad1,
864 const GstVideoMixerPad * pad2)
866 return pad1->zorder - pad2->zorder;
870 gst_videomixer_sort_pads (GstVideoMixer * mix)
872 mix->sinkpads = g_slist_sort (mix->sinkpads,
873 (GCompareFunc) pad_zorder_compare);
876 /* fill a buffer with a checkerboard pattern */
878 gst_videomixer_fill_checker (guint8 * dest, gint width, gint height)
883 static int tab[] = { 80, 160, 80, 160 };
885 stride = ROUND_UP_4 (width);
886 size = stride * height;
887 size2 = ROUND_UP_4 ((width + 1) >> 1) * ((height + 1) >> 1);
889 for (i = 0; i < height; i++) {
890 for (j = 0; j < stride; j++) {
891 *dest++ = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)];
894 memset (dest, 128, 2 * size2);
898 gst_videomixer_fill_color (guint8 * dest, gint width, gint height,
899 gint colY, gint colU, gint colV)
903 size = ROUND_UP_4 (width) * height;
904 size2 = ROUND_UP_4 ((width + 1) >> 1) * ((height + 1) >> 1);
906 memset (dest, colY, size);
907 memset (dest + size, colU, size2);
908 memset (dest + size + size2, colV, size2);
911 /* try to get a buffer on all pads. As long as the queued value is
912 * negative, we skip buffers */
914 gst_videomixer_fill_queues (GstVideoMixer * mix)
919 /* loop over all pads and fill it with a buffer */
920 walk = mix->sinkpads;
922 GstVideoMixerPad *pad = GST_VIDEO_MIXER_PAD (walk->data);
924 walk = g_slist_next (walk);
926 GST_DEBUG ("looking at pad %s", gst_pad_get_name (GST_PAD (pad)));
928 /* don't care about eos pads */
930 GST_DEBUG ("pad %s in eos, skipping", gst_pad_get_name (GST_PAD (pad)));
934 GST_DEBUG ("pad %s: buffer %p, queued %lld ",
935 gst_pad_get_name (GST_PAD (pad)), pad->buffer, pad->queued);
937 /* this pad is in need of a new buffer */
938 if (pad->buffer == NULL) {
942 /* as long as not enough buffers have been queued */
943 while (pad->queued <= 0 && !pad->eos) {
944 data = gst_pad_pull (GST_PAD (pad));
945 if (GST_IS_EVENT (data)) {
946 GstEvent *event = GST_EVENT (data);
948 switch (GST_EVENT_TYPE (event)) {
950 GST_DEBUG ("videomixer: EOS on pad %s",
951 gst_pad_get_name (GST_PAD (pad)));
954 gst_event_unref (event);
957 gst_pad_event_default (GST_PAD (pad), GST_EVENT (data));
962 buffer = GST_BUFFER (data);
963 duration = GST_BUFFER_DURATION (buffer);
964 /* no duration on the buffer, use the framerate */
965 if (duration == -1) {
966 if (pad->in_framerate == 0.0) {
967 duration = G_MAXINT64;
969 duration = GST_SECOND / pad->in_framerate;
972 pad->queued += duration;
973 /* this buffer will need to be mixed */
974 if (pad->queued > 0) {
975 pad->buffer = buffer;
977 /* skip buffer, it's too old */
978 gst_buffer_unref (buffer);
981 GST_DEBUG ("pad %s: in loop, buffer %p, queued %lld ",
982 gst_pad_get_name (GST_PAD (pad)), pad->buffer, pad->queued);
985 if (pad->buffer != NULL) {
986 /* got a buffer somewhere so were not eos */
993 /* blend all buffers present on the pads */
995 gst_videomixer_blend_buffers (GstVideoMixer * mix, GstBuffer * outbuf)
999 walk = mix->sinkpads;
1001 GstVideoMixerPad *pad = GST_VIDEO_MIXER_PAD (walk->data);
1003 walk = g_slist_next (walk);
1005 if (pad->buffer != NULL) {
1006 gst_videomixer_blend_ayuv_i420 (GST_BUFFER_DATA (pad->buffer),
1007 pad->xpos, pad->ypos,
1008 pad->in_width, pad->in_height,
1010 GST_BUFFER_DATA (outbuf), mix->out_width, mix->out_height);
1011 if (pad == mix->master) {
1012 GST_BUFFER_TIMESTAMP (outbuf) = GST_BUFFER_TIMESTAMP (pad->buffer);
1013 GST_BUFFER_DURATION (outbuf) = GST_BUFFER_DURATION (pad->buffer);
1019 /* remove buffers from the queue that were expired in the
1020 * interval of the master, we also prepare the queued value
1021 * in the pad so that we can skip and fill buffers later on */
1023 gst_videomixer_update_queues (GstVideoMixer * mix)
1028 interval = mix->master->queued;
1029 if (interval <= 0) {
1030 if (mix->in_framerate == 0.0) {
1031 interval = G_MAXINT64;
1033 interval = GST_SECOND / mix->in_framerate;
1037 walk = mix->sinkpads;
1039 GstVideoMixerPad *pad = GST_VIDEO_MIXER_PAD (walk->data);
1041 walk = g_slist_next (walk);
1043 if (pad->buffer != NULL) {
1044 pad->queued -= interval;
1045 GST_DEBUG ("queued now %s %lld", gst_pad_get_name (GST_PAD (pad)),
1047 if (pad->queued <= 0) {
1048 gst_buffer_unref (pad->buffer);
1056 * The basic idea is to get a buffer on all pads and mix them together.
1057 * Based on the framerate, buffers are removed from the queues to make room
1061 gst_videomixer_loop (GstElement * element)
1066 gint new_width, new_height;
1069 mix = GST_VIDEO_MIXER (element);
1071 eos = gst_videomixer_fill_queues (mix);
1073 gst_pad_push (mix->srcpad, GST_DATA (gst_event_new (GST_EVENT_EOS)));
1074 gst_element_set_eos (GST_ELEMENT (mix));
1078 new_width = mix->in_width;
1079 new_height = mix->in_height;
1081 if (new_width != mix->out_width ||
1082 new_height != mix->out_height || !GST_PAD_CAPS (mix->srcpad)) {
1086 gst_caps_copy (gst_pad_get_negotiated_caps (GST_PAD (mix->master)));
1087 gst_caps_set_simple (newcaps, "format", GST_TYPE_FOURCC,
1088 GST_STR_FOURCC ("I420"), "width", G_TYPE_INT, new_width, "height",
1089 G_TYPE_INT, new_height, NULL);
1091 if (GST_PAD_LINK_FAILED (gst_pad_try_set_caps (mix->srcpad, newcaps))) {
1092 GST_ELEMENT_ERROR (mix, CORE, NEGOTIATION, (NULL), (NULL));
1096 mix->out_width = new_width;
1097 mix->out_height = new_height;
1100 outsize = ROUND_UP_4 (mix->out_width) * ROUND_UP_2 (mix->out_height) +
1101 2 * ROUND_UP_8 (mix->out_width) / 2 * ROUND_UP_2 (mix->out_height);
1103 outbuf = gst_pad_alloc_buffer (mix->srcpad, GST_BUFFER_OFFSET_NONE, outsize);
1104 switch (mix->background) {
1105 case VIDEO_MIXER_BACKGROUND_CHECKER:
1106 gst_videomixer_fill_checker (GST_BUFFER_DATA (outbuf),
1107 new_width, new_height);
1109 case VIDEO_MIXER_BACKGROUND_BLACK:
1110 gst_videomixer_fill_color (GST_BUFFER_DATA (outbuf),
1111 new_width, new_height, 16, 128, 128);
1113 case VIDEO_MIXER_BACKGROUND_WHITE:
1114 gst_videomixer_fill_color (GST_BUFFER_DATA (outbuf),
1115 new_width, new_height, 240, 128, 128);
1119 gst_videomixer_blend_buffers (mix, outbuf);
1121 gst_videomixer_update_queues (mix);
1123 gst_pad_push (mix->srcpad, GST_DATA (outbuf));
1127 gst_videomixer_get_property (GObject * object,
1128 guint prop_id, GValue * value, GParamSpec * pspec)
1130 GstVideoMixer *mix = GST_VIDEO_MIXER (object);
1133 case ARG_BACKGROUND:
1134 g_value_set_enum (value, mix->background);
1137 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1143 gst_videomixer_set_property (GObject * object,
1144 guint prop_id, const GValue * value, GParamSpec * pspec)
1146 GstVideoMixer *mix = GST_VIDEO_MIXER (object);
1149 case ARG_BACKGROUND:
1150 mix->background = g_value_get_enum (value);
1153 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1158 static GstElementStateReturn
1159 gst_videomixer_change_state (GstElement * element)
1162 gint transition = GST_STATE_TRANSITION (element);
1164 g_return_val_if_fail (GST_IS_VIDEO_MIXER (element), GST_STATE_FAILURE);
1166 mix = GST_VIDEO_MIXER (element);
1168 switch (transition) {
1169 case GST_STATE_NULL_TO_READY:
1170 case GST_STATE_READY_TO_PAUSED:
1172 case GST_STATE_PAUSED_TO_PLAYING:
1173 case GST_STATE_PLAYING_TO_PAUSED:
1174 case GST_STATE_PAUSED_TO_READY:
1175 case GST_STATE_READY_TO_NULL:
1179 if (GST_ELEMENT_CLASS (parent_class)->change_state)
1180 return GST_ELEMENT_CLASS (parent_class)->change_state (element);
1182 return GST_STATE_SUCCESS;
1186 plugin_init (GstPlugin * plugin)
1188 GST_DEBUG_CATEGORY_INIT (gst_videomixer_debug, "videomixer", 0,
1191 return gst_element_register (plugin, "videomixer", GST_RANK_PRIMARY,
1192 GST_TYPE_VIDEO_MIXER);
1195 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
1198 "Video mixer", plugin_init, VERSION, GST_LICENSE, GST_PACKAGE, GST_ORIGIN)