3 * Copyright (C) 2009 Julien Isorce <julien.isorce@gmail.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 * SECTION:element-glvideomixer
23 * @title: glvideomixer
25 * Composites a number of streams into a single output scene using OpenGL in
26 * a similar fashion to compositor and videomixer. See the compositor plugin
27 * for documentation about the #GstGLVideoMixerPad properties.
31 * gst-launch-1.0 glvideomixer name=m ! glimagesink \
32 * videotestsrc ! video/x-raw, format=YUY2 ! glupload ! glcolorconvert ! m. \
33 * videotestsrc pattern=12 ! video/x-raw, format=I420, framerate=5/1, width=100, height=200 ! queue ! \
34 * glupload ! glcolorconvert ! m. \
35 * videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. \
36 * videotestsrc ! glupload ! glfiltercube ! queue ! m. \
37 * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m.
47 #include <gst/controller/gstproxycontrolbinding.h>
48 #include <gst/gl/gstglfuncs.h>
49 #include <gst/video/gstvideoaffinetransformationmeta.h>
51 #include "gstglelements.h"
52 #include "gstglvideomixer.h"
54 #include "gstglmixerbin.h"
55 #include "gstglutils.h"
57 #define GST_CAT_DEFAULT gst_gl_video_mixer_debug
58 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
60 static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
63 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
64 (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
68 #define GST_TYPE_GL_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type())
70 gst_gl_video_mixer_background_get_type (void)
72 static GType mixer_background_type = 0;
74 static const GEnumValue mixer_background[] = {
75 {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"},
76 {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"},
77 {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"},
78 {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT,
79 "Transparent Background to enable further compositing", "transparent"},
83 if (!mixer_background_type) {
84 mixer_background_type =
85 g_enum_register_static ("GstGLVideoMixerBackground", mixer_background);
87 return mixer_background_type;
90 #define GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION (gst_gl_video_mixer_blend_equation_get_type())
92 gst_gl_video_mixer_blend_equation_get_type (void)
94 static GType mixer_blend_equation_type = 0;
96 static const GEnumValue mixer_blend_equations[] = {
97 {GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD, "Add", "add"},
98 {GST_GL_VIDEO_MIXER_BLEND_EQUATION_SUBTRACT, "Subtract", "subtract"},
99 {GST_GL_VIDEO_MIXER_BLEND_EQUATION_REVERSE_SUBTRACT, "Reverse Subtract",
104 if (!mixer_blend_equation_type) {
105 mixer_blend_equation_type =
106 g_enum_register_static ("GstGLVideoMixerBlendEquation",
107 mixer_blend_equations);
109 return mixer_blend_equation_type;
112 #define GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION (gst_gl_video_mixer_blend_function_get_type())
114 gst_gl_video_mixer_blend_function_get_type (void)
116 static GType mixer_blend_function_type = 0;
118 static const GEnumValue mixer_blend_funcs[] = {
119 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ZERO, "Zero", "zero"},
120 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE, "One", "one"},
121 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_COLOR, "Source Color", "src-color"},
122 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_COLOR,
123 "One Minus Source Color", "one-minus-src-color"},
124 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_COLOR, "Destination Color",
126 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_COLOR,
127 "One Minus Destination Color", "one-minus-dst-color"},
128 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA, "Source Alpha", "src-alpha"},
129 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA,
130 "One Minus Source Alpha", "one-minus-src-alpha"},
131 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_ALPHA, "Destination Alpha",
133 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_ALPHA,
134 "One Minus Destination Alpha", "one-minus-dst-alpha"},
135 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_COLOR, "Constant Color",
137 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR,
138 "One Minus Constant Color", "one-minus-contant-color"},
139 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA, "Constant Alpha",
141 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_ALPHA,
142 "One Minus Constant Alpha", "one-minus-contant-alpha"},
143 {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE,
144 "Source Alpha Saturate", "src-alpha-saturate"},
148 if (!mixer_blend_function_type) {
149 mixer_blend_function_type =
150 g_enum_register_static ("GstGLVideoMixerBlendFunction",
153 return mixer_blend_function_type;
156 #define DEFAULT_PAD_XPOS 0
157 #define DEFAULT_PAD_YPOS 0
158 #define DEFAULT_PAD_WIDTH 0
159 #define DEFAULT_PAD_HEIGHT 0
160 #define DEFAULT_PAD_ALPHA 1.0
161 #define DEFAULT_PAD_ZORDER 0
162 #define DEFAULT_PAD_REPEAT_AFTER_EOS FALSE
163 #define DEFAULT_PAD_BLEND_EQUATION_RGB GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD
164 #define DEFAULT_PAD_BLEND_EQUATION_ALPHA GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD
165 #define DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA
166 #define DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE
167 #define DEFAULT_PAD_BLEND_FUNCTION_DST_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA
168 #define DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA
169 #define DEFAULT_PAD_CROP 0
179 PROP_INPUT_BLEND_EQUATION_RGB,
180 PROP_INPUT_BLEND_EQUATION_ALPHA,
181 PROP_INPUT_BLEND_FUNCTION_SRC_RGB,
182 PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA,
183 PROP_INPUT_BLEND_FUNCTION_DST_RGB,
184 PROP_INPUT_BLEND_FUNCTION_DST_ALPHA,
185 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_RED,
186 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_GREEN,
187 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_BLUE,
188 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA,
190 PROP_INPUT_REPEAT_AFTER_EOS,
191 PROP_INPUT_CROP_LEFT,
192 PROP_INPUT_CROP_RIGHT,
194 PROP_INPUT_CROP_BOTTOM,
197 static void gst_gl_video_mixer_input_get_property (GObject * object,
198 guint prop_id, GValue * value, GParamSpec * pspec);
199 static void gst_gl_video_mixer_input_set_property (GObject * object,
200 guint prop_id, const GValue * value, GParamSpec * pspec);
201 static gboolean gst_gl_video_mixer_src_event (GstAggregator * agg,
204 typedef struct _GstGLVideoMixerInput GstGLVideoMixerInput;
205 typedef GstGhostPadClass GstGLVideoMixerInputClass;
207 struct _GstGLVideoMixerInput
216 GType gst_gl_video_mixer_input_get_type (void);
217 G_DEFINE_TYPE (GstGLVideoMixerInput, gst_gl_video_mixer_input,
221 gst_gl_video_mixer_input_init (GstGLVideoMixerInput * self)
226 gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass)
228 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
230 gobject_class->set_property = gst_gl_video_mixer_input_set_property;
231 gobject_class->get_property = gst_gl_video_mixer_input_get_property;
233 g_object_class_install_property (gobject_class, PROP_INPUT_ZORDER,
234 g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture",
235 0, 10000, DEFAULT_PAD_ZORDER,
236 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
237 g_object_class_install_property (gobject_class, PROP_INPUT_REPEAT_AFTER_EOS,
238 g_param_spec_boolean ("repeat-after-eos", "Repeat After EOS",
239 "Aggregate the last "
240 "frame on pads that are EOS till they are released",
241 DEFAULT_PAD_REPEAT_AFTER_EOS,
242 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
243 g_object_class_install_property (gobject_class, PROP_INPUT_XPOS,
244 g_param_spec_int ("xpos", "X Position", "X Position of the picture",
245 G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
246 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
247 g_object_class_install_property (gobject_class, PROP_INPUT_YPOS,
248 g_param_spec_int ("ypos", "Y Position", "Y Position of the picture",
249 G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
250 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
251 g_object_class_install_property (gobject_class, PROP_INPUT_WIDTH,
252 g_param_spec_int ("width", "Width", "Width of the picture", G_MININT,
253 G_MAXINT, DEFAULT_PAD_WIDTH,
254 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
255 g_object_class_install_property (gobject_class, PROP_INPUT_HEIGHT,
256 g_param_spec_int ("height", "Height", "Height of the picture", G_MININT,
257 G_MAXINT, DEFAULT_PAD_HEIGHT,
258 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
259 g_object_class_install_property (gobject_class, PROP_INPUT_ALPHA,
260 g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
262 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
263 g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_EQUATION_RGB,
264 g_param_spec_enum ("blend-equation-rgb", "Blend Equation RGB",
265 "Blend Equation for RGB", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION,
266 DEFAULT_PAD_BLEND_EQUATION_RGB,
267 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
268 g_object_class_install_property (gobject_class,
269 PROP_INPUT_BLEND_EQUATION_ALPHA,
270 g_param_spec_enum ("blend-equation-alpha", "Blend Equation Alpha",
271 "Blend Equation for Alpha", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION,
272 DEFAULT_PAD_BLEND_EQUATION_ALPHA,
273 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
274 g_object_class_install_property (gobject_class,
275 PROP_INPUT_BLEND_FUNCTION_SRC_RGB,
276 g_param_spec_enum ("blend-function-src-rgb", "Blend Function Source RGB",
277 "Blend Function for Source RGB",
278 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
279 DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB,
280 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
281 g_object_class_install_property (gobject_class,
282 PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA,
283 g_param_spec_enum ("blend-function-src-alpha",
284 "Blend Function Source Alpha", "Blend Function for Source Alpha",
285 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
286 DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA,
287 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
288 g_object_class_install_property (gobject_class,
289 PROP_INPUT_BLEND_FUNCTION_DST_RGB,
290 g_param_spec_enum ("blend-function-dst-rgb",
291 "Blend Function Destination RGB",
292 "Blend Function for Destination RGB",
293 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
294 DEFAULT_PAD_BLEND_FUNCTION_DST_RGB,
295 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
296 g_object_class_install_property (gobject_class,
297 PROP_INPUT_BLEND_FUNCTION_DST_ALPHA,
298 g_param_spec_enum ("blend-function-dst-alpha",
299 "Blend Function Destination Alpha",
300 "Blend Function for Destination Alpha",
301 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
302 DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA,
303 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
304 g_object_class_install_property (gobject_class,
305 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_RED,
306 g_param_spec_double ("blend-constant-color-red",
307 "Blend Constant Color Red", "Blend Constant Color Red", 0.0, 1.0, 0.0,
308 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
309 g_object_class_install_property (gobject_class,
310 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_GREEN,
311 g_param_spec_double ("blend-constant-color-green",
312 "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0,
314 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
315 g_object_class_install_property (gobject_class,
316 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_BLUE,
317 g_param_spec_double ("blend-constant-color-blue",
318 "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0,
320 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
321 g_object_class_install_property (gobject_class,
322 PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA,
323 g_param_spec_double ("blend-constant-color-alpha",
324 "Blend Constant Color Alpha", "Blend Constant Color Alpha", 0.0, 1.0,
326 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
328 * GstGLVideoMixerInput:crop-left:
330 * Defines how many pixels of the input in input size should be cropped on
335 g_object_class_install_property (gobject_class, PROP_INPUT_CROP_LEFT,
336 g_param_spec_int ("crop-left", "Crop Left", "Crop left of the picture", 0,
337 G_MAXINT, DEFAULT_PAD_CROP,
338 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
340 * GstGLVideoMixerInput:crop-right:
342 * Defines how many pixels of the input in input size should be cropped on
347 g_object_class_install_property (gobject_class, PROP_INPUT_CROP_RIGHT,
348 g_param_spec_int ("crop-right", "Crop Right", "Crop right of the picture",
349 0, G_MAXINT, DEFAULT_PAD_CROP,
350 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
352 * GstGLVideoMixerInput:crop-top:
354 * Defines how many pixels of the input in input size should be cropped on
359 g_object_class_install_property (gobject_class, PROP_INPUT_CROP_TOP,
360 g_param_spec_int ("crop-top", "Crop Top", "Crop top of the picture", 0,
361 G_MAXINT, DEFAULT_PAD_CROP,
362 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
364 * GstGLVideoMixerInput:crop-bottom:
366 * Defines how many pixels of the input in input size should be cropped on
371 g_object_class_install_property (gobject_class, PROP_INPUT_CROP_BOTTOM,
372 g_param_spec_int ("crop-bottom", "Crop Bottom",
373 "Crop bottom of the picture", 0, G_MAXINT, DEFAULT_PAD_CROP,
374 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
378 gst_gl_video_mixer_input_get_property (GObject * object, guint prop_id,
379 GValue * value, GParamSpec * pspec)
381 GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object;
384 g_object_get_property (G_OBJECT (self->mixer_pad), pspec->name, value);
388 gst_gl_video_mixer_input_set_property (GObject * object, guint prop_id,
389 const GValue * value, GParamSpec * pspec)
391 GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object;
394 g_object_set_property (G_OBJECT (self->mixer_pad), pspec->name, value);
398 _create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad)
400 GstGLVideoMixerInput *input =
401 g_object_new (gst_gl_video_mixer_input_get_type (), "name",
402 GST_OBJECT_NAME (mixer_pad), "direction", GST_PAD_DIRECTION (mixer_pad),
405 #define ADD_BINDING(obj,ref,prop) \
406 gst_object_add_control_binding (GST_OBJECT (obj), \
407 gst_proxy_control_binding_new (GST_OBJECT (obj), prop, \
408 GST_OBJECT (ref), prop));
409 ADD_BINDING (mixer_pad, input, "zorder");
410 ADD_BINDING (mixer_pad, input, "xpos");
411 ADD_BINDING (mixer_pad, input, "ypos");
412 ADD_BINDING (mixer_pad, input, "width");
413 ADD_BINDING (mixer_pad, input, "height");
414 ADD_BINDING (mixer_pad, input, "alpha");
415 ADD_BINDING (mixer_pad, input, "blend-equation-rgb");
416 ADD_BINDING (mixer_pad, input, "blend-equation-alpha");
417 ADD_BINDING (mixer_pad, input, "blend-function-src-rgb");
418 ADD_BINDING (mixer_pad, input, "blend-function-src-alpha");
419 ADD_BINDING (mixer_pad, input, "blend-function-dst-rgb");
420 ADD_BINDING (mixer_pad, input, "blend-function-dst-alpha");
421 ADD_BINDING (mixer_pad, input, "blend-constant-color-red");
422 ADD_BINDING (mixer_pad, input, "blend-constant-color-green");
423 ADD_BINDING (mixer_pad, input, "blend-constant-color-blue");
424 ADD_BINDING (mixer_pad, input, "blend-constant-color-alpha");
427 input->mixer_pad = mixer_pad;
429 return GST_GHOST_PAD (input);
437 #define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER
439 static void gst_gl_video_mixer_bin_get_property (GObject * object,
440 guint prop_id, GValue * value, GParamSpec * pspec);
441 static void gst_gl_video_mixer_bin_set_property (GObject * object,
442 guint prop_id, const GValue * value, GParamSpec * pspec);
444 typedef GstGLMixerBin GstGLVideoMixerBin;
445 typedef GstGLMixerBinClass GstGLVideoMixerBinClass;
447 G_DEFINE_TYPE (GstGLVideoMixerBin, gst_gl_video_mixer_bin,
448 GST_TYPE_GL_MIXER_BIN);
449 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (glvideomixer, "glvideomixer",
450 GST_RANK_NONE, gst_gl_video_mixer_bin_get_type (),
451 gl_element_init (plugin));
454 gst_gl_video_mixer_bin_init (GstGLVideoMixerBin * self)
456 GstGLMixerBin *mix_bin = GST_GL_MIXER_BIN (self);
458 gst_gl_mixer_bin_finish_init_with_element (mix_bin,
459 g_object_new (GST_TYPE_GL_VIDEO_MIXER, NULL));
463 gst_gl_video_mixer_bin_class_init (GstGLVideoMixerBinClass * klass)
465 GstGLMixerBinClass *mixer_class = GST_GL_MIXER_BIN_CLASS (klass);
466 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
467 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
468 GstCaps *upload_caps;
470 mixer_class->create_input_pad = _create_video_mixer_input;
472 gobject_class->set_property = gst_gl_video_mixer_bin_set_property;
473 gobject_class->get_property = gst_gl_video_mixer_bin_get_property;
475 g_object_class_install_property (gobject_class, PROP_BIN_BACKGROUND,
476 g_param_spec_enum ("background", "Background", "Background type",
477 GST_TYPE_GL_VIDEO_MIXER_BACKGROUND,
478 DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
480 /* override the sink_%u pad template from GstGLMixerBin.
481 * We pass it the GType of our sink pad so it's properly documented when
482 * inspecting the element. */
483 upload_caps = gst_gl_upload_get_input_template_caps ();
484 gst_element_class_add_pad_template (element_class,
485 gst_pad_template_new_with_gtype ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST,
486 upload_caps, gst_gl_video_mixer_input_get_type ()));
487 gst_caps_unref (upload_caps);
489 gst_element_class_set_metadata (element_class, "OpenGL video_mixer bin",
490 "Bin/Filter/Effect/Video/Compositor", "OpenGL video_mixer bin",
491 "Matthew Waters <matthew@centricular.com>");
493 gst_type_mark_as_plugin_api (gst_gl_video_mixer_input_get_type (), 0);
497 gst_gl_video_mixer_bin_get_property (GObject * object, guint prop_id,
498 GValue * value, GParamSpec * pspec)
500 GstGLMixerBin *self = GST_GL_MIXER_BIN (object);
503 g_object_get_property (G_OBJECT (self->mixer), pspec->name, value);
507 gst_gl_video_mixer_bin_set_property (GObject * object, guint prop_id,
508 const GValue * value, GParamSpec * pspec)
510 GstGLMixerBin *self = GST_GL_MIXER_BIN (object);
513 g_object_set_property (G_OBJECT (self->mixer), pspec->name, value);
522 static void gst_gl_video_mixer_child_proxy_init (gpointer g_iface,
523 gpointer iface_data);
526 GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element");
528 #define gst_gl_video_mixer_parent_class parent_class
529 G_DEFINE_TYPE_WITH_CODE (GstGLVideoMixer, gst_gl_video_mixer, GST_TYPE_GL_MIXER,
530 G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
531 gst_gl_video_mixer_child_proxy_init); DEBUG_INIT);
532 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (glvideomixerelement,
533 "glvideomixerelement", GST_RANK_NONE, gst_gl_video_mixer_get_type (),
534 gl_element_init (plugin));
536 static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id,
537 const GValue * value, GParamSpec * pspec);
538 static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id,
539 GValue * value, GParamSpec * pspec);
541 static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps);
542 static GstCaps *_fixate_caps (GstAggregator * agg, GstCaps * caps);
543 static gboolean gst_gl_video_mixer_propose_allocation (GstAggregator *
544 agg, GstAggregatorPad * agg_pad, GstQuery * decide_query, GstQuery * query);
545 static gboolean gst_gl_video_mixer_gl_start (GstGLBaseMixer * base_mix);
546 static void gst_gl_video_mixer_gl_stop (GstGLBaseMixer * base_mix);
547 static gboolean gst_gl_video_mixer_set_caps (GstGLMixer * mixer,
550 static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mixer,
551 GstGLMemory * out_tex);
552 static gboolean gst_gl_video_mixer_callback (gpointer stuff);
556 /* fragment source */
557 static const gchar *video_mixer_f_src =
558 "uniform sampler2D texture; \n"
559 "uniform float alpha;\n"
560 "varying vec2 v_texcoord; \n"
563 " vec4 rgba = texture2D(texture, v_texcoord);\n"
564 " gl_FragColor = vec4(rgba.rgb, rgba.a * alpha);\n"
567 /* checker vertex source */
568 static const gchar *checker_v_src =
569 "attribute vec4 a_position;\n"
572 " gl_Position = a_position;\n"
575 /* checker fragment source */
576 static const gchar *checker_f_src =
577 "const float blocksize = 8.0;\n"
580 " vec4 high = vec4(0.667, 0.667, 0.667, 1.0);\n"
581 " vec4 low = vec4(0.333, 0.333, 0.333, 1.0);\n"
582 " if (mod(gl_FragCoord.x, blocksize * 2.0) >= blocksize) {\n"
583 " if (mod(gl_FragCoord.y, blocksize * 2.0) >= blocksize)\n"
584 " gl_FragColor = low;\n"
586 " gl_FragColor = high;\n"
588 " if (mod(gl_FragCoord.y, blocksize * 2.0) < blocksize)\n"
589 " gl_FragColor = low;\n"
591 " gl_FragColor = high;\n"
596 #define GST_TYPE_GL_VIDEO_MIXER_PAD (gst_gl_video_mixer_pad_get_type())
597 #define GST_GL_VIDEO_MIXER_PAD(obj) \
598 (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_VIDEO_MIXER_PAD, GstGLVideoMixerPad))
599 #define GST_GL_VIDEO_MIXER_PAD_CLASS(klass) \
600 (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_VIDEO_MIXER_PAD, GstGLVideoMixerPadClass))
601 #define GST_IS_GL_VIDEO_MIXER_PAD(obj) \
602 (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_VIDEO_MIXER_PAD))
603 #define GST_IS_GL_VIDEO_MIXER_PAD_CLASS(klass) \
604 (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_VIDEO_MIXER_PAD))
606 typedef struct _GstGLVideoMixerPad GstGLVideoMixerPad;
607 typedef struct _GstGLVideoMixerPadClass GstGLVideoMixerPadClass;
608 typedef struct _GstGLVideoMixerCollect GstGLVideoMixerCollect;
611 * GstGLVideoMixerPad:
613 * The opaque #GstGLVideoMixerPad structure.
615 struct _GstGLVideoMixerPad
617 GstGLMixerPad parent;
624 GstGLVideoMixerBlendEquation blend_equation_rgb;
625 GstGLVideoMixerBlendEquation blend_equation_alpha;
626 GstGLVideoMixerBlendFunction blend_function_src_rgb;
627 GstGLVideoMixerBlendFunction blend_function_src_alpha;
628 GstGLVideoMixerBlendFunction blend_function_dst_rgb;
629 GstGLVideoMixerBlendFunction blend_function_dst_alpha;
630 gdouble blend_constant_color_red;
631 gdouble blend_constant_color_green;
632 gdouble blend_constant_color_blue;
633 gdouble blend_constant_color_alpha;
635 gint crop_left, crop_right;
636 gint crop_top, crop_bottom;
638 gboolean geometry_change;
639 GLuint vertex_buffer;
643 struct _GstGLVideoMixerPadClass
645 GstGLMixerPadClass parent_class;
648 GType gst_gl_video_mixer_pad_get_type (void);
649 G_DEFINE_TYPE (GstGLVideoMixerPad, gst_gl_video_mixer_pad,
650 GST_TYPE_GL_MIXER_PAD);
652 static void gst_gl_video_mixer_pad_set_property (GObject * object,
653 guint prop_id, const GValue * value, GParamSpec * pspec);
654 static void gst_gl_video_mixer_pad_get_property (GObject * object,
655 guint prop_id, GValue * value, GParamSpec * pspec);
665 PROP_PAD_BLEND_EQUATION_RGB,
666 PROP_PAD_BLEND_EQUATION_ALPHA,
667 PROP_PAD_BLEND_FUNCTION_SRC_RGB,
668 PROP_PAD_BLEND_FUNCTION_SRC_ALPHA,
669 PROP_PAD_BLEND_FUNCTION_DST_RGB,
670 PROP_PAD_BLEND_FUNCTION_DST_ALPHA,
671 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED,
672 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN,
673 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE,
674 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA,
678 PROP_PAD_CROP_BOTTOM,
682 gst_gl_video_mixer_pad_init (GstGLVideoMixerPad * pad)
684 pad->alpha = DEFAULT_PAD_ALPHA;
685 pad->blend_equation_rgb = DEFAULT_PAD_BLEND_EQUATION_RGB;
686 pad->blend_equation_alpha = DEFAULT_PAD_BLEND_EQUATION_ALPHA;
687 pad->blend_function_src_rgb = DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB;
688 pad->blend_function_src_alpha = DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA;
689 pad->blend_function_dst_rgb = DEFAULT_PAD_BLEND_FUNCTION_DST_RGB;
690 pad->blend_function_dst_alpha = DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA;
691 memset (pad->m_matrix, 0, sizeof (gfloat) * 4 * 4);
692 pad->m_matrix[0] = 1.0;
693 pad->m_matrix[5] = 1.0;
694 pad->m_matrix[10] = 1.0;
695 pad->m_matrix[15] = 1.0;
699 gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass)
701 GObjectClass *gobject_class = (GObjectClass *) klass;
703 gobject_class->set_property = gst_gl_video_mixer_pad_set_property;
704 gobject_class->get_property = gst_gl_video_mixer_pad_get_property;
706 g_object_class_install_property (gobject_class, PROP_PAD_XPOS,
707 g_param_spec_int ("xpos", "X Position", "X Position of the picture",
708 G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
709 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
710 g_object_class_install_property (gobject_class, PROP_PAD_YPOS,
711 g_param_spec_int ("ypos", "Y Position", "Y Position of the picture",
712 G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
713 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
714 g_object_class_install_property (gobject_class, PROP_PAD_WIDTH,
715 g_param_spec_int ("width", "Width", "Width of the picture",
716 G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH,
717 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
718 g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT,
719 g_param_spec_int ("height", "Height", "Height of the picture",
720 G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT,
721 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
722 g_object_class_install_property (gobject_class, PROP_PAD_ALPHA,
723 g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
725 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
726 g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_EQUATION_RGB,
727 g_param_spec_enum ("blend-equation-rgb", "Blend Equation RGB",
728 "Blend Equation for RGB",
729 GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION,
730 DEFAULT_PAD_BLEND_EQUATION_RGB,
731 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
732 g_object_class_install_property (gobject_class,
733 PROP_INPUT_BLEND_EQUATION_ALPHA,
734 g_param_spec_enum ("blend-equation-alpha", "Blend Equation Alpha",
735 "Blend Equation for Alpha", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION,
736 DEFAULT_PAD_BLEND_EQUATION_ALPHA,
737 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
738 g_object_class_install_property (gobject_class,
739 PROP_INPUT_BLEND_FUNCTION_SRC_RGB,
740 g_param_spec_enum ("blend-function-src-rgb", "Blend Function Source RGB",
741 "Blend Function for Source RGB",
742 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
743 DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB,
744 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
745 g_object_class_install_property (gobject_class,
746 PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA,
747 g_param_spec_enum ("blend-function-src-alpha",
748 "Blend Function Source Alpha", "Blend Function for Source Alpha",
749 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
750 DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA,
751 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
752 g_object_class_install_property (gobject_class,
753 PROP_INPUT_BLEND_FUNCTION_DST_RGB,
754 g_param_spec_enum ("blend-function-dst-rgb",
755 "Blend Function Destination RGB",
756 "Blend Function for Destination RGB",
757 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
758 DEFAULT_PAD_BLEND_FUNCTION_DST_RGB,
759 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
760 g_object_class_install_property (gobject_class,
761 PROP_INPUT_BLEND_FUNCTION_DST_ALPHA,
762 g_param_spec_enum ("blend-function-dst-alpha",
763 "Blend Function Destination Alpha",
764 "Blend Function for Destination Alpha",
765 GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION,
766 DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA,
767 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
768 g_object_class_install_property (gobject_class,
769 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED,
770 g_param_spec_double ("blend-constant-color-red",
771 "Blend Constant Color Red", "Blend Constant Color Red", 0.0, 1.0, 0.0,
772 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
773 g_object_class_install_property (gobject_class,
774 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN,
775 g_param_spec_double ("blend-constant-color-green",
776 "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0,
778 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
779 g_object_class_install_property (gobject_class,
780 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE,
781 g_param_spec_double ("blend-constant-color-blue",
782 "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0,
784 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
785 g_object_class_install_property (gobject_class,
786 PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA,
787 g_param_spec_double ("blend-constant-color-alpha",
788 "Blend Constant Color Alpha", "Blend Constant Color Alpha", 0.0, 1.0,
790 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
792 * GstGLVideoMixerPad:crop-left:
794 * Defines how many pixels of the input in input size should be cropped on
799 g_object_class_install_property (gobject_class, PROP_PAD_CROP_LEFT,
800 g_param_spec_int ("crop-left", "Crop Left", "Crop left of the picture", 0,
801 G_MAXINT, DEFAULT_PAD_CROP,
802 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
804 * GstGLVideoMixerPad:crop-right:
806 * Defines how many pixels of the input in input size should be cropped on
811 g_object_class_install_property (gobject_class, PROP_PAD_CROP_RIGHT,
812 g_param_spec_int ("crop-right", "Crop Right", "Crop right of the picture",
813 0, G_MAXINT, DEFAULT_PAD_CROP,
814 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
816 * GstGLVideoMixerPad:crop-top:
818 * Defines how many pixels of the input in input size should be cropped on
823 g_object_class_install_property (gobject_class, PROP_PAD_CROP_TOP,
824 g_param_spec_int ("crop-top", "Crop Top", "Crop top of the picture", 0,
825 G_MAXINT, DEFAULT_PAD_CROP,
826 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
828 * GstGLVideoMixerPad:crop-bottom:
830 * Defines how many pixels of the input in input size should be cropped on
835 g_object_class_install_property (gobject_class, PROP_PAD_CROP_BOTTOM,
836 g_param_spec_int ("crop-bottom", "Crop Bottom",
837 "Crop bottom of the picture", 0, G_MAXINT, DEFAULT_PAD_CROP,
838 G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
842 gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id,
843 GValue * value, GParamSpec * pspec)
845 GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (object);
849 g_value_set_int (value, pad->xpos);
852 g_value_set_int (value, pad->ypos);
855 g_value_set_int (value, pad->width);
857 case PROP_PAD_HEIGHT:
858 g_value_set_int (value, pad->height);
861 g_value_set_double (value, pad->alpha);
863 case PROP_PAD_BLEND_EQUATION_RGB:
864 g_value_set_enum (value, pad->blend_equation_rgb);
866 case PROP_PAD_BLEND_EQUATION_ALPHA:
867 g_value_set_enum (value, pad->blend_equation_alpha);
869 case PROP_PAD_BLEND_FUNCTION_SRC_RGB:
870 g_value_set_enum (value, pad->blend_function_src_rgb);
872 case PROP_PAD_BLEND_FUNCTION_SRC_ALPHA:
873 g_value_set_enum (value, pad->blend_function_src_alpha);
875 case PROP_PAD_BLEND_FUNCTION_DST_RGB:
876 g_value_set_enum (value, pad->blend_function_dst_rgb);
878 case PROP_PAD_BLEND_FUNCTION_DST_ALPHA:
879 g_value_set_enum (value, pad->blend_function_dst_alpha);
881 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED:
882 g_value_set_double (value, pad->blend_constant_color_red);
884 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN:
885 g_value_set_double (value, pad->blend_constant_color_green);
887 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE:
888 g_value_set_double (value, pad->blend_constant_color_blue);
890 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA:
891 g_value_set_double (value, pad->blend_constant_color_alpha);
893 case PROP_PAD_CROP_LEFT:
894 g_value_set_int (value, pad->crop_left);
896 case PROP_PAD_CROP_RIGHT:
897 g_value_set_int (value, pad->crop_right);
899 case PROP_PAD_CROP_TOP:
900 g_value_set_int (value, pad->crop_top);
902 case PROP_PAD_CROP_BOTTOM:
903 g_value_set_int (value, pad->crop_bottom);
906 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
912 gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id,
913 const GValue * value, GParamSpec * pspec)
915 GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (object);
916 GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent (GST_PAD (pad)));
918 GST_OBJECT_LOCK (pad);
921 gint val = g_value_get_int (value);
922 pad->geometry_change |= val != pad->xpos;
927 gint val = g_value_get_int (value);
928 pad->geometry_change |= val != pad->ypos;
932 case PROP_PAD_WIDTH:{
933 gint val = g_value_get_int (value);
934 pad->geometry_change |= val != pad->width;
938 case PROP_PAD_HEIGHT:{
939 gint val = g_value_get_int (value);
940 pad->geometry_change |= val != pad->height;
945 pad->alpha = g_value_get_double (value);
947 case PROP_PAD_BLEND_EQUATION_RGB:
948 pad->blend_equation_rgb = g_value_get_enum (value);
950 case PROP_PAD_BLEND_EQUATION_ALPHA:
951 pad->blend_equation_alpha = g_value_get_enum (value);
953 case PROP_PAD_BLEND_FUNCTION_SRC_RGB:
954 pad->blend_function_src_rgb = g_value_get_enum (value);
956 case PROP_PAD_BLEND_FUNCTION_SRC_ALPHA:
957 pad->blend_function_src_alpha = g_value_get_enum (value);
959 case PROP_PAD_BLEND_FUNCTION_DST_RGB:
960 pad->blend_function_dst_rgb = g_value_get_enum (value);
962 case PROP_PAD_BLEND_FUNCTION_DST_ALPHA:
963 pad->blend_function_dst_alpha = g_value_get_enum (value);
965 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED:
966 pad->blend_constant_color_red = g_value_get_double (value);
968 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN:
969 pad->blend_constant_color_green = g_value_get_double (value);
971 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE:
972 pad->blend_constant_color_blue = g_value_get_double (value);
974 case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA:
975 pad->blend_constant_color_alpha = g_value_get_double (value);
977 case PROP_PAD_CROP_LEFT:{
978 gint val = g_value_get_int (value);
979 pad->geometry_change |= val != pad->crop_left;
980 pad->crop_left = val;
983 case PROP_PAD_CROP_RIGHT:{
984 gint val = g_value_get_int (value);
985 pad->geometry_change |= val != pad->crop_right;
986 pad->crop_right = val;
989 case PROP_PAD_CROP_TOP:{
990 gint val = g_value_get_int (value);
991 pad->geometry_change |= val != pad->crop_top;
995 case PROP_PAD_CROP_BOTTOM:{
996 gint val = g_value_get_int (value);
997 pad->geometry_change |= val != pad->crop_bottom;
998 pad->crop_bottom = val;
1002 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1005 GST_OBJECT_UNLOCK (pad);
1007 gst_object_unref (mix);
1011 _del_buffer (GstGLContext * context, GLuint * pBuffer)
1013 context->gl_vtable->DeleteBuffers (1, pBuffer);
1017 gst_gl_video_mixer_request_new_pad (GstElement * element,
1018 GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps)
1023 GST_ELEMENT_CLASS (parent_class)->request_new_pad (element,
1024 templ, req_name, caps);
1027 goto could_not_create;
1029 gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad),
1030 GST_OBJECT_NAME (newpad));
1036 GST_DEBUG_OBJECT (element, "could not create/add pad");
1042 gst_gl_video_mixer_release_pad (GstElement * element, GstPad * p)
1044 GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (p);
1046 gst_child_proxy_child_removed (GST_CHILD_PROXY (element), G_OBJECT (pad),
1047 GST_OBJECT_NAME (pad));
1049 /* we call the base class first as this will remove the pad from
1050 * the aggregator, thus stopping misc callbacks from being called,
1051 * one of which (process_textures) will recreate the vertex_buffer
1052 * if it is destroyed. Calling the parent may release the last ref to the pad
1053 * so we need to keep the pad alive for the follow up clean up */
1054 gst_object_ref (pad);
1055 GST_ELEMENT_CLASS (g_type_class_peek_parent (G_OBJECT_GET_CLASS (element)))
1056 ->release_pad (element, p);
1058 if (pad->vertex_buffer) {
1059 GstGLBaseMixer *mix = GST_GL_BASE_MIXER (element);
1060 gst_gl_context_thread_add (mix->context, (GstGLContextThreadFunc)
1061 _del_buffer, &pad->vertex_buffer);
1062 pad->vertex_buffer = 0;
1064 gst_object_unref (pad);
1068 gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass)
1070 GObjectClass *gobject_class;
1071 GstElementClass *element_class;
1072 GstAggregatorClass *agg_class = (GstAggregatorClass *) klass;
1073 GstVideoAggregatorClass *vagg_class = (GstVideoAggregatorClass *) klass;
1075 gobject_class = (GObjectClass *) klass;
1076 element_class = GST_ELEMENT_CLASS (klass);
1077 element_class->request_new_pad = gst_gl_video_mixer_request_new_pad;
1078 element_class->release_pad = gst_gl_video_mixer_release_pad;
1080 gobject_class->set_property = gst_gl_video_mixer_set_property;
1081 gobject_class->get_property = gst_gl_video_mixer_get_property;
1083 gst_element_class_set_metadata (element_class, "OpenGL video_mixer",
1084 "Filter/Effect/Video/Compositor", "OpenGL video_mixer",
1085 "Matthew Waters <matthew@centricular.com>");
1087 gst_element_class_add_static_pad_template_with_gtype (element_class,
1088 &sink_factory, GST_TYPE_GL_VIDEO_MIXER_PAD);
1090 g_object_class_install_property (gobject_class, PROP_BACKGROUND,
1091 g_param_spec_enum ("background", "Background", "Background type",
1092 GST_TYPE_GL_VIDEO_MIXER_BACKGROUND,
1093 DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1095 GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_video_mixer_set_caps;
1096 GST_GL_MIXER_CLASS (klass)->process_textures =
1097 gst_gl_video_mixer_process_textures;
1099 GST_GL_BASE_MIXER_CLASS (klass)->gl_stop = gst_gl_video_mixer_gl_stop;
1100 GST_GL_BASE_MIXER_CLASS (klass)->gl_start = gst_gl_video_mixer_gl_start;
1102 vagg_class->update_caps = _update_caps;
1104 agg_class->src_event = gst_gl_video_mixer_src_event;
1105 agg_class->fixate_src_caps = _fixate_caps;
1106 agg_class->propose_allocation = gst_gl_video_mixer_propose_allocation;
1108 GST_GL_BASE_MIXER_CLASS (klass)->supported_gl_api =
1109 GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2;
1111 gst_type_mark_as_plugin_api (GST_TYPE_GL_VIDEO_MIXER_BACKGROUND, 0);
1112 gst_type_mark_as_plugin_api (GST_TYPE_GL_VIDEO_MIXER_PAD, 0);
1113 gst_type_mark_as_plugin_api (GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION, 0);
1114 gst_type_mark_as_plugin_api (GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, 0);
1118 gst_gl_video_mixer_init (GstGLVideoMixer * video_mixer)
1120 video_mixer->background = DEFAULT_BACKGROUND;
1121 video_mixer->shader = NULL;
1125 gst_gl_video_mixer_set_property (GObject * object, guint prop_id,
1126 const GValue * value, GParamSpec * pspec)
1128 GstGLVideoMixer *mixer = GST_GL_VIDEO_MIXER (object);
1131 case PROP_BACKGROUND:
1132 mixer->background = g_value_get_enum (value);
1135 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1141 gst_gl_video_mixer_get_property (GObject * object, guint prop_id,
1142 GValue * value, GParamSpec * pspec)
1144 GstGLVideoMixer *mixer = GST_GL_VIDEO_MIXER (object);
1147 case PROP_BACKGROUND:
1148 g_value_set_enum (value, mixer->background);
1151 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
1157 gst_gl_video_mixer_propose_allocation (GstAggregator * agg,
1158 GstAggregatorPad * agg_pad, GstQuery * decide_query, GstQuery * query)
1160 if (!GST_AGGREGATOR_CLASS (parent_class)->propose_allocation (agg,
1161 agg_pad, decide_query, query))
1164 gst_query_add_allocation_meta (query,
1165 GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, 0);
1171 _mixer_pad_get_output_size (GstGLVideoMixer * mix,
1172 GstGLVideoMixerPad * mix_pad, gint out_par_n, gint out_par_d, gint * width,
1175 GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (mix_pad);
1176 gint pad_width, pad_height;
1179 /* FIXME: Anything better we can do here? */
1180 if (!vagg_pad->info.finfo
1181 || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) {
1182 GST_DEBUG_OBJECT (mix_pad, "Have no caps yet");
1188 if (mix_pad->width <= 0) {
1189 gint crop = mix_pad->crop_left + mix_pad->crop_right;
1191 if (GST_VIDEO_INFO_WIDTH (&vagg_pad->info) > crop)
1192 pad_width = GST_VIDEO_INFO_WIDTH (&vagg_pad->info) - crop;
1196 pad_width = mix_pad->width;
1199 if (mix_pad->height <= 0) {
1200 gint crop = mix_pad->crop_top + mix_pad->crop_bottom;
1202 if (GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) > crop)
1203 pad_height = GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) - crop;
1207 pad_height = mix_pad->height;
1210 if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height,
1211 GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
1212 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) {
1213 GST_WARNING_OBJECT (mix_pad, "Cannot calculate display aspect ratio");
1214 *width = *height = 0;
1217 GST_LOG_OBJECT (mix_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width,
1218 pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
1219 GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d);
1221 if (pad_height % dar_n == 0) {
1222 pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
1223 } else if (pad_width % dar_d == 0) {
1224 pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n);
1226 pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
1230 *height = pad_height;
1234 _update_caps (GstVideoAggregator * vagg, GstCaps * caps)
1236 GstCaps *template_caps, *ret;
1239 GST_OBJECT_LOCK (vagg);
1240 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
1241 GstVideoAggregatorPad *vaggpad = l->data;
1243 if (!vaggpad->info.finfo)
1246 if (GST_VIDEO_INFO_FORMAT (&vaggpad->info) == GST_VIDEO_FORMAT_UNKNOWN)
1249 if (GST_VIDEO_INFO_MULTIVIEW_MODE (&vaggpad->info) !=
1250 GST_VIDEO_MULTIVIEW_MODE_NONE
1251 && GST_VIDEO_INFO_MULTIVIEW_MODE (&vaggpad->info) !=
1252 GST_VIDEO_MULTIVIEW_MODE_MONO) {
1253 GST_FIXME_OBJECT (vaggpad, "Multiview support is not implemented yet");
1254 GST_OBJECT_UNLOCK (vagg);
1259 GST_OBJECT_UNLOCK (vagg);
1261 template_caps = gst_pad_get_pad_template_caps (GST_AGGREGATOR_SRC_PAD (vagg));
1262 ret = gst_caps_intersect (caps, template_caps);
1268 _fixate_caps (GstAggregator * agg, GstCaps * caps)
1270 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
1271 GstGLVideoMixer *mix = GST_GL_VIDEO_MIXER (vagg);
1272 gint best_width = 0, best_height = 0;
1273 gint best_fps_n = 0, best_fps_d = 0;
1275 gdouble best_fps = 0.;
1276 GstCaps *ret = NULL;
1280 ret = gst_caps_make_writable (caps);
1282 /* we need this to calculate how large to make the output frame */
1283 s = gst_caps_get_structure (ret, 0);
1284 if (!gst_structure_has_field (s, "pixel-aspect-ratio")) {
1285 gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL);
1287 gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1);
1288 gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d);
1290 GST_OBJECT_LOCK (vagg);
1291 for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
1292 GstVideoAggregatorPad *vaggpad = l->data;
1293 GstGLVideoMixerPad *mixer_pad = GST_GL_VIDEO_MIXER_PAD (vaggpad);
1294 gint this_width, this_height;
1299 fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info);
1300 fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info);
1301 _mixer_pad_get_output_size (mix, mixer_pad, par_n, par_d, &width, &height);
1303 if (width == 0 || height == 0)
1306 this_width = width + MAX (mixer_pad->xpos, 0);
1307 this_height = height + MAX (mixer_pad->ypos, 0);
1309 if (best_width < this_width)
1310 best_width = this_width;
1311 if (best_height < this_height)
1312 best_height = this_height;
1317 gst_util_fraction_to_double (fps_n, fps_d, &cur_fps);
1319 if (best_fps < cur_fps) {
1325 GST_OBJECT_UNLOCK (vagg);
1327 if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) {
1333 s = gst_caps_get_structure (ret, 0);
1334 gst_structure_fixate_field_nearest_int (s, "width", best_width);
1335 gst_structure_fixate_field_nearest_int (s, "height", best_height);
1336 gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n,
1338 ret = gst_caps_fixate (ret);
1344 _reset_pad_gl (GstElement * agg, GstPad * aggpad, gpointer udata)
1346 const GstGLFuncs *gl = GST_GL_BASE_MIXER (agg)->context->gl_vtable;
1347 GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (aggpad);
1349 if (pad->vertex_buffer) {
1350 gl->DeleteBuffers (1, &pad->vertex_buffer);
1351 pad->vertex_buffer = 0;
1358 _reset_gl (GstGLContext * context, GstGLVideoMixer * video_mixer)
1360 const GstGLFuncs *gl = GST_GL_BASE_MIXER (video_mixer)->context->gl_vtable;
1362 if (video_mixer->vao) {
1363 gl->DeleteVertexArrays (1, &video_mixer->vao);
1364 video_mixer->vao = 0;
1367 if (video_mixer->vbo_indices) {
1368 gl->DeleteBuffers (1, &video_mixer->vbo_indices);
1369 video_mixer->vbo_indices = 0;
1372 if (video_mixer->checker_vbo) {
1373 gl->DeleteBuffers (1, &video_mixer->checker_vbo);
1374 video_mixer->checker_vbo = 0;
1377 gst_element_foreach_sink_pad (GST_ELEMENT (video_mixer), _reset_pad_gl, NULL);
1381 is_point_contained (const GstVideoRectangle rect, const gint px, const gint py)
1383 if ((px >= rect.x) && (px <= rect.x + rect.w) &&
1384 (py >= rect.y) && (py <= rect.y + rect.h))
1390 src_pad_mouse_event (GstElement * element, GstPad * pad, gpointer user_data)
1392 GstGLVideoMixer *mix = GST_GL_VIDEO_MIXER (element);
1393 GstGLVideoMixerPad *mix_pad = GST_GL_VIDEO_MIXER_PAD (pad);
1394 GstCaps *caps = gst_pad_get_current_caps (pad);
1395 GstStructure *event_st, *caps_st;
1396 gint par_n = 1, par_d = 1;
1397 gdouble event_x, event_y;
1398 GstVideoRectangle rect;
1401 gst_structure_copy (gst_event_get_structure (GST_EVENT_CAST (user_data)));
1402 caps_st = gst_structure_copy (gst_caps_get_structure (caps, 0));
1404 gst_structure_get (event_st, "pointer_x", G_TYPE_DOUBLE, &event_x,
1405 "pointer_y", G_TYPE_DOUBLE, &event_y, NULL);
1407 /* Find output rectangle of this pad */
1408 gst_structure_get_fraction (caps_st, "pixel-aspect-ratio", &par_n, &par_d);
1409 _mixer_pad_get_output_size (mix, mix_pad, par_n, par_d, &(rect.w), &(rect.h));
1410 rect.x = mix_pad->xpos;
1411 rect.y = mix_pad->ypos;
1413 /* Translate coordinates and send event if it lies in this rectangle */
1414 if (is_point_contained (rect, event_x, event_y)) {
1415 GstVideoAggregatorPad *vpad = GST_VIDEO_AGGREGATOR_PAD_CAST (mix_pad);
1418 w = (gdouble) GST_VIDEO_INFO_WIDTH (&vpad->info);
1419 h = (gdouble) GST_VIDEO_INFO_HEIGHT (&vpad->info);
1420 x = (event_x - (gdouble) rect.x) * (w / (gdouble) rect.w);
1421 y = (event_y - (gdouble) rect.y) * (h / (gdouble) rect.h);
1423 gst_structure_set (event_st, "pointer_x", G_TYPE_DOUBLE, x,
1424 "pointer_y", G_TYPE_DOUBLE, y, NULL);
1425 gst_pad_push_event (pad, gst_event_new_navigation (event_st));
1427 gst_structure_free (event_st);
1430 gst_structure_free (caps_st);
1435 gst_gl_video_mixer_src_event (GstAggregator * agg, GstEvent * event)
1437 GstNavigationEventType event_type;
1439 switch (GST_EVENT_TYPE (event)) {
1440 case GST_EVENT_NAVIGATION:
1442 event_type = gst_navigation_event_get_type (event);
1443 switch (event_type) {
1444 case GST_NAVIGATION_EVENT_MOUSE_BUTTON_PRESS:
1445 case GST_NAVIGATION_EVENT_MOUSE_BUTTON_RELEASE:
1446 case GST_NAVIGATION_EVENT_MOUSE_MOVE:
1447 case GST_NAVIGATION_EVENT_MOUSE_SCROLL:
1448 gst_element_foreach_sink_pad (GST_ELEMENT_CAST (agg),
1449 src_pad_mouse_event, event);
1450 gst_event_unref (event);
1462 return GST_AGGREGATOR_CLASS (parent_class)->src_event (agg, event);
1466 gst_gl_video_mixer_set_caps (GstGLMixer * mixer, GstCaps * outcaps)
1468 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer);
1470 /* need reconfigure output geometry */
1471 video_mixer->output_geo_change = TRUE;
1477 gst_gl_video_mixer_gl_stop (GstGLBaseMixer * base_mix)
1479 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (base_mix);
1481 gst_clear_object (&video_mixer->shader);
1482 gst_clear_object (&video_mixer->checker);
1484 _reset_gl (base_mix->context, video_mixer);
1486 GST_GL_BASE_MIXER_CLASS (parent_class)->gl_stop (base_mix);
1490 gst_gl_video_mixer_gl_start (GstGLBaseMixer * base_mix)
1492 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (base_mix);
1494 if (!video_mixer->shader) {
1495 gchar *frag_str = g_strdup_printf ("%s%s",
1496 gst_gl_shader_string_get_highest_precision (base_mix->context,
1497 GST_GLSL_VERSION_NONE,
1498 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY),
1501 gst_gl_context_gen_shader (base_mix->context,
1502 gst_gl_shader_string_vertex_mat4_vertex_transform,
1503 frag_str, &video_mixer->shader);
1507 return GST_GL_BASE_MIXER_CLASS (parent_class)->gl_start (base_mix);
1511 _video_mixer_process_gl (GstGLContext * context, GstGLVideoMixer * video_mixer)
1513 GstGLMixer *mixer = GST_GL_MIXER (video_mixer);
1515 gst_gl_framebuffer_draw_to_texture (mixer->fbo, video_mixer->out_tex,
1516 gst_gl_video_mixer_callback, video_mixer);
1520 gst_gl_video_mixer_process_textures (GstGLMixer * mix, GstGLMemory * out_tex)
1522 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mix);
1523 GstGLContext *context = GST_GL_BASE_MIXER (mix)->context;
1525 video_mixer->out_tex = out_tex;
1527 gst_gl_context_thread_add (context,
1528 (GstGLContextThreadFunc) _video_mixer_process_gl, video_mixer);
1533 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
1536 _init_vbo_indices (GstGLVideoMixer * mixer)
1538 const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable;
1540 if (!mixer->vbo_indices) {
1541 gl->GenBuffers (1, &mixer->vbo_indices);
1542 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, mixer->vbo_indices);
1543 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
1549 _draw_checker_background (GstGLVideoMixer * video_mixer)
1551 GstGLMixer *mixer = GST_GL_MIXER (video_mixer);
1552 const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable;
1553 gint attr_position_loc = 0;
1556 gfloat v_vertices[] = {
1564 if (!video_mixer->checker) {
1568 g_strdup_printf ("%s%s",
1569 gst_gl_shader_string_get_highest_precision (GST_GL_BASE_MIXER
1570 (mixer)->context, GST_GLSL_VERSION_NONE,
1571 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY),
1574 if (!gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context,
1575 checker_v_src, frag_str, &video_mixer->checker)) {
1582 gst_gl_shader_use (video_mixer->checker);
1584 gst_gl_shader_get_attribute_location (video_mixer->checker, "a_position");
1586 _init_vbo_indices (video_mixer);
1588 if (!video_mixer->checker_vbo) {
1589 gl->GenBuffers (1, &video_mixer->checker_vbo);
1590 gl->BindBuffer (GL_ARRAY_BUFFER, video_mixer->checker_vbo);
1591 gl->BufferData (GL_ARRAY_BUFFER, 4 * 3 * sizeof (GLfloat), v_vertices,
1594 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, video_mixer->vbo_indices);
1595 gl->BindBuffer (GL_ARRAY_BUFFER, video_mixer->checker_vbo);
1598 gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT,
1599 GL_FALSE, 3 * sizeof (GLfloat), (void *) 0);
1601 gl->EnableVertexAttribArray (attr_position_loc);
1603 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
1605 gl->DisableVertexAttribArray (attr_position_loc);
1606 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1607 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1613 _draw_background (GstGLVideoMixer * video_mixer)
1615 GstGLMixer *mixer = GST_GL_MIXER (video_mixer);
1616 const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable;
1618 switch (video_mixer->background) {
1619 case GST_GL_VIDEO_MIXER_BACKGROUND_BLACK:
1620 gl->ClearColor (0.0, 0.0, 0.0, 1.0);
1621 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
1623 case GST_GL_VIDEO_MIXER_BACKGROUND_WHITE:
1624 gl->ClearColor (1.0, 1.0, 1.0, 1.0);
1625 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
1627 case GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT:
1628 gl->ClearColor (0.0, 0.0, 0.0, 0.0);
1629 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
1631 case GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER:
1632 return _draw_checker_background (video_mixer);
1642 _blend_equation_to_gl (GstGLVideoMixerBlendEquation equation)
1645 case GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD:
1647 case GST_GL_VIDEO_MIXER_BLEND_EQUATION_SUBTRACT:
1648 return GL_FUNC_SUBTRACT;
1649 case GST_GL_VIDEO_MIXER_BLEND_EQUATION_REVERSE_SUBTRACT:
1650 return GL_FUNC_REVERSE_SUBTRACT;
1652 g_assert_not_reached ();
1658 _blend_function_to_gl (GstGLVideoMixerBlendFunction equation)
1661 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ZERO:
1663 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE:
1665 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_COLOR:
1666 return GL_SRC_COLOR;
1667 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_COLOR:
1668 return GL_ONE_MINUS_SRC_COLOR;
1669 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_COLOR:
1670 return GL_DST_COLOR;
1671 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_COLOR:
1672 return GL_ONE_MINUS_DST_COLOR;
1673 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA:
1674 return GL_SRC_ALPHA;
1675 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA:
1676 return GL_ONE_MINUS_SRC_ALPHA;
1677 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_ALPHA:
1678 return GL_DST_ALPHA;
1679 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_ALPHA:
1680 return GL_ONE_MINUS_DST_ALPHA;
1681 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_COLOR:
1682 return GL_CONSTANT_COLOR;
1683 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR:
1684 return GL_ONE_MINUS_CONSTANT_COLOR;
1685 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA:
1686 return GL_CONSTANT_ALPHA;
1687 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_ALPHA:
1688 return GL_ONE_MINUS_CONSTANT_ALPHA;
1689 case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE:
1690 return GL_SRC_ALPHA_SATURATE;
1692 g_assert_not_reached ();
1698 _set_blend_state (GstGLVideoMixer * video_mixer, GstGLVideoMixerPad * mix_pad)
1700 const GstGLFuncs *gl = GST_GL_BASE_MIXER (video_mixer)->context->gl_vtable;
1701 gboolean require_separate = FALSE;
1702 guint gl_func_src_rgb, gl_func_src_alpha, gl_func_dst_rgb, gl_func_dst_alpha;
1703 guint gl_equation_rgb, gl_equation_alpha;
1706 mix_pad->blend_equation_rgb != mix_pad->blend_equation_alpha
1707 || mix_pad->blend_function_src_rgb != mix_pad->blend_function_src_alpha
1708 || mix_pad->blend_function_dst_rgb != mix_pad->blend_function_dst_alpha;
1710 if (require_separate && (!gl->BlendFuncSeparate
1711 || !gl->BlendEquationSeparate)) {
1712 GST_ERROR_OBJECT (mix_pad,
1713 "separated blend equations/functions requested however "
1714 "glBlendFuncSeparate or glBlendEquationSeparate not available");
1718 if (mix_pad->blend_function_dst_rgb ==
1719 GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE) {
1720 GST_ERROR_OBJECT (mix_pad,
1721 "Destination RGB blend function cannot be \'SRC_ALPHA_SATURATE\'");
1725 if (mix_pad->blend_function_dst_alpha ==
1726 GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE) {
1727 GST_ERROR_OBJECT (mix_pad,
1728 "Destination alpha blend function cannot be \'SRC_ALPHA_SATURATE\'");
1732 gl_equation_rgb = _blend_equation_to_gl (mix_pad->blend_equation_rgb);
1733 gl_equation_alpha = _blend_equation_to_gl (mix_pad->blend_equation_alpha);
1735 gl_func_src_rgb = _blend_function_to_gl (mix_pad->blend_function_src_rgb);
1736 gl_func_src_alpha = _blend_function_to_gl (mix_pad->blend_function_src_alpha);
1737 gl_func_dst_rgb = _blend_function_to_gl (mix_pad->blend_function_dst_rgb);
1738 gl_func_dst_alpha = _blend_function_to_gl (mix_pad->blend_function_dst_alpha);
1740 if (gl->BlendEquationSeparate)
1741 gl->BlendEquationSeparate (gl_equation_rgb, gl_equation_alpha);
1743 gl->BlendEquation (gl_equation_rgb);
1745 if (gl->BlendFuncSeparate)
1746 gl->BlendFuncSeparate (gl_func_src_rgb, gl_func_dst_rgb, gl_func_src_alpha,
1749 gl->BlendFunc (gl_func_src_rgb, gl_func_dst_rgb);
1751 gl->BlendColor (mix_pad->blend_constant_color_red,
1752 mix_pad->blend_constant_color_green, mix_pad->blend_constant_color_blue,
1753 mix_pad->blend_constant_color_alpha);
1758 /* opengl scene, params: input texture (not the output mixer->texture) */
1760 gst_gl_video_mixer_callback (gpointer stuff)
1762 GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (stuff);
1763 GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (stuff);
1764 GstGLMixer *mixer = GST_GL_MIXER (video_mixer);
1765 GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable;
1766 GLint attr_position_loc = 0;
1767 GLint attr_texture_loc = 0;
1768 guint out_width, out_height;
1771 out_width = GST_VIDEO_INFO_WIDTH (&vagg->info);
1772 out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info);
1774 gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context);
1775 gl->BindTexture (GL_TEXTURE_2D, 0);
1777 gl->Disable (GL_DEPTH_TEST);
1778 gl->Disable (GL_CULL_FACE);
1780 if (gl->GenVertexArrays) {
1781 if (!video_mixer->vao)
1782 gl->GenVertexArrays (1, &video_mixer->vao);
1783 gl->BindVertexArray (video_mixer->vao);
1786 if (!_draw_background (video_mixer))
1789 gst_gl_shader_use (video_mixer->shader);
1792 gst_gl_shader_get_attribute_location (video_mixer->shader, "a_position");
1794 gst_gl_shader_get_attribute_location (video_mixer->shader, "a_texcoord");
1796 gl->Enable (GL_BLEND);
1798 GST_OBJECT_LOCK (video_mixer);
1799 walk = GST_ELEMENT (video_mixer)->sinkpads;
1801 GstGLMixerPad *mix_pad = walk->data;
1802 GstGLVideoMixerPad *pad = walk->data;
1803 GstVideoAggregatorPad *vagg_pad = walk->data;
1804 GstVideoInfo *v_info;
1806 guint in_width, in_height;
1808 v_info = &GST_VIDEO_AGGREGATOR_PAD (pad)->info;
1809 in_width = GST_VIDEO_INFO_WIDTH (v_info);
1810 in_height = GST_VIDEO_INFO_HEIGHT (v_info);
1812 if (!mix_pad->current_texture || in_width <= 0 || in_height <= 0
1813 || pad->alpha == 0.0f) {
1814 GST_DEBUG ("skipping texture:%u pad:%p width:%u height:%u alpha:%f",
1815 mix_pad->current_texture, pad, in_width, in_height, pad->alpha);
1816 walk = g_list_next (walk);
1820 if (!_set_blend_state (video_mixer, pad)) {
1821 GST_FIXME_OBJECT (pad, "skipping due to incorrect blend parameters");
1822 walk = g_list_next (walk);
1826 in_tex = mix_pad->current_texture;
1828 _init_vbo_indices (video_mixer);
1830 if (video_mixer->output_geo_change
1831 || pad->geometry_change || !pad->vertex_buffer) {
1832 gint pad_width, pad_height;
1835 gfloat v_vertices[] = {
1836 -1.0,-1.0, 0.0f, 0.0f, 0.0f,
1837 1.0,-1.0, 0.0f, 1.0f, 0.0f,
1838 1.0, 1.0, 0.0f, 1.0f, 1.0f,
1839 -1.0, 1.0, 0.0f, 0.0f, 1.0f,
1843 _mixer_pad_get_output_size (video_mixer, pad,
1844 GST_VIDEO_INFO_PAR_N (&vagg->info),
1845 GST_VIDEO_INFO_PAR_D (&vagg->info), &pad_width, &pad_height);
1847 w = ((gfloat) pad_width / (gfloat) out_width);
1848 h = ((gfloat) pad_height / (gfloat) out_height);
1850 pad->m_matrix[0] = w;
1851 pad->m_matrix[5] = h;
1853 2. * (gfloat) pad->xpos / (gfloat) out_width - (1. - w);
1855 2. * (gfloat) pad->ypos / (gfloat) out_height - (1. - h);
1857 v_vertices[0 * 5 + 3] = v_vertices[3 * 5 + 3] =
1858 pad->crop_left ? ((gfloat) pad->crop_left) /
1859 ((gfloat) in_width) : 0.0f;
1860 v_vertices[1 * 5 + 3] = v_vertices[2 * 5 + 3] =
1861 pad->crop_right ? 1.0 -
1862 ((gfloat) pad->crop_right) / ((gfloat) in_width) : 1.0f;
1863 v_vertices[0 * 5 + 4] = v_vertices[1 * 5 + 4] =
1864 pad->crop_top ? ((gfloat) pad->crop_top) /
1865 ((gfloat) in_height) : 0.0f;
1866 v_vertices[2 * 5 + 4] = v_vertices[3 * 5 + 4] =
1867 pad->crop_bottom ? 1.0 -
1868 ((gfloat) pad->crop_bottom) / ((gfloat) in_height) : 1.0f;
1870 GST_TRACE ("processing texture:%u dimensions:%ux%u with texture "
1871 "coordinates %f:%fx%f:%f, at %f,%f %fx%f with alpha:%f "
1872 "and crop: %d:%dx%d:%d", in_tex, in_width, in_height,
1873 v_vertices[0 * 5 + 3], v_vertices[1 * 5 + 3], v_vertices[0 * 5 + 4],
1874 v_vertices[1 * 5 + 4], pad->m_matrix[12], pad->m_matrix[13],
1875 pad->m_matrix[0], pad->m_matrix[5], pad->alpha, pad->crop_left,
1876 pad->crop_right, pad->crop_top, pad->crop_bottom);
1878 if (!pad->vertex_buffer)
1879 gl->GenBuffers (1, &pad->vertex_buffer);
1881 gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer);
1882 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), v_vertices,
1885 pad->geometry_change = FALSE;
1887 gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer);
1889 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, video_mixer->vbo_indices);
1891 gl->ActiveTexture (GL_TEXTURE0);
1892 gl->BindTexture (GL_TEXTURE_2D, in_tex);
1893 gst_gl_shader_set_uniform_1i (video_mixer->shader, "texture", 0);
1894 gst_gl_shader_set_uniform_1f (video_mixer->shader, "alpha", pad->alpha);
1897 GstVideoAffineTransformationMeta *af_meta;
1899 gfloat af_matrix[16];
1901 gst_video_aggregator_pad_get_current_buffer (vagg_pad);
1903 af_meta = gst_buffer_get_video_affine_transformation_meta (buffer);
1904 gst_gl_get_affine_transformation_meta_as_ndc (af_meta, af_matrix);
1905 gst_gl_multiply_matrix4 (af_matrix, pad->m_matrix, matrix);
1906 gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader,
1907 "u_transformation", 1, FALSE, matrix);
1910 gl->EnableVertexAttribArray (attr_position_loc);
1911 gl->EnableVertexAttribArray (attr_texture_loc);
1913 gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT,
1914 GL_FALSE, 5 * sizeof (GLfloat), (void *) 0);
1916 gl->VertexAttribPointer (attr_texture_loc, 2, GL_FLOAT,
1917 GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
1919 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
1921 walk = g_list_next (walk);
1924 video_mixer->output_geo_change = FALSE;
1925 GST_OBJECT_UNLOCK (video_mixer);
1927 if (gl->GenVertexArrays) {
1928 gl->BindVertexArray (0);
1930 gl->DisableVertexAttribArray (attr_position_loc);
1931 gl->DisableVertexAttribArray (attr_texture_loc);
1933 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1934 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1935 gl->BindTexture (GL_TEXTURE_2D, 0);
1938 gl->Disable (GL_BLEND);
1940 gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context);
1945 /* GstChildProxy implementation */
1947 gst_gl_video_mixer_child_proxy_get_child_by_index (GstChildProxy * child_proxy,
1950 GstGLVideoMixer *gl_video_mixer = GST_GL_VIDEO_MIXER (child_proxy);
1951 GObject *obj = NULL;
1953 GST_OBJECT_LOCK (gl_video_mixer);
1954 obj = g_list_nth_data (GST_ELEMENT_CAST (gl_video_mixer)->sinkpads, index);
1956 gst_object_ref (obj);
1957 GST_OBJECT_UNLOCK (gl_video_mixer);
1963 gst_gl_video_mixer_child_proxy_get_children_count (GstChildProxy * child_proxy)
1966 GstGLVideoMixer *gl_video_mixer = GST_GL_VIDEO_MIXER (child_proxy);
1968 GST_OBJECT_LOCK (gl_video_mixer);
1969 count = GST_ELEMENT_CAST (gl_video_mixer)->numsinkpads;
1970 GST_OBJECT_UNLOCK (gl_video_mixer);
1971 GST_INFO_OBJECT (gl_video_mixer, "Children Count: %d", count);
1977 gst_gl_video_mixer_child_proxy_init (gpointer g_iface, gpointer iface_data)
1979 GstChildProxyInterface *iface = g_iface;
1981 iface->get_child_by_index = gst_gl_video_mixer_child_proxy_get_child_by_index;
1982 iface->get_children_count = gst_gl_video_mixer_child_proxy_get_children_count;