3 * Copyright (C) 2003 Julien Moutte <julien@moutte.net>
4 * Copyright (C) 2005,2006,2007 David A. Schleef <ds@schleef.org>
5 * Copyright (C) 2008 Julien Isorce <julien.isorce@gmail.com>
6 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Library General Public
10 * License as published by the Free Software Foundation; either
11 * version 2 of the License, or (at your option) any later version.
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Library General Public License for more details.
18 * You should have received a copy of the GNU Library General Public
19 * License along with this library; if not, write to the
20 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
21 * Boston, MA 02110-1301, USA.
25 * SECTION:element-glimagesink
27 * glimagesink renders video frames to a drawable on a local or remote
28 * display using OpenGL. This element can receive a Window ID from the
29 * application through the VideoOverlay interface and will then render video
30 * frames in this drawable.
31 * If no Window ID was provided by the application, the element will
32 * create its own internal window and render into it.
34 * See the #GstGLDisplay documentation for a list of environment variables that
35 * can override window/platform detection.
38 * <title>Scaling</title>
40 * Depends on the driver, OpenGL handles hardware accelerated
41 * scaling of video frames. This means that the element will just accept
42 * incoming video frames no matter their geometry and will then put them to the
43 * drawable scaling them on the fly. Using the #GstGLImageSink:force-aspect-ratio
44 * property it is possible to enforce scaling with a constant aspect ratio,
45 * which means drawing black borders around the video frame.
49 * <title>Events</title>
51 * Through the gl thread, glimagesink handle some events coming from the drawable
52 * to manage its appearance even when the data is not flowing (GST_STATE_PAUSED).
53 * That means that even when the element is paused, it will receive expose events
54 * from the drawable and draw the latest frame with correct borders/aspect-ratio.
58 * <title>Examples</title>
60 * gst-launch-1.0 -v videotestsrc ! video/x-raw ! glimagesink
61 * ]| A pipeline to test hardware scaling.
62 * No special opengl extension is used in this pipeline, that's why it should work
63 * with OpenGL >= 1.1. That's the case if you are using the MESA3D driver v1.3.
65 * gst-launch-1.0 -v videotestsrc ! video/x-raw,format=I420 ! glimagesink
66 * ]| A pipeline to test hardware scaling and hardware colorspace conversion.
67 * When your driver supports GLSL (OpenGL Shading Language needs OpenGL >= 2.1),
68 * the 4 following format YUY2, UYVY, I420, YV12 and AYUV are converted to RGB32
69 * through some fragment shaders and using one framebuffer (FBO extension OpenGL >= 1.4).
70 * If your driver does not support GLSL but supports MESA_YCbCr extension then
71 * the you can use YUY2 and UYVY. In this case the colorspace conversion is automatically
72 * made when loading the texture and therefore no framebuffer is used.
74 * gst-launch-1.0 -v gltestsrc ! glimagesink
75 * ]| A pipeline 100% OpenGL.
76 * No special opengl extension is used in this pipeline, that's why it should work
77 * with OpenGL >= 1.1. That's the case if you are using the MESA3D driver v1.3.
79 * gst-plugins-bas/tests/examples/gl/generic/cube
80 * ]| The graphic FPS scene can be greater than the input video FPS.
81 * The graphic scene can be written from a client code through the
82 * two glfilterapp properties.
90 #include <gst/video/videooverlay.h>
91 #include <gst/video/navigation.h>
93 #include "gstglimagesink.h"
94 #include "gstglsinkbin.h"
96 #if GST_GL_HAVE_PLATFORM_EGL
97 #include <gst/gl/egl/gsteglimagememory.h>
100 GST_DEBUG_CATEGORY (gst_debug_glimage_sink);
101 #define GST_CAT_DEFAULT gst_debug_glimage_sink
103 #define DEFAULT_SHOW_PREROLL_FRAME TRUE
104 #define DEFAULT_SYNC TRUE
105 #define DEFAULT_MAX_LATENESS -1
106 #define DEFAULT_QOS FALSE
107 #define DEFAULT_ASYNC TRUE
108 #define DEFAULT_TS_OFFSET 0
109 #define DEFAULT_BLOCKSIZE 4096
110 #define DEFAULT_RENDER_DELAY 0
111 #define DEFAULT_ENABLE_LAST_SAMPLE TRUE
112 #define DEFAULT_THROTTLE_TIME 0
113 #define DEFAULT_MAX_BITRATE 0
114 #define DEFAULT_HANDLE_EVENTS TRUE
115 #define DEFAULT_FORCE_ASPECT_RATIO TRUE
117 typedef GstGLSinkBin GstGLImageSinkBin;
118 typedef GstGLSinkBinClass GstGLImageSinkBinClass;
120 G_DEFINE_TYPE (GstGLImageSinkBin, gst_gl_image_sink_bin, GST_TYPE_GL_SINK_BIN);
125 PROP_BIN_FORCE_ASPECT_RATIO,
126 PROP_BIN_PIXEL_ASPECT_RATIO,
127 PROP_BIN_HANDLE_EVENTS,
129 PROP_BIN_SHOW_PREROLL_FRAME,
131 PROP_BIN_MAX_LATENESS,
135 PROP_BIN_ENABLE_LAST_SAMPLE,
136 PROP_BIN_LAST_SAMPLE,
138 PROP_BIN_RENDER_DELAY,
139 PROP_BIN_THROTTLE_TIME,
140 PROP_BIN_MAX_BITRATE,
146 SIGNAL_BIN_CLIENT_DRAW,
147 SIGNAL_BIN_CLIENT_RESHAPE,
151 static guint gst_gl_image_sink_bin_signals[SIGNAL_BIN_LAST] = { 0 };
154 gst_gl_image_sink_bin_set_property (GObject * object, guint prop_id,
155 const GValue * value, GParamSpec * param_spec)
157 g_object_set_property (G_OBJECT (GST_GL_SINK_BIN (object)->sink),
158 param_spec->name, value);
162 gst_gl_image_sink_bin_get_property (GObject * object, guint prop_id,
163 GValue * value, GParamSpec * param_spec)
165 g_object_get_property (G_OBJECT (GST_GL_SINK_BIN (object)->sink),
166 param_spec->name, value);
170 _on_client_reshape (GstGLImageSink * sink, GstGLContext * context,
171 guint width, guint height, gpointer data)
175 g_signal_emit (data, gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_RESHAPE],
176 0, context, width, height, &ret);
182 _on_client_draw (GstGLImageSink * sink, GstGLContext * context,
183 guint tex_id, guint width, guint height, gpointer data)
187 g_signal_emit (data, gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_DRAW], 0,
188 context, tex_id, width, height, &ret);
194 gst_gl_image_sink_bin_init (GstGLImageSinkBin * self)
196 GstGLImageSink *sink = g_object_new (GST_TYPE_GLIMAGE_SINK, NULL);
198 g_signal_connect (sink, "client-reshape", (GCallback) _on_client_reshape,
200 g_signal_connect (sink, "client-draw", (GCallback) _on_client_draw, self);
202 gst_gl_sink_bin_finish_init_with_element (GST_GL_SINK_BIN (self),
207 gst_gl_image_sink_bin_class_init (GstGLImageSinkBinClass * klass)
209 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
211 gobject_class->get_property = gst_gl_image_sink_bin_get_property;
212 gobject_class->set_property = gst_gl_image_sink_bin_set_property;
215 g_object_class_install_property (gobject_class, PROP_BIN_FORCE_ASPECT_RATIO,
216 g_param_spec_boolean ("force-aspect-ratio",
217 "Force aspect ratio",
218 "When enabled, scaling will respect original aspect ratio",
219 DEFAULT_FORCE_ASPECT_RATIO,
220 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
221 g_object_class_install_property (gobject_class, PROP_BIN_HANDLE_EVENTS,
222 g_param_spec_boolean ("ignore-alpha", "Ignore Alpha",
223 "When enabled, alpha will be ignored and converted to black",
224 DEFAULT_HANDLE_EVENTS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
225 g_object_class_install_property (gobject_class, PROP_BIN_CONTEXT,
226 g_param_spec_object ("context", "OpenGL context", "Get OpenGL context",
227 GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
228 g_object_class_install_property (gobject_class, PROP_BIN_PIXEL_ASPECT_RATIO,
229 gst_param_spec_fraction ("pixel-aspect-ratio", "Pixel Aspect Ratio",
230 "The pixel aspect ratio of the device", 0, 1, G_MAXINT, 1, 1, 1,
231 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
234 g_object_class_install_property (gobject_class, PROP_BIN_SHOW_PREROLL_FRAME,
235 g_param_spec_boolean ("show-preroll-frame", "Show preroll frame",
236 "Whether to render video frames during preroll",
237 DEFAULT_SHOW_PREROLL_FRAME,
238 G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
241 g_object_class_install_property (gobject_class, PROP_BIN_SYNC,
242 g_param_spec_boolean ("sync", "Sync", "Sync on the clock", DEFAULT_SYNC,
243 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
244 g_object_class_install_property (gobject_class, PROP_BIN_MAX_LATENESS,
245 g_param_spec_int64 ("max-lateness", "Max Lateness",
246 "Maximum number of nanoseconds that a buffer can be late before it "
247 "is dropped (-1 unlimited)", -1, G_MAXINT64, DEFAULT_MAX_LATENESS,
248 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
249 g_object_class_install_property (gobject_class, PROP_BIN_QOS,
250 g_param_spec_boolean ("qos", "Qos",
251 "Generate Quality-of-Service events upstream", DEFAULT_QOS,
252 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
253 g_object_class_install_property (gobject_class, PROP_BIN_ASYNC,
254 g_param_spec_boolean ("async", "Async",
255 "Go asynchronously to PAUSED", DEFAULT_ASYNC,
256 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
257 g_object_class_install_property (gobject_class, PROP_BIN_TS_OFFSET,
258 g_param_spec_int64 ("ts-offset", "TS Offset",
259 "Timestamp offset in nanoseconds", G_MININT64, G_MAXINT64,
260 DEFAULT_TS_OFFSET, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
261 g_object_class_install_property (gobject_class, PROP_BIN_ENABLE_LAST_SAMPLE,
262 g_param_spec_boolean ("enable-last-sample", "Enable Last Buffer",
263 "Enable the last-sample property", DEFAULT_ENABLE_LAST_SAMPLE,
264 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
265 g_object_class_install_property (gobject_class, PROP_BIN_LAST_SAMPLE,
266 g_param_spec_boxed ("last-sample", "Last Sample",
267 "The last sample received in the sink", GST_TYPE_SAMPLE,
268 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
269 g_object_class_install_property (gobject_class, PROP_BIN_BLOCKSIZE,
270 g_param_spec_uint ("blocksize", "Block size",
271 "Size in bytes to pull per buffer (0 = default)", 0, G_MAXUINT,
272 DEFAULT_BLOCKSIZE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
273 g_object_class_install_property (gobject_class, PROP_BIN_RENDER_DELAY,
274 g_param_spec_uint64 ("render-delay", "Render Delay",
275 "Additional render delay of the sink in nanoseconds", 0, G_MAXUINT64,
276 DEFAULT_RENDER_DELAY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
277 g_object_class_install_property (gobject_class, PROP_BIN_THROTTLE_TIME,
278 g_param_spec_uint64 ("throttle-time", "Throttle time",
279 "The time to keep between rendered buffers (0 = disabled)", 0,
280 G_MAXUINT64, DEFAULT_THROTTLE_TIME,
281 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
282 g_object_class_install_property (gobject_class, PROP_BIN_MAX_BITRATE,
283 g_param_spec_uint64 ("max-bitrate", "Max Bitrate",
284 "The maximum bits per second to render (0 = disabled)", 0,
285 G_MAXUINT64, DEFAULT_MAX_BITRATE,
286 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
288 gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_DRAW] =
289 g_signal_new ("client-draw", G_TYPE_FROM_CLASS (klass),
290 G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
291 G_TYPE_BOOLEAN, 4, GST_GL_TYPE_CONTEXT,
292 G_TYPE_UINT, G_TYPE_UINT, G_TYPE_UINT);
294 gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_RESHAPE] =
295 g_signal_new ("client-reshape", G_TYPE_FROM_CLASS (klass),
296 G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
297 G_TYPE_BOOLEAN, 3, GST_GL_TYPE_CONTEXT, G_TYPE_UINT, G_TYPE_UINT);
300 #define GST_GLIMAGE_SINK_GET_LOCK(glsink) \
301 (GST_GLIMAGE_SINK(glsink)->drawing_lock)
302 #define GST_GLIMAGE_SINK_LOCK(glsink) \
303 (g_mutex_lock(&GST_GLIMAGE_SINK_GET_LOCK (glsink)))
304 #define GST_GLIMAGE_SINK_UNLOCK(glsink) \
305 (g_mutex_unlock(&GST_GLIMAGE_SINK_GET_LOCK (glsink)))
307 #define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
308 #define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
309 #define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
310 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
311 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
313 #define SUPPORTED_GL_APIS GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3
315 static void gst_glimage_sink_thread_init_redisplay (GstGLImageSink * gl_sink);
316 static void gst_glimage_sink_cleanup_glthread (GstGLImageSink * gl_sink);
317 static void gst_glimage_sink_on_close (GstGLImageSink * gl_sink);
318 static void gst_glimage_sink_on_resize (GstGLImageSink * gl_sink,
319 gint width, gint height);
320 static void gst_glimage_sink_on_draw (GstGLImageSink * gl_sink);
321 static gboolean gst_glimage_sink_redisplay (GstGLImageSink * gl_sink);
323 static void gst_glimage_sink_finalize (GObject * object);
324 static void gst_glimage_sink_set_property (GObject * object, guint prop_id,
325 const GValue * value, GParamSpec * param_spec);
326 static void gst_glimage_sink_get_property (GObject * object, guint prop_id,
327 GValue * value, GParamSpec * param_spec);
329 static gboolean gst_glimage_sink_stop (GstBaseSink * bsink);
331 static gboolean gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query);
332 static void gst_glimage_sink_set_context (GstElement * element,
333 GstContext * context);
335 static GstStateChangeReturn
336 gst_glimage_sink_change_state (GstElement * element, GstStateChange transition);
338 static void gst_glimage_sink_get_times (GstBaseSink * bsink, GstBuffer * buf,
339 GstClockTime * start, GstClockTime * end);
340 static gboolean gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps);
341 static GstCaps *gst_glimage_sink_get_caps (GstBaseSink * bsink,
343 static GstFlowReturn gst_glimage_sink_prepare (GstBaseSink * bsink,
345 static GstFlowReturn gst_glimage_sink_show_frame (GstVideoSink * bsink,
347 static gboolean gst_glimage_sink_propose_allocation (GstBaseSink * bsink,
350 static void gst_glimage_sink_video_overlay_init (GstVideoOverlayInterface *
352 static void gst_glimage_sink_set_window_handle (GstVideoOverlay * overlay,
354 static void gst_glimage_sink_expose (GstVideoOverlay * overlay);
357 gst_glimage_sink_handle_events (GstVideoOverlay * overlay,
358 gboolean handle_events);
360 static GstStaticPadTemplate gst_glimage_sink_template =
361 GST_STATIC_PAD_TEMPLATE ("sink",
364 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
365 (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
373 PROP_FORCE_ASPECT_RATIO,
374 PROP_PIXEL_ASPECT_RATIO,
384 CLIENT_RESHAPE_SIGNAL,
388 static guint gst_glimage_sink_signals[LAST_SIGNAL] = { 0 };
391 gst_glimage_sink_navigation_send_event (GstNavigation * navigation, GstStructure
394 GstGLImageSink *sink = GST_GLIMAGE_SINK (navigation);
395 GstEvent *event = NULL;
399 gdouble x, y, xscale, yscale;
404 window = gst_gl_context_get_window (sink->context);
405 g_return_if_fail (GST_GL_IS_WINDOW (window));
407 width = GST_VIDEO_SINK_WIDTH (sink);
408 height = GST_VIDEO_SINK_HEIGHT (sink);
409 gst_gl_window_get_surface_dimensions (window, &width, &height);
411 event = gst_event_new_navigation (structure);
413 pad = gst_pad_get_peer (GST_VIDEO_SINK_PAD (sink));
414 /* Converting pointer coordinates to the non scaled geometry */
415 if (width != GST_VIDEO_SINK_WIDTH (sink) &&
416 width != 0 && gst_structure_get_double (structure, "pointer_x", &x)) {
417 xscale = (gdouble) GST_VIDEO_SINK_WIDTH (sink) / width;
418 gst_structure_set (structure, "pointer_x", G_TYPE_DOUBLE,
419 (gdouble) x * xscale, NULL);
421 if (height != GST_VIDEO_SINK_HEIGHT (sink) &&
422 height != 0 && gst_structure_get_double (structure, "pointer_y", &y)) {
423 yscale = (gdouble) GST_VIDEO_SINK_HEIGHT (sink) / height;
424 gst_structure_set (structure, "pointer_y", G_TYPE_DOUBLE,
425 (gdouble) y * yscale, NULL);
429 if (GST_IS_PAD (pad) && GST_IS_EVENT (event))
430 gst_pad_send_event (pad, event);
432 gst_object_unref (pad);
433 gst_object_unref (window);
437 gst_glimage_sink_navigation_interface_init (GstNavigationInterface * iface)
439 iface->send_event = gst_glimage_sink_navigation_send_event;
442 #define gst_glimage_sink_parent_class parent_class
443 G_DEFINE_TYPE_WITH_CODE (GstGLImageSink, gst_glimage_sink,
444 GST_TYPE_VIDEO_SINK, G_IMPLEMENT_INTERFACE (GST_TYPE_VIDEO_OVERLAY,
445 gst_glimage_sink_video_overlay_init);
446 G_IMPLEMENT_INTERFACE (GST_TYPE_NAVIGATION,
447 gst_glimage_sink_navigation_interface_init);
448 GST_DEBUG_CATEGORY_INIT (gst_debug_glimage_sink, "glimagesink", 0,
449 "OpenGL Video Sink"));
452 gst_glimage_sink_class_init (GstGLImageSinkClass * klass)
454 GObjectClass *gobject_class;
455 GstElementClass *gstelement_class;
456 GstBaseSinkClass *gstbasesink_class;
457 GstVideoSinkClass *gstvideosink_class;
458 GstElementClass *element_class;
460 gobject_class = (GObjectClass *) klass;
461 gstelement_class = (GstElementClass *) klass;
462 gstbasesink_class = (GstBaseSinkClass *) klass;
463 gstvideosink_class = (GstVideoSinkClass *) klass;
464 element_class = GST_ELEMENT_CLASS (klass);
466 gobject_class->set_property = gst_glimage_sink_set_property;
467 gobject_class->get_property = gst_glimage_sink_get_property;
469 g_object_class_install_property (gobject_class, ARG_DISPLAY,
470 g_param_spec_string ("display", "Display", "Display name",
471 NULL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
473 g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO,
474 g_param_spec_boolean ("force-aspect-ratio", "Force aspect ratio",
475 "When enabled, scaling will respect original aspect ratio",
476 DEFAULT_FORCE_ASPECT_RATIO,
477 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
479 g_object_class_install_property (gobject_class, PROP_PIXEL_ASPECT_RATIO,
480 gst_param_spec_fraction ("pixel-aspect-ratio", "Pixel Aspect Ratio",
481 "The pixel aspect ratio of the device", 0, 1, G_MAXINT, 1, 1, 1,
482 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
484 g_object_class_install_property (gobject_class, PROP_CONTEXT,
485 g_param_spec_object ("context", "OpenGL context", "Get OpenGL context",
486 GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
488 g_object_class_install_property (gobject_class, PROP_HANDLE_EVENTS,
489 g_param_spec_boolean ("handle-events", "Handle XEvents",
490 "When enabled, XEvents will be selected and handled", TRUE,
491 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
493 g_object_class_install_property (gobject_class, PROP_IGNORE_ALPHA,
494 g_param_spec_boolean ("ignore-alpha", "Ignore Alpha",
495 "When enabled, alpha will be ignored and converted to black", TRUE,
496 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
498 gst_element_class_set_metadata (element_class, "OpenGL video sink",
499 "Sink/Video", "A videosink based on OpenGL",
500 "Julien Isorce <julien.isorce@gmail.com>");
503 * GstGLImageSink::client-draw:
504 * @object: the #GstGLImageSink
505 * @texture: the #guint id of the texture.
506 * @width: the #guint width of the texture.
507 * @height: the #guint height of the texture.
509 * Will be emitted before actually drawing the texture. The client should
510 * redraw the surface/contents with the @texture, @width and @height and
513 * Returns: whether the texture was redrawn by the signal. If not, a
514 * default redraw will occur.
516 gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL] =
517 g_signal_new ("client-draw", G_TYPE_FROM_CLASS (klass),
518 G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
519 G_TYPE_BOOLEAN, 4, GST_GL_TYPE_CONTEXT,
520 G_TYPE_UINT, G_TYPE_UINT, G_TYPE_UINT);
523 * GstGLImageSink::client-reshape:
524 * @object: the #GstGLImageSink
525 * @width: the #guint width of the texture.
526 * @height: the #guint height of the texture.
528 * The client should resize the surface/window/viewport with the @width and
529 * @height and return %TRUE.
531 * Returns: whether the content area was resized by the signal. If not, a
532 * default viewport resize will occur.
534 gst_glimage_sink_signals[CLIENT_RESHAPE_SIGNAL] =
535 g_signal_new ("client-reshape", G_TYPE_FROM_CLASS (klass),
536 G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
537 G_TYPE_BOOLEAN, 3, GST_GL_TYPE_CONTEXT, G_TYPE_UINT, G_TYPE_UINT);
539 gst_element_class_add_pad_template (element_class,
540 gst_static_pad_template_get (&gst_glimage_sink_template));
542 gobject_class->finalize = gst_glimage_sink_finalize;
544 gstelement_class->change_state = gst_glimage_sink_change_state;
545 gstelement_class->set_context = gst_glimage_sink_set_context;
546 gstbasesink_class->query = GST_DEBUG_FUNCPTR (gst_glimage_sink_query);
547 gstbasesink_class->set_caps = gst_glimage_sink_set_caps;
548 gstbasesink_class->get_caps = gst_glimage_sink_get_caps;
549 gstbasesink_class->get_times = gst_glimage_sink_get_times;
550 gstbasesink_class->prepare = gst_glimage_sink_prepare;
551 gstbasesink_class->propose_allocation = gst_glimage_sink_propose_allocation;
552 gstbasesink_class->stop = gst_glimage_sink_stop;
554 gstvideosink_class->show_frame =
555 GST_DEBUG_FUNCPTR (gst_glimage_sink_show_frame);
559 gst_glimage_sink_init (GstGLImageSink * glimage_sink)
561 glimage_sink->display_name = NULL;
562 glimage_sink->window_id = 0;
563 glimage_sink->new_window_id = 0;
564 glimage_sink->display = NULL;
565 glimage_sink->keep_aspect_ratio = TRUE;
566 glimage_sink->par_n = 0;
567 glimage_sink->par_d = 1;
568 glimage_sink->pool = NULL;
569 glimage_sink->stored_buffer = NULL;
570 glimage_sink->redisplay_texture = 0;
571 glimage_sink->handle_events = TRUE;
572 glimage_sink->ignore_alpha = TRUE;
574 g_mutex_init (&glimage_sink->drawing_lock);
578 gst_glimage_sink_set_property (GObject * object, guint prop_id,
579 const GValue * value, GParamSpec * pspec)
581 GstGLImageSink *glimage_sink;
583 g_return_if_fail (GST_IS_GLIMAGE_SINK (object));
585 glimage_sink = GST_GLIMAGE_SINK (object);
590 g_free (glimage_sink->display_name);
591 glimage_sink->display_name = g_strdup (g_value_get_string (value));
594 case PROP_FORCE_ASPECT_RATIO:
596 glimage_sink->keep_aspect_ratio = g_value_get_boolean (value);
599 case PROP_PIXEL_ASPECT_RATIO:
601 glimage_sink->par_n = gst_value_get_fraction_numerator (value);
602 glimage_sink->par_d = gst_value_get_fraction_denominator (value);
605 case PROP_HANDLE_EVENTS:
606 gst_glimage_sink_handle_events (GST_VIDEO_OVERLAY (glimage_sink),
607 g_value_get_boolean (value));
609 case PROP_IGNORE_ALPHA:
610 glimage_sink->ignore_alpha = g_value_get_boolean (value);
613 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
619 gst_glimage_sink_finalize (GObject * object)
621 GstGLImageSink *glimage_sink;
623 g_return_if_fail (GST_IS_GLIMAGE_SINK (object));
625 glimage_sink = GST_GLIMAGE_SINK (object);
627 g_mutex_clear (&glimage_sink->drawing_lock);
629 if (glimage_sink->other_context) {
630 gst_object_unref (glimage_sink->other_context);
631 glimage_sink->other_context = NULL;
634 g_free (glimage_sink->display_name);
636 GST_DEBUG ("finalized");
637 G_OBJECT_CLASS (parent_class)->finalize (object);
641 gst_glimage_sink_get_property (GObject * object, guint prop_id,
642 GValue * value, GParamSpec * pspec)
644 GstGLImageSink *glimage_sink;
646 g_return_if_fail (GST_IS_GLIMAGE_SINK (object));
648 glimage_sink = GST_GLIMAGE_SINK (object);
652 g_value_set_string (value, glimage_sink->display_name);
654 case PROP_FORCE_ASPECT_RATIO:
655 g_value_set_boolean (value, glimage_sink->keep_aspect_ratio);
657 case PROP_PIXEL_ASPECT_RATIO:
658 gst_value_set_fraction (value, glimage_sink->par_n, glimage_sink->par_d);
661 g_value_set_object (value, glimage_sink->context);
663 case PROP_HANDLE_EVENTS:
664 g_value_set_boolean (value, glimage_sink->handle_events);
666 case PROP_IGNORE_ALPHA:
667 g_value_set_boolean (value, glimage_sink->ignore_alpha);
670 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
676 gst_glimage_sink_key_event_cb (GstGLWindow * window, char *event_name, char
677 *key_string, GstGLImageSink * gl_sink)
679 GST_DEBUG_OBJECT (gl_sink, "glimagesink event %s key %s pressed", event_name,
681 gst_navigation_send_key_event (GST_NAVIGATION (gl_sink),
682 event_name, key_string);
686 gst_glimage_sink_mouse_event_cb (GstGLWindow * window, char *event_name,
687 int button, double posx, double posy, GstGLImageSink * gl_sink)
689 GST_DEBUG_OBJECT (gl_sink, "glimagesink event %s at %g, %g", event_name, posx,
691 gst_navigation_send_mouse_event (GST_NAVIGATION (gl_sink),
692 event_name, button, posx, posy);
696 _ensure_gl_setup (GstGLImageSink * gl_sink)
698 GError *error = NULL;
700 GST_DEBUG_OBJECT (gl_sink, "Ensuring setup");
702 if (!gl_sink->context) {
704 GstGLContext *other_context;
707 if (gl_sink->context)
708 gst_object_unref (gl_sink->context);
710 GST_DEBUG_OBJECT (gl_sink,
711 "No current context, creating one for %" GST_PTR_FORMAT,
714 gl_sink->context = gst_gl_context_new (gl_sink->display);
715 if (!gl_sink->context)
716 goto context_creation_error;
718 window = gst_gl_context_get_window (gl_sink->context);
720 GST_DEBUG_OBJECT (gl_sink, "got window %" GST_PTR_FORMAT, window);
722 if (!gl_sink->window_id && !gl_sink->new_window_id)
723 gst_video_overlay_prepare_window_handle (GST_VIDEO_OVERLAY (gl_sink));
725 GST_DEBUG_OBJECT (gl_sink,
726 "window_id : %" G_GUINTPTR_FORMAT " , new_window_id : %"
727 G_GUINTPTR_FORMAT, gl_sink->window_id, gl_sink->new_window_id);
729 if (gl_sink->window_id != gl_sink->new_window_id) {
730 gl_sink->window_id = gl_sink->new_window_id;
731 GST_DEBUG_OBJECT (gl_sink, "Setting window handle on gl window");
732 gst_gl_window_set_window_handle (window, gl_sink->window_id);
735 if (gl_sink->other_context) {
736 other_context = gst_object_ref (gl_sink->other_context);
739 gst_gl_display_get_gl_context_for_thread (gl_sink->display, NULL);
742 GST_DEBUG_OBJECT (gl_sink,
743 "creating context %" GST_PTR_FORMAT " from other context %"
744 GST_PTR_FORMAT, gl_sink->context, other_context);
746 if (!gst_gl_context_create (gl_sink->context, other_context, &error)) {
748 gst_object_unref (other_context);
749 gst_object_unref (window);
753 gst_gl_window_handle_events (window, gl_sink->handle_events);
755 /* setup callbacks */
756 gst_gl_window_set_resize_callback (window,
757 GST_GL_WINDOW_RESIZE_CB (gst_glimage_sink_on_resize),
758 gst_object_ref (gl_sink), (GDestroyNotify) gst_object_unref);
759 gst_gl_window_set_draw_callback (window,
760 GST_GL_WINDOW_CB (gst_glimage_sink_on_draw),
761 gst_object_ref (gl_sink), (GDestroyNotify) gst_object_unref);
762 gst_gl_window_set_close_callback (window,
763 GST_GL_WINDOW_CB (gst_glimage_sink_on_close),
764 gst_object_ref (gl_sink), (GDestroyNotify) gst_object_unref);
765 gl_sink->key_sig_id = g_signal_connect (window, "key-event", G_CALLBACK
766 (gst_glimage_sink_key_event_cb), gl_sink);
767 gl_sink->mouse_sig_id =
768 g_signal_connect (window, "mouse-event",
769 G_CALLBACK (gst_glimage_sink_mouse_event_cb), gl_sink);
772 gst_object_unref (other_context);
773 gst_object_unref (window);
774 } while (!gst_gl_display_add_context (gl_sink->display, gl_sink->context));
776 GST_DEBUG_OBJECT (gl_sink, "Already have a context");
780 context_creation_error:
782 GST_ELEMENT_ERROR (gl_sink, RESOURCE, NOT_FOUND,
783 ("Failed to create GL context"), (NULL));
789 GST_ELEMENT_ERROR (gl_sink, RESOURCE, NOT_FOUND, ("%s", error->message),
791 gst_object_unref (gl_sink->context);
792 gl_sink->context = NULL;
798 gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query)
800 GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (bsink);
801 gboolean res = FALSE;
803 switch (GST_QUERY_TYPE (query)) {
804 case GST_QUERY_CONTEXT:
806 const gchar *context_type;
807 GstContext *context, *old_context;
811 gst_gl_handle_context_query ((GstElement *) glimage_sink, query,
812 &glimage_sink->display, &glimage_sink->other_context);
813 if (glimage_sink->display)
814 gst_gl_display_filter_gl_api (glimage_sink->display, SUPPORTED_GL_APIS);
816 gst_query_parse_context_type (query, &context_type);
818 if (g_strcmp0 (context_type, "gst.gl.local_context") == 0) {
821 gst_query_parse_context (query, &old_context);
824 context = gst_context_copy (old_context);
826 context = gst_context_new ("gst.gl.local_context", FALSE);
828 s = gst_context_writable_structure (context);
829 gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT,
830 glimage_sink->context, NULL);
831 gst_query_set_context (query, context);
832 gst_context_unref (context);
834 ret = glimage_sink->context != NULL;
836 GST_DEBUG_OBJECT (glimage_sink, "context query of type %s %i",
841 case GST_QUERY_DRAIN:
843 GstBuffer *buf = NULL;
845 GST_GLIMAGE_SINK_LOCK (glimage_sink);
846 glimage_sink->redisplay_texture = 0;
847 buf = glimage_sink->stored_buffer;
848 glimage_sink->stored_buffer = NULL;
849 GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
852 gst_buffer_unref (buf);
854 gst_buffer_replace (&glimage_sink->next_buffer, NULL);
856 res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
860 res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
868 gst_glimage_sink_stop (GstBaseSink * bsink)
870 GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (bsink);
872 if (glimage_sink->pool) {
873 gst_object_unref (glimage_sink->pool);
874 glimage_sink->pool = NULL;
881 gst_glimage_sink_set_context (GstElement * element, GstContext * context)
883 GstGLImageSink *gl_sink = GST_GLIMAGE_SINK (element);
885 gst_gl_handle_set_context (element, context, &gl_sink->display,
886 &gl_sink->other_context);
888 if (gl_sink->display)
889 gst_gl_display_filter_gl_api (gl_sink->display, SUPPORTED_GL_APIS);
892 static GstStateChangeReturn
893 gst_glimage_sink_change_state (GstElement * element, GstStateChange transition)
895 GstGLImageSink *glimage_sink;
896 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
898 GST_DEBUG ("changing state: %s => %s",
899 gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)),
900 gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition)));
902 glimage_sink = GST_GLIMAGE_SINK (element);
904 switch (transition) {
905 case GST_STATE_CHANGE_NULL_TO_READY:
906 if (!gst_gl_ensure_element_data (glimage_sink, &glimage_sink->display,
907 &glimage_sink->other_context))
908 return GST_STATE_CHANGE_FAILURE;
910 gst_gl_display_filter_gl_api (glimage_sink->display, SUPPORTED_GL_APIS);
912 case GST_STATE_CHANGE_READY_TO_PAUSED:
913 g_atomic_int_set (&glimage_sink->to_quit, 0);
915 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
921 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
922 if (ret == GST_STATE_CHANGE_FAILURE)
925 switch (transition) {
926 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
928 case GST_STATE_CHANGE_PAUSED_TO_READY:
930 /* mark the redisplay_texture as unavailable (=0)
933 GST_GLIMAGE_SINK_LOCK (glimage_sink);
934 glimage_sink->redisplay_texture = 0;
935 if (glimage_sink->stored_buffer) {
936 gst_buffer_unref (glimage_sink->stored_buffer);
937 glimage_sink->stored_buffer = NULL;
939 GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
940 gst_buffer_replace (&glimage_sink->next_buffer, NULL);
942 glimage_sink->window_id = 0;
943 /* but do not reset glimage_sink->new_window_id */
945 GST_VIDEO_SINK_WIDTH (glimage_sink) = 1;
946 GST_VIDEO_SINK_HEIGHT (glimage_sink) = 1;
947 if (glimage_sink->context) {
948 GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);
950 gst_gl_window_send_message (window,
951 GST_GL_WINDOW_CB (gst_glimage_sink_cleanup_glthread), glimage_sink);
953 gst_gl_window_set_resize_callback (window, NULL, NULL, NULL);
954 gst_gl_window_set_draw_callback (window, NULL, NULL, NULL);
955 gst_gl_window_set_close_callback (window, NULL, NULL, NULL);
957 gst_object_unref (window);
958 gst_object_unref (glimage_sink->context);
959 glimage_sink->context = NULL;
962 if (glimage_sink->display) {
963 gst_object_unref (glimage_sink->display);
964 glimage_sink->display = NULL;
968 case GST_STATE_CHANGE_READY_TO_NULL:
978 gst_glimage_sink_get_times (GstBaseSink * bsink, GstBuffer * buf,
979 GstClockTime * start, GstClockTime * end)
981 GstGLImageSink *glimagesink;
983 glimagesink = GST_GLIMAGE_SINK (bsink);
985 if (GST_BUFFER_TIMESTAMP_IS_VALID (buf)) {
986 *start = GST_BUFFER_TIMESTAMP (buf);
987 if (GST_BUFFER_DURATION_IS_VALID (buf))
988 *end = *start + GST_BUFFER_DURATION (buf);
990 if (GST_VIDEO_INFO_FPS_N (&glimagesink->info) > 0) {
992 gst_util_uint64_scale_int (GST_SECOND,
993 GST_VIDEO_INFO_FPS_D (&glimagesink->info),
994 GST_VIDEO_INFO_FPS_N (&glimagesink->info));
1001 gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter)
1003 GstCaps *tmp = NULL;
1004 GstCaps *result = NULL;
1006 tmp = gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA");
1009 result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
1010 gst_caps_unref (tmp);
1015 GST_DEBUG_OBJECT (bsink, "returning caps: %" GST_PTR_FORMAT, result);
1021 gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
1023 GstGLImageSink *glimage_sink;
1028 gint display_par_n, display_par_d;
1029 guint display_ratio_num, display_ratio_den;
1032 GST_DEBUG_OBJECT (bsink, "set caps with %" GST_PTR_FORMAT, caps);
1034 glimage_sink = GST_GLIMAGE_SINK (bsink);
1036 ok = gst_video_info_from_caps (&vinfo, caps);
1040 width = GST_VIDEO_INFO_WIDTH (&vinfo);
1041 height = GST_VIDEO_INFO_HEIGHT (&vinfo);
1043 par_n = GST_VIDEO_INFO_PAR_N (&vinfo);
1044 par_d = GST_VIDEO_INFO_PAR_D (&vinfo);
1049 /* get display's PAR */
1050 if (glimage_sink->par_n != 0 && glimage_sink->par_d != 0) {
1051 display_par_n = glimage_sink->par_n;
1052 display_par_d = glimage_sink->par_d;
1058 ok = gst_video_calculate_display_ratio (&display_ratio_num,
1059 &display_ratio_den, width, height, par_n, par_d, display_par_n,
1065 GST_TRACE ("PAR: %u/%u DAR:%u/%u", par_n, par_d, display_par_n,
1068 if (height % display_ratio_den == 0) {
1069 GST_DEBUG ("keeping video height");
1070 GST_VIDEO_SINK_WIDTH (glimage_sink) = (guint)
1071 gst_util_uint64_scale_int (height, display_ratio_num,
1073 GST_VIDEO_SINK_HEIGHT (glimage_sink) = height;
1074 } else if (width % display_ratio_num == 0) {
1075 GST_DEBUG ("keeping video width");
1076 GST_VIDEO_SINK_WIDTH (glimage_sink) = width;
1077 GST_VIDEO_SINK_HEIGHT (glimage_sink) = (guint)
1078 gst_util_uint64_scale_int (width, display_ratio_den, display_ratio_num);
1080 GST_DEBUG ("approximating while keeping video height");
1081 GST_VIDEO_SINK_WIDTH (glimage_sink) = (guint)
1082 gst_util_uint64_scale_int (height, display_ratio_num,
1084 GST_VIDEO_SINK_HEIGHT (glimage_sink) = height;
1086 GST_DEBUG ("scaling to %dx%d", GST_VIDEO_SINK_WIDTH (glimage_sink),
1087 GST_VIDEO_SINK_HEIGHT (glimage_sink));
1089 glimage_sink->info = vinfo;
1090 if (!_ensure_gl_setup (glimage_sink))
1093 glimage_sink->caps_change = TRUE;
1098 static GstFlowReturn
1099 gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
1101 GstGLImageSink *glimage_sink;
1102 GstVideoFrame gl_frame;
1104 glimage_sink = GST_GLIMAGE_SINK (bsink);
1106 GST_TRACE ("preparing buffer:%p", buf);
1108 if (GST_VIDEO_SINK_WIDTH (glimage_sink) < 1 ||
1109 GST_VIDEO_SINK_HEIGHT (glimage_sink) < 1) {
1110 return GST_FLOW_NOT_NEGOTIATED;
1113 if (!_ensure_gl_setup (glimage_sink))
1114 return GST_FLOW_NOT_NEGOTIATED;
1116 if (!gst_video_frame_map (&gl_frame, &glimage_sink->info, buf,
1117 GST_MAP_READ | GST_MAP_GL)) {
1121 glimage_sink->next_tex = *(guint *) gl_frame.data[0];
1123 gst_buffer_replace (&glimage_sink->next_buffer, buf);
1125 gst_video_frame_unmap (&gl_frame);
1127 if (glimage_sink->window_id != glimage_sink->new_window_id) {
1128 GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);
1130 glimage_sink->window_id = glimage_sink->new_window_id;
1131 gst_gl_window_set_window_handle (window, glimage_sink->window_id);
1133 gst_object_unref (window);
1140 GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND,
1141 ("%s", "Failed to upload buffer"), (NULL));
1142 return GST_FLOW_ERROR;
1146 static GstFlowReturn
1147 gst_glimage_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
1149 GstGLImageSink *glimage_sink;
1150 GstBuffer *stored_buffer;
1152 GST_TRACE ("rendering buffer:%p", buf);
1154 glimage_sink = GST_GLIMAGE_SINK (vsink);
1156 GST_TRACE ("redisplay texture:%u of size:%ux%u, window size:%ux%u",
1157 glimage_sink->next_tex, GST_VIDEO_INFO_WIDTH (&glimage_sink->info),
1158 GST_VIDEO_INFO_HEIGHT (&glimage_sink->info),
1159 GST_VIDEO_SINK_WIDTH (glimage_sink),
1160 GST_VIDEO_SINK_HEIGHT (glimage_sink));
1162 /* Avoid to release the texture while drawing */
1163 GST_GLIMAGE_SINK_LOCK (glimage_sink);
1164 glimage_sink->redisplay_texture = glimage_sink->next_tex;
1165 stored_buffer = glimage_sink->stored_buffer;
1166 glimage_sink->stored_buffer = gst_buffer_ref (glimage_sink->next_buffer);
1167 GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
1169 /* Ask the underlying window to redraw its content */
1170 if (!gst_glimage_sink_redisplay (glimage_sink))
1171 goto redisplay_failed;
1173 GST_TRACE ("post redisplay");
1176 gst_buffer_unref (stored_buffer);
1178 if (g_atomic_int_get (&glimage_sink->to_quit) != 0) {
1179 GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND,
1180 ("%s", gst_gl_context_get_error ()), (NULL));
1181 return GST_FLOW_ERROR;
1189 GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND,
1190 ("%s", gst_gl_context_get_error ()), (NULL));
1191 return GST_FLOW_ERROR;
1196 gst_glimage_sink_video_overlay_init (GstVideoOverlayInterface * iface)
1198 iface->set_window_handle = gst_glimage_sink_set_window_handle;
1199 iface->handle_events = gst_glimage_sink_handle_events;
1200 iface->expose = gst_glimage_sink_expose;
1204 gst_glimage_sink_set_window_handle (GstVideoOverlay * overlay, guintptr id)
1206 GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (overlay);
1207 guintptr window_id = (guintptr) id;
1209 g_return_if_fail (GST_IS_GLIMAGE_SINK (overlay));
1211 GST_DEBUG ("set_xwindow_id %" G_GUINT64_FORMAT, (guint64) window_id);
1213 glimage_sink->new_window_id = window_id;
1218 gst_glimage_sink_expose (GstVideoOverlay * overlay)
1220 GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (overlay);
1222 /* redisplay opengl scene */
1223 if (glimage_sink->display && glimage_sink->window_id) {
1225 if (glimage_sink->window_id != glimage_sink->new_window_id) {
1226 GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);
1228 glimage_sink->window_id = glimage_sink->new_window_id;
1229 gst_gl_window_set_window_handle (window, glimage_sink->window_id);
1231 gst_object_unref (window);
1234 gst_glimage_sink_redisplay (glimage_sink);
1239 gst_glimage_sink_handle_events (GstVideoOverlay * overlay,
1240 gboolean handle_events)
1242 GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (overlay);
1244 glimage_sink->handle_events = handle_events;
1245 if (G_LIKELY (glimage_sink->context)) {
1246 GstGLWindow *window;
1247 window = gst_gl_context_get_window (glimage_sink->context);
1248 gst_gl_window_handle_events (window, handle_events);
1249 gst_object_unref (window);
1254 gst_glimage_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
1256 GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (bsink);
1257 GstStructure *config;
1262 if (!_ensure_gl_setup (glimage_sink))
1265 gst_query_parse_allocation (query, &caps, &need_pool);
1273 if (!gst_video_info_from_caps (&info, caps))
1276 /* the normal size of a frame */
1279 if (glimage_sink->pool) {
1282 /* we had a pool, check caps */
1283 GST_DEBUG_OBJECT (glimage_sink, "check existing pool caps");
1284 config = gst_buffer_pool_get_config (glimage_sink->pool);
1285 gst_buffer_pool_config_get_params (config, &pcaps, &size, NULL, NULL);
1287 if (!gst_caps_is_equal (caps, pcaps)) {
1288 GST_DEBUG_OBJECT (glimage_sink, "pool has different caps");
1289 /* different caps, we can't use this pool */
1290 gst_object_unref (glimage_sink->pool);
1291 glimage_sink->pool = NULL;
1293 gst_structure_free (config);
1296 if (glimage_sink->pool == NULL) {
1297 GST_DEBUG_OBJECT (glimage_sink, "create new pool");
1299 glimage_sink->pool = gst_gl_buffer_pool_new (glimage_sink->context);
1300 config = gst_buffer_pool_get_config (glimage_sink->pool);
1301 gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
1303 if (!gst_buffer_pool_set_config (glimage_sink->pool, config))
1307 /* we need at least 2 buffer because we hold on to the last one */
1308 gst_query_add_allocation_pool (query, glimage_sink->pool, size, 2, 0);
1311 if (glimage_sink->context->gl_vtable->FenceSync)
1312 gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0);
1319 GST_WARNING_OBJECT (bsink, "no caps specified");
1324 GST_WARNING_OBJECT (bsink, "invalid caps specified");
1329 GST_WARNING_OBJECT (bsink, "failed setting config");
1335 static const GLfloat vertices[] = {
1336 1.0f, 1.0f, 0.0f, 1.0f, 0.0f,
1337 -1.0f, 1.0f, 0.0f, 0.0f, 0.0f,
1338 -1.0f, -1.0f, 0.0f, 0.0f, 1.0f,
1339 1.0f, -1.0f, 0.0f, 1.0f, 1.0f
1344 _bind_buffer (GstGLImageSink * gl_sink)
1346 const GstGLFuncs *gl = gl_sink->context->gl_vtable;
1348 gl->BindBuffer (GL_ARRAY_BUFFER, gl_sink->vertex_buffer);
1349 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
1352 /* Load the vertex position */
1353 gl->VertexAttribPointer (gl_sink->attr_position, 3, GL_FLOAT, GL_FALSE,
1354 5 * sizeof (GLfloat), (void *) 0);
1356 /* Load the texture coordinate */
1357 gl->VertexAttribPointer (gl_sink->attr_texture, 2, GL_FLOAT, GL_FALSE,
1358 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
1360 gl->EnableVertexAttribArray (gl_sink->attr_position);
1361 gl->EnableVertexAttribArray (gl_sink->attr_texture);
1365 _unbind_buffer (GstGLImageSink * gl_sink)
1367 const GstGLFuncs *gl = gl_sink->context->gl_vtable;
1369 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1371 gl->DisableVertexAttribArray (gl_sink->attr_position);
1372 gl->DisableVertexAttribArray (gl_sink->attr_texture);
1375 /* Called in the gl thread */
1377 gst_glimage_sink_thread_init_redisplay (GstGLImageSink * gl_sink)
1379 const GstGLFuncs *gl = gl_sink->context->gl_vtable;
1381 gl_sink->redisplay_shader = gst_gl_shader_new (gl_sink->context);
1383 if (!gst_gl_shader_compile_with_default_vf_and_check
1384 (gl_sink->redisplay_shader, &gl_sink->attr_position,
1385 &gl_sink->attr_texture))
1386 gst_glimage_sink_cleanup_glthread (gl_sink);
1388 if (gl->GenVertexArrays) {
1389 gl->GenVertexArrays (1, &gl_sink->vao);
1390 gl->BindVertexArray (gl_sink->vao);
1393 gl->GenBuffers (1, &gl_sink->vertex_buffer);
1394 _bind_buffer (gl_sink);
1396 if (gl->GenVertexArrays) {
1397 gl->BindVertexArray (0);
1398 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1400 _unbind_buffer (gl_sink);
1405 gst_glimage_sink_cleanup_glthread (GstGLImageSink * gl_sink)
1407 const GstGLFuncs *gl = gl_sink->context->gl_vtable;
1409 if (gl_sink->redisplay_shader) {
1410 gst_object_unref (gl_sink->redisplay_shader);
1411 gl_sink->redisplay_shader = NULL;
1415 gl->DeleteVertexArrays (1, &gl_sink->vao);
1421 gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height)
1423 /* Here gl_sink members (ex:gl_sink->info) have a life time of set_caps.
1424 * It means that they cannot not change between two set_caps
1426 const GstGLFuncs *gl = gl_sink->context->gl_vtable;
1427 gboolean do_reshape;
1429 GST_TRACE ("GL Window resized to %ux%u", width, height);
1431 /* check if a client reshape callback is registered */
1432 g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_RESHAPE_SIGNAL], 0,
1433 gl_sink->context, width, height, &do_reshape);
1435 width = MAX (1, width);
1436 height = MAX (1, height);
1438 gl_sink->window_width = width;
1439 gl_sink->window_height = height;
1441 /* default reshape */
1443 if (gl_sink->keep_aspect_ratio) {
1444 GstVideoRectangle src, dst, result;
1448 src.w = GST_VIDEO_SINK_WIDTH (gl_sink);
1449 src.h = GST_VIDEO_SINK_HEIGHT (gl_sink);
1456 gst_video_sink_center_rect (src, dst, &result, TRUE);
1457 gl->Viewport (result.x, result.y, result.w, result.h);
1459 gl->Viewport (0, 0, width, height);
1465 gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
1467 /* Here gl_sink members (ex:gl_sink->info) have a life time of set_caps.
1468 * It means that they cannot not change between two set_caps as well as
1469 * for the redisplay_texture size.
1470 * Whereas redisplay_texture id changes every sink_render
1473 const GstGLFuncs *gl = NULL;
1474 GstGLWindow *window = NULL;
1475 gboolean do_redisplay;
1476 GstGLSyncMeta *sync_meta;
1478 g_return_if_fail (GST_IS_GLIMAGE_SINK (gl_sink));
1480 gl = gl_sink->context->gl_vtable;
1482 GST_GLIMAGE_SINK_LOCK (gl_sink);
1484 /* check if texture is ready for being drawn */
1485 if (!gl_sink->redisplay_texture) {
1486 GST_GLIMAGE_SINK_UNLOCK (gl_sink);
1490 window = gst_gl_context_get_window (gl_sink->context);
1491 window->is_drawing = TRUE;
1494 GST_TRACE ("redrawing texture:%u", gl_sink->redisplay_texture);
1496 if (gl_sink->caps_change && gl_sink->window_width > 0
1497 && gl_sink->window_height > 0) {
1498 GST_GLIMAGE_SINK_UNLOCK (gl_sink);
1499 gst_glimage_sink_on_resize (gl_sink, gl_sink->window_width,
1500 gl_sink->window_height);
1501 GST_GLIMAGE_SINK_LOCK (gl_sink);
1502 gl_sink->caps_change = FALSE;
1505 sync_meta = gst_buffer_get_gl_sync_meta (gl_sink->stored_buffer);
1507 gst_gl_sync_meta_wait (sync_meta, gl_sink->context);
1509 /* make sure that the environnement is clean */
1510 gst_gl_context_clear_shader (gl_sink->context);
1512 #if GST_GL_HAVE_OPENGL
1513 if (USING_OPENGL (gl_sink->context))
1514 gl->Disable (GL_TEXTURE_2D);
1517 gl->BindTexture (GL_TEXTURE_2D, 0);
1519 g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL], 0,
1521 gl_sink->redisplay_texture, GST_VIDEO_INFO_WIDTH (&gl_sink->info),
1522 GST_VIDEO_INFO_HEIGHT (&gl_sink->info), &do_redisplay);
1524 if (!do_redisplay) {
1525 gfloat alpha = gl_sink->ignore_alpha ? 1.0f : 0.0f;
1526 GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
1528 gl->ClearColor (0.0, 0.0, 0.0, alpha);
1529 gl->Clear (GL_COLOR_BUFFER_BIT);
1531 if (gl_sink->ignore_alpha) {
1532 gl->BlendColor (0.0, 0.0, 0.0, alpha);
1533 gl->BlendFunc (GL_SRC_ALPHA, GL_CONSTANT_COLOR);
1534 gl->BlendEquation (GL_FUNC_ADD);
1535 gl->Enable (GL_BLEND);
1538 gst_gl_shader_use (gl_sink->redisplay_shader);
1540 if (gl->GenVertexArrays)
1541 gl->BindVertexArray (gl_sink->vao);
1543 _bind_buffer (gl_sink);
1545 gl->ActiveTexture (GL_TEXTURE0);
1546 gl->BindTexture (GL_TEXTURE_2D, gl_sink->redisplay_texture);
1547 gst_gl_shader_set_uniform_1i (gl_sink->redisplay_shader, "tex", 0);
1549 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices);
1551 gst_gl_context_clear_shader (gl_sink->context);
1553 if (gl->GenVertexArrays)
1554 gl->BindVertexArray (0);
1556 _unbind_buffer (gl_sink);
1558 if (gl_sink->ignore_alpha)
1559 gl->Disable (GL_BLEND);
1561 /* end default opengl scene */
1562 window->is_drawing = FALSE;
1563 gst_object_unref (window);
1565 GST_GLIMAGE_SINK_UNLOCK (gl_sink);
1569 gst_glimage_sink_on_close (GstGLImageSink * gl_sink)
1571 GstGLWindow *window;
1573 gst_gl_context_set_error (gl_sink->context, "Output window was closed");
1575 window = gst_gl_context_get_window (gl_sink->context);
1577 g_signal_handler_disconnect (window, gl_sink->key_sig_id);
1578 g_signal_handler_disconnect (window, gl_sink->mouse_sig_id);
1580 g_atomic_int_set (&gl_sink->to_quit, 1);
1582 gst_object_unref (window);
1586 gst_glimage_sink_redisplay (GstGLImageSink * gl_sink)
1588 GstGLWindow *window;
1591 window = gst_gl_context_get_window (gl_sink->context);
1595 if (gst_gl_window_is_running (window)) {
1596 if (G_UNLIKELY (!gl_sink->redisplay_shader)) {
1597 gst_gl_window_send_message (window,
1598 GST_GL_WINDOW_CB (gst_glimage_sink_thread_init_redisplay), gl_sink);
1600 /* if the shader is still null it means it failed to be useable */
1601 if (G_UNLIKELY (!gl_sink->redisplay_shader)) {
1602 gst_object_unref (window);
1606 gst_gl_window_set_preferred_size (window, GST_VIDEO_SINK_WIDTH (gl_sink),
1607 GST_VIDEO_SINK_HEIGHT (gl_sink));
1608 gst_gl_window_show (window);
1611 /* Drawing is asynchronous: gst_gl_window_draw is not blocking
1612 * It means that it does not wait for stuff to be executed in other threads
1614 gst_gl_window_draw (window);
1616 alive = gst_gl_window_is_running (window);
1617 gst_object_unref (window);