2 * gstvaapisink.c - VA-API video sink
4 * gstreamer-vaapi (C) 2010-2011 Splitted-Desktop Systems
5 * Copyright (C) 2011 Intel Corporation
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public License
9 * as published by the Free Software Foundation; either version 2.1
10 * of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
24 * SECTION:gstvaapisink
25 * @short_description: A VA-API based videosink
27 * vaapisink renders video frames to a drawable (X #Window) on a local
28 * display using the Video Acceleration (VA) API. The element will
29 * create its own internal window and render into it.
34 #include <gst/gstutils_version.h>
35 #include <gst/video/video.h>
36 #include <gst/video/videocontext.h>
37 #include <gst/vaapi/gstvaapivideobuffer.h>
38 #include <gst/vaapi/gstvaapivideosink.h>
39 #include <gst/vaapi/gstvaapidisplay_x11.h>
40 #include <gst/vaapi/gstvaapiwindow_x11.h>
42 #include <gst/vaapi/gstvaapidisplay_glx.h>
43 #include <gst/vaapi/gstvaapiwindow_glx.h>
46 /* Supported interfaces */
47 #include <gst/interfaces/xoverlay.h>
49 #include "gstvaapisink.h"
50 #include "gstvaapipluginutil.h"
52 #define HAVE_GST_XOVERLAY_SET_WINDOW_HANDLE \
53 GST_PLUGINS_BASE_CHECK_VERSION(0,10,31)
55 #define GST_PLUGIN_NAME "vaapisink"
56 #define GST_PLUGIN_DESC "A VA-API based videosink"
58 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapisink);
59 #define GST_CAT_DEFAULT gst_debug_vaapisink
61 /* ElementFactory information */
62 static const GstElementDetails gst_vaapisink_details =
67 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
69 /* Default template */
70 static GstStaticPadTemplate gst_vaapisink_sink_factory =
71 GST_STATIC_PAD_TEMPLATE(
75 GST_STATIC_CAPS(GST_VAAPI_SURFACE_CAPS));
77 static void gst_vaapisink_iface_init(GType type);
84 gst_vaapisink_iface_init);
95 /* GstImplementsInterface interface */
98 gst_vaapisink_implements_interface_supported(
99 GstImplementsInterface *iface,
103 return (type == GST_TYPE_VIDEO_CONTEXT ||
104 type == GST_TYPE_X_OVERLAY);
108 gst_vaapisink_implements_iface_init(GstImplementsInterfaceClass *iface)
110 iface->supported = gst_vaapisink_implements_interface_supported;
113 /* GstVaapiVideoSink interface */
116 gst_vaapisink_set_video_context(GstVideoContext *context, const gchar *type,
119 GstVaapiSink *sink = GST_VAAPISINK (context);
120 gst_vaapi_set_display (type, value, &sink->display);
124 gst_vaapisink_video_context_iface_init(GstVideoContextInterface *iface)
126 iface->set_context = gst_vaapisink_set_video_context;
129 /* GstXOverlay interface */
132 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id);
135 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *buffer);
138 _gst_vaapisink_xoverlay_set_xid(GstXOverlay *overlay, guintptr window_id)
140 GstVaapiSink * const sink = GST_VAAPISINK(overlay);
142 /* Disable GLX rendering when vaapisink is using a foreign X
143 window. It's pretty much useless */
144 sink->use_glx = FALSE;
146 sink->foreign_window = TRUE;
147 gst_vaapisink_ensure_window_xid(sink, window_id);
150 #if HAVE_GST_XOVERLAY_SET_WINDOW_HANDLE
152 gst_vaapisink_xoverlay_set_window_handle(GstXOverlay *overlay, guintptr window_id)
154 _gst_vaapisink_xoverlay_set_xid(overlay, window_id);
158 gst_vaapisink_xoverlay_set_xid(GstXOverlay *overlay, XID xid)
160 _gst_vaapisink_xoverlay_set_xid(overlay, xid);
165 gst_vaapisink_xoverlay_expose(GstXOverlay *overlay)
167 GstBaseSink * const base_sink = GST_BASE_SINK(overlay);
170 buffer = gst_base_sink_get_last_buffer(base_sink);
172 gst_vaapisink_show_frame(base_sink, buffer);
173 gst_buffer_unref(buffer);
178 gst_vaapisink_xoverlay_iface_init(GstXOverlayClass *iface)
180 #if HAVE_GST_XOVERLAY_SET_WINDOW_HANDLE
181 iface->set_window_handle = gst_vaapisink_xoverlay_set_window_handle;
183 iface->set_xwindow_id = gst_vaapisink_xoverlay_set_xid;
185 iface->expose = gst_vaapisink_xoverlay_expose;
189 gst_vaapisink_iface_init(GType type)
191 const GType g_define_type_id = type;
193 G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
194 gst_vaapisink_implements_iface_init);
195 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
196 gst_vaapisink_video_context_iface_init);
197 G_IMPLEMENT_INTERFACE(GST_TYPE_X_OVERLAY,
198 gst_vaapisink_xoverlay_iface_init);
202 gst_vaapisink_destroy(GstVaapiSink *sink)
205 g_object_unref(sink->display);
206 sink->display = NULL;
210 /* Checks whether a ConfigureNotify event is in the queue */
211 typedef struct _ConfigureNotifyEventPendingArgs ConfigureNotifyEventPendingArgs;
212 struct _ConfigureNotifyEventPendingArgs {
220 configure_notify_event_pending_cb(Display *dpy, XEvent *xev, XPointer arg)
222 ConfigureNotifyEventPendingArgs * const args =
223 (ConfigureNotifyEventPendingArgs *)arg;
225 if (xev->type == ConfigureNotify &&
226 xev->xconfigure.window == args->window &&
227 xev->xconfigure.width == args->width &&
228 xev->xconfigure.height == args->height)
231 /* XXX: this is a hack to traverse the whole queue because we
232 can't use XPeekIfEvent() since it could block */
237 configure_notify_event_pending(
244 ConfigureNotifyEventPendingArgs args;
247 args.window = window;
249 args.height = height;
252 /* XXX: don't use XPeekIfEvent() because it might block */
254 gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
256 configure_notify_event_pending_cb, (XPointer)&args
262 gst_vaapisink_ensure_render_rect(GstVaapiSink *sink, guint width, guint height)
264 GstVaapiRectangle * const display_rect = &sink->display_rect;
265 guint num, den, display_par_n, display_par_d;
266 double display_ratio;
269 GST_DEBUG("ensure render rect within %ux%u bounds", width, height);
271 gst_vaapi_display_get_pixel_aspect_ratio(
273 &display_par_n, &display_par_d
275 GST_DEBUG("display pixel-aspect-ratio %d/%d",
276 display_par_n, display_par_d);
278 success = gst_video_calculate_display_ratio(
280 sink->video_width, sink->video_height,
281 sink->video_par_n, sink->video_par_d,
282 display_par_n, display_par_d
286 GST_DEBUG("video size %dx%d, calculated ratio %d/%d",
287 sink->video_width, sink->video_height, num, den);
289 if ((sink->video_height % den) == 0) {
290 GST_DEBUG("keeping video height");
291 display_rect->width =
292 gst_util_uint64_scale_int(sink->video_height, num, den);
293 display_rect->height = sink->video_height;
295 else if ((sink->video_width % num) == 0) {
296 GST_DEBUG("keeping video width");
297 display_rect->width = sink->video_width;
298 display_rect->height =
299 gst_util_uint64_scale_int(sink->video_width, den, num);
302 GST_DEBUG("approximating while keeping video height");
303 display_rect->width =
304 gst_util_uint64_scale_int(sink->video_height, num, den);
305 display_rect->height = sink->video_height;
307 display_ratio = (gdouble)display_rect->width / display_rect->height;
308 GST_DEBUG("scaling to %ux%u", display_rect->width, display_rect->height);
310 if (sink->fullscreen || sink->foreign_window ||
311 display_rect->width > width || display_rect->height > height) {
312 if (sink->video_width > sink->video_height) {
313 display_rect->width = width;
314 display_rect->height = width / display_ratio;
317 display_rect->width = height * display_ratio;
318 display_rect->height = height;
322 if (sink->fullscreen) {
323 display_rect->x = (width - display_rect->width) / 2;
324 display_rect->y = (height - display_rect->height) / 2;
331 GST_DEBUG("render rect (%d,%d):%ux%u",
332 display_rect->x, display_rect->y,
333 display_rect->width, display_rect->height);
337 static inline gboolean
338 gst_vaapisink_ensure_window(GstVaapiSink *sink, guint width, guint height)
340 GstVaapiDisplay * const display = sink->display;
343 #if USE_VAAPISINK_GLX
345 sink->window = gst_vaapi_window_glx_new(display, width, height);
348 sink->window = gst_vaapi_window_x11_new(display, width, height);
350 gst_x_overlay_got_xwindow_id(
352 gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window))
355 return sink->window != NULL;
359 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id)
362 unsigned int width, height, border_width, depth;
366 if (!gst_vaapi_ensure_display(sink, &sink->display))
369 gst_vaapi_display_lock(sink->display);
371 gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
374 &x, &y, &width, &height, &border_width, &depth
376 gst_vaapi_display_unlock(sink->display);
378 if ((width != sink->window_width || height != sink->window_height) &&
379 !configure_notify_event_pending(sink, xid, width, height)) {
380 if (!gst_vaapisink_ensure_render_rect(sink, width, height))
382 sink->window_width = width;
383 sink->window_height = height;
387 gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window)) == xid)
391 g_object_unref(sink->window);
395 #if USE_VAAPISINK_GLX
397 sink->window = gst_vaapi_window_glx_new_with_xid(sink->display, xid);
400 sink->window = gst_vaapi_window_x11_new_with_xid(sink->display, xid);
401 return sink->window != NULL;
405 gst_vaapisink_start(GstBaseSink *base_sink)
407 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
413 gst_vaapisink_stop(GstBaseSink *base_sink)
415 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
418 g_object_unref(sink->window);
423 g_object_unref(sink->display);
424 sink->display = NULL;
430 gst_vaapisink_set_caps(GstBaseSink *base_sink, GstCaps *caps)
432 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
433 GstStructure * const structure = gst_caps_get_structure(caps, 0);
434 guint display_width, display_height, win_width, win_height;
435 gint video_width, video_height, video_par_n = 1, video_par_d = 1;
439 if (!gst_structure_get_int(structure, "width", &video_width))
441 if (!gst_structure_get_int(structure, "height", &video_height))
443 sink->video_width = video_width;
444 sink->video_height = video_height;
446 gst_video_parse_caps_pixel_aspect_ratio(caps, &video_par_n, &video_par_d);
447 sink->video_par_n = video_par_n;
448 sink->video_par_d = video_par_d;
449 GST_DEBUG("video pixel-aspect-ratio %d/%d", video_par_n, video_par_d);
451 if (!gst_vaapi_ensure_display(sink, &sink->display))
454 gst_vaapi_display_get_size(sink->display, &display_width, &display_height);
455 if (!gst_vaapisink_ensure_render_rect(sink, display_width, display_height))
458 if (sink->fullscreen) {
459 win_width = display_width;
460 win_height = display_height;
463 win_width = sink->display_rect.width;
464 win_height = sink->display_rect.height;
468 gst_vaapi_window_set_size(sink->window, win_width, win_height);
470 gst_vaapi_display_lock(sink->display);
471 gst_x_overlay_prepare_xwindow_id(GST_X_OVERLAY(sink));
472 gst_vaapi_display_unlock(sink->display);
475 if (!gst_vaapisink_ensure_window(sink, win_width, win_height))
477 gst_vaapi_window_set_fullscreen(sink->window, sink->fullscreen);
478 gst_vaapi_window_show(sink->window);
480 sink->window_width = win_width;
481 sink->window_height = win_height;
485 #if USE_VAAPISINK_GLX
487 render_background(GstVaapiSink *sink)
489 /* Original code from Mirco Muller (MacSlow):
490 <http://cgit.freedesktop.org/~macslow/gl-gst-player/> */
491 GLfloat fStartX = 0.0f;
492 GLfloat fStartY = 0.0f;
493 GLfloat fWidth = (GLfloat)sink->window_width;
494 GLfloat fHeight = (GLfloat)sink->window_height;
496 glClear(GL_COLOR_BUFFER_BIT);
499 /* top third, darker grey to white */
500 glColor3f(0.85f, 0.85f, 0.85f);
501 glVertex3f(fStartX, fStartY, 0.0f);
502 glColor3f(0.85f, 0.85f, 0.85f);
503 glVertex3f(fStartX + fWidth, fStartY, 0.0f);
504 glColor3f(1.0f, 1.0f, 1.0f);
505 glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
506 glColor3f(1.0f, 1.0f, 1.0f);
507 glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
509 /* middle third, just plain white */
510 glColor3f(1.0f, 1.0f, 1.0f);
511 glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
512 glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
513 glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
514 glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
516 /* bottom third, white to lighter grey */
517 glColor3f(1.0f, 1.0f, 1.0f);
518 glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
519 glColor3f(1.0f, 1.0f, 1.0f);
520 glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
521 glColor3f(0.62f, 0.66f, 0.69f);
522 glVertex3f(fStartX + fWidth, fStartY + fHeight, 0.0f);
523 glColor3f(0.62f, 0.66f, 0.69f);
524 glVertex3f(fStartX, fStartY + fHeight, 0.0f);
530 render_frame(GstVaapiSink *sink)
532 const guint x1 = sink->display_rect.x;
533 const guint x2 = sink->display_rect.x + sink->display_rect.width;
534 const guint y1 = sink->display_rect.y;
535 const guint y2 = sink->display_rect.y + sink->display_rect.height;
537 glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
540 glTexCoord2f(0.0f, 0.0f); glVertex2i(x1, y1);
541 glTexCoord2f(0.0f, 1.0f); glVertex2i(x1, y2);
542 glTexCoord2f(1.0f, 1.0f); glVertex2i(x2, y2);
543 glTexCoord2f(1.0f, 0.0f); glVertex2i(x2, y1);
549 render_reflection(GstVaapiSink *sink)
551 const guint x1 = sink->display_rect.x;
552 const guint x2 = sink->display_rect.x + sink->display_rect.width;
553 const guint y1 = sink->display_rect.y;
554 const guint rh = sink->display_rect.height / 5;
555 GLfloat ry = 1.0f - (GLfloat)rh / (GLfloat)sink->display_rect.height;
559 glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
560 glTexCoord2f(0.0f, 1.0f); glVertex2i(x1, y1);
561 glTexCoord2f(1.0f, 1.0f); glVertex2i(x2, y1);
563 glColor4f(1.0f, 1.0f, 1.0f, 0.0f);
564 glTexCoord2f(1.0f, ry); glVertex2i(x2, y1 + rh);
565 glTexCoord2f(0.0f, ry); glVertex2i(x1, y1 + rh);
571 gst_vaapisink_show_frame_glx(
573 GstVaapiSurface *surface,
577 GstVaapiWindowGLX * const window = GST_VAAPI_WINDOW_GLX(sink->window);
581 gst_vaapi_window_glx_make_current(window);
582 if (!sink->texture) {
583 sink->texture = gst_vaapi_texture_new(
591 goto error_create_texture;
593 if (!gst_vaapi_texture_put_surface(sink->texture, surface, flags))
594 goto error_transfer_surface;
596 target = gst_vaapi_texture_get_target(sink->texture);
597 texture = gst_vaapi_texture_get_id(sink->texture);
598 if (target != GL_TEXTURE_2D || !texture)
601 if (sink->use_reflection)
602 render_background(sink);
605 glBindTexture(target, texture);
607 if (sink->use_reflection) {
609 glRotatef(20.0f, 0.0f, 1.0f, 0.0f);
610 glTranslatef(50.0f, 0.0f, 0.0f);
613 if (sink->use_reflection) {
615 glTranslatef(0.0, (GLfloat)sink->display_rect.height + 5.0f, 0.0f);
616 render_reflection(sink);
621 glBindTexture(target, 0);
623 gst_vaapi_window_glx_swap_buffers(window);
627 error_create_texture:
629 GST_DEBUG("could not create VA/GLX texture");
632 error_transfer_surface:
634 GST_DEBUG("could not transfer VA surface to texture");
640 static inline gboolean
641 gst_vaapisink_show_frame_x11(
643 GstVaapiSurface *surface,
647 if (!gst_vaapi_window_put_surface(sink->window, surface,
648 NULL, &sink->display_rect, flags)) {
649 GST_DEBUG("could not render VA surface");
656 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *buffer)
658 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
659 GstVaapiVideoBuffer * const vbuffer = GST_VAAPI_VIDEO_BUFFER(buffer);
660 GstVaapiSurface *surface;
663 GstVideoOverlayComposition * const composition =
664 gst_video_buffer_get_overlay_composition(buffer);
666 if (sink->display != gst_vaapi_video_buffer_get_display (vbuffer)) {
668 g_object_unref (sink->display);
669 sink->display = g_object_ref (gst_vaapi_video_buffer_get_display (vbuffer));
673 return GST_FLOW_UNEXPECTED;
675 surface = gst_vaapi_video_buffer_get_surface(vbuffer);
677 return GST_FLOW_UNEXPECTED;
679 GST_DEBUG("render surface %" GST_VAAPI_ID_FORMAT,
680 GST_VAAPI_ID_ARGS(gst_vaapi_surface_get_id(surface)));
682 flags = GST_VAAPI_PICTURE_STRUCTURE_FRAME;
684 if (!gst_vaapi_surface_set_subpictures_from_composition(surface, composition))
685 GST_WARNING("could not update subtitles");
687 #if USE_VAAPISINK_GLX
689 success = gst_vaapisink_show_frame_glx(sink, surface, flags);
692 success = gst_vaapisink_show_frame_x11(sink, surface, flags);
693 return success ? GST_FLOW_OK : GST_FLOW_UNEXPECTED;
697 gst_vaapisink_query(GstBaseSink *base_sink, GstQuery *query)
699 GstVaapiSink *sink = GST_VAAPISINK(base_sink);
700 GST_DEBUG ("sharing display %p", sink->display);
701 return gst_vaapi_reply_to_query (query, sink->display);
705 gst_vaapisink_finalize(GObject *object)
707 gst_vaapisink_destroy(GST_VAAPISINK(object));
709 G_OBJECT_CLASS(parent_class)->finalize(object);
713 gst_vaapisink_set_property(
720 GstVaapiSink * const sink = GST_VAAPISINK(object);
724 sink->use_glx = g_value_get_boolean(value);
726 case PROP_FULLSCREEN:
727 sink->fullscreen = g_value_get_boolean(value);
729 case PROP_SYNCHRONOUS:
730 sink->synchronous = g_value_get_boolean(value);
732 case PROP_USE_REFLECTION:
733 sink->use_reflection = g_value_get_boolean(value);
736 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
742 gst_vaapisink_get_property(
749 GstVaapiSink * const sink = GST_VAAPISINK(object);
753 g_value_set_boolean(value, sink->use_glx);
755 case PROP_FULLSCREEN:
756 g_value_set_boolean(value, sink->fullscreen);
758 case PROP_SYNCHRONOUS:
759 g_value_set_boolean(value, sink->synchronous);
761 case PROP_USE_REFLECTION:
762 g_value_set_boolean(value, sink->use_reflection);
765 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
771 gst_vaapisink_base_init(gpointer klass)
773 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
775 gst_element_class_set_details(element_class, &gst_vaapisink_details);
777 gst_element_class_add_pad_template(
779 gst_static_pad_template_get(&gst_vaapisink_sink_factory)
784 gst_vaapisink_class_init(GstVaapiSinkClass *klass)
786 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
787 GstBaseSinkClass * const basesink_class = GST_BASE_SINK_CLASS(klass);
789 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapisink,
790 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
792 object_class->finalize = gst_vaapisink_finalize;
793 object_class->set_property = gst_vaapisink_set_property;
794 object_class->get_property = gst_vaapisink_get_property;
796 basesink_class->start = gst_vaapisink_start;
797 basesink_class->stop = gst_vaapisink_stop;
798 basesink_class->set_caps = gst_vaapisink_set_caps;
799 basesink_class->preroll = gst_vaapisink_show_frame;
800 basesink_class->render = gst_vaapisink_show_frame;
801 basesink_class->query = gst_vaapisink_query;
803 #if USE_VAAPISINK_GLX
804 g_object_class_install_property
807 g_param_spec_boolean("use-glx",
809 "Enables GLX rendering",
813 g_object_class_install_property
816 g_param_spec_boolean("use-reflection",
818 "Enables OpenGL reflection effect",
823 g_object_class_install_property
826 g_param_spec_boolean("fullscreen",
828 "Requests window in fullscreen state",
833 * GstVaapiSink:synchronous:
835 * When enabled, runs the X display in synchronous mode. Note that
836 * this is used only for debugging.
838 g_object_class_install_property
841 g_param_spec_boolean("synchronous",
843 "Toggles X display synchronous mode",
849 gst_vaapisink_init(GstVaapiSink *sink, GstVaapiSinkClass *klass)
851 sink->display = NULL;
853 sink->window_width = 0;
854 sink->window_height = 0;
855 sink->texture = NULL;
856 sink->video_width = 0;
857 sink->video_height = 0;
858 sink->video_par_n = 1;
859 sink->video_par_d = 1;
860 sink->foreign_window = FALSE;
861 sink->fullscreen = FALSE;
862 sink->synchronous = FALSE;
863 sink->use_glx = USE_VAAPISINK_GLX;
864 sink->use_reflection = FALSE;