2 * gstvaapisink.c - VA-API video sink
4 * Copyright (C) 2010-2011 Splitted-Desktop Systems
5 * Copyright (C) 2011-2013 Intel Corporation
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public License
9 * as published by the Free Software Foundation; either version 2.1
10 * of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
24 * SECTION:gstvaapisink
25 * @short_description: A VA-API based videosink
27 * vaapisink renders video frames to a drawable (X #Window) on a local
28 * display using the Video Acceleration (VA) API. The element will
29 * create its own internal window and render into it.
32 #include "gst/vaapi/sysdeps.h"
34 #include <gst/video/video.h>
35 #if !GST_CHECK_VERSION(1,1,0)
36 #include <gst/video/videocontext.h>
39 #include <gst/vaapi/gstvaapivalue.h>
41 # include <gst/vaapi/gstvaapidisplay_drm.h>
44 # include <gst/vaapi/gstvaapidisplay_x11.h>
45 # include <gst/vaapi/gstvaapiwindow_x11.h>
48 # include <gst/vaapi/gstvaapidisplay_glx.h>
49 # include <gst/vaapi/gstvaapiwindow_glx.h>
52 # include <gst/vaapi/gstvaapidisplay_wayland.h>
53 # include <gst/vaapi/gstvaapiwindow_wayland.h>
56 /* Supported interfaces */
57 #if GST_CHECK_VERSION(1,0,0)
58 # include <gst/video/videooverlay.h>
60 # include <gst/interfaces/xoverlay.h>
62 # define GST_TYPE_VIDEO_OVERLAY GST_TYPE_X_OVERLAY
63 # define GST_VIDEO_OVERLAY GST_X_OVERLAY
64 # define GstVideoOverlay GstXOverlay
65 # define GstVideoOverlayInterface GstXOverlayClass
67 # define gst_video_overlay_prepare_window_handle(sink) \
68 gst_x_overlay_prepare_xwindow_id(sink)
69 # define gst_video_overlay_got_window_handle(sink, window_handle) \
70 gst_x_overlay_got_window_handle(sink, window_handle)
73 #include "gstvaapisink.h"
74 #include "gstvaapipluginutil.h"
75 #include "gstvaapivideometa.h"
76 #if GST_CHECK_VERSION(1,0,0)
77 #include "gstvaapivideobufferpool.h"
78 #include "gstvaapivideomemory.h"
81 #define GST_PLUGIN_NAME "vaapisink"
82 #define GST_PLUGIN_DESC "A VA-API based videosink"
84 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapisink);
85 #define GST_CAT_DEFAULT gst_debug_vaapisink
87 /* Default template */
88 static const char gst_vaapisink_sink_caps_str[] =
89 #if GST_CHECK_VERSION(1,0,0)
90 GST_VIDEO_CAPS_MAKE(GST_VIDEO_FORMATS_ALL) "; "
93 "width = (int) [ 1, MAX ], "
94 "height = (int) [ 1, MAX ]; "
96 GST_VAAPI_SURFACE_CAPS;
98 static GstStaticPadTemplate gst_vaapisink_sink_factory =
99 GST_STATIC_PAD_TEMPLATE(
103 GST_STATIC_CAPS(gst_vaapisink_sink_caps_str));
105 /* GstImplementsInterface interface */
106 #if !GST_CHECK_VERSION(1,0,0)
108 gst_vaapisink_implements_interface_supported(
109 GstImplementsInterface *iface,
113 return (type == GST_TYPE_VIDEO_CONTEXT ||
114 type == GST_TYPE_VIDEO_OVERLAY);
118 gst_vaapisink_implements_iface_init(GstImplementsInterfaceClass *iface)
120 iface->supported = gst_vaapisink_implements_interface_supported;
124 /* GstVideoContext interface */
125 #if !GST_CHECK_VERSION(1,1,0)
127 gst_vaapisink_set_video_context(GstVideoContext *context, const gchar *type,
130 GstVaapiSink *sink = GST_VAAPISINK (context);
131 gst_vaapi_set_display (type, value, &sink->display);
135 gst_vaapisink_video_context_iface_init(GstVideoContextInterface *iface)
137 iface->set_context = gst_vaapisink_set_video_context;
142 gst_vaapisink_video_overlay_iface_init(GstVideoOverlayInterface *iface);
144 G_DEFINE_TYPE_WITH_CODE(
148 #if !GST_CHECK_VERSION(1,0,0)
149 G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
150 gst_vaapisink_implements_iface_init);
152 #if !GST_CHECK_VERSION(1,1,0)
153 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
154 gst_vaapisink_video_context_iface_init);
156 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_OVERLAY,
157 gst_vaapisink_video_overlay_iface_init))
168 PROP_FORCE_ASPECT_RATIO,
171 #define DEFAULT_DISPLAY_TYPE GST_VAAPI_DISPLAY_TYPE_ANY
172 #define DEFAULT_ROTATION GST_VAAPI_ROTATION_0
174 /* GstVideoOverlay interface */
178 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id);
182 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *buffer);
185 gst_vaapisink_video_overlay_set_window_handle(GstVideoOverlay *overlay, guintptr window)
187 GstVaapiSink * const sink = GST_VAAPISINK(overlay);
189 /* Disable GLX rendering when vaapisink is using a foreign X
190 window. It's pretty much useless */
191 if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_GLX)
192 sink->display_type = GST_VAAPI_DISPLAY_TYPE_X11;
194 sink->foreign_window = TRUE;
196 switch (sink->display_type) {
198 case GST_VAAPI_DISPLAY_TYPE_X11:
199 gst_vaapisink_ensure_window_xid(sink, window);
208 gst_vaapisink_video_overlay_set_render_rectangle(
209 GstVideoOverlay *overlay,
216 GstVaapiSink * const sink = GST_VAAPISINK(overlay);
217 GstVaapiRectangle * const display_rect = &sink->display_rect;
221 display_rect->width = width;
222 display_rect->height = height;
224 GST_DEBUG("render rect (%d,%d):%ux%u",
225 display_rect->x, display_rect->y,
226 display_rect->width, display_rect->height);
230 gst_vaapisink_video_overlay_expose(GstVideoOverlay *overlay)
232 GstVaapiSink * const sink = GST_VAAPISINK(overlay);
233 GstBaseSink * const base_sink = GST_BASE_SINK(overlay);
236 if (sink->use_overlay)
237 buffer = sink->video_buffer ? gst_buffer_ref(sink->video_buffer) : NULL;
239 #if GST_CHECK_VERSION(1,0,0)
240 GstSample * const sample = gst_base_sink_get_last_sample(base_sink);
243 buffer = gst_buffer_ref(gst_sample_get_buffer(sample));
244 gst_sample_unref(sample);
246 buffer = gst_base_sink_get_last_buffer(base_sink);
250 gst_vaapisink_show_frame(base_sink, buffer);
251 gst_buffer_unref(buffer);
256 gst_vaapisink_video_overlay_iface_init(GstVideoOverlayInterface *iface)
258 iface->set_window_handle = gst_vaapisink_video_overlay_set_window_handle;
259 iface->set_render_rectangle = gst_vaapisink_video_overlay_set_render_rectangle;
260 iface->expose = gst_vaapisink_video_overlay_expose;
264 gst_vaapisink_destroy(GstVaapiSink *sink)
266 gst_buffer_replace(&sink->video_buffer, NULL);
268 gst_vaapi_texture_replace(&sink->texture, NULL);
270 gst_vaapi_display_replace(&sink->display, NULL);
271 g_clear_object(&sink->uploader);
273 gst_caps_replace(&sink->caps, NULL);
277 /* Checks whether a ConfigureNotify event is in the queue */
278 typedef struct _ConfigureNotifyEventPendingArgs ConfigureNotifyEventPendingArgs;
279 struct _ConfigureNotifyEventPendingArgs {
287 configure_notify_event_pending_cb(Display *dpy, XEvent *xev, XPointer arg)
289 ConfigureNotifyEventPendingArgs * const args =
290 (ConfigureNotifyEventPendingArgs *)arg;
292 if (xev->type == ConfigureNotify &&
293 xev->xconfigure.window == args->window &&
294 xev->xconfigure.width == args->width &&
295 xev->xconfigure.height == args->height)
298 /* XXX: this is a hack to traverse the whole queue because we
299 can't use XPeekIfEvent() since it could block */
304 configure_notify_event_pending(
311 ConfigureNotifyEventPendingArgs args;
314 args.window = window;
316 args.height = height;
319 /* XXX: don't use XPeekIfEvent() because it might block */
321 gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
323 configure_notify_event_pending_cb, (XPointer)&args
330 get_display_type_name(GstVaapiDisplayType display_type)
332 gpointer const klass = g_type_class_peek(GST_VAAPI_TYPE_DISPLAY_TYPE);
333 GEnumValue * const e = g_enum_get_value(klass, display_type);
336 return e->value_name;
337 return "<unknown-type>";
340 static inline gboolean
341 gst_vaapisink_ensure_display(GstVaapiSink *sink)
343 GstVaapiDisplayType display_type;
344 GstVaapiRenderMode render_mode;
345 const gboolean had_display = sink->display != NULL;
347 if (!gst_vaapi_ensure_display(sink, sink->display_type, &sink->display))
350 display_type = gst_vaapi_display_get_display_type(sink->display);
351 if (display_type != sink->display_type || (!had_display && sink->display)) {
352 GST_INFO("created %s %p", get_display_type_name(display_type),
354 sink->display_type = display_type;
357 gst_vaapi_display_get_render_mode(sink->display, &render_mode) &&
358 render_mode == GST_VAAPI_RENDER_MODE_OVERLAY;
359 GST_DEBUG("use %s rendering mode", sink->use_overlay ? "overlay" : "texture");
361 sink->use_rotation = gst_vaapi_display_has_property(
362 sink->display, GST_VAAPI_DISPLAY_PROP_ROTATION);
368 gst_vaapisink_ensure_uploader(GstVaapiSink *sink)
370 if (!gst_vaapisink_ensure_display(sink))
373 if (!sink->uploader) {
374 sink->uploader = gst_vaapi_uploader_new(sink->display);
382 gst_vaapisink_ensure_render_rect(GstVaapiSink *sink, guint width, guint height)
384 GstVaapiRectangle * const display_rect = &sink->display_rect;
385 guint num, den, display_par_n, display_par_d;
388 /* Return success if caps are not set yet */
392 if (!sink->keep_aspect) {
393 display_rect->width = width;
394 display_rect->height = height;
398 GST_DEBUG("force-aspect-ratio is false; distorting while scaling video");
399 GST_DEBUG("render rect (%d,%d):%ux%u",
400 display_rect->x, display_rect->y,
401 display_rect->width, display_rect->height);
405 GST_DEBUG("ensure render rect within %ux%u bounds", width, height);
407 gst_vaapi_display_get_pixel_aspect_ratio(
409 &display_par_n, &display_par_d
411 GST_DEBUG("display pixel-aspect-ratio %d/%d",
412 display_par_n, display_par_d);
414 success = gst_video_calculate_display_ratio(
416 sink->video_width, sink->video_height,
417 sink->video_par_n, sink->video_par_d,
418 display_par_n, display_par_d
422 GST_DEBUG("video size %dx%d, calculated ratio %d/%d",
423 sink->video_width, sink->video_height, num, den);
425 display_rect->width = gst_util_uint64_scale_int(height, num, den);
426 if (display_rect->width <= width) {
427 GST_DEBUG("keeping window height");
428 display_rect->height = height;
431 GST_DEBUG("keeping window width");
432 display_rect->width = width;
433 display_rect->height =
434 gst_util_uint64_scale_int(width, den, num);
436 GST_DEBUG("scaling video to %ux%u", display_rect->width, display_rect->height);
438 g_assert(display_rect->width <= width);
439 g_assert(display_rect->height <= height);
441 display_rect->x = (width - display_rect->width) / 2;
442 display_rect->y = (height - display_rect->height) / 2;
444 GST_DEBUG("render rect (%d,%d):%ux%u",
445 display_rect->x, display_rect->y,
446 display_rect->width, display_rect->height);
451 gst_vaapisink_ensure_window_size(GstVaapiSink *sink, guint *pwidth, guint *pheight)
453 GstVideoRectangle src_rect, dst_rect, out_rect;
454 guint num, den, display_width, display_height, display_par_n, display_par_d;
455 gboolean success, scale;
457 if (sink->foreign_window) {
458 *pwidth = sink->window_width;
459 *pheight = sink->window_height;
463 gst_vaapi_display_get_size(sink->display, &display_width, &display_height);
464 if (sink->fullscreen) {
465 *pwidth = display_width;
466 *pheight = display_height;
470 gst_vaapi_display_get_pixel_aspect_ratio(
472 &display_par_n, &display_par_d
475 success = gst_video_calculate_display_ratio(
477 sink->video_width, sink->video_height,
478 sink->video_par_n, sink->video_par_d,
479 display_par_n, display_par_d
482 num = sink->video_par_n;
483 den = sink->video_par_d;
488 src_rect.w = gst_util_uint64_scale_int(sink->video_height, num, den);
489 src_rect.h = sink->video_height;
492 dst_rect.w = display_width;
493 dst_rect.h = display_height;
494 scale = (src_rect.w > dst_rect.w || src_rect.h > dst_rect.h);
495 gst_video_sink_center_rect(src_rect, dst_rect, &out_rect, scale);
496 *pwidth = out_rect.w;
497 *pheight = out_rect.h;
500 static inline gboolean
501 gst_vaapisink_ensure_window(GstVaapiSink *sink, guint width, guint height)
503 GstVaapiDisplay * const display = sink->display;
506 switch (sink->display_type) {
508 case GST_VAAPI_DISPLAY_TYPE_GLX:
509 sink->window = gst_vaapi_window_glx_new(display, width, height);
510 goto notify_video_overlay_interface;
513 case GST_VAAPI_DISPLAY_TYPE_X11:
514 sink->window = gst_vaapi_window_x11_new(display, width, height);
515 notify_video_overlay_interface:
518 gst_video_overlay_got_window_handle(
519 GST_VIDEO_OVERLAY(sink),
520 gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window))
525 case GST_VAAPI_DISPLAY_TYPE_WAYLAND:
526 sink->window = gst_vaapi_window_wayland_new(display, width, height);
530 GST_ERROR("unsupported display type %d", sink->display_type);
534 return sink->window != NULL;
539 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id)
542 unsigned int width, height, border_width, depth;
546 if (!gst_vaapisink_ensure_display(sink))
549 gst_vaapi_display_lock(sink->display);
551 gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
554 &x, &y, &width, &height, &border_width, &depth
556 gst_vaapi_display_unlock(sink->display);
558 if ((width != sink->window_width || height != sink->window_height) &&
559 !configure_notify_event_pending(sink, xid, width, height)) {
560 if (!gst_vaapisink_ensure_render_rect(sink, width, height))
562 sink->window_width = width;
563 sink->window_height = height;
567 gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window)) == xid)
570 gst_vaapi_window_replace(&sink->window, NULL);
572 switch (sink->display_type) {
574 case GST_VAAPI_DISPLAY_TYPE_GLX:
575 sink->window = gst_vaapi_window_glx_new_with_xid(sink->display, xid);
578 case GST_VAAPI_DISPLAY_TYPE_X11:
579 sink->window = gst_vaapi_window_x11_new_with_xid(sink->display, xid);
582 GST_ERROR("unsupported display type %d", sink->display_type);
585 return sink->window != NULL;
590 gst_vaapisink_ensure_rotation(GstVaapiSink *sink, gboolean recalc_display_rect)
592 gboolean success = FALSE;
594 g_return_val_if_fail(sink->display, FALSE);
596 if (sink->rotation == sink->rotation_req)
599 if (!sink->use_rotation) {
600 GST_WARNING("VA display does not support rotation");
604 gst_vaapi_display_lock(sink->display);
605 success = gst_vaapi_display_set_rotation(sink->display, sink->rotation_req);
606 gst_vaapi_display_unlock(sink->display);
608 GST_ERROR("failed to change VA display rotation mode");
612 if (((sink->rotation + sink->rotation_req) % 180) == 90) {
613 /* Orientation changed */
614 G_PRIMITIVE_SWAP(guint, sink->video_width, sink->video_height);
615 G_PRIMITIVE_SWAP(gint, sink->video_par_n, sink->video_par_d);
618 if (recalc_display_rect && !sink->foreign_window)
619 gst_vaapisink_ensure_render_rect(sink, sink->window_width,
620 sink->window_height);
624 sink->rotation = sink->rotation_req;
629 gst_vaapisink_ensure_video_buffer_pool(GstVaapiSink *sink, GstCaps *caps)
631 #if GST_CHECK_VERSION(1,0,0)
634 GstStructure *config;
638 if (!gst_vaapisink_ensure_display(sink))
641 if (sink->video_buffer_pool) {
642 config = gst_buffer_pool_get_config(sink->video_buffer_pool);
643 gst_buffer_pool_config_get_params(config, &pool_caps, NULL, NULL, NULL);
644 need_pool = !gst_caps_is_equal(caps, pool_caps);
645 gst_structure_free(config);
648 g_clear_object(&sink->video_buffer_pool);
649 sink->video_buffer_size = 0;
652 pool = gst_vaapi_video_buffer_pool_new(sink->display);
654 goto error_create_pool;
656 gst_video_info_init(&vi);
657 gst_video_info_from_caps(&vi, caps);
658 if (GST_VIDEO_INFO_FORMAT(&vi) == GST_VIDEO_FORMAT_ENCODED) {
659 GST_DEBUG("assume video buffer pool format is NV12");
660 gst_video_info_set_format(&vi, GST_VIDEO_FORMAT_NV12,
661 GST_VIDEO_INFO_WIDTH(&vi), GST_VIDEO_INFO_HEIGHT(&vi));
663 sink->video_buffer_size = vi.size;
665 config = gst_buffer_pool_get_config(pool);
666 gst_buffer_pool_config_set_params(config, caps, sink->video_buffer_size,
668 gst_buffer_pool_config_add_option(config,
669 GST_BUFFER_POOL_OPTION_VAAPI_VIDEO_META);
670 gst_buffer_pool_config_add_option(config,
671 GST_BUFFER_POOL_OPTION_VIDEO_META);
672 if (!gst_buffer_pool_set_config(pool, config))
673 goto error_pool_config;
674 sink->video_buffer_pool = pool;
680 GST_ERROR("failed to create buffer pool");
685 GST_ERROR("failed to reset buffer pool config");
686 gst_object_unref(pool);
695 gst_vaapisink_start(GstBaseSink *base_sink)
697 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
699 return gst_vaapisink_ensure_uploader(sink);
703 gst_vaapisink_stop(GstBaseSink *base_sink)
705 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
707 gst_buffer_replace(&sink->video_buffer, NULL);
708 #if GST_CHECK_VERSION(1,0,0)
709 g_clear_object(&sink->video_buffer_pool);
711 gst_vaapi_window_replace(&sink->window, NULL);
712 gst_vaapi_display_replace(&sink->display, NULL);
713 g_clear_object(&sink->uploader);
719 gst_vaapisink_get_caps_impl(GstBaseSink *base_sink)
721 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
722 GstCaps *out_caps, *yuv_caps;
724 out_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS);
728 if (gst_vaapisink_ensure_uploader(sink)) {
729 yuv_caps = gst_vaapi_uploader_get_caps(sink->uploader);
731 gst_caps_append(out_caps, gst_caps_copy(yuv_caps));
736 #if GST_CHECK_VERSION(1,0,0)
737 static inline GstCaps *
738 gst_vaapisink_get_caps(GstBaseSink *base_sink, GstCaps *filter)
740 GstCaps *caps, *out_caps;
742 caps = gst_vaapisink_get_caps_impl(base_sink);
743 if (caps && filter) {
744 out_caps = gst_caps_intersect_full(caps, filter,
745 GST_CAPS_INTERSECT_FIRST);
746 gst_caps_unref(caps);
753 #define gst_vaapisink_get_caps gst_vaapisink_get_caps_impl
757 gst_vaapisink_set_caps(GstBaseSink *base_sink, GstCaps *caps)
759 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
760 GstVideoInfo * const vip = &sink->video_info;
761 guint win_width, win_height;
764 if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_DRM)
768 if (!gst_vaapisink_ensure_video_buffer_pool(sink, caps))
771 if (!gst_video_info_from_caps(vip, caps))
773 sink->use_video_raw = GST_VIDEO_INFO_IS_YUV(vip);
774 sink->video_width = GST_VIDEO_INFO_WIDTH(vip);
775 sink->video_height = GST_VIDEO_INFO_HEIGHT(vip);
776 sink->video_par_n = GST_VIDEO_INFO_PAR_N(vip);
777 sink->video_par_d = GST_VIDEO_INFO_PAR_D(vip);
778 GST_DEBUG("video pixel-aspect-ratio %d/%d",
779 sink->video_par_n, sink->video_par_d);
781 gst_caps_replace(&sink->caps, caps);
783 if (!gst_vaapisink_ensure_display(sink))
786 #if !GST_CHECK_VERSION(1,0,0)
787 if (sink->use_video_raw) {
788 /* Ensure the uploader is set up for upstream allocated buffers */
789 if (!gst_vaapi_uploader_ensure_display(sink->uploader, sink->display))
791 if (!gst_vaapi_uploader_ensure_caps(sink->uploader, caps, NULL))
796 gst_vaapisink_ensure_rotation(sink, FALSE);
798 gst_vaapisink_ensure_window_size(sink, &win_width, &win_height);
800 if (!sink->foreign_window || sink->fullscreen)
801 gst_vaapi_window_set_size(sink->window, win_width, win_height);
804 gst_vaapi_display_lock(sink->display);
805 gst_video_overlay_prepare_window_handle(GST_VIDEO_OVERLAY(sink));
806 gst_vaapi_display_unlock(sink->display);
809 if (!gst_vaapisink_ensure_window(sink, win_width, win_height))
811 gst_vaapi_window_set_fullscreen(sink->window, sink->fullscreen);
812 gst_vaapi_window_show(sink->window);
813 gst_vaapi_window_get_size(sink->window, &win_width, &win_height);
815 sink->window_width = win_width;
816 sink->window_height = win_height;
817 GST_DEBUG("window size %ux%u", win_width, win_height);
819 return gst_vaapisink_ensure_render_rect(sink, win_width, win_height);
824 render_background(GstVaapiSink *sink)
826 /* Original code from Mirco Muller (MacSlow):
827 <http://cgit.freedesktop.org/~macslow/gl-gst-player/> */
828 GLfloat fStartX = 0.0f;
829 GLfloat fStartY = 0.0f;
830 GLfloat fWidth = (GLfloat)sink->window_width;
831 GLfloat fHeight = (GLfloat)sink->window_height;
833 glClear(GL_COLOR_BUFFER_BIT);
836 /* top third, darker grey to white */
837 glColor3f(0.85f, 0.85f, 0.85f);
838 glVertex3f(fStartX, fStartY, 0.0f);
839 glColor3f(0.85f, 0.85f, 0.85f);
840 glVertex3f(fStartX + fWidth, fStartY, 0.0f);
841 glColor3f(1.0f, 1.0f, 1.0f);
842 glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
843 glColor3f(1.0f, 1.0f, 1.0f);
844 glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
846 /* middle third, just plain white */
847 glColor3f(1.0f, 1.0f, 1.0f);
848 glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
849 glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
850 glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
851 glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
853 /* bottom third, white to lighter grey */
854 glColor3f(1.0f, 1.0f, 1.0f);
855 glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
856 glColor3f(1.0f, 1.0f, 1.0f);
857 glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
858 glColor3f(0.62f, 0.66f, 0.69f);
859 glVertex3f(fStartX + fWidth, fStartY + fHeight, 0.0f);
860 glColor3f(0.62f, 0.66f, 0.69f);
861 glVertex3f(fStartX, fStartY + fHeight, 0.0f);
867 render_frame(GstVaapiSink *sink, GstVaapiSurface *surface,
868 const GstVaapiRectangle *surface_rect)
870 const guint x1 = sink->display_rect.x;
871 const guint x2 = sink->display_rect.x + sink->display_rect.width;
872 const guint y1 = sink->display_rect.y;
873 const guint y2 = sink->display_rect.y + sink->display_rect.height;
874 gfloat tx1, tx2, ty1, ty2;
878 gst_vaapi_surface_get_size(surface, &width, &height);
879 tx1 = (gfloat)surface_rect->x / width;
880 ty1 = (gfloat)surface_rect->y / height;
881 tx2 = (gfloat)(surface_rect->x + surface_rect->width) / width;
882 ty2 = (gfloat)(surface_rect->y + surface_rect->height) / height;
891 glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
894 glTexCoord2f(tx1, ty1); glVertex2i(x1, y1);
895 glTexCoord2f(tx1, ty2); glVertex2i(x1, y2);
896 glTexCoord2f(tx2, ty2); glVertex2i(x2, y2);
897 glTexCoord2f(tx2, ty1); glVertex2i(x2, y1);
903 render_reflection(GstVaapiSink *sink)
905 const guint x1 = sink->display_rect.x;
906 const guint x2 = sink->display_rect.x + sink->display_rect.width;
907 const guint y1 = sink->display_rect.y;
908 const guint rh = sink->display_rect.height / 5;
909 GLfloat ry = 1.0f - (GLfloat)rh / (GLfloat)sink->display_rect.height;
913 glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
914 glTexCoord2f(0.0f, 1.0f); glVertex2i(x1, y1);
915 glTexCoord2f(1.0f, 1.0f); glVertex2i(x2, y1);
917 glColor4f(1.0f, 1.0f, 1.0f, 0.0f);
918 glTexCoord2f(1.0f, ry); glVertex2i(x2, y1 + rh);
919 glTexCoord2f(0.0f, ry); glVertex2i(x1, y1 + rh);
925 gst_vaapisink_ensure_texture(GstVaapiSink *sink, GstVaapiSurface *surface)
927 GstVideoRectangle tex_rect, dis_rect, out_rect;
933 gst_vaapi_surface_get_size(surface, &width, &height);
939 gst_vaapi_display_get_size(sink->display, &width, &height);
945 gst_video_sink_center_rect(tex_rect, dis_rect, &out_rect, TRUE);
947 /* XXX: use surface size for now since some VA drivers have issues
948 with downscaling to the provided texture size. i.e. we should be
949 using the resulting out_rect size, which preserves the aspect
950 ratio of the surface */
953 GST_INFO("texture size %ux%u", width, height);
955 sink->texture = gst_vaapi_texture_new(sink->display,
956 GL_TEXTURE_2D, GL_BGRA, width, height);
957 return sink->texture != NULL;
961 gst_vaapisink_show_frame_glx(
963 GstVaapiSurface *surface,
964 const GstVaapiRectangle *surface_rect,
968 GstVaapiWindowGLX * const window = GST_VAAPI_WINDOW_GLX(sink->window);
972 gst_vaapi_window_glx_make_current(window);
973 if (!gst_vaapisink_ensure_texture(sink, surface))
974 goto error_create_texture;
975 if (!gst_vaapi_texture_put_surface(sink->texture, surface, flags))
976 goto error_transfer_surface;
978 target = gst_vaapi_texture_get_target(sink->texture);
979 texture = gst_vaapi_texture_get_id(sink->texture);
980 if (target != GL_TEXTURE_2D || !texture)
983 if (sink->use_reflection)
984 render_background(sink);
987 glBindTexture(target, texture);
989 if (sink->use_reflection) {
991 glRotatef(20.0f, 0.0f, 1.0f, 0.0f);
992 glTranslatef(50.0f, 0.0f, 0.0f);
994 render_frame(sink, surface, surface_rect);
995 if (sink->use_reflection) {
997 glTranslatef(0.0, (GLfloat)sink->display_rect.height + 5.0f, 0.0f);
998 render_reflection(sink);
1003 glBindTexture(target, 0);
1005 gst_vaapi_window_glx_swap_buffers(window);
1009 error_create_texture:
1011 GST_DEBUG("could not create VA/GLX texture");
1014 error_transfer_surface:
1016 GST_DEBUG("could not transfer VA surface to texture");
1022 static inline gboolean
1023 gst_vaapisink_put_surface(
1025 GstVaapiSurface *surface,
1026 const GstVaapiRectangle *surface_rect,
1030 if (!gst_vaapi_window_put_surface(sink->window, surface,
1031 surface_rect, &sink->display_rect, flags)) {
1032 GST_DEBUG("could not render VA surface");
1038 #if GST_CHECK_VERSION(1,0,0)
1039 static GstFlowReturn
1040 gst_vaapisink_get_render_buffer(GstVaapiSink *sink, GstBuffer *src_buffer,
1041 GstBuffer **out_buffer_ptr)
1043 GstVaapiVideoMeta *meta;
1044 GstBuffer *out_buffer;
1045 GstBufferPoolAcquireParams params = { 0, };
1046 GstVideoFrame src_frame, out_frame;
1049 meta = gst_buffer_get_vaapi_video_meta(src_buffer);
1051 *out_buffer_ptr = gst_buffer_ref(src_buffer);
1055 if (!sink->use_video_raw) {
1056 GST_ERROR("unsupported video buffer");
1057 return GST_FLOW_EOS;
1060 GST_DEBUG("buffer %p not from our pool, copying", src_buffer);
1062 *out_buffer_ptr = NULL;
1063 if (!sink->video_buffer_pool)
1066 if (!gst_buffer_pool_set_active(sink->video_buffer_pool, TRUE))
1067 goto error_activate_pool;
1069 params.flags = GST_BUFFER_POOL_ACQUIRE_FLAG_DONTWAIT;
1070 ret = gst_buffer_pool_acquire_buffer(sink->video_buffer_pool, &out_buffer,
1072 if (ret != GST_FLOW_OK)
1073 goto error_create_buffer;
1075 if (!gst_video_frame_map(&src_frame, &sink->video_info, src_buffer,
1077 goto error_map_src_buffer;
1079 if (!gst_video_frame_map(&out_frame, &sink->video_info, out_buffer,
1081 goto error_map_dst_buffer;
1083 gst_video_frame_copy(&out_frame, &src_frame);
1084 gst_video_frame_unmap(&out_frame);
1085 gst_video_frame_unmap(&src_frame);
1087 *out_buffer_ptr = out_buffer;
1092 GST_ERROR("no buffer pool was negotiated");
1093 return GST_FLOW_ERROR;
1094 error_activate_pool:
1095 GST_ERROR("failed to activate buffer pool");
1096 return GST_FLOW_ERROR;
1097 error_create_buffer:
1098 GST_WARNING("failed to create image. Skipping this frame");
1100 error_map_dst_buffer:
1101 gst_video_frame_unmap(&src_frame);
1103 error_map_src_buffer:
1104 GST_WARNING("failed to map buffer. Skipping this frame");
1105 gst_buffer_unref(out_buffer);
1109 static GstFlowReturn
1110 gst_vaapisink_get_render_buffer(GstVaapiSink *sink, GstBuffer *src_buffer,
1111 GstBuffer **out_buffer_ptr)
1113 GstVaapiVideoMeta *meta;
1114 GstBuffer *out_buffer;
1116 *out_buffer_ptr = NULL;
1117 meta = gst_buffer_get_vaapi_video_meta(src_buffer);
1119 out_buffer = gst_buffer_ref(src_buffer);
1120 else if (sink->use_video_raw) {
1121 out_buffer = gst_vaapi_uploader_get_buffer(sink->uploader);
1123 goto error_create_buffer;
1126 GST_ERROR("unsupported video buffer");
1127 return GST_FLOW_EOS;
1130 if (sink->use_video_raw &&
1131 !gst_vaapi_uploader_process(sink->uploader, src_buffer, out_buffer))
1132 goto error_copy_buffer;
1134 *out_buffer_ptr = out_buffer;
1138 error_create_buffer:
1139 GST_WARNING("failed to create buffer. Skipping this frame");
1142 GST_WARNING("failed to copy buffers. Skipping this frame");
1143 gst_buffer_unref(out_buffer);
1148 static GstFlowReturn
1149 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *src_buffer)
1151 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1152 GstVaapiVideoMeta *meta;
1153 GstVaapiSurface *surface;
1157 GstVaapiRectangle *surface_rect = NULL;
1158 #if GST_CHECK_VERSION(1,0,0)
1159 GstVaapiRectangle tmp_rect;
1163 #if GST_CHECK_VERSION(1,0,0)
1164 GstVideoCropMeta * const crop_meta =
1165 gst_buffer_get_video_crop_meta(src_buffer);
1167 surface_rect = &tmp_rect;
1168 surface_rect->x = crop_meta->x;
1169 surface_rect->y = crop_meta->y;
1170 surface_rect->width = crop_meta->width;
1171 surface_rect->height = crop_meta->height;
1175 ret = gst_vaapisink_get_render_buffer(sink, src_buffer, &buffer);
1176 if (ret != GST_FLOW_OK || !buffer)
1179 meta = gst_buffer_get_vaapi_video_meta(buffer);
1180 if (sink->display != gst_vaapi_video_meta_get_display(meta))
1181 gst_vaapi_display_replace(&sink->display,
1182 gst_vaapi_video_meta_get_display(meta));
1187 gst_vaapisink_ensure_rotation(sink, TRUE);
1189 surface = gst_vaapi_video_meta_get_surface(meta);
1193 GST_DEBUG("render surface %" GST_VAAPI_ID_FORMAT,
1194 GST_VAAPI_ID_ARGS(gst_vaapi_surface_get_id(surface)));
1197 surface_rect = (GstVaapiRectangle *)
1198 gst_vaapi_video_meta_get_render_rect(meta);
1201 GST_DEBUG("render rect (%d,%d), size %ux%u",
1202 surface_rect->x, surface_rect->y,
1203 surface_rect->width, surface_rect->height);
1205 flags = gst_vaapi_video_meta_get_render_flags(meta);
1207 if (!gst_vaapi_apply_composition(surface, src_buffer))
1208 GST_WARNING("could not update subtitles");
1210 switch (sink->display_type) {
1212 case GST_VAAPI_DISPLAY_TYPE_DRM:
1217 case GST_VAAPI_DISPLAY_TYPE_GLX:
1219 goto put_surface_x11;
1220 success = gst_vaapisink_show_frame_glx(sink, surface, surface_rect,
1225 case GST_VAAPI_DISPLAY_TYPE_X11:
1227 success = gst_vaapisink_put_surface(sink, surface, surface_rect, flags);
1231 case GST_VAAPI_DISPLAY_TYPE_WAYLAND:
1232 success = gst_vaapisink_put_surface(sink, surface, surface_rect, flags);
1236 GST_ERROR("unsupported display type %d", sink->display_type);
1243 /* Retain VA surface until the next one is displayed */
1244 if (sink->use_overlay)
1245 gst_buffer_replace(&sink->video_buffer, buffer);
1246 gst_buffer_unref(buffer);
1250 gst_buffer_unref(buffer);
1251 return GST_FLOW_EOS;
1254 #if GST_CHECK_VERSION(1,0,0)
1256 gst_vaapisink_propose_allocation(GstBaseSink *base_sink, GstQuery *query)
1258 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1259 GstCaps *caps = NULL;
1262 gst_query_parse_allocation(query, &caps, &need_pool);
1267 if (!gst_vaapisink_ensure_video_buffer_pool(sink, caps))
1269 gst_query_add_allocation_pool(query, sink->video_buffer_pool,
1270 sink->video_buffer_size, 0, 0);
1273 gst_query_add_allocation_meta(query,
1274 GST_VAAPI_VIDEO_META_API_TYPE, NULL);
1275 gst_query_add_allocation_meta(query,
1276 GST_VIDEO_META_API_TYPE, NULL);
1277 gst_query_add_allocation_meta(query,
1278 GST_VIDEO_CROP_META_API_TYPE, NULL);
1279 gst_query_add_allocation_meta(query,
1280 GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL);
1286 GST_ERROR("no caps specified");
1291 static GstFlowReturn
1292 gst_vaapisink_buffer_alloc(
1293 GstBaseSink *base_sink,
1300 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1306 if (!sink->use_video_raw) {
1307 /* Note: this code path is rarely used but for raw YUV formats
1308 from custom pipeline. Otherwise, GstBaseSink::set_caps() is
1309 called first, and GstBaseSink::buffer_alloc() is not called
1310 in VA surface format mode */
1311 if (!gst_video_info_from_caps(&vi, caps))
1312 return GST_FLOW_NOT_SUPPORTED;
1313 if (!GST_VIDEO_INFO_IS_YUV(&vi))
1317 if (!gst_vaapi_uploader_ensure_display(sink->uploader, sink->display))
1318 return GST_FLOW_NOT_SUPPORTED;
1319 if (!gst_vaapi_uploader_ensure_caps(sink->uploader, caps, NULL))
1320 return GST_FLOW_NOT_SUPPORTED;
1322 buf = gst_vaapi_uploader_get_buffer(sink->uploader);
1324 GST_WARNING("failed to allocate resources for raw YUV buffer");
1325 return GST_FLOW_NOT_SUPPORTED;
1334 gst_vaapisink_query(GstBaseSink *base_sink, GstQuery *query)
1336 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1338 if (gst_vaapi_reply_to_query(query, sink->display)) {
1339 GST_DEBUG("sharing display %p", sink->display);
1342 return GST_BASE_SINK_CLASS(gst_vaapisink_parent_class)->query(base_sink,
1347 gst_vaapisink_finalize(GObject *object)
1349 gst_vaapisink_destroy(GST_VAAPISINK(object));
1351 G_OBJECT_CLASS(gst_vaapisink_parent_class)->finalize(object);
1355 gst_vaapisink_set_property(
1358 const GValue *value,
1362 GstVaapiSink * const sink = GST_VAAPISINK(object);
1365 case PROP_DISPLAY_TYPE:
1366 sink->display_type = g_value_get_enum(value);
1368 case PROP_FULLSCREEN:
1369 sink->fullscreen = g_value_get_boolean(value);
1371 case PROP_SYNCHRONOUS:
1372 sink->synchronous = g_value_get_boolean(value);
1375 sink->use_glx = g_value_get_boolean(value);
1377 case PROP_USE_REFLECTION:
1378 sink->use_reflection = g_value_get_boolean(value);
1381 sink->rotation_req = g_value_get_enum(value);
1383 case PROP_FORCE_ASPECT_RATIO:
1384 sink->keep_aspect = g_value_get_boolean(value);
1387 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
1393 gst_vaapisink_get_property(
1400 GstVaapiSink * const sink = GST_VAAPISINK(object);
1403 case PROP_DISPLAY_TYPE:
1404 g_value_set_enum(value, sink->display_type);
1406 case PROP_FULLSCREEN:
1407 g_value_set_boolean(value, sink->fullscreen);
1409 case PROP_SYNCHRONOUS:
1410 g_value_set_boolean(value, sink->synchronous);
1413 g_value_set_boolean(value, sink->use_glx);
1415 case PROP_USE_REFLECTION:
1416 g_value_set_boolean(value, sink->use_reflection);
1419 g_value_set_enum(value, sink->rotation);
1421 case PROP_FORCE_ASPECT_RATIO:
1422 g_value_set_boolean(value, sink->keep_aspect);
1425 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
1431 gst_vaapisink_class_init(GstVaapiSinkClass *klass)
1433 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
1434 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
1435 GstBaseSinkClass * const basesink_class = GST_BASE_SINK_CLASS(klass);
1436 GstPadTemplate *pad_template;
1438 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapisink,
1439 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
1441 object_class->finalize = gst_vaapisink_finalize;
1442 object_class->set_property = gst_vaapisink_set_property;
1443 object_class->get_property = gst_vaapisink_get_property;
1445 basesink_class->start = gst_vaapisink_start;
1446 basesink_class->stop = gst_vaapisink_stop;
1447 basesink_class->get_caps = gst_vaapisink_get_caps;
1448 basesink_class->set_caps = gst_vaapisink_set_caps;
1449 basesink_class->preroll = gst_vaapisink_show_frame;
1450 basesink_class->render = gst_vaapisink_show_frame;
1451 basesink_class->query = gst_vaapisink_query;
1452 #if GST_CHECK_VERSION(1,0,0)
1453 basesink_class->propose_allocation = gst_vaapisink_propose_allocation;
1455 basesink_class->buffer_alloc = gst_vaapisink_buffer_alloc;
1458 gst_element_class_set_static_metadata(element_class,
1462 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
1464 pad_template = gst_static_pad_template_get(&gst_vaapisink_sink_factory);
1465 gst_element_class_add_pad_template(element_class, pad_template);
1467 g_object_class_install_property
1470 g_param_spec_enum("display",
1472 "display type to use",
1473 GST_VAAPI_TYPE_DISPLAY_TYPE,
1474 GST_VAAPI_DISPLAY_TYPE_ANY,
1475 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1478 g_object_class_install_property
1481 g_param_spec_boolean("use-glx",
1483 "Enables OpenGL rendering",
1485 G_PARAM_READWRITE));
1487 g_object_class_install_property
1489 PROP_USE_REFLECTION,
1490 g_param_spec_boolean("use-reflection",
1491 "Reflection effect",
1492 "Enables OpenGL reflection effect",
1494 G_PARAM_READWRITE));
1497 g_object_class_install_property
1500 g_param_spec_boolean("fullscreen",
1502 "Requests window in fullscreen state",
1504 G_PARAM_READWRITE));
1507 * GstVaapiSink:synchronous:
1509 * When enabled, runs the X display in synchronous mode. Note that
1510 * this is used only for debugging.
1512 g_object_class_install_property
1515 g_param_spec_boolean("synchronous",
1517 "Toggles X display synchronous mode",
1519 G_PARAM_READWRITE));
1522 * GstVaapiSink:rotation:
1524 * The VA display rotation mode, expressed as a #GstVaapiRotation.
1526 g_object_class_install_property
1529 g_param_spec_enum(GST_VAAPI_DISPLAY_PROP_ROTATION,
1531 "The display rotation mode",
1532 GST_VAAPI_TYPE_ROTATION,
1534 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1537 * GstVaapiSink:force-aspect-ratio:
1539 * When enabled, scaling respects video aspect ratio; when disabled, the
1540 * video is distorted to fit the window.
1542 g_object_class_install_property
1544 PROP_FORCE_ASPECT_RATIO,
1545 g_param_spec_boolean("force-aspect-ratio",
1546 "Force aspect ratio",
1547 "When enabled, scaling will respect original aspect ratio",
1549 G_PARAM_READWRITE));
1553 gst_vaapisink_init(GstVaapiSink *sink)
1556 sink->display = NULL;
1557 sink->window = NULL;
1558 sink->window_width = 0;
1559 sink->window_height = 0;
1560 sink->texture = NULL;
1561 sink->video_buffer = NULL;
1562 sink->video_width = 0;
1563 sink->video_height = 0;
1564 sink->video_par_n = 1;
1565 sink->video_par_d = 1;
1566 sink->foreign_window = FALSE;
1567 sink->fullscreen = FALSE;
1568 sink->synchronous = FALSE;
1569 sink->display_type = DEFAULT_DISPLAY_TYPE;
1570 sink->rotation = DEFAULT_ROTATION;
1571 sink->rotation_req = DEFAULT_ROTATION;
1572 sink->use_reflection = FALSE;
1573 sink->use_overlay = FALSE;
1574 sink->use_rotation = FALSE;
1575 sink->keep_aspect = TRUE;
1576 gst_video_info_init(&sink->video_info);