2 * gstvaapisink.c - VA-API video sink
4 * Copyright (C) 2010-2011 Splitted-Desktop Systems
5 * Copyright (C) 2011-2013 Intel Corporation
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public License
9 * as published by the Free Software Foundation; either version 2.1
10 * of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
24 * SECTION:gstvaapisink
25 * @short_description: A VA-API based videosink
27 * vaapisink renders video frames to a drawable (X #Window) on a local
28 * display using the Video Acceleration (VA) API. The element will
29 * create its own internal window and render into it.
32 #include "gst/vaapi/sysdeps.h"
34 #include <gst/video/video.h>
36 #include <gst/vaapi/gstvaapivalue.h>
38 # include <gst/vaapi/gstvaapidisplay_drm.h>
41 # include <gst/vaapi/gstvaapidisplay_x11.h>
42 # include <gst/vaapi/gstvaapiwindow_x11.h>
45 # include <gst/vaapi/gstvaapidisplay_glx.h>
46 # include <gst/vaapi/gstvaapiwindow_glx.h>
49 # include <gst/vaapi/gstvaapidisplay_wayland.h>
50 # include <gst/vaapi/gstvaapiwindow_wayland.h>
53 /* Supported interfaces */
54 #if GST_CHECK_VERSION(1,0,0)
55 # include <gst/video/videooverlay.h>
57 # include <gst/interfaces/xoverlay.h>
59 # define GST_TYPE_VIDEO_OVERLAY GST_TYPE_X_OVERLAY
60 # define GST_VIDEO_OVERLAY GST_X_OVERLAY
61 # define GstVideoOverlay GstXOverlay
62 # define GstVideoOverlayInterface GstXOverlayClass
64 # define gst_video_overlay_prepare_window_handle(sink) \
65 gst_x_overlay_prepare_xwindow_id(sink)
66 # define gst_video_overlay_got_window_handle(sink, window_handle) \
67 gst_x_overlay_got_window_handle(sink, window_handle)
70 #include "gstvaapisink.h"
71 #include "gstvaapipluginutil.h"
72 #include "gstvaapivideocontext.h"
73 #include "gstvaapivideometa.h"
74 #if GST_CHECK_VERSION(1,0,0)
75 #include "gstvaapivideobufferpool.h"
76 #include "gstvaapivideomemory.h"
79 #define GST_PLUGIN_NAME "vaapisink"
80 #define GST_PLUGIN_DESC "A VA-API based videosink"
82 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapisink);
83 #define GST_CAT_DEFAULT gst_debug_vaapisink
85 /* Default template */
86 static const char gst_vaapisink_sink_caps_str[] =
87 #if GST_CHECK_VERSION(1,1,0)
88 GST_VIDEO_CAPS_MAKE_WITH_FEATURES(
89 GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE, GST_VIDEO_FORMATS_ALL) ";"
90 GST_VIDEO_CAPS_MAKE_WITH_FEATURES(
91 GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META,
92 GST_VIDEO_FORMATS_ALL);
94 #if GST_CHECK_VERSION(1,0,0)
95 GST_VIDEO_CAPS_MAKE(GST_VIDEO_FORMATS_ALL) "; "
98 "width = (int) [ 1, MAX ], "
99 "height = (int) [ 1, MAX ]; "
101 GST_VAAPI_SURFACE_CAPS;
104 static GstStaticPadTemplate gst_vaapisink_sink_factory =
105 GST_STATIC_PAD_TEMPLATE(
109 GST_STATIC_CAPS(gst_vaapisink_sink_caps_str));
111 /* GstImplementsInterface interface */
112 #if !GST_CHECK_VERSION(1,0,0)
114 gst_vaapisink_implements_interface_supported(
115 GstImplementsInterface *iface,
119 return (type == GST_TYPE_VIDEO_CONTEXT ||
120 type == GST_TYPE_VIDEO_OVERLAY);
124 gst_vaapisink_implements_iface_init(GstImplementsInterfaceClass *iface)
126 iface->supported = gst_vaapisink_implements_interface_supported;
130 /* GstVideoContext interface */
131 #if !GST_CHECK_VERSION(1,1,0)
133 gst_vaapisink_set_video_context(GstVideoContext *context, const gchar *type,
136 GstVaapiSink *sink = GST_VAAPISINK (context);
137 gst_vaapi_set_display (type, value, &sink->display);
141 gst_vaapisink_video_context_iface_init(GstVideoContextInterface *iface)
143 iface->set_context = gst_vaapisink_set_video_context;
148 gst_vaapisink_video_overlay_iface_init(GstVideoOverlayInterface *iface);
150 G_DEFINE_TYPE_WITH_CODE(
154 #if !GST_CHECK_VERSION(1,0,0)
155 G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
156 gst_vaapisink_implements_iface_init);
158 #if !GST_CHECK_VERSION(1,1,0)
159 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
160 gst_vaapisink_video_context_iface_init);
162 G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_OVERLAY,
163 gst_vaapisink_video_overlay_iface_init))
174 PROP_FORCE_ASPECT_RATIO,
177 #define DEFAULT_DISPLAY_TYPE GST_VAAPI_DISPLAY_TYPE_ANY
178 #define DEFAULT_ROTATION GST_VAAPI_ROTATION_0
180 /* GstVideoOverlay interface */
184 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id);
188 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *buffer);
191 gst_vaapisink_video_overlay_set_window_handle(GstVideoOverlay *overlay, guintptr window)
193 GstVaapiSink * const sink = GST_VAAPISINK(overlay);
195 /* Disable GLX rendering when vaapisink is using a foreign X
196 window. It's pretty much useless */
197 if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_GLX)
198 sink->display_type = GST_VAAPI_DISPLAY_TYPE_X11;
200 sink->foreign_window = TRUE;
202 switch (sink->display_type) {
204 case GST_VAAPI_DISPLAY_TYPE_X11:
205 gst_vaapisink_ensure_window_xid(sink, window);
214 gst_vaapisink_video_overlay_set_render_rectangle(
215 GstVideoOverlay *overlay,
222 GstVaapiSink * const sink = GST_VAAPISINK(overlay);
223 GstVaapiRectangle * const display_rect = &sink->display_rect;
227 display_rect->width = width;
228 display_rect->height = height;
230 GST_DEBUG("render rect (%d,%d):%ux%u",
231 display_rect->x, display_rect->y,
232 display_rect->width, display_rect->height);
236 gst_vaapisink_video_overlay_expose(GstVideoOverlay *overlay)
238 GstVaapiSink * const sink = GST_VAAPISINK(overlay);
239 GstBaseSink * const base_sink = GST_BASE_SINK(overlay);
242 if (sink->use_overlay)
243 buffer = sink->video_buffer ? gst_buffer_ref(sink->video_buffer) : NULL;
245 #if GST_CHECK_VERSION(1,0,0)
246 GstSample * const sample = gst_base_sink_get_last_sample(base_sink);
249 buffer = gst_buffer_ref(gst_sample_get_buffer(sample));
250 gst_sample_unref(sample);
252 buffer = gst_base_sink_get_last_buffer(base_sink);
256 gst_vaapisink_show_frame(base_sink, buffer);
257 gst_buffer_unref(buffer);
262 gst_vaapisink_video_overlay_iface_init(GstVideoOverlayInterface *iface)
264 iface->set_window_handle = gst_vaapisink_video_overlay_set_window_handle;
265 iface->set_render_rectangle = gst_vaapisink_video_overlay_set_render_rectangle;
266 iface->expose = gst_vaapisink_video_overlay_expose;
270 gst_vaapisink_destroy(GstVaapiSink *sink)
272 gst_buffer_replace(&sink->video_buffer, NULL);
274 gst_vaapi_texture_replace(&sink->texture, NULL);
276 gst_vaapi_display_replace(&sink->display, NULL);
277 g_clear_object(&sink->uploader);
279 gst_caps_replace(&sink->caps, NULL);
283 /* Checks whether a ConfigureNotify event is in the queue */
284 typedef struct _ConfigureNotifyEventPendingArgs ConfigureNotifyEventPendingArgs;
285 struct _ConfigureNotifyEventPendingArgs {
293 configure_notify_event_pending_cb(Display *dpy, XEvent *xev, XPointer arg)
295 ConfigureNotifyEventPendingArgs * const args =
296 (ConfigureNotifyEventPendingArgs *)arg;
298 if (xev->type == ConfigureNotify &&
299 xev->xconfigure.window == args->window &&
300 xev->xconfigure.width == args->width &&
301 xev->xconfigure.height == args->height)
304 /* XXX: this is a hack to traverse the whole queue because we
305 can't use XPeekIfEvent() since it could block */
310 configure_notify_event_pending(
317 ConfigureNotifyEventPendingArgs args;
320 args.window = window;
322 args.height = height;
325 /* XXX: don't use XPeekIfEvent() because it might block */
327 gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
329 configure_notify_event_pending_cb, (XPointer)&args
336 get_display_type_name(GstVaapiDisplayType display_type)
338 gpointer const klass = g_type_class_peek(GST_VAAPI_TYPE_DISPLAY_TYPE);
339 GEnumValue * const e = g_enum_get_value(klass, display_type);
342 return e->value_name;
343 return "<unknown-type>";
346 static inline gboolean
347 gst_vaapisink_ensure_display(GstVaapiSink *sink)
349 GstVaapiDisplayType display_type;
350 GstVaapiRenderMode render_mode;
351 const gboolean had_display = sink->display != NULL;
353 if (!gst_vaapi_ensure_display(sink, sink->display_type, &sink->display))
356 display_type = gst_vaapi_display_get_display_type(sink->display);
357 if (display_type != sink->display_type || (!had_display && sink->display)) {
358 GST_INFO("created %s %p", get_display_type_name(display_type),
360 sink->display_type = display_type;
363 gst_vaapi_display_get_render_mode(sink->display, &render_mode) &&
364 render_mode == GST_VAAPI_RENDER_MODE_OVERLAY;
365 GST_DEBUG("use %s rendering mode", sink->use_overlay ? "overlay" : "texture");
367 sink->use_rotation = gst_vaapi_display_has_property(
368 sink->display, GST_VAAPI_DISPLAY_PROP_ROTATION);
374 gst_vaapisink_ensure_uploader(GstVaapiSink *sink)
376 if (!gst_vaapisink_ensure_display(sink))
379 if (!sink->uploader) {
380 sink->uploader = gst_vaapi_uploader_new(sink->display);
388 gst_vaapisink_ensure_render_rect(GstVaapiSink *sink, guint width, guint height)
390 GstVaapiRectangle * const display_rect = &sink->display_rect;
391 guint num, den, display_par_n, display_par_d;
394 /* Return success if caps are not set yet */
398 if (!sink->keep_aspect) {
399 display_rect->width = width;
400 display_rect->height = height;
404 GST_DEBUG("force-aspect-ratio is false; distorting while scaling video");
405 GST_DEBUG("render rect (%d,%d):%ux%u",
406 display_rect->x, display_rect->y,
407 display_rect->width, display_rect->height);
411 GST_DEBUG("ensure render rect within %ux%u bounds", width, height);
413 gst_vaapi_display_get_pixel_aspect_ratio(
415 &display_par_n, &display_par_d
417 GST_DEBUG("display pixel-aspect-ratio %d/%d",
418 display_par_n, display_par_d);
420 success = gst_video_calculate_display_ratio(
422 sink->video_width, sink->video_height,
423 sink->video_par_n, sink->video_par_d,
424 display_par_n, display_par_d
428 GST_DEBUG("video size %dx%d, calculated ratio %d/%d",
429 sink->video_width, sink->video_height, num, den);
431 display_rect->width = gst_util_uint64_scale_int(height, num, den);
432 if (display_rect->width <= width) {
433 GST_DEBUG("keeping window height");
434 display_rect->height = height;
437 GST_DEBUG("keeping window width");
438 display_rect->width = width;
439 display_rect->height =
440 gst_util_uint64_scale_int(width, den, num);
442 GST_DEBUG("scaling video to %ux%u", display_rect->width, display_rect->height);
444 g_assert(display_rect->width <= width);
445 g_assert(display_rect->height <= height);
447 display_rect->x = (width - display_rect->width) / 2;
448 display_rect->y = (height - display_rect->height) / 2;
450 GST_DEBUG("render rect (%d,%d):%ux%u",
451 display_rect->x, display_rect->y,
452 display_rect->width, display_rect->height);
457 gst_vaapisink_ensure_window_size(GstVaapiSink *sink, guint *pwidth, guint *pheight)
459 GstVideoRectangle src_rect, dst_rect, out_rect;
460 guint num, den, display_width, display_height, display_par_n, display_par_d;
461 gboolean success, scale;
463 if (sink->foreign_window) {
464 *pwidth = sink->window_width;
465 *pheight = sink->window_height;
469 gst_vaapi_display_get_size(sink->display, &display_width, &display_height);
470 if (sink->fullscreen) {
471 *pwidth = display_width;
472 *pheight = display_height;
476 gst_vaapi_display_get_pixel_aspect_ratio(
478 &display_par_n, &display_par_d
481 success = gst_video_calculate_display_ratio(
483 sink->video_width, sink->video_height,
484 sink->video_par_n, sink->video_par_d,
485 display_par_n, display_par_d
488 num = sink->video_par_n;
489 den = sink->video_par_d;
494 src_rect.w = gst_util_uint64_scale_int(sink->video_height, num, den);
495 src_rect.h = sink->video_height;
498 dst_rect.w = display_width;
499 dst_rect.h = display_height;
500 scale = (src_rect.w > dst_rect.w || src_rect.h > dst_rect.h);
501 gst_video_sink_center_rect(src_rect, dst_rect, &out_rect, scale);
502 *pwidth = out_rect.w;
503 *pheight = out_rect.h;
506 static inline gboolean
507 gst_vaapisink_ensure_window(GstVaapiSink *sink, guint width, guint height)
509 GstVaapiDisplay * const display = sink->display;
512 switch (sink->display_type) {
514 case GST_VAAPI_DISPLAY_TYPE_GLX:
515 sink->window = gst_vaapi_window_glx_new(display, width, height);
516 goto notify_video_overlay_interface;
519 case GST_VAAPI_DISPLAY_TYPE_X11:
520 sink->window = gst_vaapi_window_x11_new(display, width, height);
521 notify_video_overlay_interface:
524 gst_video_overlay_got_window_handle(
525 GST_VIDEO_OVERLAY(sink),
526 gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window))
531 case GST_VAAPI_DISPLAY_TYPE_WAYLAND:
532 sink->window = gst_vaapi_window_wayland_new(display, width, height);
536 GST_ERROR("unsupported display type %d", sink->display_type);
540 return sink->window != NULL;
545 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id)
548 unsigned int width, height, border_width, depth;
552 if (!gst_vaapisink_ensure_display(sink))
555 gst_vaapi_display_lock(sink->display);
557 gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
560 &x, &y, &width, &height, &border_width, &depth
562 gst_vaapi_display_unlock(sink->display);
564 if ((width != sink->window_width || height != sink->window_height) &&
565 !configure_notify_event_pending(sink, xid, width, height)) {
566 if (!gst_vaapisink_ensure_render_rect(sink, width, height))
568 sink->window_width = width;
569 sink->window_height = height;
573 gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window)) == xid)
576 gst_vaapi_window_replace(&sink->window, NULL);
578 switch (sink->display_type) {
580 case GST_VAAPI_DISPLAY_TYPE_GLX:
581 sink->window = gst_vaapi_window_glx_new_with_xid(sink->display, xid);
584 case GST_VAAPI_DISPLAY_TYPE_X11:
585 sink->window = gst_vaapi_window_x11_new_with_xid(sink->display, xid);
588 GST_ERROR("unsupported display type %d", sink->display_type);
591 return sink->window != NULL;
596 gst_vaapisink_ensure_rotation(GstVaapiSink *sink, gboolean recalc_display_rect)
598 gboolean success = FALSE;
600 g_return_val_if_fail(sink->display, FALSE);
602 if (sink->rotation == sink->rotation_req)
605 if (!sink->use_rotation) {
606 GST_WARNING("VA display does not support rotation");
610 gst_vaapi_display_lock(sink->display);
611 success = gst_vaapi_display_set_rotation(sink->display, sink->rotation_req);
612 gst_vaapi_display_unlock(sink->display);
614 GST_ERROR("failed to change VA display rotation mode");
618 if (((sink->rotation + sink->rotation_req) % 180) == 90) {
619 /* Orientation changed */
620 G_PRIMITIVE_SWAP(guint, sink->video_width, sink->video_height);
621 G_PRIMITIVE_SWAP(gint, sink->video_par_n, sink->video_par_d);
624 if (recalc_display_rect && !sink->foreign_window)
625 gst_vaapisink_ensure_render_rect(sink, sink->window_width,
626 sink->window_height);
630 sink->rotation = sink->rotation_req;
635 gst_vaapisink_ensure_video_buffer_pool(GstVaapiSink *sink, GstCaps *caps)
637 #if GST_CHECK_VERSION(1,0,0)
640 GstStructure *config;
644 if (!gst_vaapisink_ensure_display(sink))
647 if (sink->video_buffer_pool) {
648 config = gst_buffer_pool_get_config(sink->video_buffer_pool);
649 gst_buffer_pool_config_get_params(config, &pool_caps, NULL, NULL, NULL);
650 need_pool = !gst_caps_is_equal(caps, pool_caps);
651 gst_structure_free(config);
654 g_clear_object(&sink->video_buffer_pool);
655 sink->video_buffer_size = 0;
658 pool = gst_vaapi_video_buffer_pool_new(sink->display);
660 goto error_create_pool;
662 gst_video_info_init(&vi);
663 gst_video_info_from_caps(&vi, caps);
664 if (GST_VIDEO_INFO_FORMAT(&vi) == GST_VIDEO_FORMAT_ENCODED) {
665 GST_DEBUG("assume video buffer pool format is NV12");
666 gst_video_info_set_format(&vi, GST_VIDEO_FORMAT_NV12,
667 GST_VIDEO_INFO_WIDTH(&vi), GST_VIDEO_INFO_HEIGHT(&vi));
669 sink->video_buffer_size = vi.size;
671 config = gst_buffer_pool_get_config(pool);
672 gst_buffer_pool_config_set_params(config, caps, sink->video_buffer_size,
674 gst_buffer_pool_config_add_option(config,
675 GST_BUFFER_POOL_OPTION_VAAPI_VIDEO_META);
676 gst_buffer_pool_config_add_option(config,
677 GST_BUFFER_POOL_OPTION_VIDEO_META);
678 if (!gst_buffer_pool_set_config(pool, config))
679 goto error_pool_config;
680 sink->video_buffer_pool = pool;
686 GST_ERROR("failed to create buffer pool");
691 GST_ERROR("failed to reset buffer pool config");
692 gst_object_unref(pool);
701 gst_vaapisink_start(GstBaseSink *base_sink)
703 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
705 return gst_vaapisink_ensure_uploader(sink);
709 gst_vaapisink_stop(GstBaseSink *base_sink)
711 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
713 gst_buffer_replace(&sink->video_buffer, NULL);
714 #if GST_CHECK_VERSION(1,0,0)
715 g_clear_object(&sink->video_buffer_pool);
717 gst_vaapi_window_replace(&sink->window, NULL);
718 gst_vaapi_display_replace(&sink->display, NULL);
719 g_clear_object(&sink->uploader);
725 gst_vaapisink_get_caps_impl(GstBaseSink *base_sink)
727 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
728 GstCaps *out_caps, *yuv_caps;
730 #if GST_CHECK_VERSION(1,1,0)
731 out_caps = gst_static_pad_template_get_caps(&gst_vaapisink_sink_factory);
733 out_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS);
738 if (gst_vaapisink_ensure_uploader(sink)) {
739 yuv_caps = gst_vaapi_uploader_get_caps(sink->uploader);
741 out_caps = gst_caps_make_writable(out_caps);
742 gst_caps_append(out_caps, gst_caps_copy(yuv_caps));
748 #if GST_CHECK_VERSION(1,0,0)
749 static inline GstCaps *
750 gst_vaapisink_get_caps(GstBaseSink *base_sink, GstCaps *filter)
752 GstCaps *caps, *out_caps;
754 caps = gst_vaapisink_get_caps_impl(base_sink);
755 if (caps && filter) {
756 out_caps = gst_caps_intersect_full(caps, filter,
757 GST_CAPS_INTERSECT_FIRST);
758 gst_caps_unref(caps);
765 #define gst_vaapisink_get_caps gst_vaapisink_get_caps_impl
769 gst_vaapisink_set_caps(GstBaseSink *base_sink, GstCaps *caps)
771 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
772 GstVideoInfo * const vip = &sink->video_info;
773 guint win_width, win_height;
776 if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_DRM)
780 if (!gst_vaapisink_ensure_video_buffer_pool(sink, caps))
783 if (!gst_video_info_from_caps(vip, caps))
785 sink->use_video_raw = GST_VIDEO_INFO_IS_YUV(vip);
786 sink->video_width = GST_VIDEO_INFO_WIDTH(vip);
787 sink->video_height = GST_VIDEO_INFO_HEIGHT(vip);
788 sink->video_par_n = GST_VIDEO_INFO_PAR_N(vip);
789 sink->video_par_d = GST_VIDEO_INFO_PAR_D(vip);
790 GST_DEBUG("video pixel-aspect-ratio %d/%d",
791 sink->video_par_n, sink->video_par_d);
793 gst_caps_replace(&sink->caps, caps);
795 if (!gst_vaapisink_ensure_display(sink))
798 #if !GST_CHECK_VERSION(1,0,0)
799 if (sink->use_video_raw) {
800 /* Ensure the uploader is set up for upstream allocated buffers */
801 if (!gst_vaapi_uploader_ensure_display(sink->uploader, sink->display))
803 if (!gst_vaapi_uploader_ensure_caps(sink->uploader, caps, NULL))
808 gst_vaapisink_ensure_rotation(sink, FALSE);
810 gst_vaapisink_ensure_window_size(sink, &win_width, &win_height);
812 if (!sink->foreign_window || sink->fullscreen)
813 gst_vaapi_window_set_size(sink->window, win_width, win_height);
816 gst_vaapi_display_lock(sink->display);
817 gst_video_overlay_prepare_window_handle(GST_VIDEO_OVERLAY(sink));
818 gst_vaapi_display_unlock(sink->display);
821 if (!gst_vaapisink_ensure_window(sink, win_width, win_height))
823 gst_vaapi_window_set_fullscreen(sink->window, sink->fullscreen);
824 gst_vaapi_window_show(sink->window);
825 gst_vaapi_window_get_size(sink->window, &win_width, &win_height);
827 sink->window_width = win_width;
828 sink->window_height = win_height;
829 GST_DEBUG("window size %ux%u", win_width, win_height);
831 return gst_vaapisink_ensure_render_rect(sink, win_width, win_height);
836 render_background(GstVaapiSink *sink)
838 /* Original code from Mirco Muller (MacSlow):
839 <http://cgit.freedesktop.org/~macslow/gl-gst-player/> */
840 GLfloat fStartX = 0.0f;
841 GLfloat fStartY = 0.0f;
842 GLfloat fWidth = (GLfloat)sink->window_width;
843 GLfloat fHeight = (GLfloat)sink->window_height;
845 glClear(GL_COLOR_BUFFER_BIT);
848 /* top third, darker grey to white */
849 glColor3f(0.85f, 0.85f, 0.85f);
850 glVertex3f(fStartX, fStartY, 0.0f);
851 glColor3f(0.85f, 0.85f, 0.85f);
852 glVertex3f(fStartX + fWidth, fStartY, 0.0f);
853 glColor3f(1.0f, 1.0f, 1.0f);
854 glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
855 glColor3f(1.0f, 1.0f, 1.0f);
856 glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
858 /* middle third, just plain white */
859 glColor3f(1.0f, 1.0f, 1.0f);
860 glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
861 glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
862 glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
863 glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
865 /* bottom third, white to lighter grey */
866 glColor3f(1.0f, 1.0f, 1.0f);
867 glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
868 glColor3f(1.0f, 1.0f, 1.0f);
869 glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
870 glColor3f(0.62f, 0.66f, 0.69f);
871 glVertex3f(fStartX + fWidth, fStartY + fHeight, 0.0f);
872 glColor3f(0.62f, 0.66f, 0.69f);
873 glVertex3f(fStartX, fStartY + fHeight, 0.0f);
879 render_frame(GstVaapiSink *sink, GstVaapiSurface *surface,
880 const GstVaapiRectangle *surface_rect)
882 const guint x1 = sink->display_rect.x;
883 const guint x2 = sink->display_rect.x + sink->display_rect.width;
884 const guint y1 = sink->display_rect.y;
885 const guint y2 = sink->display_rect.y + sink->display_rect.height;
886 gfloat tx1, tx2, ty1, ty2;
890 gst_vaapi_surface_get_size(surface, &width, &height);
891 tx1 = (gfloat)surface_rect->x / width;
892 ty1 = (gfloat)surface_rect->y / height;
893 tx2 = (gfloat)(surface_rect->x + surface_rect->width) / width;
894 ty2 = (gfloat)(surface_rect->y + surface_rect->height) / height;
903 glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
906 glTexCoord2f(tx1, ty1); glVertex2i(x1, y1);
907 glTexCoord2f(tx1, ty2); glVertex2i(x1, y2);
908 glTexCoord2f(tx2, ty2); glVertex2i(x2, y2);
909 glTexCoord2f(tx2, ty1); glVertex2i(x2, y1);
915 render_reflection(GstVaapiSink *sink)
917 const guint x1 = sink->display_rect.x;
918 const guint x2 = sink->display_rect.x + sink->display_rect.width;
919 const guint y1 = sink->display_rect.y;
920 const guint rh = sink->display_rect.height / 5;
921 GLfloat ry = 1.0f - (GLfloat)rh / (GLfloat)sink->display_rect.height;
925 glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
926 glTexCoord2f(0.0f, 1.0f); glVertex2i(x1, y1);
927 glTexCoord2f(1.0f, 1.0f); glVertex2i(x2, y1);
929 glColor4f(1.0f, 1.0f, 1.0f, 0.0f);
930 glTexCoord2f(1.0f, ry); glVertex2i(x2, y1 + rh);
931 glTexCoord2f(0.0f, ry); glVertex2i(x1, y1 + rh);
937 gst_vaapisink_ensure_texture(GstVaapiSink *sink, GstVaapiSurface *surface)
939 GstVideoRectangle tex_rect, dis_rect, out_rect;
945 gst_vaapi_surface_get_size(surface, &width, &height);
951 gst_vaapi_display_get_size(sink->display, &width, &height);
957 gst_video_sink_center_rect(tex_rect, dis_rect, &out_rect, TRUE);
959 /* XXX: use surface size for now since some VA drivers have issues
960 with downscaling to the provided texture size. i.e. we should be
961 using the resulting out_rect size, which preserves the aspect
962 ratio of the surface */
965 GST_INFO("texture size %ux%u", width, height);
967 sink->texture = gst_vaapi_texture_new(sink->display,
968 GL_TEXTURE_2D, GL_BGRA, width, height);
969 return sink->texture != NULL;
973 gst_vaapisink_show_frame_glx(
975 GstVaapiSurface *surface,
976 const GstVaapiRectangle *surface_rect,
980 GstVaapiWindowGLX * const window = GST_VAAPI_WINDOW_GLX(sink->window);
984 gst_vaapi_window_glx_make_current(window);
985 if (!gst_vaapisink_ensure_texture(sink, surface))
986 goto error_create_texture;
987 if (!gst_vaapi_texture_put_surface(sink->texture, surface, flags))
988 goto error_transfer_surface;
990 target = gst_vaapi_texture_get_target(sink->texture);
991 texture = gst_vaapi_texture_get_id(sink->texture);
992 if (target != GL_TEXTURE_2D || !texture)
995 if (sink->use_reflection)
996 render_background(sink);
999 glBindTexture(target, texture);
1001 if (sink->use_reflection) {
1003 glRotatef(20.0f, 0.0f, 1.0f, 0.0f);
1004 glTranslatef(50.0f, 0.0f, 0.0f);
1006 render_frame(sink, surface, surface_rect);
1007 if (sink->use_reflection) {
1009 glTranslatef(0.0, (GLfloat)sink->display_rect.height + 5.0f, 0.0f);
1010 render_reflection(sink);
1015 glBindTexture(target, 0);
1017 gst_vaapi_window_glx_swap_buffers(window);
1021 error_create_texture:
1023 GST_DEBUG("could not create VA/GLX texture");
1026 error_transfer_surface:
1028 GST_DEBUG("could not transfer VA surface to texture");
1034 static inline gboolean
1035 gst_vaapisink_put_surface(
1037 GstVaapiSurface *surface,
1038 const GstVaapiRectangle *surface_rect,
1042 if (!gst_vaapi_window_put_surface(sink->window, surface,
1043 surface_rect, &sink->display_rect, flags)) {
1044 GST_DEBUG("could not render VA surface");
1050 #if GST_CHECK_VERSION(1,0,0)
1051 static GstFlowReturn
1052 gst_vaapisink_get_render_buffer(GstVaapiSink *sink, GstBuffer *src_buffer,
1053 GstBuffer **out_buffer_ptr)
1055 GstVaapiVideoMeta *meta;
1056 GstBuffer *out_buffer;
1057 GstBufferPoolAcquireParams params = { 0, };
1058 GstVideoFrame src_frame, out_frame;
1061 meta = gst_buffer_get_vaapi_video_meta(src_buffer);
1063 *out_buffer_ptr = gst_buffer_ref(src_buffer);
1067 if (!sink->use_video_raw) {
1068 GST_ERROR("unsupported video buffer");
1069 return GST_FLOW_EOS;
1072 GST_DEBUG("buffer %p not from our pool, copying", src_buffer);
1074 *out_buffer_ptr = NULL;
1075 if (!sink->video_buffer_pool)
1078 if (!gst_buffer_pool_set_active(sink->video_buffer_pool, TRUE))
1079 goto error_activate_pool;
1081 params.flags = GST_BUFFER_POOL_ACQUIRE_FLAG_DONTWAIT;
1082 ret = gst_buffer_pool_acquire_buffer(sink->video_buffer_pool, &out_buffer,
1084 if (ret != GST_FLOW_OK)
1085 goto error_create_buffer;
1087 if (!gst_video_frame_map(&src_frame, &sink->video_info, src_buffer,
1089 goto error_map_src_buffer;
1091 if (!gst_video_frame_map(&out_frame, &sink->video_info, out_buffer,
1093 goto error_map_dst_buffer;
1095 gst_video_frame_copy(&out_frame, &src_frame);
1096 gst_video_frame_unmap(&out_frame);
1097 gst_video_frame_unmap(&src_frame);
1099 *out_buffer_ptr = out_buffer;
1104 GST_ERROR("no buffer pool was negotiated");
1105 return GST_FLOW_ERROR;
1106 error_activate_pool:
1107 GST_ERROR("failed to activate buffer pool");
1108 return GST_FLOW_ERROR;
1109 error_create_buffer:
1110 GST_WARNING("failed to create image. Skipping this frame");
1112 error_map_dst_buffer:
1113 gst_video_frame_unmap(&src_frame);
1115 error_map_src_buffer:
1116 GST_WARNING("failed to map buffer. Skipping this frame");
1117 gst_buffer_unref(out_buffer);
1121 static GstFlowReturn
1122 gst_vaapisink_get_render_buffer(GstVaapiSink *sink, GstBuffer *src_buffer,
1123 GstBuffer **out_buffer_ptr)
1125 GstVaapiVideoMeta *meta;
1126 GstBuffer *out_buffer;
1128 *out_buffer_ptr = NULL;
1129 meta = gst_buffer_get_vaapi_video_meta(src_buffer);
1131 out_buffer = gst_buffer_ref(src_buffer);
1132 else if (sink->use_video_raw) {
1133 out_buffer = gst_vaapi_uploader_get_buffer(sink->uploader);
1135 goto error_create_buffer;
1138 GST_ERROR("unsupported video buffer");
1139 return GST_FLOW_EOS;
1142 if (sink->use_video_raw &&
1143 !gst_vaapi_uploader_process(sink->uploader, src_buffer, out_buffer))
1144 goto error_copy_buffer;
1146 *out_buffer_ptr = out_buffer;
1150 error_create_buffer:
1151 GST_WARNING("failed to create buffer. Skipping this frame");
1154 GST_WARNING("failed to copy buffers. Skipping this frame");
1155 gst_buffer_unref(out_buffer);
1160 static GstFlowReturn
1161 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *src_buffer)
1163 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1164 GstVaapiVideoMeta *meta;
1165 GstVaapiSurface *surface;
1169 GstVaapiRectangle *surface_rect = NULL;
1170 #if GST_CHECK_VERSION(1,0,0)
1171 GstVaapiRectangle tmp_rect;
1175 #if GST_CHECK_VERSION(1,0,0)
1176 GstVideoCropMeta * const crop_meta =
1177 gst_buffer_get_video_crop_meta(src_buffer);
1179 surface_rect = &tmp_rect;
1180 surface_rect->x = crop_meta->x;
1181 surface_rect->y = crop_meta->y;
1182 surface_rect->width = crop_meta->width;
1183 surface_rect->height = crop_meta->height;
1187 ret = gst_vaapisink_get_render_buffer(sink, src_buffer, &buffer);
1188 if (ret != GST_FLOW_OK || !buffer)
1191 meta = gst_buffer_get_vaapi_video_meta(buffer);
1192 if (sink->display != gst_vaapi_video_meta_get_display(meta))
1193 gst_vaapi_display_replace(&sink->display,
1194 gst_vaapi_video_meta_get_display(meta));
1199 gst_vaapisink_ensure_rotation(sink, TRUE);
1201 surface = gst_vaapi_video_meta_get_surface(meta);
1205 GST_DEBUG("render surface %" GST_VAAPI_ID_FORMAT,
1206 GST_VAAPI_ID_ARGS(gst_vaapi_surface_get_id(surface)));
1209 surface_rect = (GstVaapiRectangle *)
1210 gst_vaapi_video_meta_get_render_rect(meta);
1213 GST_DEBUG("render rect (%d,%d), size %ux%u",
1214 surface_rect->x, surface_rect->y,
1215 surface_rect->width, surface_rect->height);
1217 flags = gst_vaapi_video_meta_get_render_flags(meta);
1219 if (!gst_vaapi_apply_composition(surface, src_buffer))
1220 GST_WARNING("could not update subtitles");
1222 switch (sink->display_type) {
1224 case GST_VAAPI_DISPLAY_TYPE_DRM:
1229 case GST_VAAPI_DISPLAY_TYPE_GLX:
1231 goto put_surface_x11;
1232 success = gst_vaapisink_show_frame_glx(sink, surface, surface_rect,
1237 case GST_VAAPI_DISPLAY_TYPE_X11:
1239 success = gst_vaapisink_put_surface(sink, surface, surface_rect, flags);
1243 case GST_VAAPI_DISPLAY_TYPE_WAYLAND:
1244 success = gst_vaapisink_put_surface(sink, surface, surface_rect, flags);
1248 GST_ERROR("unsupported display type %d", sink->display_type);
1255 /* Retain VA surface until the next one is displayed */
1256 if (sink->use_overlay)
1257 gst_buffer_replace(&sink->video_buffer, buffer);
1258 gst_buffer_unref(buffer);
1262 gst_buffer_unref(buffer);
1263 return GST_FLOW_EOS;
1266 #if GST_CHECK_VERSION(1,0,0)
1268 gst_vaapisink_propose_allocation(GstBaseSink *base_sink, GstQuery *query)
1270 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1271 GstCaps *caps = NULL;
1274 gst_query_parse_allocation(query, &caps, &need_pool);
1279 if (!gst_vaapisink_ensure_video_buffer_pool(sink, caps))
1281 gst_query_add_allocation_pool(query, sink->video_buffer_pool,
1282 sink->video_buffer_size, 0, 0);
1285 gst_query_add_allocation_meta(query,
1286 GST_VAAPI_VIDEO_META_API_TYPE, NULL);
1287 gst_query_add_allocation_meta(query,
1288 GST_VIDEO_META_API_TYPE, NULL);
1289 gst_query_add_allocation_meta(query,
1290 GST_VIDEO_CROP_META_API_TYPE, NULL);
1291 gst_query_add_allocation_meta(query,
1292 GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL);
1298 GST_ERROR("no caps specified");
1303 static GstFlowReturn
1304 gst_vaapisink_buffer_alloc(
1305 GstBaseSink *base_sink,
1312 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1318 if (!sink->use_video_raw) {
1319 /* Note: this code path is rarely used but for raw YUV formats
1320 from custom pipeline. Otherwise, GstBaseSink::set_caps() is
1321 called first, and GstBaseSink::buffer_alloc() is not called
1322 in VA surface format mode */
1323 if (!gst_video_info_from_caps(&vi, caps))
1324 return GST_FLOW_NOT_SUPPORTED;
1325 if (!GST_VIDEO_INFO_IS_YUV(&vi))
1329 if (!gst_vaapi_uploader_ensure_display(sink->uploader, sink->display))
1330 return GST_FLOW_NOT_SUPPORTED;
1331 if (!gst_vaapi_uploader_ensure_caps(sink->uploader, caps, NULL))
1332 return GST_FLOW_NOT_SUPPORTED;
1334 buf = gst_vaapi_uploader_get_buffer(sink->uploader);
1336 GST_WARNING("failed to allocate resources for raw YUV buffer");
1337 return GST_FLOW_NOT_SUPPORTED;
1345 #if GST_CHECK_VERSION(1,1,0)
1347 gst_vaapisink_set_context(GstElement *element, GstContext *context)
1349 GstVaapiSink * const sink = GST_VAAPISINK(element);
1350 GstVaapiDisplay *display = NULL;
1352 if (gst_vaapi_video_context_get_display(context, &display)) {
1353 GST_INFO_OBJECT(element, "set display %p", display);
1354 gst_vaapi_display_replace(&sink->display, display);
1360 gst_vaapisink_query(GstBaseSink *base_sink, GstQuery *query)
1362 GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1364 GST_INFO_OBJECT(sink, "query type %s", GST_QUERY_TYPE_NAME(query));
1366 if (gst_vaapi_reply_to_query(query, sink->display)) {
1367 GST_DEBUG("sharing display %p", sink->display);
1371 switch(GST_QUERY_TYPE(query)) {
1372 #if GST_CHECK_VERSION(1,1,0)
1373 case GST_QUERY_CONTEXT: {
1374 const gchar *context_type = NULL;
1376 if (gst_query_parse_context_type(query, &context_type) &&
1377 !g_strcmp0(context_type, GST_VAAPI_DISPLAY_CONTEXT_TYPE_NAME) &&
1379 GstContext *context;
1381 context = gst_vaapi_video_context_new_with_display(
1382 sink->display, FALSE);
1383 gst_query_set_context(query, context);
1384 gst_context_unref(context);
1394 return GST_BASE_SINK_CLASS(gst_vaapisink_parent_class)->query(base_sink,
1399 gst_vaapisink_finalize(GObject *object)
1401 gst_vaapisink_destroy(GST_VAAPISINK(object));
1403 G_OBJECT_CLASS(gst_vaapisink_parent_class)->finalize(object);
1407 gst_vaapisink_set_property(
1410 const GValue *value,
1414 GstVaapiSink * const sink = GST_VAAPISINK(object);
1417 case PROP_DISPLAY_TYPE:
1418 sink->display_type = g_value_get_enum(value);
1420 case PROP_FULLSCREEN:
1421 sink->fullscreen = g_value_get_boolean(value);
1423 case PROP_SYNCHRONOUS:
1424 sink->synchronous = g_value_get_boolean(value);
1427 sink->use_glx = g_value_get_boolean(value);
1429 case PROP_USE_REFLECTION:
1430 sink->use_reflection = g_value_get_boolean(value);
1433 sink->rotation_req = g_value_get_enum(value);
1435 case PROP_FORCE_ASPECT_RATIO:
1436 sink->keep_aspect = g_value_get_boolean(value);
1439 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
1445 gst_vaapisink_get_property(
1452 GstVaapiSink * const sink = GST_VAAPISINK(object);
1455 case PROP_DISPLAY_TYPE:
1456 g_value_set_enum(value, sink->display_type);
1458 case PROP_FULLSCREEN:
1459 g_value_set_boolean(value, sink->fullscreen);
1461 case PROP_SYNCHRONOUS:
1462 g_value_set_boolean(value, sink->synchronous);
1465 g_value_set_boolean(value, sink->use_glx);
1467 case PROP_USE_REFLECTION:
1468 g_value_set_boolean(value, sink->use_reflection);
1471 g_value_set_enum(value, sink->rotation);
1473 case PROP_FORCE_ASPECT_RATIO:
1474 g_value_set_boolean(value, sink->keep_aspect);
1477 G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
1483 gst_vaapisink_class_init(GstVaapiSinkClass *klass)
1485 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
1486 GstElementClass * const element_class = GST_ELEMENT_CLASS(klass);
1487 GstBaseSinkClass * const basesink_class = GST_BASE_SINK_CLASS(klass);
1488 GstPadTemplate *pad_template;
1490 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapisink,
1491 GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
1493 object_class->finalize = gst_vaapisink_finalize;
1494 object_class->set_property = gst_vaapisink_set_property;
1495 object_class->get_property = gst_vaapisink_get_property;
1497 basesink_class->start = gst_vaapisink_start;
1498 basesink_class->stop = gst_vaapisink_stop;
1499 basesink_class->get_caps = gst_vaapisink_get_caps;
1500 basesink_class->set_caps = gst_vaapisink_set_caps;
1501 basesink_class->preroll = gst_vaapisink_show_frame;
1502 basesink_class->render = gst_vaapisink_show_frame;
1503 basesink_class->query = gst_vaapisink_query;
1504 #if GST_CHECK_VERSION(1,0,0)
1505 basesink_class->propose_allocation = gst_vaapisink_propose_allocation;
1507 basesink_class->buffer_alloc = gst_vaapisink_buffer_alloc;
1510 #if GST_CHECK_VERSION(1,1,0)
1511 element_class->set_context = gst_vaapisink_set_context;
1514 gst_element_class_set_static_metadata(element_class,
1518 "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
1520 pad_template = gst_static_pad_template_get(&gst_vaapisink_sink_factory);
1521 gst_element_class_add_pad_template(element_class, pad_template);
1523 g_object_class_install_property
1526 g_param_spec_enum("display",
1528 "display type to use",
1529 GST_VAAPI_TYPE_DISPLAY_TYPE,
1530 GST_VAAPI_DISPLAY_TYPE_ANY,
1531 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1534 g_object_class_install_property
1537 g_param_spec_boolean("use-glx",
1539 "Enables OpenGL rendering",
1541 G_PARAM_READWRITE));
1543 g_object_class_install_property
1545 PROP_USE_REFLECTION,
1546 g_param_spec_boolean("use-reflection",
1547 "Reflection effect",
1548 "Enables OpenGL reflection effect",
1550 G_PARAM_READWRITE));
1553 g_object_class_install_property
1556 g_param_spec_boolean("fullscreen",
1558 "Requests window in fullscreen state",
1560 G_PARAM_READWRITE));
1563 * GstVaapiSink:synchronous:
1565 * When enabled, runs the X display in synchronous mode. Note that
1566 * this is used only for debugging.
1568 g_object_class_install_property
1571 g_param_spec_boolean("synchronous",
1573 "Toggles X display synchronous mode",
1575 G_PARAM_READWRITE));
1578 * GstVaapiSink:rotation:
1580 * The VA display rotation mode, expressed as a #GstVaapiRotation.
1582 g_object_class_install_property
1585 g_param_spec_enum(GST_VAAPI_DISPLAY_PROP_ROTATION,
1587 "The display rotation mode",
1588 GST_VAAPI_TYPE_ROTATION,
1590 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1593 * GstVaapiSink:force-aspect-ratio:
1595 * When enabled, scaling respects video aspect ratio; when disabled, the
1596 * video is distorted to fit the window.
1598 g_object_class_install_property
1600 PROP_FORCE_ASPECT_RATIO,
1601 g_param_spec_boolean("force-aspect-ratio",
1602 "Force aspect ratio",
1603 "When enabled, scaling will respect original aspect ratio",
1605 G_PARAM_READWRITE));
1609 gst_vaapisink_init(GstVaapiSink *sink)
1612 sink->display = NULL;
1613 sink->window = NULL;
1614 sink->window_width = 0;
1615 sink->window_height = 0;
1616 sink->texture = NULL;
1617 sink->video_buffer = NULL;
1618 sink->video_width = 0;
1619 sink->video_height = 0;
1620 sink->video_par_n = 1;
1621 sink->video_par_d = 1;
1622 sink->foreign_window = FALSE;
1623 sink->fullscreen = FALSE;
1624 sink->synchronous = FALSE;
1625 sink->display_type = DEFAULT_DISPLAY_TYPE;
1626 sink->rotation = DEFAULT_ROTATION;
1627 sink->rotation_req = DEFAULT_ROTATION;
1628 sink->use_reflection = FALSE;
1629 sink->use_overlay = FALSE;
1630 sink->use_rotation = FALSE;
1631 sink->keep_aspect = TRUE;
1632 gst_video_info_init(&sink->video_info);