plugins: use new video buffer pools.
[platform/upstream/gstreamer-vaapi.git] / gst / vaapi / gstvaapisink.c
1 /*
2  *  gstvaapisink.c - VA-API video sink
3  *
4  *  Copyright (C) 2010-2011 Splitted-Desktop Systems
5  *  Copyright (C) 2011-2013 Intel Corporation
6  *
7  *  This library is free software; you can redistribute it and/or
8  *  modify it under the terms of the GNU Lesser General Public License
9  *  as published by the Free Software Foundation; either version 2.1
10  *  of the License, or (at your option) any later version.
11  *
12  *  This library is distributed in the hope that it will be useful,
13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  *  Lesser General Public License for more details.
16  *
17  *  You should have received a copy of the GNU Lesser General Public
18  *  License along with this library; if not, write to the Free
19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20  *  Boston, MA 02110-1301 USA
21  */
22
23 /**
24  * SECTION:gstvaapisink
25  * @short_description: A VA-API based videosink
26  *
27  * vaapisink renders video frames to a drawable (X #Window) on a local
28  * display using the Video Acceleration (VA) API. The element will
29  * create its own internal window and render into it.
30  */
31
32 #include "gst/vaapi/sysdeps.h"
33 #include <gst/gst.h>
34 #include <gst/video/video.h>
35 #include <gst/video/videocontext.h>
36 #include <gst/vaapi/gstvaapivalue.h>
37 #if USE_DRM
38 # include <gst/vaapi/gstvaapidisplay_drm.h>
39 #endif
40 #if USE_X11
41 # include <gst/vaapi/gstvaapidisplay_x11.h>
42 # include <gst/vaapi/gstvaapiwindow_x11.h>
43 #endif
44 #if USE_GLX
45 # include <gst/vaapi/gstvaapidisplay_glx.h>
46 # include <gst/vaapi/gstvaapiwindow_glx.h>
47 #endif
48 #if USE_WAYLAND
49 # include <gst/vaapi/gstvaapidisplay_wayland.h>
50 # include <gst/vaapi/gstvaapiwindow_wayland.h>
51 #endif
52
53 /* Supported interfaces */
54 #if GST_CHECK_VERSION(1,0,0)
55 # include <gst/video/videooverlay.h>
56 #else
57 # include <gst/interfaces/xoverlay.h>
58
59 # define GST_TYPE_VIDEO_OVERLAY         GST_TYPE_X_OVERLAY
60 # define GST_VIDEO_OVERLAY              GST_X_OVERLAY
61 # define GstVideoOverlay                GstXOverlay
62 # define GstVideoOverlayInterface       GstXOverlayClass
63
64 # define gst_video_overlay_prepare_window_handle(sink) \
65     gst_x_overlay_prepare_xwindow_id(sink)
66 # define gst_video_overlay_got_window_handle(sink, window_handle) \
67     gst_x_overlay_got_window_handle(sink, window_handle)
68 #endif
69
70 #include "gstvaapisink.h"
71 #include "gstvaapipluginutil.h"
72 #include "gstvaapivideometa.h"
73 #include "gstvaapivideobufferpool.h"
74 #include "gstvaapivideomemory.h"
75
76 #define GST_PLUGIN_NAME "vaapisink"
77 #define GST_PLUGIN_DESC "A VA-API based videosink"
78
79 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapisink);
80 #define GST_CAT_DEFAULT gst_debug_vaapisink
81
82 /* Default template */
83 static const char gst_vaapisink_sink_caps_str[] =
84 #if !GST_CHECK_VERSION(1,0,0)
85     "video/x-raw-yuv, "
86     "width  = (int) [ 1, MAX ], "
87     "height = (int) [ 1, MAX ]; "
88 #endif
89     GST_VAAPI_SURFACE_CAPS;
90
91 static GstStaticPadTemplate gst_vaapisink_sink_factory =
92     GST_STATIC_PAD_TEMPLATE(
93         "sink",
94         GST_PAD_SINK,
95         GST_PAD_ALWAYS,
96         GST_STATIC_CAPS(gst_vaapisink_sink_caps_str));
97
98 /* GstImplementsInterface interface */
99 #if !GST_CHECK_VERSION(1,0,0)
100 static gboolean
101 gst_vaapisink_implements_interface_supported(
102     GstImplementsInterface *iface,
103     GType                   type
104 )
105 {
106     return (type == GST_TYPE_VIDEO_CONTEXT ||
107             type == GST_TYPE_VIDEO_OVERLAY);
108 }
109
110 static void
111 gst_vaapisink_implements_iface_init(GstImplementsInterfaceClass *iface)
112 {
113     iface->supported = gst_vaapisink_implements_interface_supported;
114 }
115 #endif
116
117 /* GstVideoContext interface */
118 static void
119 gst_vaapisink_set_video_context(GstVideoContext *context, const gchar *type,
120     const GValue *value)
121 {
122   GstVaapiSink *sink = GST_VAAPISINK (context);
123   gst_vaapi_set_display (type, value, &sink->display);
124 }
125
126 static void
127 gst_vaapisink_video_context_iface_init(GstVideoContextInterface *iface)
128 {
129     iface->set_context = gst_vaapisink_set_video_context;
130 }
131
132 static void
133 gst_vaapisink_video_overlay_iface_init(GstVideoOverlayInterface *iface);
134
135 G_DEFINE_TYPE_WITH_CODE(
136     GstVaapiSink,
137     gst_vaapisink,
138     GST_TYPE_VIDEO_SINK,
139 #if !GST_CHECK_VERSION(1,0,0)
140     G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
141                           gst_vaapisink_implements_iface_init);
142 #endif
143     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
144                           gst_vaapisink_video_context_iface_init);
145     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_OVERLAY,
146                           gst_vaapisink_video_overlay_iface_init))
147
148 enum {
149     PROP_0,
150
151     PROP_DISPLAY_TYPE,
152     PROP_FULLSCREEN,
153     PROP_SYNCHRONOUS,
154     PROP_USE_REFLECTION,
155     PROP_ROTATION,
156 };
157
158 #define DEFAULT_DISPLAY_TYPE            GST_VAAPI_DISPLAY_TYPE_ANY
159 #define DEFAULT_ROTATION                GST_VAAPI_ROTATION_0
160
161 /* GstVideoOverlay interface */
162
163 #if USE_X11
164 static gboolean
165 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id);
166 #endif
167
168 static GstFlowReturn
169 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *buffer);
170
171 static void
172 gst_vaapisink_video_overlay_set_window_handle(GstVideoOverlay *overlay, guintptr window)
173 {
174     GstVaapiSink * const sink = GST_VAAPISINK(overlay);
175
176     /* Disable GLX rendering when vaapisink is using a foreign X
177        window. It's pretty much useless */
178     if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_GLX)
179         sink->display_type = GST_VAAPI_DISPLAY_TYPE_X11;
180
181     sink->foreign_window = TRUE;
182
183     switch (sink->display_type) {
184 #if USE_X11
185     case GST_VAAPI_DISPLAY_TYPE_X11:
186         gst_vaapisink_ensure_window_xid(sink, window);
187         break;
188 #endif
189     default:
190         break;
191     }
192 }
193
194 static void
195 gst_vaapisink_video_overlay_set_render_rectangle(
196     GstVideoOverlay *overlay,
197     gint         x,
198     gint         y,
199     gint         width,
200     gint         height
201 )
202 {
203     GstVaapiSink * const sink = GST_VAAPISINK(overlay);
204     GstVaapiRectangle * const display_rect = &sink->display_rect;
205
206     display_rect->x      = x;
207     display_rect->y      = y;
208     display_rect->width  = width;
209     display_rect->height = height;
210     
211     GST_DEBUG("render rect (%d,%d):%ux%u",
212               display_rect->x, display_rect->y,
213               display_rect->width, display_rect->height);
214 }
215
216 static void
217 gst_vaapisink_video_overlay_expose(GstVideoOverlay *overlay)
218 {
219     GstBaseSink * const base_sink = GST_BASE_SINK(overlay);
220     GstBuffer *buffer;
221
222 #if GST_CHECK_VERSION(1,0,0)
223     GstSample * const sample = gst_base_sink_get_last_sample(base_sink);
224     if (!sample)
225         return;
226     buffer = gst_sample_get_buffer(sample);
227 #else
228     buffer = gst_base_sink_get_last_buffer(base_sink);
229 #endif
230     if (buffer) {
231         gst_vaapisink_show_frame(base_sink, buffer);
232         gst_buffer_unref(buffer);
233     }
234 }
235
236 static void
237 gst_vaapisink_video_overlay_iface_init(GstVideoOverlayInterface *iface)
238 {
239     iface->set_window_handle    = gst_vaapisink_video_overlay_set_window_handle;
240     iface->set_render_rectangle = gst_vaapisink_video_overlay_set_render_rectangle;
241     iface->expose               = gst_vaapisink_video_overlay_expose;
242 }
243
244 static void
245 gst_vaapisink_destroy(GstVaapiSink *sink)
246 {
247     gst_buffer_replace(&sink->video_buffer, NULL);
248     g_clear_object(&sink->texture);
249     g_clear_object(&sink->display);
250     g_clear_object(&sink->uploader);
251
252     gst_caps_replace(&sink->caps, NULL);
253 }
254
255 #if USE_X11
256 /* Checks whether a ConfigureNotify event is in the queue */
257 typedef struct _ConfigureNotifyEventPendingArgs ConfigureNotifyEventPendingArgs;
258 struct _ConfigureNotifyEventPendingArgs {
259     Window      window;
260     guint       width;
261     guint       height;
262     gboolean    match;
263 };
264
265 static Bool
266 configure_notify_event_pending_cb(Display *dpy, XEvent *xev, XPointer arg)
267 {
268     ConfigureNotifyEventPendingArgs * const args =
269         (ConfigureNotifyEventPendingArgs *)arg;
270
271     if (xev->type == ConfigureNotify &&
272         xev->xconfigure.window == args->window &&
273         xev->xconfigure.width  == args->width  &&
274         xev->xconfigure.height == args->height)
275         args->match = TRUE;
276
277     /* XXX: this is a hack to traverse the whole queue because we
278        can't use XPeekIfEvent() since it could block */
279     return False;
280 }
281
282 static gboolean
283 configure_notify_event_pending(
284     GstVaapiSink *sink,
285     Window        window,
286     guint         width,
287     guint         height
288 )
289 {
290     ConfigureNotifyEventPendingArgs args;
291     XEvent xev;
292
293     args.window = window;
294     args.width  = width;
295     args.height = height;
296     args.match  = FALSE;
297
298     /* XXX: don't use XPeekIfEvent() because it might block */
299     XCheckIfEvent(
300         gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
301         &xev,
302         configure_notify_event_pending_cb, (XPointer)&args
303     );
304     return args.match;
305 }
306 #endif
307
308 static const gchar *
309 get_display_type_name(GstVaapiDisplayType display_type)
310 {
311     gpointer const klass = g_type_class_peek(GST_VAAPI_TYPE_DISPLAY_TYPE);
312     GEnumValue * const e = g_enum_get_value(klass, display_type);
313
314     if (e)
315         return e->value_name;
316     return "<unknown-type>";
317 }
318
319 static inline gboolean
320 gst_vaapisink_ensure_display(GstVaapiSink *sink)
321 {
322     GstVaapiDisplayType display_type;
323     GstVaapiRenderMode render_mode;
324
325     if (!gst_vaapi_ensure_display(sink, sink->display_type, &sink->display))
326         return FALSE;
327
328     display_type = gst_vaapi_display_get_display_type(sink->display);
329     if (display_type != sink->display_type) {
330         GST_INFO("created %s %p", get_display_type_name(display_type),
331             sink->display);
332         sink->display_type = display_type;
333
334         sink->use_overlay =
335             gst_vaapi_display_get_render_mode(sink->display, &render_mode) &&
336             render_mode == GST_VAAPI_RENDER_MODE_OVERLAY;
337         GST_DEBUG("use %s rendering mode", sink->use_overlay ? "overlay" : "texture");
338
339         sink->use_rotation = gst_vaapi_display_has_property(
340             sink->display, GST_VAAPI_DISPLAY_PROP_ROTATION);
341     }
342     return TRUE;
343 }
344
345 static gboolean
346 gst_vaapisink_ensure_uploader(GstVaapiSink *sink)
347 {
348     if (!gst_vaapisink_ensure_display(sink))
349         return FALSE;
350
351     if (!sink->uploader) {
352         sink->uploader = gst_vaapi_uploader_new(sink->display);
353         if (!sink->uploader)
354             return FALSE;
355     }
356     return TRUE;
357 }
358
359 static gboolean
360 gst_vaapisink_ensure_render_rect(GstVaapiSink *sink, guint width, guint height)
361 {
362     GstVaapiRectangle * const display_rect = &sink->display_rect;
363     guint num, den, display_par_n, display_par_d;
364     gboolean success;
365
366     /* Return success if caps are not set yet */
367     if (!sink->caps)
368         return TRUE;
369
370     GST_DEBUG("ensure render rect within %ux%u bounds", width, height);
371
372     gst_vaapi_display_get_pixel_aspect_ratio(
373         sink->display,
374         &display_par_n, &display_par_d
375     );
376     GST_DEBUG("display pixel-aspect-ratio %d/%d",
377               display_par_n, display_par_d);
378
379     success = gst_video_calculate_display_ratio(
380         &num, &den,
381         sink->video_width, sink->video_height,
382         sink->video_par_n, sink->video_par_d,
383         display_par_n, display_par_d
384     );
385     if (!success)
386         return FALSE;
387     GST_DEBUG("video size %dx%d, calculated ratio %d/%d",
388               sink->video_width, sink->video_height, num, den);
389
390     display_rect->width = gst_util_uint64_scale_int(height, num, den);
391     if (display_rect->width <= width) {
392         GST_DEBUG("keeping window height");
393         display_rect->height = height;
394     }
395     else {
396         GST_DEBUG("keeping window width");
397         display_rect->width  = width;
398         display_rect->height =
399             gst_util_uint64_scale_int(width, den, num);
400     }
401     GST_DEBUG("scaling video to %ux%u", display_rect->width, display_rect->height);
402
403     g_assert(display_rect->width  <= width);
404     g_assert(display_rect->height <= height);
405
406     display_rect->x = (width  - display_rect->width)  / 2;
407     display_rect->y = (height - display_rect->height) / 2;
408
409     GST_DEBUG("render rect (%d,%d):%ux%u",
410               display_rect->x, display_rect->y,
411               display_rect->width, display_rect->height);
412     return TRUE;
413 }
414
415 static void
416 gst_vaapisink_ensure_window_size(GstVaapiSink *sink, guint *pwidth, guint *pheight)
417 {
418     GstVideoRectangle src_rect, dst_rect, out_rect;
419     guint num, den, display_width, display_height, display_par_n, display_par_d;
420     gboolean success, scale;
421
422     if (sink->foreign_window) {
423         *pwidth  = sink->window_width;
424         *pheight = sink->window_height;
425         return;
426     }
427
428     gst_vaapi_display_get_size(sink->display, &display_width, &display_height);
429     if (sink->fullscreen) {
430         *pwidth  = display_width;
431         *pheight = display_height;
432         return;
433     }
434
435     gst_vaapi_display_get_pixel_aspect_ratio(
436         sink->display,
437         &display_par_n, &display_par_d
438     );
439
440     success = gst_video_calculate_display_ratio(
441         &num, &den,
442         sink->video_width, sink->video_height,
443         sink->video_par_n, sink->video_par_d,
444         display_par_n, display_par_d
445     );
446     if (!success) {
447         num = sink->video_par_n;
448         den = sink->video_par_d;
449     }
450
451     src_rect.x = 0;
452     src_rect.y = 0;
453     src_rect.w = gst_util_uint64_scale_int(sink->video_height, num, den);
454     src_rect.h = sink->video_height;
455     dst_rect.x = 0;
456     dst_rect.y = 0;
457     dst_rect.w = display_width;
458     dst_rect.h = display_height;
459     scale      = (src_rect.w > dst_rect.w || src_rect.h > dst_rect.h);
460     gst_video_sink_center_rect(src_rect, dst_rect, &out_rect, scale);
461     *pwidth    = out_rect.w;
462     *pheight   = out_rect.h;
463 }
464
465 static inline gboolean
466 gst_vaapisink_ensure_window(GstVaapiSink *sink, guint width, guint height)
467 {
468     GstVaapiDisplay * const display = sink->display;
469
470     if (!sink->window) {
471         switch (sink->display_type) {
472 #if USE_GLX
473         case GST_VAAPI_DISPLAY_TYPE_GLX:
474             sink->window = gst_vaapi_window_glx_new(display, width, height);
475             goto notify_video_overlay_interface;
476 #endif
477 #if USE_X11
478         case GST_VAAPI_DISPLAY_TYPE_X11:
479             sink->window = gst_vaapi_window_x11_new(display, width, height);
480         notify_video_overlay_interface:
481             if (!sink->window)
482                 break;
483             gst_video_overlay_got_window_handle(
484                 GST_VIDEO_OVERLAY(sink),
485                 gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window))
486             );
487             break;
488 #endif
489 #if USE_WAYLAND
490         case GST_VAAPI_DISPLAY_TYPE_WAYLAND:
491             sink->window = gst_vaapi_window_wayland_new(display, width, height);
492             break;
493 #endif
494         default:
495             GST_ERROR("unsupported display type %d", sink->display_type);
496             return FALSE;
497         }
498     }
499     return sink->window != NULL;
500 }
501
502 #if USE_X11
503 static gboolean
504 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id)
505 {
506     Window rootwin;
507     unsigned int width, height, border_width, depth;
508     int x, y;
509     XID xid = window_id;
510
511     if (!gst_vaapisink_ensure_display(sink))
512         return FALSE;
513
514     gst_vaapi_display_lock(sink->display);
515     XGetGeometry(
516         gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
517         xid,
518         &rootwin,
519         &x, &y, &width, &height, &border_width, &depth
520     );
521     gst_vaapi_display_unlock(sink->display);
522
523     if ((width != sink->window_width || height != sink->window_height) &&
524         !configure_notify_event_pending(sink, xid, width, height)) {
525         if (!gst_vaapisink_ensure_render_rect(sink, width, height))
526             return FALSE;
527         sink->window_width  = width;
528         sink->window_height = height;
529     }
530
531     if (sink->window &&
532         gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window)) == xid)
533         return TRUE;
534
535     g_clear_object(&sink->window);
536
537     switch (sink->display_type) {
538 #if USE_GLX
539     case GST_VAAPI_DISPLAY_TYPE_GLX:
540         sink->window = gst_vaapi_window_glx_new_with_xid(sink->display, xid);
541         break;
542 #endif
543     case GST_VAAPI_DISPLAY_TYPE_X11:
544         sink->window = gst_vaapi_window_x11_new_with_xid(sink->display, xid);
545         break;
546     default:
547         GST_ERROR("unsupported display type %d", sink->display_type);
548         return FALSE;
549     }
550     return sink->window != NULL;
551 }
552 #endif
553
554 static gboolean
555 gst_vaapisink_ensure_rotation(GstVaapiSink *sink, gboolean recalc_display_rect)
556 {
557     gboolean success = FALSE;
558
559     g_return_val_if_fail(sink->display, FALSE);
560
561     if (sink->rotation == sink->rotation_req)
562         return TRUE;
563
564     if (!sink->use_rotation) {
565         GST_WARNING("VA display does not support rotation");
566         goto end;
567     }
568
569     gst_vaapi_display_lock(sink->display);
570     success = gst_vaapi_display_set_rotation(sink->display, sink->rotation_req);
571     gst_vaapi_display_unlock(sink->display);
572     if (!success) {
573         GST_ERROR("failed to change VA display rotation mode");
574         goto end;
575     }
576
577     if (((sink->rotation + sink->rotation_req) % 180) == 90) {
578         /* Orientation changed */
579         G_PRIMITIVE_SWAP(guint, sink->video_width, sink->video_height);
580         G_PRIMITIVE_SWAP(gint, sink->video_par_n, sink->video_par_d);
581     }
582
583     if (recalc_display_rect && !sink->foreign_window)
584         gst_vaapisink_ensure_render_rect(sink, sink->window_width,
585             sink->window_height);
586     success = TRUE;
587
588 end:
589     sink->rotation = sink->rotation_req;
590     return success;
591 }
592
593 static gboolean
594 gst_vaapisink_ensure_video_buffer_pool(GstVaapiSink *sink, GstCaps *caps)
595 {
596     GstBufferPool *pool;
597     GstCaps *pool_caps;
598     GstStructure *config;
599     GstVideoInfo vi;
600     gboolean need_pool;
601
602     if (!gst_vaapisink_ensure_display(sink))
603         return FALSE;
604
605     if (sink->video_buffer_pool) {
606         config = gst_buffer_pool_get_config(sink->video_buffer_pool);
607         gst_buffer_pool_config_get_params(config, &pool_caps, NULL, NULL, NULL);
608         need_pool = !gst_caps_is_equal(caps, pool_caps);
609         gst_structure_free(config);
610         if (!need_pool)
611             return TRUE;
612         g_clear_object(&sink->video_buffer_pool);
613         sink->video_buffer_size = 0;
614     }
615
616     pool = gst_vaapi_video_buffer_pool_new(sink->display);
617     if (!pool)
618         goto error_create_pool;
619
620     gst_video_info_init(&vi);
621     gst_video_info_from_caps(&vi, caps);
622     if (GST_VIDEO_INFO_FORMAT(&vi) == GST_VIDEO_FORMAT_ENCODED) {
623         GST_DEBUG("assume video buffer pool format is NV12");
624         gst_video_info_set_format(&vi, GST_VIDEO_FORMAT_NV12,
625             GST_VIDEO_INFO_WIDTH(&vi), GST_VIDEO_INFO_HEIGHT(&vi));
626     }
627     sink->video_buffer_size = vi.size;
628
629     config = gst_buffer_pool_get_config(pool);
630     gst_buffer_pool_config_set_params(config, caps, sink->video_buffer_size,
631         0, 0);
632     gst_buffer_pool_config_add_option(config,
633         GST_BUFFER_POOL_OPTION_VAAPI_VIDEO_META);
634     if (!gst_buffer_pool_set_config(pool, config))
635         goto error_pool_config;
636     sink->video_buffer_pool = pool;
637     return TRUE;
638
639     /* ERRORS */
640 error_create_pool:
641     {
642         GST_ERROR("failed to create buffer pool");
643         return FALSE;
644     }
645 error_pool_config:
646     {
647         GST_ERROR("failed to reset buffer pool config");
648         g_object_unref(pool);
649         return FALSE;
650     }
651 }
652
653 static gboolean
654 gst_vaapisink_start(GstBaseSink *base_sink)
655 {
656     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
657
658     if (!gst_vaapisink_ensure_display(sink))
659         return FALSE;
660
661     sink->uploader = gst_vaapi_uploader_new(sink->display);
662     if (!sink->uploader)
663         return FALSE;
664     return TRUE;
665 }
666
667 static gboolean
668 gst_vaapisink_stop(GstBaseSink *base_sink)
669 {
670     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
671
672     gst_buffer_replace(&sink->video_buffer, NULL);
673     g_clear_object(&sink->video_buffer_pool);
674     g_clear_object(&sink->window);
675     g_clear_object(&sink->display);
676     g_clear_object(&sink->uploader);
677
678     return TRUE;
679 }
680
681 static GstCaps *
682 gst_vaapisink_get_caps(GstBaseSink *base_sink, GstCaps *filter)
683 {
684     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
685     GstCaps *out_caps, *yuv_caps;
686
687     out_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS);
688     if (!out_caps)
689         return NULL;
690
691     if (gst_vaapisink_ensure_uploader(sink)) {
692         yuv_caps = gst_vaapi_uploader_get_caps(sink->uploader);
693         if (yuv_caps)
694             gst_caps_append(out_caps, gst_caps_copy(yuv_caps));
695     }
696     return out_caps;
697 }
698
699 static gboolean
700 gst_vaapisink_set_caps(GstBaseSink *base_sink, GstCaps *caps)
701 {
702     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
703     GstVideoInfo vi;
704     guint win_width, win_height;
705
706 #if USE_DRM
707     if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_DRM)
708         return TRUE;
709 #endif
710
711     if (!gst_vaapisink_ensure_video_buffer_pool(sink, caps))
712         return FALSE;
713
714     if (!gst_video_info_from_caps(&vi, caps))
715         return FALSE;
716     sink->use_video_raw = GST_VIDEO_INFO_IS_YUV(&vi);
717     sink->video_width   = GST_VIDEO_INFO_WIDTH(&vi);
718     sink->video_height  = GST_VIDEO_INFO_HEIGHT(&vi);
719     sink->video_par_n   = GST_VIDEO_INFO_PAR_N(&vi);
720     sink->video_par_d   = GST_VIDEO_INFO_PAR_D(&vi);
721     GST_DEBUG("video pixel-aspect-ratio %d/%d",
722               sink->video_par_n, sink->video_par_d);
723
724     gst_caps_replace(&sink->caps, caps);
725
726     if (!gst_vaapisink_ensure_display(sink))
727         return FALSE;
728
729     gst_vaapisink_ensure_rotation(sink, FALSE);
730
731     gst_vaapisink_ensure_window_size(sink, &win_width, &win_height);
732     if (sink->window) {
733         if (!sink->foreign_window || sink->fullscreen)
734             gst_vaapi_window_set_size(sink->window, win_width, win_height);
735     }
736     else {
737         gst_vaapi_display_lock(sink->display);
738         gst_video_overlay_prepare_window_handle(GST_VIDEO_OVERLAY(sink));
739         gst_vaapi_display_unlock(sink->display);
740         if (sink->window)
741             return TRUE;
742         if (!gst_vaapisink_ensure_window(sink, win_width, win_height))
743             return FALSE;
744         gst_vaapi_window_set_fullscreen(sink->window, sink->fullscreen);
745         gst_vaapi_window_show(sink->window);
746         gst_vaapi_window_get_size(sink->window, &win_width, &win_height);
747     }
748     sink->window_width  = win_width;
749     sink->window_height = win_height;
750     GST_DEBUG("window size %ux%u", win_width, win_height);
751
752     return gst_vaapisink_ensure_render_rect(sink, win_width, win_height);
753 }
754
755 #if USE_GLX
756 static void
757 render_background(GstVaapiSink *sink)
758 {
759     /* Original code from Mirco Muller (MacSlow):
760        <http://cgit.freedesktop.org/~macslow/gl-gst-player/> */
761     GLfloat fStartX = 0.0f;
762     GLfloat fStartY = 0.0f;
763     GLfloat fWidth  = (GLfloat)sink->window_width;
764     GLfloat fHeight = (GLfloat)sink->window_height;
765
766     glClear(GL_COLOR_BUFFER_BIT);
767     glBegin(GL_QUADS);
768     {
769         /* top third, darker grey to white */
770         glColor3f(0.85f, 0.85f, 0.85f);
771         glVertex3f(fStartX, fStartY, 0.0f);
772         glColor3f(0.85f, 0.85f, 0.85f);
773         glVertex3f(fStartX + fWidth, fStartY, 0.0f);
774         glColor3f(1.0f, 1.0f, 1.0f);
775         glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
776         glColor3f(1.0f, 1.0f, 1.0f);
777         glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
778
779         /* middle third, just plain white */
780         glColor3f(1.0f, 1.0f, 1.0f);
781         glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
782         glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
783         glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
784         glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
785
786         /* bottom third, white to lighter grey */
787         glColor3f(1.0f, 1.0f, 1.0f);
788         glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
789         glColor3f(1.0f, 1.0f, 1.0f);
790         glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
791         glColor3f(0.62f, 0.66f, 0.69f);
792         glVertex3f(fStartX + fWidth, fStartY + fHeight, 0.0f);
793         glColor3f(0.62f, 0.66f, 0.69f);
794         glVertex3f(fStartX, fStartY + fHeight, 0.0f);
795     }
796     glEnd();
797 }
798
799 static void
800 render_frame(GstVaapiSink *sink)
801 {
802     const guint x1 = sink->display_rect.x;
803     const guint x2 = sink->display_rect.x + sink->display_rect.width;
804     const guint y1 = sink->display_rect.y;
805     const guint y2 = sink->display_rect.y + sink->display_rect.height;
806
807     glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
808     glBegin(GL_QUADS);
809     {
810         glTexCoord2f(0.0f, 0.0f); glVertex2i(x1, y1);
811         glTexCoord2f(0.0f, 1.0f); glVertex2i(x1, y2);
812         glTexCoord2f(1.0f, 1.0f); glVertex2i(x2, y2);
813         glTexCoord2f(1.0f, 0.0f); glVertex2i(x2, y1);
814     }
815     glEnd();
816 }
817
818 static void
819 render_reflection(GstVaapiSink *sink)
820 {
821     const guint x1 = sink->display_rect.x;
822     const guint x2 = sink->display_rect.x + sink->display_rect.width;
823     const guint y1 = sink->display_rect.y;
824     const guint rh = sink->display_rect.height / 5;
825     GLfloat     ry = 1.0f - (GLfloat)rh / (GLfloat)sink->display_rect.height;
826
827     glBegin(GL_QUADS);
828     {
829         glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
830         glTexCoord2f(0.0f, 1.0f); glVertex2i(x1, y1);
831         glTexCoord2f(1.0f, 1.0f); glVertex2i(x2, y1);
832
833         glColor4f(1.0f, 1.0f, 1.0f, 0.0f);
834         glTexCoord2f(1.0f, ry); glVertex2i(x2, y1 + rh);
835         glTexCoord2f(0.0f, ry); glVertex2i(x1, y1 + rh);
836     }
837     glEnd();
838 }
839
840 static gboolean
841 gst_vaapisink_show_frame_glx(
842     GstVaapiSink    *sink,
843     GstVaapiSurface *surface,
844     guint            flags
845 )
846 {
847     GstVaapiWindowGLX * const window = GST_VAAPI_WINDOW_GLX(sink->window);
848     GLenum target;
849     GLuint texture;
850
851     gst_vaapi_window_glx_make_current(window);
852     if (!sink->texture) {
853         sink->texture = gst_vaapi_texture_new(
854             sink->display,
855             GL_TEXTURE_2D,
856             GL_BGRA,
857             sink->video_width,
858             sink->video_height
859         );
860         if (!sink->texture)
861             goto error_create_texture;
862     }
863     if (!gst_vaapi_texture_put_surface(sink->texture, surface, flags))
864         goto error_transfer_surface;
865
866     target  = gst_vaapi_texture_get_target(sink->texture);
867     texture = gst_vaapi_texture_get_id(sink->texture);
868     if (target != GL_TEXTURE_2D || !texture)
869         return FALSE;
870
871     if (sink->use_reflection)
872         render_background(sink);
873
874     glEnable(target);
875     glBindTexture(target, texture);
876     {
877         if (sink->use_reflection) {
878             glPushMatrix();
879             glRotatef(20.0f, 0.0f, 1.0f, 0.0f);
880             glTranslatef(50.0f, 0.0f, 0.0f);
881         }
882         render_frame(sink);
883         if (sink->use_reflection) {
884             glPushMatrix();
885             glTranslatef(0.0, (GLfloat)sink->display_rect.height + 5.0f, 0.0f);
886             render_reflection(sink);
887             glPopMatrix();
888             glPopMatrix();
889         }
890     }
891     glBindTexture(target, 0);
892     glDisable(target);
893     gst_vaapi_window_glx_swap_buffers(window);
894     return TRUE;
895
896     /* ERRORS */
897 error_create_texture:
898     {
899         GST_DEBUG("could not create VA/GLX texture");
900         return FALSE;
901     }
902 error_transfer_surface:
903     {
904         GST_DEBUG("could not transfer VA surface to texture");
905         return FALSE;
906     }
907 }
908 #endif
909
910 static inline gboolean
911 gst_vaapisink_put_surface(
912     GstVaapiSink    *sink,
913     GstVaapiSurface *surface,
914     guint            flags
915 )
916 {
917     if (!gst_vaapi_window_put_surface(sink->window, surface,
918                 NULL, &sink->display_rect, flags)) {
919         GST_DEBUG("could not render VA surface");
920         return FALSE;
921     }
922     return TRUE;
923 }
924
925 static GstFlowReturn
926 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *src_buffer)
927 {
928     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
929     GstVaapiVideoMeta *meta;
930     GstVaapiSurface *surface;
931     GstBuffer *buffer;
932     guint flags;
933     gboolean success;
934
935     meta = gst_buffer_get_vaapi_video_meta(src_buffer);
936 #if GST_CHECK_VERSION(1,0,0)
937     if (!meta)
938         return GST_FLOW_EOS;
939     buffer = gst_buffer_ref(src_buffer);
940 #else
941     if (meta)
942         buffer = gst_buffer_ref(src_buffer);
943     else if (sink->use_video_raw) {
944         buffer = gst_vaapi_uploader_get_buffer(sink->uploader);
945         if (!buffer)
946             return GST_FLOW_EOS;
947         meta = gst_buffer_get_vaapi_video_meta(buffer);
948         if (!meta)
949             goto error;
950     }
951     else
952         return GST_FLOW_EOS;
953
954     if (sink->use_video_raw &&
955         !gst_vaapi_uploader_process(sink->uploader, src_buffer, buffer)) {
956         GST_WARNING("failed to process raw YUV buffer");
957         goto error;
958     }
959 #endif
960
961     if (sink->display != gst_vaapi_video_meta_get_display(meta)) {
962         g_clear_object(&sink->display);
963         sink->display = g_object_ref(gst_vaapi_video_meta_get_display(meta));
964     }
965
966     if (!sink->window)
967         goto error;
968
969     gst_vaapisink_ensure_rotation(sink, TRUE);
970
971     surface = gst_vaapi_video_meta_get_surface(meta);
972     if (!surface)
973         goto error;
974
975     GST_DEBUG("render surface %" GST_VAAPI_ID_FORMAT,
976               GST_VAAPI_ID_ARGS(gst_vaapi_surface_get_id(surface)));
977
978     flags = gst_vaapi_video_meta_get_render_flags(meta);
979
980     if (!gst_vaapi_apply_composition(surface, src_buffer))
981         GST_WARNING("could not update subtitles");
982
983     switch (sink->display_type) {
984 #if USE_GLX
985     case GST_VAAPI_DISPLAY_TYPE_GLX:
986         success = gst_vaapisink_show_frame_glx(sink, surface, flags);
987         break;
988 #endif
989 #if USE_DRM
990     case GST_VAAPI_DISPLAY_TYPE_DRM:
991         success = TRUE;
992         break;
993 #endif
994 #if USE_X11
995     case GST_VAAPI_DISPLAY_TYPE_X11:
996         success = gst_vaapisink_put_surface(sink, surface, flags);
997         break;
998 #endif
999 #if USE_WAYLAND
1000     case GST_VAAPI_DISPLAY_TYPE_WAYLAND:
1001         success = gst_vaapisink_put_surface(sink, surface, flags);
1002         break;
1003 #endif
1004     default:
1005         GST_ERROR("unsupported display type %d", sink->display_type);
1006         success = FALSE;
1007         break;
1008     }
1009     if (!success)
1010         goto error;
1011
1012     /* Retain VA surface until the next one is displayed */
1013     if (sink->use_overlay)
1014         gst_buffer_replace(&sink->video_buffer, buffer);
1015     gst_buffer_unref(buffer);
1016     return GST_FLOW_OK;
1017
1018 error:
1019     gst_buffer_unref(buffer);
1020     return GST_FLOW_EOS;
1021 }
1022
1023 #if GST_CHECK_VERSION(1,0,0)
1024 static gboolean
1025 gst_vaapisink_propose_allocation(GstBaseSink *base_sink, GstQuery *query)
1026 {
1027     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1028     GstCaps *caps = NULL;
1029     gboolean need_pool;
1030
1031     gst_query_parse_allocation(query, &caps, &need_pool);
1032
1033     if (need_pool) {
1034         if (!caps)
1035             goto error_no_caps;
1036         if (!gst_vaapisink_ensure_video_buffer_pool(sink, caps))
1037             return FALSE;
1038         gst_query_add_allocation_pool(query, sink->video_buffer_pool,
1039             sink->video_buffer_size, 0, 0);
1040     }
1041
1042     gst_query_add_allocation_meta(query,
1043         GST_VAAPI_VIDEO_META_API_TYPE, NULL);
1044     gst_query_add_allocation_meta(query,
1045         GST_VIDEO_META_API_TYPE, NULL);
1046     gst_query_add_allocation_meta(query,
1047         GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL);
1048     return TRUE;
1049
1050     /* ERRORS */
1051 error_no_caps:
1052     {
1053         GST_ERROR("no caps specified");
1054         return FALSE;
1055     }
1056 }
1057 #else
1058 static GstFlowReturn
1059 gst_vaapisink_buffer_alloc(
1060     GstBaseSink        *base_sink,
1061     guint64             offset,
1062     guint               size,
1063     GstCaps            *caps,
1064     GstBuffer         **pbuf
1065 )
1066 {
1067     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1068     GstVideoInfo vi;
1069     GstBuffer *buf;
1070
1071     *pbuf = NULL;
1072
1073     if (!sink->use_video_raw) {
1074         /* Note: this code path is rarely used but for raw YUV formats
1075            from custom pipeline. Otherwise, GstBaseSink::set_caps() is
1076            called first, and GstBaseSink::buffer_alloc() is not called
1077            in VA surface format mode */
1078         if (!gst_video_info_from_caps(&vi, caps))
1079             return GST_FLOW_NOT_SUPPORTED;
1080         if (!GST_VIDEO_INFO_IS_YUV(&vi))
1081             return GST_FLOW_OK;
1082     }
1083
1084     if (!gst_vaapi_uploader_ensure_display(sink->uploader, sink->display))
1085         return GST_FLOW_NOT_SUPPORTED;
1086     if (!gst_vaapi_uploader_ensure_caps(sink->uploader, caps, NULL))
1087         return GST_FLOW_NOT_SUPPORTED;
1088
1089     buf = gst_vaapi_uploader_get_buffer(sink->uploader);
1090     if (!buf) {
1091         GST_WARNING("failed to allocate resources for raw YUV buffer");
1092         return GST_FLOW_NOT_SUPPORTED;
1093     }
1094
1095     *pbuf = buf;
1096     return GST_FLOW_OK;
1097 }
1098 #endif
1099
1100 static gboolean
1101 gst_vaapisink_query(GstBaseSink *base_sink, GstQuery *query)
1102 {
1103     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
1104
1105     if (gst_vaapi_reply_to_query(query, sink->display)) {
1106         GST_DEBUG("sharing display %p", sink->display);
1107         return TRUE;
1108     }
1109     return GST_BASE_SINK_CLASS(gst_vaapisink_parent_class)->query(base_sink,
1110         query);
1111 }
1112
1113 static void
1114 gst_vaapisink_finalize(GObject *object)
1115 {
1116     gst_vaapisink_destroy(GST_VAAPISINK(object));
1117
1118     G_OBJECT_CLASS(gst_vaapisink_parent_class)->finalize(object);
1119 }
1120
1121 static void
1122 gst_vaapisink_set_property(
1123     GObject      *object,
1124     guint         prop_id,
1125     const GValue *value,
1126     GParamSpec   *pspec
1127 )
1128 {
1129     GstVaapiSink * const sink = GST_VAAPISINK(object);
1130
1131     switch (prop_id) {
1132     case PROP_DISPLAY_TYPE:
1133         sink->display_type = g_value_get_enum(value);
1134         break;
1135     case PROP_FULLSCREEN:
1136         sink->fullscreen = g_value_get_boolean(value);
1137         break;
1138     case PROP_SYNCHRONOUS:
1139         sink->synchronous = g_value_get_boolean(value);
1140         break;
1141     case PROP_USE_REFLECTION:
1142         sink->use_reflection = g_value_get_boolean(value);
1143         break;
1144     case PROP_ROTATION:
1145         sink->rotation_req = g_value_get_enum(value);
1146         break;
1147     default:
1148         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
1149         break;
1150     }
1151 }
1152
1153 static void
1154 gst_vaapisink_get_property(
1155     GObject    *object,
1156     guint       prop_id,
1157     GValue     *value,
1158     GParamSpec *pspec
1159 )
1160 {
1161     GstVaapiSink * const sink = GST_VAAPISINK(object);
1162
1163     switch (prop_id) {
1164     case PROP_DISPLAY_TYPE:
1165         g_value_set_enum(value, sink->display_type);
1166         break;
1167     case PROP_FULLSCREEN:
1168         g_value_set_boolean(value, sink->fullscreen);
1169         break;
1170     case PROP_SYNCHRONOUS:
1171         g_value_set_boolean(value, sink->synchronous);
1172         break;
1173     case PROP_USE_REFLECTION:
1174         g_value_set_boolean(value, sink->use_reflection);
1175         break;
1176     case PROP_ROTATION:
1177         g_value_set_enum(value, sink->rotation);
1178         break;
1179     default:
1180         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
1181         break;
1182     }
1183 }
1184
1185 static void
1186 gst_vaapisink_class_init(GstVaapiSinkClass *klass)
1187 {
1188     GObjectClass * const     object_class   = G_OBJECT_CLASS(klass);
1189     GstElementClass * const  element_class  = GST_ELEMENT_CLASS(klass);
1190     GstBaseSinkClass * const basesink_class = GST_BASE_SINK_CLASS(klass);
1191     GstPadTemplate *pad_template;
1192
1193     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapisink,
1194                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
1195
1196     object_class->finalize       = gst_vaapisink_finalize;
1197     object_class->set_property   = gst_vaapisink_set_property;
1198     object_class->get_property   = gst_vaapisink_get_property;
1199
1200     basesink_class->start        = gst_vaapisink_start;
1201     basesink_class->stop         = gst_vaapisink_stop;
1202     basesink_class->get_caps     = gst_vaapisink_get_caps;
1203     basesink_class->set_caps     = gst_vaapisink_set_caps;
1204     basesink_class->preroll      = gst_vaapisink_show_frame;
1205     basesink_class->render       = gst_vaapisink_show_frame;
1206     basesink_class->query        = gst_vaapisink_query;
1207 #if GST_CHECK_VERSION(1,0,0)
1208     basesink_class->propose_allocation = gst_vaapisink_propose_allocation;
1209 #else
1210     basesink_class->buffer_alloc = gst_vaapisink_buffer_alloc;
1211 #endif
1212
1213     gst_element_class_set_static_metadata(element_class,
1214         "VA-API sink",
1215         "Sink/Video",
1216         GST_PLUGIN_DESC,
1217         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
1218
1219     pad_template = gst_static_pad_template_get(&gst_vaapisink_sink_factory);
1220     gst_element_class_add_pad_template(element_class, pad_template);
1221
1222     g_object_class_install_property
1223         (object_class,
1224          PROP_DISPLAY_TYPE,
1225          g_param_spec_enum("display",
1226                            "display type",
1227                            "display type to use",
1228                            GST_VAAPI_TYPE_DISPLAY_TYPE,
1229                            GST_VAAPI_DISPLAY_TYPE_ANY,
1230                            G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1231
1232 #if USE_GLX
1233     g_object_class_install_property
1234         (object_class,
1235          PROP_USE_REFLECTION,
1236          g_param_spec_boolean("use-reflection",
1237                               "Reflection effect",
1238                               "Enables OpenGL reflection effect",
1239                               FALSE,
1240                               G_PARAM_READWRITE));
1241 #endif
1242
1243     g_object_class_install_property
1244         (object_class,
1245          PROP_FULLSCREEN,
1246          g_param_spec_boolean("fullscreen",
1247                               "Fullscreen",
1248                               "Requests window in fullscreen state",
1249                               FALSE,
1250                               G_PARAM_READWRITE));
1251
1252     /**
1253      * GstVaapiSink:synchronous:
1254      *
1255      * When enabled, runs the X display in synchronous mode. Note that
1256      * this is used only for debugging.
1257      */
1258     g_object_class_install_property
1259         (object_class,
1260          PROP_SYNCHRONOUS,
1261          g_param_spec_boolean("synchronous",
1262                               "Synchronous mode",
1263                               "Toggles X display synchronous mode",
1264                               FALSE,
1265                               G_PARAM_READWRITE));
1266
1267     /**
1268      * GstVaapiSink:rotation:
1269      *
1270      * The VA display rotation mode, expressed as a #GstVaapiRotation.
1271      */
1272     g_object_class_install_property
1273         (object_class,
1274          PROP_ROTATION,
1275          g_param_spec_enum(GST_VAAPI_DISPLAY_PROP_ROTATION,
1276                            "rotation",
1277                            "The display rotation mode",
1278                            GST_VAAPI_TYPE_ROTATION,
1279                            DEFAULT_ROTATION,
1280                            G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1281 }
1282
1283 static void
1284 gst_vaapisink_init(GstVaapiSink *sink)
1285 {
1286     sink->caps           = NULL;
1287     sink->display        = NULL;
1288     sink->window         = NULL;
1289     sink->window_width   = 0;
1290     sink->window_height  = 0;
1291     sink->texture        = NULL;
1292     sink->video_buffer   = NULL;
1293     sink->video_width    = 0;
1294     sink->video_height   = 0;
1295     sink->video_par_n    = 1;
1296     sink->video_par_d    = 1;
1297     sink->foreign_window = FALSE;
1298     sink->fullscreen     = FALSE;
1299     sink->synchronous    = FALSE;
1300     sink->display_type   = DEFAULT_DISPLAY_TYPE;
1301     sink->rotation       = DEFAULT_ROTATION;
1302     sink->rotation_req   = DEFAULT_ROTATION;
1303     sink->use_reflection = FALSE;
1304     sink->use_overlay    = FALSE;
1305     sink->use_rotation   = FALSE;
1306 }