gst/vaapi/gstvaapisink.c

   1 /*
   2  *  gstvaapisink.c - VA-API video sink
   3  *
   4  *  Copyright (C) 2010-2011 Splitted-Desktop Systems
   5  *  Copyright (C) 2011-2013 Intel Corporation
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public License
   9  *  as published by the Free Software Foundation; either version 2.1
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free
  19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  *  Boston, MA 02110-1301 USA
  21  */
  22
  23 /**
  24  * SECTION:gstvaapisink
  25  * @short_description: A VA-API based videosink
  26  *
  27  * vaapisink renders video frames to a drawable (X #Window) on a local
  28  * display using the Video Acceleration (VA) API. The element will
  29  * create its own internal window and render into it.
  30  */
  31
  32 #include "gst/vaapi/sysdeps.h"
  33 #include <gst/gst.h>
  34 #include <gst/video/video.h>
  35 #include <gst/video/videocontext.h>
  36 #include <gst/vaapi/gstvaapivalue.h>
  37 #include <gst/vaapi/gstvaapivideometa.h>
  38 #if USE_DRM
  39 # include <gst/vaapi/gstvaapidisplay_drm.h>
  40 #endif
  41 #if USE_X11
  42 # include <gst/vaapi/gstvaapidisplay_x11.h>
  43 # include <gst/vaapi/gstvaapiwindow_x11.h>
  44 #endif
  45 #if USE_GLX
  46 # include <gst/vaapi/gstvaapidisplay_glx.h>
  47 # include <gst/vaapi/gstvaapiwindow_glx.h>
  48 #endif
  49 #if USE_WAYLAND
  50 # include <gst/vaapi/gstvaapidisplay_wayland.h>
  51 # include <gst/vaapi/gstvaapiwindow_wayland.h>
  52 #endif
  53
  54 /* Supported interfaces */
  55 #include <gst/interfaces/xoverlay.h>
  56
  57 #include "gstvaapisink.h"
  58 #include "gstvaapipluginutil.h"
  59
  60 #define GST_PLUGIN_NAME "vaapisink"
  61 #define GST_PLUGIN_DESC "A VA-API based videosink"
  62
  63 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapisink);
  64 #define GST_CAT_DEFAULT gst_debug_vaapisink
  65
  66 /* ElementFactory information */
  67 static const GstElementDetails gst_vaapisink_details =
  68     GST_ELEMENT_DETAILS(
  69         "VA-API sink",
  70         "Sink/Video",
  71         GST_PLUGIN_DESC,
  72         "Gwenole Beauchesne <gwenole.beauchesne@intel.com>");
  73
  74 /* Default template */
  75 static const char gst_vaapisink_sink_caps_str[] =
  76     "video/x-raw-yuv, "
  77     "width  = (int) [ 1, MAX ], "
  78     "height = (int) [ 1, MAX ]; "
  79     GST_VAAPI_SURFACE_CAPS;
  80
  81 static GstStaticPadTemplate gst_vaapisink_sink_factory =
  82     GST_STATIC_PAD_TEMPLATE(
  83         "sink",
  84         GST_PAD_SINK,
  85         GST_PAD_ALWAYS,
  86         GST_STATIC_CAPS(gst_vaapisink_sink_caps_str));
  87
  88 static void
  89 gst_vaapisink_implements_iface_init(GstImplementsInterfaceClass *iface);
  90
  91 static void
  92 gst_vaapisink_video_context_iface_init(GstVideoContextInterface *iface);
  93
  94 static void
  95 gst_vaapisink_xoverlay_iface_init(GstXOverlayClass *iface);
  96
  97 G_DEFINE_TYPE_WITH_CODE(
  98     GstVaapiSink,
  99     gst_vaapisink,
 100     GST_TYPE_VIDEO_SINK,
 101     G_IMPLEMENT_INTERFACE(GST_TYPE_IMPLEMENTS_INTERFACE,
 102                           gst_vaapisink_implements_iface_init);
 103     G_IMPLEMENT_INTERFACE(GST_TYPE_VIDEO_CONTEXT,
 104                           gst_vaapisink_video_context_iface_init);
 105     G_IMPLEMENT_INTERFACE(GST_TYPE_X_OVERLAY,
 106                           gst_vaapisink_xoverlay_iface_init))
 107
 108 enum {
 109     PROP_0,
 110
 111     PROP_DISPLAY_TYPE,
 112     PROP_FULLSCREEN,
 113     PROP_SYNCHRONOUS,
 114     PROP_USE_REFLECTION,
 115     PROP_ROTATION,
 116 };
 117
 118 #define DEFAULT_DISPLAY_TYPE            GST_VAAPI_DISPLAY_TYPE_ANY
 119 #define DEFAULT_ROTATION                GST_VAAPI_ROTATION_0
 120
 121 /* GstImplementsInterface interface */
 122
 123 static gboolean
 124 gst_vaapisink_implements_interface_supported(
 125     GstImplementsInterface *iface,
 126     GType                   type
 127 )
 128 {
 129     return (type == GST_TYPE_VIDEO_CONTEXT ||
 130             type == GST_TYPE_X_OVERLAY);
 131 }
 132
 133 static void
 134 gst_vaapisink_implements_iface_init(GstImplementsInterfaceClass *iface)
 135 {
 136     iface->supported = gst_vaapisink_implements_interface_supported;
 137 }
 138
 139 /* GstVideoContext interface */
 140
 141 static void
 142 gst_vaapisink_set_video_context(GstVideoContext *context, const gchar *type,
 143     const GValue *value)
 144 {
 145   GstVaapiSink *sink = GST_VAAPISINK (context);
 146   gst_vaapi_set_display (type, value, &sink->display);
 147 }
 148
 149 static void
 150 gst_vaapisink_video_context_iface_init(GstVideoContextInterface *iface)
 151 {
 152     iface->set_context = gst_vaapisink_set_video_context;
 153 }
 154
 155 /* GstXOverlay interface */
 156
 157 #if USE_X11
 158 static gboolean
 159 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id);
 160 #endif
 161
 162 static GstFlowReturn
 163 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *buffer);
 164
 165 static void
 166 gst_vaapisink_xoverlay_set_window_handle(GstXOverlay *overlay, guintptr window)
 167 {
 168     GstVaapiSink * const sink = GST_VAAPISINK(overlay);
 169
 170     /* Disable GLX rendering when vaapisink is using a foreign X
 171        window. It's pretty much useless */
 172     if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_GLX)
 173         sink->display_type = GST_VAAPI_DISPLAY_TYPE_X11;
 174
 175     sink->foreign_window = TRUE;
 176
 177     switch (sink->display_type) {
 178 #if USE_X11
 179     case GST_VAAPI_DISPLAY_TYPE_X11:
 180         gst_vaapisink_ensure_window_xid(sink, window);
 181         break;
 182 #endif
 183     default:
 184         break;
 185     }
 186 }
 187
 188 static void
 189 gst_vaapisink_xoverlay_set_render_rectangle(
 190     GstXOverlay *overlay,
 191     gint         x,
 192     gint         y,
 193     gint         width,
 194     gint         height
 195 )
 196 {
 197     GstVaapiSink * const sink = GST_VAAPISINK(overlay);
 198     GstVaapiRectangle * const display_rect = &sink->display_rect;
 199
 200     display_rect->x      = x;
 201     display_rect->y      = y;
 202     display_rect->width  = width;
 203     display_rect->height = height;
 204
 205     GST_DEBUG("render rect (%d,%d):%ux%u",
 206               display_rect->x, display_rect->y,
 207               display_rect->width, display_rect->height);
 208 }
 209
 210 static void
 211 gst_vaapisink_xoverlay_expose(GstXOverlay *overlay)
 212 {
 213     GstBaseSink * const base_sink = GST_BASE_SINK(overlay);
 214     GstBuffer *buffer;
 215
 216     buffer = gst_base_sink_get_last_buffer(base_sink);
 217     if (buffer) {
 218         gst_vaapisink_show_frame(base_sink, buffer);
 219         gst_buffer_unref(buffer);
 220     }
 221 }
 222
 223 static void
 224 gst_vaapisink_xoverlay_iface_init(GstXOverlayClass *iface)
 225 {
 226     iface->set_window_handle    = gst_vaapisink_xoverlay_set_window_handle;
 227     iface->set_render_rectangle = gst_vaapisink_xoverlay_set_render_rectangle;
 228     iface->expose               = gst_vaapisink_xoverlay_expose;
 229 }
 230
 231 static void
 232 gst_vaapisink_destroy(GstVaapiSink *sink)
 233 {
 234     gst_buffer_replace(&sink->video_buffer, NULL);
 235     g_clear_object(&sink->texture);
 236     g_clear_object(&sink->display);
 237     g_clear_object(&sink->uploader);
 238
 239     gst_caps_replace(&sink->caps, NULL);
 240 }
 241
 242 #if USE_X11
 243 /* Checks whether a ConfigureNotify event is in the queue */
 244 typedef struct _ConfigureNotifyEventPendingArgs ConfigureNotifyEventPendingArgs;
 245 struct _ConfigureNotifyEventPendingArgs {
 246     Window      window;
 247     guint       width;
 248     guint       height;
 249     gboolean    match;
 250 };
 251
 252 static Bool
 253 configure_notify_event_pending_cb(Display *dpy, XEvent *xev, XPointer arg)
 254 {
 255     ConfigureNotifyEventPendingArgs * const args =
 256         (ConfigureNotifyEventPendingArgs *)arg;
 257
 258     if (xev->type == ConfigureNotify &&
 259         xev->xconfigure.window == args->window &&
 260         xev->xconfigure.width  == args->width  &&
 261         xev->xconfigure.height == args->height)
 262         args->match = TRUE;
 263
 264     /* XXX: this is a hack to traverse the whole queue because we
 265        can't use XPeekIfEvent() since it could block */
 266     return False;
 267 }
 268
 269 static gboolean
 270 configure_notify_event_pending(
 271     GstVaapiSink *sink,
 272     Window        window,
 273     guint         width,
 274     guint         height
 275 )
 276 {
 277     ConfigureNotifyEventPendingArgs args;
 278     XEvent xev;
 279
 280     args.window = window;
 281     args.width  = width;
 282     args.height = height;
 283     args.match  = FALSE;
 284
 285     /* XXX: don't use XPeekIfEvent() because it might block */
 286     XCheckIfEvent(
 287         gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
 288         &xev,
 289         configure_notify_event_pending_cb, (XPointer)&args
 290     );
 291     return args.match;
 292 }
 293 #endif
 294
 295 static const gchar *
 296 get_display_type_name(GstVaapiDisplayType display_type)
 297 {
 298     gpointer const klass = g_type_class_peek(GST_VAAPI_TYPE_DISPLAY_TYPE);
 299     GEnumValue * const e = g_enum_get_value(klass, display_type);
 300
 301     if (e)
 302         return e->value_name;
 303     return "<unknown-type>";
 304 }
 305
 306 static inline gboolean
 307 gst_vaapisink_ensure_display(GstVaapiSink *sink)
 308 {
 309     GstVaapiDisplayType display_type;
 310     GstVaapiRenderMode render_mode;
 311
 312     if (!gst_vaapi_ensure_display(sink, sink->display_type, &sink->display))
 313         return FALSE;
 314
 315     display_type = gst_vaapi_display_get_display_type(sink->display);
 316     if (display_type != sink->display_type) {
 317         GST_INFO("created %s %p", get_display_type_name(display_type),
 318             sink->display);
 319         sink->display_type = display_type;
 320
 321         sink->use_overlay =
 322             gst_vaapi_display_get_render_mode(sink->display, &render_mode) &&
 323             render_mode == GST_VAAPI_RENDER_MODE_OVERLAY;
 324         GST_DEBUG("use %s rendering mode", sink->use_overlay ? "overlay" : "texture");
 325
 326         sink->use_rotation = gst_vaapi_display_has_property(
 327             sink->display, GST_VAAPI_DISPLAY_PROP_ROTATION);
 328     }
 329     return TRUE;
 330 }
 331
 332 static gboolean
 333 gst_vaapisink_ensure_uploader(GstVaapiSink *sink)
 334 {
 335     if (!gst_vaapisink_ensure_display(sink))
 336         return FALSE;
 337
 338     if (!sink->uploader) {
 339         sink->uploader = gst_vaapi_uploader_new(sink->display);
 340         if (!sink->uploader)
 341             return FALSE;
 342     }
 343     return TRUE;
 344 }
 345
 346 static gboolean
 347 gst_vaapisink_ensure_render_rect(GstVaapiSink *sink, guint width, guint height)
 348 {
 349     GstVaapiRectangle * const display_rect = &sink->display_rect;
 350     guint num, den, display_par_n, display_par_d;
 351     gboolean success;
 352
 353     /* Return success if caps are not set yet */
 354     if (!sink->caps)
 355         return TRUE;
 356
 357     GST_DEBUG("ensure render rect within %ux%u bounds", width, height);
 358
 359     gst_vaapi_display_get_pixel_aspect_ratio(
 360         sink->display,
 361         &display_par_n, &display_par_d
 362     );
 363     GST_DEBUG("display pixel-aspect-ratio %d/%d",
 364               display_par_n, display_par_d);
 365
 366     success = gst_video_calculate_display_ratio(
 367         &num, &den,
 368         sink->video_width, sink->video_height,
 369         sink->video_par_n, sink->video_par_d,
 370         display_par_n, display_par_d
 371     );
 372     if (!success)
 373         return FALSE;
 374     GST_DEBUG("video size %dx%d, calculated ratio %d/%d",
 375               sink->video_width, sink->video_height, num, den);
 376
 377     display_rect->width = gst_util_uint64_scale_int(height, num, den);
 378     if (display_rect->width <= width) {
 379         GST_DEBUG("keeping window height");
 380         display_rect->height = height;
 381     }
 382     else {
 383         GST_DEBUG("keeping window width");
 384         display_rect->width  = width;
 385         display_rect->height =
 386             gst_util_uint64_scale_int(width, den, num);
 387     }
 388     GST_DEBUG("scaling video to %ux%u", display_rect->width, display_rect->height);
 389
 390     g_assert(display_rect->width  <= width);
 391     g_assert(display_rect->height <= height);
 392
 393     display_rect->x = (width  - display_rect->width)  / 2;
 394     display_rect->y = (height - display_rect->height) / 2;
 395
 396     GST_DEBUG("render rect (%d,%d):%ux%u",
 397               display_rect->x, display_rect->y,
 398               display_rect->width, display_rect->height);
 399     return TRUE;
 400 }
 401
 402 static void
 403 gst_vaapisink_ensure_window_size(GstVaapiSink *sink, guint *pwidth, guint *pheight)
 404 {
 405     GstVideoRectangle src_rect, dst_rect, out_rect;
 406     guint num, den, display_width, display_height, display_par_n, display_par_d;
 407     gboolean success, scale;
 408
 409     if (sink->foreign_window) {
 410         *pwidth  = sink->window_width;
 411         *pheight = sink->window_height;
 412         return;
 413     }
 414
 415     gst_vaapi_display_get_size(sink->display, &display_width, &display_height);
 416     if (sink->fullscreen) {
 417         *pwidth  = display_width;
 418         *pheight = display_height;
 419         return;
 420     }
 421
 422     gst_vaapi_display_get_pixel_aspect_ratio(
 423         sink->display,
 424         &display_par_n, &display_par_d
 425     );
 426
 427     success = gst_video_calculate_display_ratio(
 428         &num, &den,
 429         sink->video_width, sink->video_height,
 430         sink->video_par_n, sink->video_par_d,
 431         display_par_n, display_par_d
 432     );
 433     if (!success) {
 434         num = sink->video_par_n;
 435         den = sink->video_par_d;
 436     }
 437
 438     src_rect.x = 0;
 439     src_rect.y = 0;
 440     src_rect.w = gst_util_uint64_scale_int(sink->video_height, num, den);
 441     src_rect.h = sink->video_height;
 442     dst_rect.x = 0;
 443     dst_rect.y = 0;
 444     dst_rect.w = display_width;
 445     dst_rect.h = display_height;
 446     scale      = (src_rect.w > dst_rect.w || src_rect.h > dst_rect.h);
 447     gst_video_sink_center_rect(src_rect, dst_rect, &out_rect, scale);
 448     *pwidth    = out_rect.w;
 449     *pheight   = out_rect.h;
 450 }
 451
 452 static inline gboolean
 453 gst_vaapisink_ensure_window(GstVaapiSink *sink, guint width, guint height)
 454 {
 455     GstVaapiDisplay * const display = sink->display;
 456
 457     if (!sink->window) {
 458         switch (sink->display_type) {
 459 #if USE_GLX
 460         case GST_VAAPI_DISPLAY_TYPE_GLX:
 461             sink->window = gst_vaapi_window_glx_new(display, width, height);
 462             goto notify_xoverlay_interface;
 463 #endif
 464 #if USE_X11
 465         case GST_VAAPI_DISPLAY_TYPE_X11:
 466             sink->window = gst_vaapi_window_x11_new(display, width, height);
 467         notify_xoverlay_interface:
 468             if (!sink->window)
 469                 break;
 470             gst_x_overlay_got_window_handle(
 471                 GST_X_OVERLAY(sink),
 472                 gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window))
 473             );
 474             break;
 475 #endif
 476 #if USE_WAYLAND
 477         case GST_VAAPI_DISPLAY_TYPE_WAYLAND:
 478             sink->window = gst_vaapi_window_wayland_new(display, width, height);
 479             break;
 480 #endif
 481         default:
 482             GST_ERROR("unsupported display type %d", sink->display_type);
 483             return FALSE;
 484         }
 485     }
 486     return sink->window != NULL;
 487 }
 488
 489 #if USE_X11
 490 static gboolean
 491 gst_vaapisink_ensure_window_xid(GstVaapiSink *sink, guintptr window_id)
 492 {
 493     Window rootwin;
 494     unsigned int width, height, border_width, depth;
 495     int x, y;
 496     XID xid = window_id;
 497
 498     if (!gst_vaapisink_ensure_display(sink))
 499         return FALSE;
 500
 501     gst_vaapi_display_lock(sink->display);
 502     XGetGeometry(
 503         gst_vaapi_display_x11_get_display(GST_VAAPI_DISPLAY_X11(sink->display)),
 504         xid,
 505         &rootwin,
 506         &x, &y, &width, &height, &border_width, &depth
 507     );
 508     gst_vaapi_display_unlock(sink->display);
 509
 510     if ((width != sink->window_width || height != sink->window_height) &&
 511         !configure_notify_event_pending(sink, xid, width, height)) {
 512         if (!gst_vaapisink_ensure_render_rect(sink, width, height))
 513             return FALSE;
 514         sink->window_width  = width;
 515         sink->window_height = height;
 516     }
 517
 518     if (sink->window &&
 519         gst_vaapi_window_x11_get_xid(GST_VAAPI_WINDOW_X11(sink->window)) == xid)
 520         return TRUE;
 521
 522     g_clear_object(&sink->window);
 523
 524     switch (sink->display_type) {
 525 #if USE_GLX
 526     case GST_VAAPI_DISPLAY_TYPE_GLX:
 527         sink->window = gst_vaapi_window_glx_new_with_xid(sink->display, xid);
 528         break;
 529 #endif
 530     case GST_VAAPI_DISPLAY_TYPE_X11:
 531         sink->window = gst_vaapi_window_x11_new_with_xid(sink->display, xid);
 532         break;
 533     default:
 534         GST_ERROR("unsupported display type %d", sink->display_type);
 535         return FALSE;
 536     }
 537     return sink->window != NULL;
 538 }
 539 #endif
 540
 541 static gboolean
 542 gst_vaapisink_ensure_rotation(GstVaapiSink *sink, gboolean recalc_display_rect)
 543 {
 544     gboolean success = FALSE;
 545
 546     g_return_val_if_fail(sink->display, FALSE);
 547
 548     if (sink->rotation == sink->rotation_req)
 549         return TRUE;
 550
 551     if (!sink->use_rotation) {
 552         GST_WARNING("VA display does not support rotation");
 553         goto end;
 554     }
 555
 556     gst_vaapi_display_lock(sink->display);
 557     success = gst_vaapi_display_set_rotation(sink->display, sink->rotation_req);
 558     gst_vaapi_display_unlock(sink->display);
 559     if (!success) {
 560         GST_ERROR("failed to change VA display rotation mode");
 561         goto end;
 562     }
 563
 564     if (((sink->rotation + sink->rotation_req) % 180) == 90) {
 565         /* Orientation changed */
 566         G_PRIMITIVE_SWAP(guint, sink->video_width, sink->video_height);
 567         G_PRIMITIVE_SWAP(gint, sink->video_par_n, sink->video_par_d);
 568     }
 569
 570     if (recalc_display_rect && !sink->foreign_window)
 571         gst_vaapisink_ensure_render_rect(sink, sink->window_width,
 572             sink->window_height);
 573     success = TRUE;
 574
 575 end:
 576     sink->rotation = sink->rotation_req;
 577     return success;
 578 }
 579
 580 static gboolean
 581 gst_vaapisink_start(GstBaseSink *base_sink)
 582 {
 583     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
 584
 585     if (!gst_vaapisink_ensure_display(sink))
 586         return FALSE;
 587
 588     sink->uploader = gst_vaapi_uploader_new(sink->display);
 589     if (!sink->uploader)
 590         return FALSE;
 591     return TRUE;
 592 }
 593
 594 static gboolean
 595 gst_vaapisink_stop(GstBaseSink *base_sink)
 596 {
 597     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
 598
 599     gst_buffer_replace(&sink->video_buffer, NULL);
 600     g_clear_object(&sink->window);
 601     g_clear_object(&sink->display);
 602     g_clear_object(&sink->uploader);
 603
 604     return TRUE;
 605 }
 606
 607 static GstCaps *
 608 gst_vaapisink_get_caps(GstBaseSink *base_sink)
 609 {
 610     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
 611     GstCaps *out_caps, *yuv_caps;
 612
 613     out_caps = gst_caps_from_string(GST_VAAPI_SURFACE_CAPS);
 614     if (!out_caps)
 615         return NULL;
 616
 617     if (gst_vaapisink_ensure_uploader(sink)) {
 618         yuv_caps = gst_vaapi_uploader_get_caps(sink->uploader);
 619         if (yuv_caps)
 620             gst_caps_append(out_caps, gst_caps_copy(yuv_caps));
 621     }
 622     return out_caps;
 623 }
 624
 625 static gboolean
 626 gst_vaapisink_set_caps(GstBaseSink *base_sink, GstCaps *caps)
 627 {
 628     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
 629     GstStructure * const structure = gst_caps_get_structure(caps, 0);
 630     GstVideoInfo vi;
 631     guint win_width, win_height;
 632
 633 #if USE_DRM
 634     if (sink->display_type == GST_VAAPI_DISPLAY_TYPE_DRM)
 635         return TRUE;
 636 #endif
 637
 638     if (!gst_video_info_from_caps(&vi, caps))
 639         return FALSE;
 640     sink->video_width  = GST_VIDEO_INFO_WIDTH(&vi);
 641     sink->video_height = GST_VIDEO_INFO_HEIGHT(&vi);
 642     sink->video_par_n  = GST_VIDEO_INFO_PAR_N(&vi);
 643     sink->video_par_d  = GST_VIDEO_INFO_PAR_D(&vi);
 644     GST_DEBUG("video pixel-aspect-ratio %d/%d",
 645               sink->video_par_n, sink->video_par_d);
 646
 647     if (gst_structure_has_name(structure, "video/x-raw-yuv"))
 648         sink->use_video_raw = TRUE;
 649
 650     gst_caps_replace(&sink->caps, caps);
 651
 652     if (!gst_vaapisink_ensure_display(sink))
 653         return FALSE;
 654
 655     gst_vaapisink_ensure_rotation(sink, FALSE);
 656
 657     gst_vaapisink_ensure_window_size(sink, &win_width, &win_height);
 658     if (sink->window) {
 659         if (!sink->foreign_window || sink->fullscreen)
 660             gst_vaapi_window_set_size(sink->window, win_width, win_height);
 661     }
 662     else {
 663         gst_vaapi_display_lock(sink->display);
 664         gst_x_overlay_prepare_xwindow_id(GST_X_OVERLAY(sink));
 665         gst_vaapi_display_unlock(sink->display);
 666         if (sink->window)
 667             return TRUE;
 668         if (!gst_vaapisink_ensure_window(sink, win_width, win_height))
 669             return FALSE;
 670         gst_vaapi_window_set_fullscreen(sink->window, sink->fullscreen);
 671         gst_vaapi_window_show(sink->window);
 672         gst_vaapi_window_get_size(sink->window, &win_width, &win_height);
 673     }
 674     sink->window_width  = win_width;
 675     sink->window_height = win_height;
 676     GST_DEBUG("window size %ux%u", win_width, win_height);
 677
 678     return gst_vaapisink_ensure_render_rect(sink, win_width, win_height);
 679 }
 680
 681 #if USE_GLX
 682 static void
 683 render_background(GstVaapiSink *sink)
 684 {
 685     /* Original code from Mirco Muller (MacSlow):
 686        <http://cgit.freedesktop.org/~macslow/gl-gst-player/> */
 687     GLfloat fStartX = 0.0f;
 688     GLfloat fStartY = 0.0f;
 689     GLfloat fWidth  = (GLfloat)sink->window_width;
 690     GLfloat fHeight = (GLfloat)sink->window_height;
 691
 692     glClear(GL_COLOR_BUFFER_BIT);
 693     glBegin(GL_QUADS);
 694     {
 695         /* top third, darker grey to white */
 696         glColor3f(0.85f, 0.85f, 0.85f);
 697         glVertex3f(fStartX, fStartY, 0.0f);
 698         glColor3f(0.85f, 0.85f, 0.85f);
 699         glVertex3f(fStartX + fWidth, fStartY, 0.0f);
 700         glColor3f(1.0f, 1.0f, 1.0f);
 701         glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
 702         glColor3f(1.0f, 1.0f, 1.0f);
 703         glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
 704
 705         /* middle third, just plain white */
 706         glColor3f(1.0f, 1.0f, 1.0f);
 707         glVertex3f(fStartX, fStartY + fHeight / 3.0f, 0.0f);
 708         glVertex3f(fStartX + fWidth, fStartY + fHeight / 3.0f, 0.0f);
 709         glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
 710         glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
 711
 712         /* bottom third, white to lighter grey */
 713         glColor3f(1.0f, 1.0f, 1.0f);
 714         glVertex3f(fStartX, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
 715         glColor3f(1.0f, 1.0f, 1.0f);
 716         glVertex3f(fStartX + fWidth, fStartY + 2.0f * fHeight / 3.0f, 0.0f);
 717         glColor3f(0.62f, 0.66f, 0.69f);
 718         glVertex3f(fStartX + fWidth, fStartY + fHeight, 0.0f);
 719         glColor3f(0.62f, 0.66f, 0.69f);
 720         glVertex3f(fStartX, fStartY + fHeight, 0.0f);
 721     }
 722     glEnd();
 723 }
 724
 725 static void
 726 render_frame(GstVaapiSink *sink)
 727 {
 728     const guint x1 = sink->display_rect.x;
 729     const guint x2 = sink->display_rect.x + sink->display_rect.width;
 730     const guint y1 = sink->display_rect.y;
 731     const guint y2 = sink->display_rect.y + sink->display_rect.height;
 732
 733     glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
 734     glBegin(GL_QUADS);
 735     {
 736         glTexCoord2f(0.0f, 0.0f); glVertex2i(x1, y1);
 737         glTexCoord2f(0.0f, 1.0f); glVertex2i(x1, y2);
 738         glTexCoord2f(1.0f, 1.0f); glVertex2i(x2, y2);
 739         glTexCoord2f(1.0f, 0.0f); glVertex2i(x2, y1);
 740     }
 741     glEnd();
 742 }
 743
 744 static void
 745 render_reflection(GstVaapiSink *sink)
 746 {
 747     const guint x1 = sink->display_rect.x;
 748     const guint x2 = sink->display_rect.x + sink->display_rect.width;
 749     const guint y1 = sink->display_rect.y;
 750     const guint rh = sink->display_rect.height / 5;
 751     GLfloat     ry = 1.0f - (GLfloat)rh / (GLfloat)sink->display_rect.height;
 752
 753     glBegin(GL_QUADS);
 754     {
 755         glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
 756         glTexCoord2f(0.0f, 1.0f); glVertex2i(x1, y1);
 757         glTexCoord2f(1.0f, 1.0f); glVertex2i(x2, y1);
 758
 759         glColor4f(1.0f, 1.0f, 1.0f, 0.0f);
 760         glTexCoord2f(1.0f, ry); glVertex2i(x2, y1 + rh);
 761         glTexCoord2f(0.0f, ry); glVertex2i(x1, y1 + rh);
 762     }
 763     glEnd();
 764 }
 765
 766 static gboolean
 767 gst_vaapisink_show_frame_glx(
 768     GstVaapiSink    *sink,
 769     GstVaapiSurface *surface,
 770     guint            flags
 771 )
 772 {
 773     GstVaapiWindowGLX * const window = GST_VAAPI_WINDOW_GLX(sink->window);
 774     GLenum target;
 775     GLuint texture;
 776
 777     gst_vaapi_window_glx_make_current(window);
 778     if (!sink->texture) {
 779         sink->texture = gst_vaapi_texture_new(
 780             sink->display,
 781             GL_TEXTURE_2D,
 782             GL_BGRA,
 783             sink->video_width,
 784             sink->video_height
 785         );
 786         if (!sink->texture)
 787             goto error_create_texture;
 788     }
 789     if (!gst_vaapi_texture_put_surface(sink->texture, surface, flags))
 790         goto error_transfer_surface;
 791
 792     target  = gst_vaapi_texture_get_target(sink->texture);
 793     texture = gst_vaapi_texture_get_id(sink->texture);
 794     if (target != GL_TEXTURE_2D || !texture)
 795         return FALSE;
 796
 797     if (sink->use_reflection)
 798         render_background(sink);
 799
 800     glEnable(target);
 801     glBindTexture(target, texture);
 802     {
 803         if (sink->use_reflection) {
 804             glPushMatrix();
 805             glRotatef(20.0f, 0.0f, 1.0f, 0.0f);
 806             glTranslatef(50.0f, 0.0f, 0.0f);
 807         }
 808         render_frame(sink);
 809         if (sink->use_reflection) {
 810             glPushMatrix();
 811             glTranslatef(0.0, (GLfloat)sink->display_rect.height + 5.0f, 0.0f);
 812             render_reflection(sink);
 813             glPopMatrix();
 814             glPopMatrix();
 815         }
 816     }
 817     glBindTexture(target, 0);
 818     glDisable(target);
 819     gst_vaapi_window_glx_swap_buffers(window);
 820     return TRUE;
 821
 822     /* ERRORS */
 823 error_create_texture:
 824     {
 825         GST_DEBUG("could not create VA/GLX texture");
 826         return FALSE;
 827     }
 828 error_transfer_surface:
 829     {
 830         GST_DEBUG("could not transfer VA surface to texture");
 831         return FALSE;
 832     }
 833 }
 834 #endif
 835
 836 static inline gboolean
 837 gst_vaapisink_put_surface(
 838     GstVaapiSink    *sink,
 839     GstVaapiSurface *surface,
 840     guint            flags
 841 )
 842 {
 843     if (!gst_vaapi_window_put_surface(sink->window, surface,
 844                 NULL, &sink->display_rect, flags)) {
 845         GST_DEBUG("could not render VA surface");
 846         return FALSE;
 847     }
 848     return TRUE;
 849 }
 850
 851 static GstFlowReturn
 852 gst_vaapisink_show_frame(GstBaseSink *base_sink, GstBuffer *src_buffer)
 853 {
 854     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
 855     GstVideoOverlayComposition *composition;
 856     GstVaapiVideoMeta *meta;
 857     GstVaapiSurface *surface;
 858     GstBuffer *buffer;
 859     guint flags;
 860     gboolean success;
 861
 862     meta = gst_buffer_get_vaapi_video_meta(src_buffer);
 863     if (meta)
 864         buffer = gst_buffer_ref(src_buffer);
 865     else if (sink->use_video_raw) {
 866         buffer = gst_vaapi_uploader_get_buffer(sink->uploader);
 867         if (!buffer)
 868             return GST_FLOW_UNEXPECTED;
 869         if (!gst_vaapi_uploader_process(sink->uploader, src_buffer, buffer))
 870             goto error;
 871         meta = gst_buffer_get_vaapi_video_meta(buffer);
 872         if (!meta)
 873             goto error;
 874     }
 875     else
 876         return GST_FLOW_UNEXPECTED;
 877
 878     if (sink->display != gst_vaapi_video_meta_get_display(meta)) {
 879         g_clear_object(&sink->display);
 880         sink->display = g_object_ref(gst_vaapi_video_meta_get_display(meta));
 881     }
 882
 883     if (!sink->window)
 884         goto error;
 885
 886     gst_vaapisink_ensure_rotation(sink, TRUE);
 887
 888     surface = gst_vaapi_video_meta_get_surface(meta);
 889     if (!surface)
 890         goto error;
 891
 892     GST_DEBUG("render surface %" GST_VAAPI_ID_FORMAT,
 893               GST_VAAPI_ID_ARGS(gst_vaapi_surface_get_id(surface)));
 894
 895     flags = gst_vaapi_video_meta_get_render_flags(meta);
 896
 897     composition = gst_video_buffer_get_overlay_composition(src_buffer);
 898     if (!gst_vaapi_surface_set_subpictures_from_composition(surface,
 899              composition, TRUE))
 900         GST_WARNING("could not update subtitles");
 901
 902     switch (sink->display_type) {
 903 #if USE_GLX
 904     case GST_VAAPI_DISPLAY_TYPE_GLX:
 905         success = gst_vaapisink_show_frame_glx(sink, surface, flags);
 906         break;
 907 #endif
 908 #if USE_DRM
 909     case GST_VAAPI_DISPLAY_TYPE_DRM:
 910         success = TRUE;
 911         break;
 912 #endif
 913 #if USE_X11
 914     case GST_VAAPI_DISPLAY_TYPE_X11:
 915         success = gst_vaapisink_put_surface(sink, surface, flags);
 916         break;
 917 #endif
 918 #if USE_WAYLAND
 919     case GST_VAAPI_DISPLAY_TYPE_WAYLAND:
 920         success = gst_vaapisink_put_surface(sink, surface, flags);
 921         break;
 922 #endif
 923     default:
 924         GST_ERROR("unsupported display type %d", sink->display_type);
 925         success = FALSE;
 926         break;
 927     }
 928     if (!success)
 929         goto error;
 930
 931     /* Retain VA surface until the next one is displayed */
 932     if (sink->use_overlay)
 933         gst_buffer_replace(&sink->video_buffer, buffer);
 934     gst_buffer_unref(buffer);
 935     return GST_FLOW_OK;
 936
 937 error:
 938     gst_buffer_unref(buffer);
 939     return GST_FLOW_UNEXPECTED;
 940 }
 941
 942 static GstFlowReturn
 943 gst_vaapisink_buffer_alloc(
 944     GstBaseSink        *base_sink,
 945     guint64             offset,
 946     guint               size,
 947     GstCaps            *caps,
 948     GstBuffer         **pbuf
 949 )
 950 {
 951     GstVaapiSink * const sink = GST_VAAPISINK(base_sink);
 952     GstStructure *structure;
 953     GstBuffer *buf;
 954
 955     *pbuf = NULL;
 956
 957     structure = gst_caps_get_structure(caps, 0);
 958     if (!gst_structure_has_name(structure, "video/x-raw-yuv"))
 959         return GST_FLOW_OK;
 960
 961     if (!gst_vaapi_uploader_ensure_display(sink->uploader, sink->display))
 962         return GST_FLOW_NOT_SUPPORTED;
 963     if (!gst_vaapi_uploader_ensure_caps(sink->uploader, caps, NULL))
 964         return GST_FLOW_NOT_SUPPORTED;
 965
 966     buf = gst_vaapi_uploader_get_buffer(sink->uploader);
 967     if (!buf) {
 968         GST_WARNING("failed to allocate resources for raw YUV buffer");
 969         return GST_FLOW_NOT_SUPPORTED;
 970     }
 971
 972     *pbuf = buf;
 973     return GST_FLOW_OK;
 974 }
 975
 976 static gboolean
 977 gst_vaapisink_query(GstBaseSink *base_sink, GstQuery *query)
 978 {
 979     GstVaapiSink *sink = GST_VAAPISINK(base_sink);
 980     GST_DEBUG ("sharing display %p", sink->display);
 981     return gst_vaapi_reply_to_query (query, sink->display);
 982 }
 983
 984 static void
 985 gst_vaapisink_finalize(GObject *object)
 986 {
 987     gst_vaapisink_destroy(GST_VAAPISINK(object));
 988
 989     G_OBJECT_CLASS(gst_vaapisink_parent_class)->finalize(object);
 990 }
 991
 992 static void
 993 gst_vaapisink_set_property(
 994     GObject      *object,
 995     guint         prop_id,
 996     const GValue *value,
 997     GParamSpec   *pspec
 998 )
 999 {
1000     GstVaapiSink * const sink = GST_VAAPISINK(object);
1001
1002     switch (prop_id) {
1003     case PROP_DISPLAY_TYPE:
1004         sink->display_type = g_value_get_enum(value);
1005         break;
1006     case PROP_FULLSCREEN:
1007         sink->fullscreen = g_value_get_boolean(value);
1008         break;
1009     case PROP_SYNCHRONOUS:
1010         sink->synchronous = g_value_get_boolean(value);
1011         break;
1012     case PROP_USE_REFLECTION:
1013         sink->use_reflection = g_value_get_boolean(value);
1014         break;
1015     case PROP_ROTATION:
1016         sink->rotation_req = g_value_get_enum(value);
1017         break;
1018     default:
1019         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
1020         break;
1021     }
1022 }
1023
1024 static void
1025 gst_vaapisink_get_property(
1026     GObject    *object,
1027     guint       prop_id,
1028     GValue     *value,
1029     GParamSpec *pspec
1030 )
1031 {
1032     GstVaapiSink * const sink = GST_VAAPISINK(object);
1033
1034     switch (prop_id) {
1035     case PROP_DISPLAY_TYPE:
1036         g_value_set_enum(value, sink->display_type);
1037         break;
1038     case PROP_FULLSCREEN:
1039         g_value_set_boolean(value, sink->fullscreen);
1040         break;
1041     case PROP_SYNCHRONOUS:
1042         g_value_set_boolean(value, sink->synchronous);
1043         break;
1044     case PROP_USE_REFLECTION:
1045         g_value_set_boolean(value, sink->use_reflection);
1046         break;
1047     case PROP_ROTATION:
1048         g_value_set_enum(value, sink->rotation);
1049         break;
1050     default:
1051         G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
1052         break;
1053     }
1054 }
1055
1056 static void
1057 gst_vaapisink_class_init(GstVaapiSinkClass *klass)
1058 {
1059     GObjectClass * const     object_class   = G_OBJECT_CLASS(klass);
1060     GstElementClass * const  element_class  = GST_ELEMENT_CLASS(klass);
1061     GstBaseSinkClass * const basesink_class = GST_BASE_SINK_CLASS(klass);
1062     GstPadTemplate *pad_template;
1063
1064     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapisink,
1065                             GST_PLUGIN_NAME, 0, GST_PLUGIN_DESC);
1066
1067     object_class->finalize       = gst_vaapisink_finalize;
1068     object_class->set_property   = gst_vaapisink_set_property;
1069     object_class->get_property   = gst_vaapisink_get_property;
1070
1071     basesink_class->start        = gst_vaapisink_start;
1072     basesink_class->stop         = gst_vaapisink_stop;
1073     basesink_class->get_caps     = gst_vaapisink_get_caps;
1074     basesink_class->set_caps     = gst_vaapisink_set_caps;
1075     basesink_class->preroll      = gst_vaapisink_show_frame;
1076     basesink_class->render       = gst_vaapisink_show_frame;
1077     basesink_class->query        = gst_vaapisink_query;
1078     basesink_class->buffer_alloc = gst_vaapisink_buffer_alloc;
1079
1080     gst_element_class_set_details_simple(
1081         element_class,
1082         gst_vaapisink_details.longname,
1083         gst_vaapisink_details.klass,
1084         gst_vaapisink_details.description,
1085         gst_vaapisink_details.author
1086     );
1087
1088     pad_template = gst_static_pad_template_get(&gst_vaapisink_sink_factory);
1089     gst_element_class_add_pad_template(element_class, pad_template);
1090     gst_object_unref(pad_template);
1091
1092     g_object_class_install_property
1093         (object_class,
1094          PROP_DISPLAY_TYPE,
1095          g_param_spec_enum("display",
1096                            "display type",
1097                            "display type to use",
1098                            GST_VAAPI_TYPE_DISPLAY_TYPE,
1099                            GST_VAAPI_DISPLAY_TYPE_ANY,
1100                            G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1101
1102 #if USE_GLX
1103     g_object_class_install_property
1104         (object_class,
1105          PROP_USE_REFLECTION,
1106          g_param_spec_boolean("use-reflection",
1107                               "Reflection effect",
1108                               "Enables OpenGL reflection effect",
1109                               FALSE,
1110                               G_PARAM_READWRITE));
1111 #endif
1112
1113     g_object_class_install_property
1114         (object_class,
1115          PROP_FULLSCREEN,
1116          g_param_spec_boolean("fullscreen",
1117                               "Fullscreen",
1118                               "Requests window in fullscreen state",
1119                               FALSE,
1120                               G_PARAM_READWRITE));
1121
1122     /**
1123      * GstVaapiSink:synchronous:
1124      *
1125      * When enabled, runs the X display in synchronous mode. Note that
1126      * this is used only for debugging.
1127      */
1128     g_object_class_install_property
1129         (object_class,
1130          PROP_SYNCHRONOUS,
1131          g_param_spec_boolean("synchronous",
1132                               "Synchronous mode",
1133                               "Toggles X display synchronous mode",
1134                               FALSE,
1135                               G_PARAM_READWRITE));
1136
1137     /**
1138      * GstVaapiSink:rotation:
1139      *
1140      * The VA display rotation mode, expressed as a #GstVaapiRotation.
1141      */
1142     g_object_class_install_property
1143         (object_class,
1144          PROP_ROTATION,
1145          g_param_spec_enum(GST_VAAPI_DISPLAY_PROP_ROTATION,
1146                            "rotation",
1147                            "The display rotation mode",
1148                            GST_VAAPI_TYPE_ROTATION,
1149                            DEFAULT_ROTATION,
1150                            G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
1151 }
1152
1153 static void
1154 gst_vaapisink_init(GstVaapiSink *sink)
1155 {
1156     sink->caps           = NULL;
1157     sink->display        = NULL;
1158     sink->window         = NULL;
1159     sink->window_width   = 0;
1160     sink->window_height  = 0;
1161     sink->texture        = NULL;
1162     sink->video_buffer   = NULL;
1163     sink->video_width    = 0;
1164     sink->video_height   = 0;
1165     sink->video_par_n    = 1;
1166     sink->video_par_d    = 1;
1167     sink->foreign_window = FALSE;
1168     sink->fullscreen     = FALSE;
1169     sink->synchronous    = FALSE;
1170     sink->display_type   = DEFAULT_DISPLAY_TYPE;
1171     sink->rotation       = DEFAULT_ROTATION;
1172     sink->rotation_req   = DEFAULT_ROTATION;
1173     sink->use_reflection = FALSE;
1174     sink->use_overlay    = FALSE;
1175     sink->use_rotation   = FALSE;
1176 }