va/glx/va_glx_impl.c

   1 /*
   2  * Copyright (C) 2009 Splitted-Desktop Systems. All Rights Reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sub license, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the
  13  * next paragraph) shall be included in all copies or substantial portions
  14  * of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  19  * IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
  20  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  22  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #define _GNU_SOURCE 1
  26 #include "va_glx_private.h"
  27 #include "va_glx_impl.h"
  28 #include <stdio.h>
  29 #include <stdlib.h>
  30 #include <stdarg.h>
  31 #include <string.h>
  32 #include <assert.h>
  33 #include <dlfcn.h>
  34
  35 static void va_glx_error_message(const char *format, ...)
  36 {
  37     va_list args;
  38     va_start(args, format);
  39     fprintf(stderr, "libva-glx error: ");
  40     vfprintf(stderr, format, args);
  41     va_end(args);
  42 }
  43
  44 // X error trap
  45 static int x11_error_code = 0;
  46 static int (*old_error_handler)(Display *, XErrorEvent *);
  47
  48 static int error_handler(Display *dpy, XErrorEvent *error)
  49 {
  50     x11_error_code = error->error_code;
  51     return 0;
  52 }
  53
  54 static void x11_trap_errors(void)
  55 {
  56     x11_error_code    = 0;
  57     old_error_handler = XSetErrorHandler(error_handler);
  58 }
  59
  60 static int x11_untrap_errors(void)
  61 {
  62     XSetErrorHandler(old_error_handler);
  63     return x11_error_code;
  64 }
  65
  66 // Returns a string representation of an OpenGL error
  67 static const char *gl_get_error_string(GLenum error)
  68 {
  69     static const struct {
  70         GLenum val;
  71         const char *str;
  72     }
  73     gl_errors[] = {
  74         { GL_NO_ERROR,          "no error" },
  75         { GL_INVALID_ENUM,      "invalid enumerant" },
  76         { GL_INVALID_VALUE,     "invalid value" },
  77         { GL_INVALID_OPERATION, "invalid operation" },
  78         { GL_STACK_OVERFLOW,    "stack overflow" },
  79         { GL_STACK_UNDERFLOW,   "stack underflow" },
  80         { GL_OUT_OF_MEMORY,     "out of memory" },
  81 #ifdef GL_INVALID_FRAMEBUFFER_OPERATION_EXT
  82         { GL_INVALID_FRAMEBUFFER_OPERATION_EXT, "invalid framebuffer operation" },
  83 #endif
  84         { ~0, NULL }
  85     };
  86
  87     int i;
  88     for (i = 0; gl_errors[i].str; i++) {
  89         if (gl_errors[i].val == error)
  90             return gl_errors[i].str;
  91     }
  92     return "unknown";
  93 }
  94
  95 static inline int gl_do_check_error(int report)
  96 {
  97     GLenum error;
  98     int is_error = 0;
  99     while ((error = glGetError()) != GL_NO_ERROR) {
 100         if (report)
 101             va_glx_error_message("glError: %s caught\n",
 102                                  gl_get_error_string(error));
 103         is_error = 1;
 104     }
 105     return is_error;
 106 }
 107
 108 static inline void gl_purge_errors(void)
 109 {
 110     gl_do_check_error(0);
 111 }
 112
 113 static inline int gl_check_error(void)
 114 {
 115     return gl_do_check_error(1);
 116 }
 117
 118 // glGetTexLevelParameteriv() wrapper
 119 static int gl_get_texture_param(GLenum param, unsigned int *pval)
 120 {
 121     GLint val;
 122
 123     gl_purge_errors();
 124     glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, param, &val);
 125     if (gl_check_error())
 126         return 0;
 127     if (pval)
 128         *pval = val;
 129     return 1;
 130 }
 131
 132 // Returns the OpenGL VTable
 133 static inline VAOpenGLVTableP gl_get_vtable(VADriverContextP ctx)
 134 {
 135     return &VA_DRIVER_CONTEXT_GLX(ctx)->gl_vtable;
 136 }
 137
 138 // Lookup for a GLX function
 139 typedef void (*GLFuncPtr)(void);
 140 typedef GLFuncPtr (*GLXGetProcAddressProc)(const char *);
 141
 142 static GLFuncPtr get_proc_address_default(const char *name)
 143 {
 144     return NULL;
 145 }
 146
 147 static GLXGetProcAddressProc get_proc_address_func(void)
 148 {
 149     GLXGetProcAddressProc get_proc_func;
 150
 151     dlerror();
 152     get_proc_func = (GLXGetProcAddressProc)
 153         dlsym(RTLD_DEFAULT, "glXGetProcAddress");
 154     if (!dlerror())
 155         return get_proc_func;
 156
 157     get_proc_func = (GLXGetProcAddressProc)
 158         dlsym(RTLD_DEFAULT, "glXGetProcAddressARB");
 159     if (!dlerror())
 160         return get_proc_func;
 161
 162     return get_proc_address_default;
 163 }
 164
 165 static inline GLFuncPtr get_proc_address(const char *name)
 166 {
 167     static GLXGetProcAddressProc get_proc_func = NULL;
 168     if (!get_proc_func)
 169         get_proc_func = get_proc_address_func();
 170     return get_proc_func(name);
 171 }
 172
 173 // Check for GLX extensions (TFP, FBO)
 174 static int check_extension(const char *name, const char *ext)
 175 {
 176     const char *end;
 177     int name_len, n;
 178
 179     if (!name || !ext)
 180         return 0;
 181
 182     end = ext + strlen(ext);
 183     name_len = strlen(name);
 184     while (ext < end) {
 185         n = strcspn(ext, " ");
 186         if (n == name_len && strncmp(name, ext, n) == 0)
 187             return 1;
 188         ext += (n + 1);
 189     }
 190     return 0;
 191 }
 192
 193 static int check_tfp_extensions(VADriverContextP ctx)
 194 {
 195     const char *gl_extensions;
 196     const char *glx_extensions;
 197
 198     gl_extensions = (const char *)glGetString(GL_EXTENSIONS);
 199     if (!check_extension("GL_ARB_texture_non_power_of_two", gl_extensions))
 200         return 0;
 201
 202     glx_extensions = glXQueryExtensionsString(ctx->native_dpy, ctx->x11_screen);
 203     if (!check_extension("GLX_EXT_texture_from_pixmap", glx_extensions))
 204         return 0;
 205     return 1;
 206 }
 207
 208 static int check_fbo_extensions(VADriverContextP ctx)
 209 {
 210     const char *gl_extensions;
 211
 212     gl_extensions = (const char *)glGetString(GL_EXTENSIONS);
 213     if (check_extension("GL_ARB_framebuffer_object", gl_extensions))
 214         return 1;
 215     if (check_extension("GL_EXT_framebuffer_object", gl_extensions))
 216         return 1;
 217     return 0;
 218 }
 219
 220 // Load GLX extensions
 221 static int load_tfp_extensions(VADriverContextP ctx)
 222 {
 223     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 224
 225     pOpenGLVTable->glx_create_pixmap = (PFNGLXCREATEPIXMAPPROC)
 226         get_proc_address("glXCreatePixmap");
 227     if (!pOpenGLVTable->glx_create_pixmap)
 228         return 0;
 229     pOpenGLVTable->glx_destroy_pixmap = (PFNGLXDESTROYPIXMAPPROC)
 230         get_proc_address("glXDestroyPixmap");
 231     if (!pOpenGLVTable->glx_destroy_pixmap)
 232         return 0;
 233     pOpenGLVTable->glx_bind_tex_image = (PFNGLXBINDTEXIMAGEEXTPROC)
 234         get_proc_address("glXBindTexImageEXT");
 235     if (!pOpenGLVTable->glx_bind_tex_image)
 236         return 0;
 237     pOpenGLVTable->glx_release_tex_image = (PFNGLXRELEASETEXIMAGEEXTPROC)
 238         get_proc_address("glXReleaseTexImageEXT");
 239     if (!pOpenGLVTable->glx_release_tex_image)
 240         return 0;
 241     return 1;
 242 }
 243
 244 static int load_fbo_extensions(VADriverContextP ctx)
 245 {
 246     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 247
 248     pOpenGLVTable->gl_gen_framebuffers = (PFNGLGENFRAMEBUFFERSEXTPROC)
 249         get_proc_address("glGenFramebuffersEXT");
 250     if (!pOpenGLVTable->gl_gen_framebuffers)
 251         return 0;
 252     pOpenGLVTable->gl_delete_framebuffers = (PFNGLDELETEFRAMEBUFFERSEXTPROC)
 253         get_proc_address("glDeleteFramebuffersEXT");
 254     if (!pOpenGLVTable->gl_delete_framebuffers)
 255         return 0;
 256     pOpenGLVTable->gl_bind_framebuffer = (PFNGLBINDFRAMEBUFFEREXTPROC)
 257         get_proc_address("glBindFramebufferEXT");
 258     if (!pOpenGLVTable->gl_bind_framebuffer)
 259         return 0;
 260     pOpenGLVTable->gl_gen_renderbuffers = (PFNGLGENRENDERBUFFERSEXTPROC)
 261         get_proc_address("glGenRenderbuffersEXT");
 262     if (!pOpenGLVTable->gl_gen_renderbuffers)
 263         return 0;
 264     pOpenGLVTable->gl_delete_renderbuffers = (PFNGLDELETERENDERBUFFERSEXTPROC)
 265         get_proc_address("glDeleteRenderbuffersEXT");
 266     if (!pOpenGLVTable->gl_delete_renderbuffers)
 267         return 0;
 268     pOpenGLVTable->gl_bind_renderbuffer = (PFNGLBINDRENDERBUFFEREXTPROC)
 269         get_proc_address("glBindRenderbufferEXT");
 270     if (!pOpenGLVTable->gl_bind_renderbuffer)
 271         return 0;
 272     pOpenGLVTable->gl_renderbuffer_storage = (PFNGLRENDERBUFFERSTORAGEEXTPROC)
 273         get_proc_address("glRenderbufferStorageEXT");
 274     if (!pOpenGLVTable->gl_renderbuffer_storage)
 275         return 0;
 276     pOpenGLVTable->gl_framebuffer_renderbuffer = (PFNGLFRAMEBUFFERRENDERBUFFEREXTPROC)
 277         get_proc_address("glFramebufferRenderbufferEXT");
 278     if (!pOpenGLVTable->gl_framebuffer_renderbuffer)
 279         return 0;
 280     pOpenGLVTable->gl_framebuffer_texture_2d = (PFNGLFRAMEBUFFERTEXTURE2DEXTPROC)
 281         get_proc_address("glFramebufferTexture2DEXT");
 282     if (!pOpenGLVTable->gl_framebuffer_texture_2d)
 283         return 0;
 284     pOpenGLVTable->gl_check_framebuffer_status = (PFNGLCHECKFRAMEBUFFERSTATUSEXTPROC)
 285         get_proc_address("glCheckFramebufferStatusEXT");
 286     if (!pOpenGLVTable->gl_check_framebuffer_status)
 287         return 0;
 288     return 1;
 289 }
 290
 291
 292 /* ========================================================================= */
 293 /* === VA/GLX helpers                                                    === */
 294 /* ========================================================================= */
 295
 296 // OpenGL context state
 297 typedef struct OpenGLContextState *OpenGLContextStateP;
 298
 299 struct OpenGLContextState {
 300     Display     *display;
 301     Window       window;
 302     GLXContext   context;
 303 };
 304
 305 static void
 306 gl_destroy_context(OpenGLContextStateP cs)
 307 {
 308     if (!cs)
 309         return;
 310
 311     if (cs->display && cs->context) {
 312         if (glXGetCurrentContext() == cs->context)
 313             glXMakeCurrent(cs->display, None, NULL);
 314         glXDestroyContext(cs->display, cs->context);
 315         cs->display = NULL;
 316         cs->context = NULL;
 317     }
 318     free(cs);
 319 }
 320
 321 static OpenGLContextStateP
 322 gl_create_context(VADriverContextP ctx, OpenGLContextStateP parent)
 323 {
 324     OpenGLContextStateP cs;
 325     GLXFBConfig *fbconfigs = NULL;
 326     int fbconfig_id, val, n, n_fbconfigs;
 327     Status status;
 328
 329     static GLint fbconfig_attrs[] = {
 330         GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT,
 331         GLX_RENDER_TYPE,   GLX_RGBA_BIT,
 332         GLX_DOUBLEBUFFER,  True,
 333         GLX_RED_SIZE,      8,
 334         GLX_GREEN_SIZE,    8,
 335         GLX_BLUE_SIZE,     8,
 336         None
 337     };
 338
 339     cs = malloc(sizeof(*cs));
 340     if (!cs)
 341         goto error;
 342
 343     if (parent) {
 344         cs->display = parent->display;
 345         cs->window  = parent->window;
 346     }
 347     else {
 348         cs->display = ctx->native_dpy;
 349         cs->window  = None;
 350     }
 351     cs->context = NULL;
 352
 353     if (parent && parent->context) {
 354         status = glXQueryContext(
 355             parent->display,
 356             parent->context,
 357             GLX_FBCONFIG_ID, &fbconfig_id
 358         );
 359         if (status != Success)
 360             goto error;
 361
 362         if (fbconfig_id == GLX_DONT_CARE)
 363             goto choose_fbconfig;
 364
 365         fbconfigs = glXGetFBConfigs(
 366             parent->display,
 367             DefaultScreen(parent->display),
 368             &n_fbconfigs
 369         );
 370         if (!fbconfigs)
 371             goto error;
 372
 373         /* Find out a GLXFBConfig compatible with the parent context */
 374         for (n = 0; n < n_fbconfigs; n++) {
 375             status = glXGetFBConfigAttrib(
 376                 cs->display,
 377                 fbconfigs[n],
 378                 GLX_FBCONFIG_ID, &val
 379             );
 380             if (status == Success && val == fbconfig_id)
 381                 break;
 382         }
 383         if (n == n_fbconfigs)
 384             goto error;
 385     }
 386     else {
 387     choose_fbconfig:
 388         fbconfigs = glXChooseFBConfig(
 389             ctx->native_dpy,
 390             ctx->x11_screen,
 391             fbconfig_attrs, &n_fbconfigs
 392         );
 393         if (!fbconfigs)
 394             goto error;
 395
 396         /* Select the first one */
 397         n = 0;
 398     }
 399
 400     cs->context = glXCreateNewContext(
 401         cs->display,
 402         fbconfigs[n],
 403         GLX_RGBA_TYPE,
 404         parent ? parent->context : NULL,
 405         True
 406     );
 407     if (cs->context)
 408         goto end;
 409
 410 error:
 411     gl_destroy_context(cs);
 412     cs = NULL;
 413 end:
 414     if (fbconfigs)
 415         XFree(fbconfigs);
 416     return cs;
 417 }
 418
 419 static void gl_get_current_context(OpenGLContextStateP cs)
 420 {
 421     cs->display = glXGetCurrentDisplay();
 422     cs->window  = glXGetCurrentDrawable();
 423     cs->context = glXGetCurrentContext();
 424 }
 425
 426 static int
 427 gl_set_current_context(OpenGLContextStateP new_cs, OpenGLContextStateP old_cs)
 428 {
 429     /* If display is NULL, this could be that new_cs was retrieved from
 430        gl_get_current_context() with none set previously. If that case,
 431        the other fields are also NULL and we don't return an error */
 432     if (!new_cs->display)
 433         return !new_cs->window && !new_cs->context;
 434
 435     if (old_cs) {
 436         if (old_cs == new_cs)
 437             return 1;
 438         gl_get_current_context(old_cs);
 439         if (old_cs->display == new_cs->display &&
 440             old_cs->window  == new_cs->window  &&
 441             old_cs->context == new_cs->context)
 442             return 1;
 443     }
 444     return glXMakeCurrent(new_cs->display, new_cs->window, new_cs->context);
 445 }
 446
 447 /** Unique VASurfaceGLX identifier */
 448 #define VA_SURFACE_GLX_MAGIC VA_FOURCC('V','A','G','L')
 449
 450 struct VASurfaceGLX {
 451     uint32_t            magic;      ///< Magic number identifying a VASurfaceGLX
 452     GLenum              target;     ///< GL target to which the texture is bound
 453     GLuint              texture;    ///< GL texture
 454     VASurfaceID         surface;    ///< Associated VA surface
 455     unsigned int        width;
 456     unsigned int        height;
 457     OpenGLContextStateP gl_context;
 458     int                 is_bound;
 459     Pixmap              pixmap;
 460     GLuint              pix_texture;
 461     GLXPixmap           glx_pixmap;
 462     GLuint              fbo;
 463 };
 464
 465 // Create Pixmaps for GLX texture-from-pixmap extension
 466 static int create_tfp_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 467 {
 468     VAOpenGLVTableP const pOpenGLVTable = gl_get_vtable(ctx);
 469     const unsigned int    width         = pSurfaceGLX->width;
 470     const unsigned int    height        = pSurfaceGLX->height;
 471     Pixmap                pixmap        = None;
 472     GLXFBConfig          *fbconfig      = NULL;
 473     GLXPixmap             glx_pixmap    = None;
 474     Window                root_window;
 475     XWindowAttributes     wattr;
 476     int                  *attrib;
 477     int                   n_fbconfig_attrs;
 478
 479     root_window = RootWindow(ctx->native_dpy, ctx->x11_screen);
 480     XGetWindowAttributes(ctx->native_dpy, root_window, &wattr);
 481     if (wattr.depth != 24 && wattr.depth != 32)
 482         return 0;
 483     pixmap = XCreatePixmap(
 484         ctx->native_dpy,
 485         root_window,
 486         width,
 487         height,
 488         wattr.depth
 489     );
 490     if (!pixmap)
 491         return 0;
 492     pSurfaceGLX->pixmap = pixmap;
 493
 494     int fbconfig_attrs[32] = {
 495         GLX_DRAWABLE_TYPE,      GLX_PIXMAP_BIT,
 496         GLX_DOUBLEBUFFER,       GL_TRUE,
 497         GLX_RENDER_TYPE,        GLX_RGBA_BIT,
 498         GLX_X_RENDERABLE,       GL_TRUE,
 499         GLX_Y_INVERTED_EXT,     GL_TRUE,
 500         GLX_RED_SIZE,           8,
 501         GLX_GREEN_SIZE,         8,
 502         GLX_BLUE_SIZE,          8,
 503         /*
 504          * depth test isn't enabled in the implementaion of VA GLX,
 505          * so depth buffer is unnecessary. However to workaround a
 506          * bug in older verson of xorg-server, always require a depth
 507          * buffer.
 508          *
 509          * See https://bugs.freedesktop.org/show_bug.cgi?id=76755
 510          */
 511         GLX_DEPTH_SIZE,         1,
 512         GL_NONE,
 513     };
 514     for (attrib = fbconfig_attrs; *attrib != GL_NONE; attrib += 2)
 515         ;
 516     if (wattr.depth == 32) {
 517     *attrib++ = GLX_ALPHA_SIZE;                 *attrib++ = 8;
 518     *attrib++ = GLX_BIND_TO_TEXTURE_RGBA_EXT;   *attrib++ = GL_TRUE;
 519     }
 520     else {
 521     *attrib++ = GLX_BIND_TO_TEXTURE_RGB_EXT;    *attrib++ = GL_TRUE;
 522     }
 523     *attrib++ = GL_NONE;
 524
 525     fbconfig = glXChooseFBConfig(
 526         ctx->native_dpy,
 527         ctx->x11_screen,
 528         fbconfig_attrs,
 529         &n_fbconfig_attrs
 530     );
 531     if (!fbconfig)
 532         return 0;
 533
 534     int pixmap_attrs[10] = {
 535         GLX_TEXTURE_TARGET_EXT, GLX_TEXTURE_2D_EXT,
 536         GLX_MIPMAP_TEXTURE_EXT, GL_FALSE,
 537         GL_NONE,
 538     };
 539     for (attrib = pixmap_attrs; *attrib != GL_NONE; attrib += 2)
 540         ;
 541     *attrib++ = GLX_TEXTURE_FORMAT_EXT;
 542     if (wattr.depth == 32)
 543     *attrib++ = GLX_TEXTURE_FORMAT_RGBA_EXT;
 544     else
 545     *attrib++ = GLX_TEXTURE_FORMAT_RGB_EXT;
 546     *attrib++ = GL_NONE;
 547
 548     x11_trap_errors();
 549     glx_pixmap = pOpenGLVTable->glx_create_pixmap(
 550         ctx->native_dpy,
 551         fbconfig[0],
 552         pixmap,
 553         pixmap_attrs
 554     );
 555     free(fbconfig);
 556     if (x11_untrap_errors() != 0)
 557         return 0;
 558     pSurfaceGLX->glx_pixmap = glx_pixmap;
 559
 560     glGenTextures(1, &pSurfaceGLX->pix_texture);
 561     glBindTexture(GL_TEXTURE_2D, pSurfaceGLX->pix_texture);
 562     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 563     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 564     return 1;
 565 }
 566
 567 // Destroy Pixmaps used for TFP
 568 static void destroy_tfp_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 569 {
 570     VAOpenGLVTableP const pOpenGLVTable = gl_get_vtable(ctx);
 571
 572     if (pSurfaceGLX->pix_texture) {
 573         glDeleteTextures(1, &pSurfaceGLX->pix_texture);
 574         pSurfaceGLX->pix_texture = 0;
 575     }
 576
 577     if (pSurfaceGLX->glx_pixmap) {
 578         pOpenGLVTable->glx_destroy_pixmap(ctx->native_dpy, pSurfaceGLX->glx_pixmap);
 579         pSurfaceGLX->glx_pixmap = None;
 580     }
 581
 582     if (pSurfaceGLX->pixmap) {
 583         XFreePixmap(ctx->native_dpy, pSurfaceGLX->pixmap);
 584         pSurfaceGLX->pixmap = None;
 585     }
 586 }
 587
 588 // Bind GLX Pixmap to texture
 589 static int bind_pixmap(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 590 {
 591     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 592
 593     if (pSurfaceGLX->is_bound)
 594         return 1;
 595
 596     glBindTexture(GL_TEXTURE_2D, pSurfaceGLX->pix_texture);
 597
 598     x11_trap_errors();
 599     pOpenGLVTable->glx_bind_tex_image(
 600         ctx->native_dpy,
 601         pSurfaceGLX->glx_pixmap,
 602         GLX_FRONT_LEFT_EXT,
 603         NULL
 604     );
 605     XSync(ctx->native_dpy, False);
 606     if (x11_untrap_errors() != 0) {
 607         va_glx_error_message("failed to bind pixmap\n");
 608         return 0;
 609     }
 610
 611     pSurfaceGLX->is_bound = 1;
 612     return 1;
 613 }
 614
 615 // Release GLX Pixmap from texture
 616 static int unbind_pixmap(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 617 {
 618     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 619
 620     if (!pSurfaceGLX->is_bound)
 621         return 1;
 622
 623     x11_trap_errors();
 624     pOpenGLVTable->glx_release_tex_image(
 625         ctx->native_dpy,
 626         pSurfaceGLX->glx_pixmap,
 627         GLX_FRONT_LEFT_EXT
 628     );
 629     XSync(ctx->native_dpy, False);
 630     if (x11_untrap_errors() != 0) {
 631         va_glx_error_message("failed to release pixmap\n");
 632         return 0;
 633     }
 634
 635     glBindTexture(GL_TEXTURE_2D, 0);
 636
 637     pSurfaceGLX->is_bound = 0;
 638     return 1;
 639 }
 640
 641 // Render GLX Pixmap to texture
 642 static void render_pixmap(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 643 {
 644     const unsigned int w = pSurfaceGLX->width;
 645     const unsigned int h = pSurfaceGLX->height;
 646
 647     glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
 648     glBegin(GL_QUADS);
 649     {
 650         glTexCoord2f(0.0f, 0.0f); glVertex2i(0, 0);
 651         glTexCoord2f(0.0f, 1.0f); glVertex2i(0, h);
 652         glTexCoord2f(1.0f, 1.0f); glVertex2i(w, h);
 653         glTexCoord2f(1.0f, 0.0f); glVertex2i(w, 0);
 654     }
 655     glEnd();
 656 }
 657
 658 // Create offscreen surface
 659 static int create_fbo_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 660 {
 661     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 662     GLuint fbo;
 663     GLenum status;
 664
 665     pOpenGLVTable->gl_gen_framebuffers(1, &fbo);
 666     pOpenGLVTable->gl_bind_framebuffer(GL_FRAMEBUFFER_EXT, fbo);
 667     pOpenGLVTable->gl_framebuffer_texture_2d(
 668         GL_FRAMEBUFFER_EXT,
 669         GL_COLOR_ATTACHMENT0_EXT,
 670         GL_TEXTURE_2D,
 671         pSurfaceGLX->texture,
 672         0
 673     );
 674
 675     status = pOpenGLVTable->gl_check_framebuffer_status(GL_DRAW_FRAMEBUFFER_EXT);
 676     pOpenGLVTable->gl_bind_framebuffer(GL_FRAMEBUFFER_EXT, 0);
 677     if (status != GL_FRAMEBUFFER_COMPLETE_EXT)
 678         return 0;
 679
 680     pSurfaceGLX->fbo = fbo;
 681     return 1;
 682 }
 683
 684 // Destroy offscreen surface
 685 static void destroy_fbo_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 686 {
 687     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 688
 689     if (pSurfaceGLX->fbo) {
 690         pOpenGLVTable->gl_delete_framebuffers(1, &pSurfaceGLX->fbo);
 691         pSurfaceGLX->fbo = 0;
 692     }
 693 }
 694
 695 // Setup matrices to match the FBO texture dimensions
 696 static void fbo_enter(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 697 {
 698     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 699     const unsigned int width  = pSurfaceGLX->width;
 700     const unsigned int height = pSurfaceGLX->height;
 701
 702     pOpenGLVTable->gl_bind_framebuffer(GL_FRAMEBUFFER_EXT, pSurfaceGLX->fbo);
 703     glPushAttrib(GL_VIEWPORT_BIT);
 704     glMatrixMode(GL_PROJECTION);
 705     glPushMatrix();
 706     glLoadIdentity();
 707     glMatrixMode(GL_MODELVIEW);
 708     glPushMatrix();
 709     glLoadIdentity();
 710     glViewport(0, 0, width, height);
 711     glTranslatef(-1.0f, -1.0f, 0.0f);
 712     glScalef(2.0f / width, 2.0f / height, 1.0f);
 713 }
 714
 715 // Restore original OpenGL matrices
 716 static void fbo_leave(VADriverContextP ctx)
 717 {
 718     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 719
 720     glPopAttrib();
 721     glMatrixMode(GL_PROJECTION);
 722     glPopMatrix();
 723     glMatrixMode(GL_MODELVIEW);
 724     glPopMatrix();
 725     pOpenGLVTable->gl_bind_framebuffer(GL_FRAMEBUFFER_EXT, 0);
 726 }
 727
 728 // Check internal texture format is supported
 729 static int is_supported_internal_format(GLenum format)
 730 {
 731     /* XXX: we don't support other textures than RGBA */
 732     switch (format) {
 733     case 4:
 734     case GL_RGBA:
 735     case GL_RGBA8:
 736         return 1;
 737     }
 738     return 0;
 739 }
 740
 741 // Destroy VA/GLX surface
 742 static void
 743 destroy_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 744 {
 745     unbind_pixmap(ctx, pSurfaceGLX);
 746     destroy_fbo_surface(ctx, pSurfaceGLX);
 747     destroy_tfp_surface(ctx, pSurfaceGLX);
 748     free(pSurfaceGLX);
 749 }
 750
 751 // Create VA/GLX surface
 752 static VASurfaceGLXP
 753 create_surface(VADriverContextP ctx, GLenum target, GLuint texture)
 754 {
 755     VASurfaceGLXP pSurfaceGLX = NULL;
 756     unsigned int internal_format, border_width, width, height;
 757     int is_error = 1;
 758
 759     pSurfaceGLX = malloc(sizeof(*pSurfaceGLX));
 760     if (!pSurfaceGLX)
 761         goto end;
 762
 763     pSurfaceGLX->magic          = VA_SURFACE_GLX_MAGIC;
 764     pSurfaceGLX->target         = target;
 765     pSurfaceGLX->texture        = texture;
 766     pSurfaceGLX->surface        = VA_INVALID_SURFACE;
 767     pSurfaceGLX->gl_context     = NULL;
 768     pSurfaceGLX->is_bound       = 0;
 769     pSurfaceGLX->pixmap         = None;
 770     pSurfaceGLX->pix_texture    = 0;
 771     pSurfaceGLX->glx_pixmap     = None;
 772     pSurfaceGLX->fbo            = 0;
 773
 774     glEnable(target);
 775     glBindTexture(target, texture);
 776     if (!gl_get_texture_param(GL_TEXTURE_INTERNAL_FORMAT, &internal_format))
 777         goto end;
 778     if (!is_supported_internal_format(internal_format))
 779         goto end;
 780
 781     /* Check texture dimensions */
 782     if (!gl_get_texture_param(GL_TEXTURE_BORDER, &border_width))
 783         goto end;
 784     if (!gl_get_texture_param(GL_TEXTURE_WIDTH, &width))
 785         goto end;
 786     if (!gl_get_texture_param(GL_TEXTURE_HEIGHT, &height))
 787         goto end;
 788
 789     width  -= 2 * border_width;
 790     height -= 2 * border_width;
 791     if (width == 0 || height == 0)
 792         goto end;
 793
 794     pSurfaceGLX->width  = width;
 795     pSurfaceGLX->height = height;
 796
 797     /* Create TFP objects */
 798     if (!create_tfp_surface(ctx, pSurfaceGLX))
 799         goto end;
 800
 801     /* Create FBO objects */
 802     if (!create_fbo_surface(ctx, pSurfaceGLX))
 803         goto end;
 804
 805     is_error = 0;
 806 end:
 807     if (is_error && pSurfaceGLX) {
 808         destroy_surface(ctx, pSurfaceGLX);
 809         pSurfaceGLX = NULL;
 810     }
 811     return pSurfaceGLX;
 812 }
 813
 814
 815 /* ========================================================================= */
 816 /* === VA/GLX implementation from the driver (fordward calls)            === */
 817 /* ========================================================================= */
 818
 819 #define INVOKE(ctx, func, args) do {                    \
 820         VADriverVTableGLXP vtable = (ctx)->vtable_glx;  \
 821         if (!vtable->va##func##GLX)                     \
 822             return VA_STATUS_ERROR_UNIMPLEMENTED;       \
 823                                                         \
 824         VAStatus status = vtable->va##func##GLX args;   \
 825         if (status != VA_STATUS_SUCCESS)                \
 826             return status;                              \
 827     } while (0)
 828
 829 static VAStatus
 830 vaCreateSurfaceGLX_impl_driver(
 831     VADriverContextP    ctx,
 832     GLenum              target,
 833     GLuint              texture,
 834     void              **gl_surface
 835 )
 836 {
 837     INVOKE(ctx, CreateSurface, (ctx, target, texture, gl_surface));
 838     return VA_STATUS_SUCCESS;
 839 }
 840
 841 static VAStatus
 842 vaDestroySurfaceGLX_impl_driver(VADriverContextP ctx, void *gl_surface)
 843 {
 844     INVOKE(ctx, DestroySurface, (ctx, gl_surface));
 845     return VA_STATUS_SUCCESS;
 846 }
 847
 848 static VAStatus
 849 vaCopySurfaceGLX_impl_driver(
 850     VADriverContextP    ctx,
 851     void               *gl_surface,
 852     VASurfaceID         surface,
 853     unsigned int        flags
 854 )
 855 {
 856     INVOKE(ctx, CopySurface, (ctx, gl_surface, surface, flags));
 857     return VA_STATUS_SUCCESS;
 858 }
 859
 860 #undef INVOKE
 861
 862
 863 /* ========================================================================= */
 864 /* === VA/GLX implementation from libVA (generic and suboptimal path)    === */
 865 /* ========================================================================= */
 866
 867 #define INIT_SURFACE(surface, surface_arg) do {         \
 868         surface = (VASurfaceGLXP)(surface_arg);         \
 869         if (!check_surface(surface))                    \
 870             return VA_STATUS_ERROR_INVALID_SURFACE;     \
 871     } while (0)
 872
 873 // Check VASurfaceGLX is valid
 874 static inline int check_surface(VASurfaceGLXP pSurfaceGLX)
 875 {
 876     return pSurfaceGLX && pSurfaceGLX->magic == VA_SURFACE_GLX_MAGIC;
 877 }
 878
 879 static VAStatus
 880 vaCreateSurfaceGLX_impl_libva(
 881     VADriverContextP    ctx,
 882     GLenum              target,
 883     GLuint              texture,
 884     void              **gl_surface
 885 )
 886 {
 887     VASurfaceGLXP pSurfaceGLX;
 888     struct OpenGLContextState old_cs, *new_cs;
 889
 890     gl_get_current_context(&old_cs);
 891     new_cs = gl_create_context(ctx, &old_cs);
 892     if (!new_cs)
 893         goto error;
 894     if (!gl_set_current_context(new_cs, NULL))
 895         goto error;
 896
 897     pSurfaceGLX = create_surface(ctx, target, texture);
 898     if (!pSurfaceGLX)
 899         goto error;
 900
 901     pSurfaceGLX->gl_context = new_cs;
 902     *gl_surface = pSurfaceGLX;
 903
 904     gl_set_current_context(&old_cs, NULL);
 905     return VA_STATUS_SUCCESS;
 906
 907 error:
 908     if (new_cs)
 909         gl_destroy_context(new_cs);
 910
 911     return VA_STATUS_ERROR_ALLOCATION_FAILED;
 912 }
 913
 914 static VAStatus
 915 vaDestroySurfaceGLX_impl_libva(VADriverContextP ctx, void *gl_surface)
 916 {
 917     VASurfaceGLXP pSurfaceGLX;
 918     struct OpenGLContextState old_cs, *new_cs;
 919
 920     INIT_SURFACE(pSurfaceGLX, gl_surface);
 921
 922     new_cs = pSurfaceGLX->gl_context;
 923     if (!gl_set_current_context(new_cs, &old_cs))
 924         return VA_STATUS_ERROR_OPERATION_FAILED;
 925
 926     destroy_surface(ctx, pSurfaceGLX);
 927
 928     gl_destroy_context(new_cs);
 929     gl_set_current_context(&old_cs, NULL);
 930     return VA_STATUS_SUCCESS;
 931 }
 932
 933 static inline VAStatus
 934 deassociate_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 935 {
 936     if (!unbind_pixmap(ctx, pSurfaceGLX))
 937         return VA_STATUS_ERROR_OPERATION_FAILED;
 938
 939     pSurfaceGLX->surface = VA_INVALID_SURFACE;
 940     return VA_STATUS_SUCCESS;
 941 }
 942
 943 static VAStatus
 944 associate_surface(
 945     VADriverContextP    ctx,
 946     VASurfaceGLXP       pSurfaceGLX,
 947     VASurfaceID         surface,
 948     unsigned int        flags
 949 )
 950 {
 951     VAStatus status;
 952
 953     /* XXX: optimise case where we are associating the same VA surface
 954        as before an no changed occurred to it */
 955     status = deassociate_surface(ctx, pSurfaceGLX);
 956     if (status != VA_STATUS_SUCCESS)
 957         return status;
 958
 959     x11_trap_errors();
 960     status = ctx->vtable->vaPutSurface(
 961         ctx,
 962         surface,
 963         (void *)pSurfaceGLX->pixmap,
 964         0, 0, pSurfaceGLX->width, pSurfaceGLX->height,
 965         0, 0, pSurfaceGLX->width, pSurfaceGLX->height,
 966         NULL, 0,
 967         flags
 968     );
 969     XSync(ctx->native_dpy, False);
 970     if (x11_untrap_errors() != 0)
 971         return VA_STATUS_ERROR_OPERATION_FAILED;
 972     if (status != VA_STATUS_SUCCESS)
 973         return status;
 974
 975     pSurfaceGLX->surface = surface;
 976     return VA_STATUS_SUCCESS;
 977 }
 978
 979 static inline VAStatus
 980 sync_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 981 {
 982     if (pSurfaceGLX->surface == VA_INVALID_SURFACE)
 983         return VA_STATUS_ERROR_INVALID_SURFACE;
 984
 985     return ctx->vtable->vaSyncSurface(ctx, pSurfaceGLX->surface);
 986 }
 987
 988 static inline VAStatus
 989 begin_render_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 990 {
 991     VAStatus status;
 992
 993     status = sync_surface(ctx, pSurfaceGLX);
 994     if (status != VA_STATUS_SUCCESS)
 995         return status;
 996
 997     if (!bind_pixmap(ctx, pSurfaceGLX))
 998         return VA_STATUS_ERROR_OPERATION_FAILED;
 999
1000     return VA_STATUS_SUCCESS;
1001 }
1002
1003 static inline VAStatus
1004 end_render_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
1005 {
1006     if (!unbind_pixmap(ctx, pSurfaceGLX))
1007         return VA_STATUS_ERROR_OPERATION_FAILED;
1008
1009     return VA_STATUS_SUCCESS;
1010 }
1011
1012 static VAStatus
1013 copy_surface(
1014     VADriverContextP    ctx,
1015     VASurfaceGLXP       pSurfaceGLX,
1016     VASurfaceID         surface,
1017     unsigned int        flags
1018 )
1019 {
1020     VAStatus status;
1021
1022     /* Associate VA surface */
1023     status = associate_surface(ctx, pSurfaceGLX, surface, flags);
1024     if (status != VA_STATUS_SUCCESS)
1025         return status;
1026
1027     /* Render to FBO */
1028     fbo_enter(ctx, pSurfaceGLX);
1029     status = begin_render_surface(ctx, pSurfaceGLX);
1030     if (status == VA_STATUS_SUCCESS) {
1031         render_pixmap(ctx, pSurfaceGLX);
1032         status = end_render_surface(ctx, pSurfaceGLX);
1033     }
1034     fbo_leave(ctx);
1035     if (status != VA_STATUS_SUCCESS)
1036         return status;
1037
1038     return deassociate_surface(ctx, pSurfaceGLX);
1039 }
1040
1041 static VAStatus
1042 vaCopySurfaceGLX_impl_libva(
1043     VADriverContextP    ctx,
1044     void               *gl_surface,
1045     VASurfaceID         surface,
1046     unsigned int        flags
1047 )
1048 {
1049     VASurfaceGLXP pSurfaceGLX;
1050     VAStatus status;
1051     struct OpenGLContextState old_cs;
1052
1053     INIT_SURFACE(pSurfaceGLX, gl_surface);
1054
1055     if (!gl_set_current_context(pSurfaceGLX->gl_context, &old_cs))
1056         return VA_STATUS_ERROR_OPERATION_FAILED;
1057
1058     status = copy_surface(ctx, pSurfaceGLX, surface, flags);
1059
1060     gl_set_current_context(&old_cs, NULL);
1061     return status;
1062 }
1063
1064 #undef INIT_SURFACE
1065
1066
1067 /* ========================================================================= */
1068 /* === Private VA/GLX vtable initialization                              === */
1069 /* ========================================================================= */
1070
1071 // Initialize GLX driver context
1072 VAStatus va_glx_init_context(VADriverContextP ctx)
1073 {
1074     VADriverContextGLXP glx_ctx = VA_DRIVER_CONTEXT_GLX(ctx);
1075     VADriverVTableGLXP  vtable  = &glx_ctx->vtable;
1076     int glx_major, glx_minor;
1077
1078     if (glx_ctx->is_initialized)
1079         return VA_STATUS_SUCCESS;
1080
1081     if (ctx->vtable_glx && ctx->vtable_glx->vaCopySurfaceGLX) {
1082         vtable->vaCreateSurfaceGLX      = vaCreateSurfaceGLX_impl_driver;
1083         vtable->vaDestroySurfaceGLX     = vaDestroySurfaceGLX_impl_driver;
1084         vtable->vaCopySurfaceGLX        = vaCopySurfaceGLX_impl_driver;
1085     }
1086     else {
1087         vtable->vaCreateSurfaceGLX      = vaCreateSurfaceGLX_impl_libva;
1088         vtable->vaDestroySurfaceGLX     = vaDestroySurfaceGLX_impl_libva;
1089         vtable->vaCopySurfaceGLX        = vaCopySurfaceGLX_impl_libva;
1090
1091         if (!glXQueryVersion(ctx->native_dpy, &glx_major, &glx_minor))
1092             return VA_STATUS_ERROR_UNIMPLEMENTED;
1093
1094         if (!check_tfp_extensions(ctx) || !load_tfp_extensions(ctx))
1095             return VA_STATUS_ERROR_UNIMPLEMENTED;
1096
1097         if (!check_fbo_extensions(ctx) || !load_fbo_extensions(ctx))
1098             return VA_STATUS_ERROR_UNIMPLEMENTED;
1099     }
1100
1101     glx_ctx->is_initialized = 1;
1102     return VA_STATUS_SUCCESS;
1103 }