va/glx/va_glx_impl.c

   1 /*
   2  * Copyright (C) 2009 Splitted-Desktop Systems. All Rights Reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the
   6  * "Software"), to deal in the Software without restriction, including
   7  * without limitation the rights to use, copy, modify, merge, publish,
   8  * distribute, sub license, and/or sell copies of the Software, and to
   9  * permit persons to whom the Software is furnished to do so, subject to
  10  * the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the
  13  * next paragraph) shall be included in all copies or substantial portions
  14  * of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  18  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  19  * IN NO EVENT SHALL PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR
  20  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  21  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  22  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  23  */
  24
  25 #define _GNU_SOURCE 1
  26 #include "sysdeps.h"
  27 #include "va_glx_private.h"
  28 #include "va_glx_impl.h"
  29 #include <stdio.h>
  30 #include <stdlib.h>
  31 #include <stdarg.h>
  32 #include <string.h>
  33 #include <assert.h>
  34 #include <dlfcn.h>
  35
  36 static void va_glx_error_message(const char *format, ...)
  37 {
  38     va_list args;
  39     va_start(args, format);
  40     fprintf(stderr, "libva-glx error: ");
  41     vfprintf(stderr, format, args);
  42     va_end(args);
  43 }
  44
  45 // X error trap
  46 static int x11_error_code = 0;
  47 static int (*old_error_handler)(Display *, XErrorEvent *);
  48
  49 static int error_handler(Display *dpy, XErrorEvent *error)
  50 {
  51     x11_error_code = error->error_code;
  52     return 0;
  53 }
  54
  55 static void x11_trap_errors(void)
  56 {
  57     x11_error_code    = 0;
  58     old_error_handler = XSetErrorHandler(error_handler);
  59 }
  60
  61 static int x11_untrap_errors(void)
  62 {
  63     XSetErrorHandler(old_error_handler);
  64     return x11_error_code;
  65 }
  66
  67 // Returns a string representation of an OpenGL error
  68 static const char *gl_get_error_string(GLenum error)
  69 {
  70     static const struct {
  71         GLenum val;
  72         const char *str;
  73     }
  74     gl_errors[] = {
  75         { GL_NO_ERROR,          "no error" },
  76         { GL_INVALID_ENUM,      "invalid enumerant" },
  77         { GL_INVALID_VALUE,     "invalid value" },
  78         { GL_INVALID_OPERATION, "invalid operation" },
  79         { GL_STACK_OVERFLOW,    "stack overflow" },
  80         { GL_STACK_UNDERFLOW,   "stack underflow" },
  81         { GL_OUT_OF_MEMORY,     "out of memory" },
  82 #ifdef GL_INVALID_FRAMEBUFFER_OPERATION_EXT
  83         { GL_INVALID_FRAMEBUFFER_OPERATION_EXT, "invalid framebuffer operation" },
  84 #endif
  85         { ~0, NULL }
  86     };
  87
  88     int i;
  89     for (i = 0; gl_errors[i].str; i++) {
  90         if (gl_errors[i].val == error)
  91             return gl_errors[i].str;
  92     }
  93     return "unknown";
  94 }
  95
  96 static inline int gl_do_check_error(int report)
  97 {
  98     GLenum error;
  99     int is_error = 0;
 100     while ((error = glGetError()) != GL_NO_ERROR) {
 101         if (report)
 102             va_glx_error_message("glError: %s caught\n",
 103                                  gl_get_error_string(error));
 104         is_error = 1;
 105     }
 106     return is_error;
 107 }
 108
 109 static inline void gl_purge_errors(void)
 110 {
 111     gl_do_check_error(0);
 112 }
 113
 114 static inline int gl_check_error(void)
 115 {
 116     return gl_do_check_error(1);
 117 }
 118
 119 // glGetTexLevelParameteriv() wrapper
 120 static int gl_get_texture_param(GLenum param, unsigned int *pval)
 121 {
 122     GLint val;
 123
 124     gl_purge_errors();
 125     glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, param, &val);
 126     if (gl_check_error())
 127         return 0;
 128     if (pval)
 129         *pval = val;
 130     return 1;
 131 }
 132
 133 // Returns the OpenGL VTable
 134 static inline VAOpenGLVTableP gl_get_vtable(VADriverContextP ctx)
 135 {
 136     return &VA_DRIVER_CONTEXT_GLX(ctx)->gl_vtable;
 137 }
 138
 139 // Lookup for a GLX function
 140 typedef void (*GLFuncPtr)(void);
 141 typedef GLFuncPtr (*GLXGetProcAddressProc)(const char *);
 142
 143 static GLFuncPtr get_proc_address_default(const char *name)
 144 {
 145     return NULL;
 146 }
 147
 148 static GLXGetProcAddressProc get_proc_address_func(void)
 149 {
 150     GLXGetProcAddressProc get_proc_func;
 151
 152     dlerror();
 153     get_proc_func = (GLXGetProcAddressProc)
 154         dlsym(RTLD_DEFAULT, "glXGetProcAddress");
 155     if (!dlerror())
 156         return get_proc_func;
 157
 158     get_proc_func = (GLXGetProcAddressProc)
 159         dlsym(RTLD_DEFAULT, "glXGetProcAddressARB");
 160     if (!dlerror())
 161         return get_proc_func;
 162
 163     return get_proc_address_default;
 164 }
 165
 166 static inline GLFuncPtr get_proc_address(const char *name)
 167 {
 168     static GLXGetProcAddressProc get_proc_func = NULL;
 169     if (!get_proc_func)
 170         get_proc_func = get_proc_address_func();
 171     return get_proc_func(name);
 172 }
 173
 174 // Check for GLX extensions (TFP, FBO)
 175 static int check_extension(const char *name, const char *ext)
 176 {
 177     const char *end;
 178     int name_len, n;
 179
 180     if (!name || !ext)
 181         return 0;
 182
 183     end = ext + strlen(ext);
 184     name_len = strlen(name);
 185     while (ext < end) {
 186         n = strcspn(ext, " ");
 187         if (n == name_len && strncmp(name, ext, n) == 0)
 188             return 1;
 189         ext += (n + 1);
 190     }
 191     return 0;
 192 }
 193
 194 static int check_extension3(const char *name)
 195 {
 196     int nbExtensions, i;
 197     PFNGLGETSTRINGIPROC glGetStringi = 0;
 198
 199     glGetStringi = (PFNGLGETSTRINGIPROC) get_proc_address("glGetStringi");
 200     if(!glGetStringi)
 201         return 0;
 202
 203
 204     glGetIntegerv(GL_NUM_EXTENSIONS, &nbExtensions);
 205     for(i = 0; i < nbExtensions; i++)
 206     {
 207         const GLubyte *strExtension = glGetStringi(GL_EXTENSIONS, i);
 208         if(strcmp(strExtension, (const GLubyte *)name) == 0)
 209             return 1;
 210     }
 211
 212     return 0;
 213 }
 214
 215 static int check_tfp_extensions(VADriverContextP ctx)
 216 {
 217     const char *gl_extensions;
 218     const char *glx_extensions;
 219
 220     gl_extensions = (const char *)glGetString(GL_EXTENSIONS);
 221     if (!check_extension("GL_ARB_texture_non_power_of_two", gl_extensions) && !check_extension3("GL_ARB_texture_non_power_of_two"))
 222         return 0;
 223
 224     glx_extensions = glXQueryExtensionsString(ctx->native_dpy, ctx->x11_screen);
 225     if (!check_extension("GLX_EXT_texture_from_pixmap", glx_extensions))
 226         return 0;
 227
 228     return 1;
 229 }
 230
 231 static int check_fbo_extensions(VADriverContextP ctx)
 232 {
 233     const char *gl_extensions;
 234
 235     gl_extensions = (const char *)glGetString(GL_EXTENSIONS);
 236     if (check_extension("GL_ARB_framebuffer_object", gl_extensions) || check_extension3("GL_ARB_framebuffer_object"))
 237         return 1;
 238     if (check_extension("GL_EXT_framebuffer_object", gl_extensions) || check_extension3("GL_EXT_framebuffer_object"))
 239         return 1;
 240
 241     return 0;
 242 }
 243
 244 // Load GLX extensions
 245 static int load_tfp_extensions(VADriverContextP ctx)
 246 {
 247     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 248
 249     pOpenGLVTable->glx_create_pixmap = (PFNGLXCREATEPIXMAPPROC)
 250         get_proc_address("glXCreatePixmap");
 251     if (!pOpenGLVTable->glx_create_pixmap)
 252         return 0;
 253     pOpenGLVTable->glx_destroy_pixmap = (PFNGLXDESTROYPIXMAPPROC)
 254         get_proc_address("glXDestroyPixmap");
 255     if (!pOpenGLVTable->glx_destroy_pixmap)
 256         return 0;
 257     pOpenGLVTable->glx_bind_tex_image = (PFNGLXBINDTEXIMAGEEXTPROC)
 258         get_proc_address("glXBindTexImageEXT");
 259     if (!pOpenGLVTable->glx_bind_tex_image)
 260         return 0;
 261     pOpenGLVTable->glx_release_tex_image = (PFNGLXRELEASETEXIMAGEEXTPROC)
 262         get_proc_address("glXReleaseTexImageEXT");
 263     if (!pOpenGLVTable->glx_release_tex_image)
 264         return 0;
 265     return 1;
 266 }
 267
 268 static int load_fbo_extensions(VADriverContextP ctx)
 269 {
 270     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 271
 272     pOpenGLVTable->gl_gen_framebuffers = (PFNGLGENFRAMEBUFFERSEXTPROC)
 273         get_proc_address("glGenFramebuffersEXT");
 274     if (!pOpenGLVTable->gl_gen_framebuffers)
 275         return 0;
 276     pOpenGLVTable->gl_delete_framebuffers = (PFNGLDELETEFRAMEBUFFERSEXTPROC)
 277         get_proc_address("glDeleteFramebuffersEXT");
 278     if (!pOpenGLVTable->gl_delete_framebuffers)
 279         return 0;
 280     pOpenGLVTable->gl_bind_framebuffer = (PFNGLBINDFRAMEBUFFEREXTPROC)
 281         get_proc_address("glBindFramebufferEXT");
 282     if (!pOpenGLVTable->gl_bind_framebuffer)
 283         return 0;
 284     pOpenGLVTable->gl_gen_renderbuffers = (PFNGLGENRENDERBUFFERSEXTPROC)
 285         get_proc_address("glGenRenderbuffersEXT");
 286     if (!pOpenGLVTable->gl_gen_renderbuffers)
 287         return 0;
 288     pOpenGLVTable->gl_delete_renderbuffers = (PFNGLDELETERENDERBUFFERSEXTPROC)
 289         get_proc_address("glDeleteRenderbuffersEXT");
 290     if (!pOpenGLVTable->gl_delete_renderbuffers)
 291         return 0;
 292     pOpenGLVTable->gl_bind_renderbuffer = (PFNGLBINDRENDERBUFFEREXTPROC)
 293         get_proc_address("glBindRenderbufferEXT");
 294     if (!pOpenGLVTable->gl_bind_renderbuffer)
 295         return 0;
 296     pOpenGLVTable->gl_renderbuffer_storage = (PFNGLRENDERBUFFERSTORAGEEXTPROC)
 297         get_proc_address("glRenderbufferStorageEXT");
 298     if (!pOpenGLVTable->gl_renderbuffer_storage)
 299         return 0;
 300     pOpenGLVTable->gl_framebuffer_renderbuffer = (PFNGLFRAMEBUFFERRENDERBUFFEREXTPROC)
 301         get_proc_address("glFramebufferRenderbufferEXT");
 302     if (!pOpenGLVTable->gl_framebuffer_renderbuffer)
 303         return 0;
 304     pOpenGLVTable->gl_framebuffer_texture_2d = (PFNGLFRAMEBUFFERTEXTURE2DEXTPROC)
 305         get_proc_address("glFramebufferTexture2DEXT");
 306     if (!pOpenGLVTable->gl_framebuffer_texture_2d)
 307         return 0;
 308     pOpenGLVTable->gl_check_framebuffer_status = (PFNGLCHECKFRAMEBUFFERSTATUSEXTPROC)
 309         get_proc_address("glCheckFramebufferStatusEXT");
 310     if (!pOpenGLVTable->gl_check_framebuffer_status)
 311         return 0;
 312     return 1;
 313 }
 314
 315
 316 /* ========================================================================= */
 317 /* === VA/GLX helpers                                                    === */
 318 /* ========================================================================= */
 319
 320 // OpenGL context state
 321 typedef struct OpenGLContextState *OpenGLContextStateP;
 322
 323 struct OpenGLContextState {
 324     Display     *display;
 325     Window       window;
 326     GLXContext   context;
 327 };
 328
 329 static void
 330 gl_destroy_context(OpenGLContextStateP cs)
 331 {
 332     if (!cs)
 333         return;
 334
 335     if (cs->display && cs->context) {
 336         if (glXGetCurrentContext() == cs->context)
 337             glXMakeCurrent(cs->display, None, NULL);
 338         glXDestroyContext(cs->display, cs->context);
 339         cs->display = NULL;
 340         cs->context = NULL;
 341     }
 342     free(cs);
 343 }
 344
 345 static OpenGLContextStateP
 346 gl_create_context(VADriverContextP ctx, OpenGLContextStateP parent)
 347 {
 348     OpenGLContextStateP cs;
 349     GLXFBConfig *fbconfigs = NULL;
 350     int fbconfig_id, val, n, n_fbconfigs;
 351     Status status;
 352
 353     static GLint fbconfig_attrs[] = {
 354         GLX_DRAWABLE_TYPE, GLX_WINDOW_BIT,
 355         GLX_RENDER_TYPE,   GLX_RGBA_BIT,
 356         GLX_DOUBLEBUFFER,  True,
 357         GLX_RED_SIZE,      8,
 358         GLX_GREEN_SIZE,    8,
 359         GLX_BLUE_SIZE,     8,
 360         None
 361     };
 362
 363     cs = malloc(sizeof(*cs));
 364     if (!cs)
 365         goto error;
 366
 367     if (parent) {
 368         cs->display = parent->display;
 369         cs->window  = parent->window;
 370     }
 371     else {
 372         cs->display = ctx->native_dpy;
 373         cs->window  = None;
 374     }
 375     cs->context = NULL;
 376
 377     if (parent && parent->context) {
 378         status = glXQueryContext(
 379             parent->display,
 380             parent->context,
 381             GLX_FBCONFIG_ID, &fbconfig_id
 382         );
 383         if (status != Success)
 384             goto error;
 385
 386         if (fbconfig_id == GLX_DONT_CARE)
 387             goto choose_fbconfig;
 388
 389         fbconfigs = glXGetFBConfigs(
 390             parent->display,
 391             DefaultScreen(parent->display),
 392             &n_fbconfigs
 393         );
 394         if (!fbconfigs)
 395             goto error;
 396
 397         /* Find out a GLXFBConfig compatible with the parent context */
 398         for (n = 0; n < n_fbconfigs; n++) {
 399             status = glXGetFBConfigAttrib(
 400                 cs->display,
 401                 fbconfigs[n],
 402                 GLX_FBCONFIG_ID, &val
 403             );
 404             if (status == Success && val == fbconfig_id)
 405                 break;
 406         }
 407         if (n == n_fbconfigs)
 408             goto error;
 409     }
 410     else {
 411     choose_fbconfig:
 412         fbconfigs = glXChooseFBConfig(
 413             ctx->native_dpy,
 414             ctx->x11_screen,
 415             fbconfig_attrs, &n_fbconfigs
 416         );
 417         if (!fbconfigs)
 418             goto error;
 419
 420         /* Select the first one */
 421         n = 0;
 422     }
 423
 424     cs->context = glXCreateNewContext(
 425         cs->display,
 426         fbconfigs[n],
 427         GLX_RGBA_TYPE,
 428         parent ? parent->context : NULL,
 429         True
 430     );
 431     if (cs->context)
 432         goto end;
 433
 434 error:
 435     gl_destroy_context(cs);
 436     cs = NULL;
 437 end:
 438     if (fbconfigs)
 439         XFree(fbconfigs);
 440     return cs;
 441 }
 442
 443 static void gl_get_current_context(OpenGLContextStateP cs)
 444 {
 445     cs->display = glXGetCurrentDisplay();
 446     cs->window  = glXGetCurrentDrawable();
 447     cs->context = glXGetCurrentContext();
 448 }
 449
 450 static int
 451 gl_set_current_context(OpenGLContextStateP new_cs, OpenGLContextStateP old_cs)
 452 {
 453     /* If display is NULL, this could be that new_cs was retrieved from
 454        gl_get_current_context() with none set previously. If that case,
 455        the other fields are also NULL and we don't return an error */
 456     if (!new_cs->display)
 457         return !new_cs->window && !new_cs->context;
 458
 459     if (old_cs) {
 460         if (old_cs == new_cs)
 461             return 1;
 462         gl_get_current_context(old_cs);
 463         if (old_cs->display == new_cs->display &&
 464             old_cs->window  == new_cs->window  &&
 465             old_cs->context == new_cs->context)
 466             return 1;
 467     }
 468     return glXMakeCurrent(new_cs->display, new_cs->window, new_cs->context);
 469 }
 470
 471 /** Unique VASurfaceGLX identifier */
 472 #define VA_SURFACE_GLX_MAGIC VA_FOURCC('V','A','G','L')
 473
 474 struct VASurfaceGLX {
 475     uint32_t            magic;      ///< Magic number identifying a VASurfaceGLX
 476     GLenum              target;     ///< GL target to which the texture is bound
 477     GLuint              texture;    ///< GL texture
 478     VASurfaceID         surface;    ///< Associated VA surface
 479     unsigned int        width;
 480     unsigned int        height;
 481     OpenGLContextStateP gl_context;
 482     int                 is_bound;
 483     Pixmap              pixmap;
 484     GLuint              pix_texture;
 485     GLXPixmap           glx_pixmap;
 486     GLuint              fbo;
 487 };
 488
 489 // Create Pixmaps for GLX texture-from-pixmap extension
 490 static int create_tfp_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 491 {
 492     VAOpenGLVTableP const pOpenGLVTable = gl_get_vtable(ctx);
 493     const unsigned int    width         = pSurfaceGLX->width;
 494     const unsigned int    height        = pSurfaceGLX->height;
 495     Pixmap                pixmap        = None;
 496     GLXFBConfig          *fbconfig      = NULL;
 497     GLXPixmap             glx_pixmap    = None;
 498     Window                root_window;
 499     XWindowAttributes     wattr;
 500     int                  *attrib;
 501     int                   n_fbconfig_attrs;
 502
 503     root_window = RootWindow(ctx->native_dpy, ctx->x11_screen);
 504     XGetWindowAttributes(ctx->native_dpy, root_window, &wattr);
 505     if (wattr.depth != 24 && wattr.depth != 32)
 506         return 0;
 507     pixmap = XCreatePixmap(
 508         ctx->native_dpy,
 509         root_window,
 510         width,
 511         height,
 512         wattr.depth
 513     );
 514     if (!pixmap)
 515         return 0;
 516     pSurfaceGLX->pixmap = pixmap;
 517
 518     int fbconfig_attrs[32] = {
 519         GLX_DRAWABLE_TYPE,      GLX_PIXMAP_BIT,
 520         GLX_DOUBLEBUFFER,       GL_TRUE,
 521         GLX_RENDER_TYPE,        GLX_RGBA_BIT,
 522         GLX_X_RENDERABLE,       GL_TRUE,
 523         GLX_Y_INVERTED_EXT,     GL_TRUE,
 524         GLX_RED_SIZE,           8,
 525         GLX_GREEN_SIZE,         8,
 526         GLX_BLUE_SIZE,          8,
 527         /*
 528          * depth test isn't enabled in the implementaion of VA GLX,
 529          * so depth buffer is unnecessary. However to workaround a
 530          * bug in older verson of xorg-server, always require a depth
 531          * buffer.
 532          *
 533          * See https://bugs.freedesktop.org/show_bug.cgi?id=76755
 534          */
 535         GLX_DEPTH_SIZE,         1,
 536         GL_NONE,
 537     };
 538     for (attrib = fbconfig_attrs; *attrib != GL_NONE; attrib += 2)
 539         ;
 540     if (wattr.depth == 32) {
 541     *attrib++ = GLX_ALPHA_SIZE;                 *attrib++ = 8;
 542     *attrib++ = GLX_BIND_TO_TEXTURE_RGBA_EXT;   *attrib++ = GL_TRUE;
 543     }
 544     else {
 545     *attrib++ = GLX_BIND_TO_TEXTURE_RGB_EXT;    *attrib++ = GL_TRUE;
 546     }
 547     *attrib++ = GL_NONE;
 548
 549     fbconfig = glXChooseFBConfig(
 550         ctx->native_dpy,
 551         ctx->x11_screen,
 552         fbconfig_attrs,
 553         &n_fbconfig_attrs
 554     );
 555     if (!fbconfig)
 556         return 0;
 557
 558     int pixmap_attrs[10] = {
 559         GLX_TEXTURE_TARGET_EXT, GLX_TEXTURE_2D_EXT,
 560         GLX_MIPMAP_TEXTURE_EXT, GL_FALSE,
 561         GL_NONE,
 562     };
 563     for (attrib = pixmap_attrs; *attrib != GL_NONE; attrib += 2)
 564         ;
 565     *attrib++ = GLX_TEXTURE_FORMAT_EXT;
 566     if (wattr.depth == 32)
 567     *attrib++ = GLX_TEXTURE_FORMAT_RGBA_EXT;
 568     else
 569     *attrib++ = GLX_TEXTURE_FORMAT_RGB_EXT;
 570     *attrib++ = GL_NONE;
 571
 572     x11_trap_errors();
 573     glx_pixmap = pOpenGLVTable->glx_create_pixmap(
 574         ctx->native_dpy,
 575         fbconfig[0],
 576         pixmap,
 577         pixmap_attrs
 578     );
 579     free(fbconfig);
 580     if (x11_untrap_errors() != 0)
 581         return 0;
 582     pSurfaceGLX->glx_pixmap = glx_pixmap;
 583
 584     glGenTextures(1, &pSurfaceGLX->pix_texture);
 585     glBindTexture(GL_TEXTURE_2D, pSurfaceGLX->pix_texture);
 586     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
 587     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 588     return 1;
 589 }
 590
 591 // Destroy Pixmaps used for TFP
 592 static void destroy_tfp_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 593 {
 594     VAOpenGLVTableP const pOpenGLVTable = gl_get_vtable(ctx);
 595
 596     if (pSurfaceGLX->pix_texture) {
 597         glDeleteTextures(1, &pSurfaceGLX->pix_texture);
 598         pSurfaceGLX->pix_texture = 0;
 599     }
 600
 601     if (pSurfaceGLX->glx_pixmap) {
 602         pOpenGLVTable->glx_destroy_pixmap(ctx->native_dpy, pSurfaceGLX->glx_pixmap);
 603         pSurfaceGLX->glx_pixmap = None;
 604     }
 605
 606     if (pSurfaceGLX->pixmap) {
 607         XFreePixmap(ctx->native_dpy, pSurfaceGLX->pixmap);
 608         pSurfaceGLX->pixmap = None;
 609     }
 610 }
 611
 612 // Bind GLX Pixmap to texture
 613 static int bind_pixmap(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 614 {
 615     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 616
 617     if (pSurfaceGLX->is_bound)
 618         return 1;
 619
 620     glBindTexture(GL_TEXTURE_2D, pSurfaceGLX->pix_texture);
 621
 622     x11_trap_errors();
 623     pOpenGLVTable->glx_bind_tex_image(
 624         ctx->native_dpy,
 625         pSurfaceGLX->glx_pixmap,
 626         GLX_FRONT_LEFT_EXT,
 627         NULL
 628     );
 629     XSync(ctx->native_dpy, False);
 630     if (x11_untrap_errors() != 0) {
 631         va_glx_error_message("failed to bind pixmap\n");
 632         return 0;
 633     }
 634
 635     pSurfaceGLX->is_bound = 1;
 636     return 1;
 637 }
 638
 639 // Release GLX Pixmap from texture
 640 static int unbind_pixmap(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 641 {
 642     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 643
 644     if (!pSurfaceGLX->is_bound)
 645         return 1;
 646
 647     x11_trap_errors();
 648     pOpenGLVTable->glx_release_tex_image(
 649         ctx->native_dpy,
 650         pSurfaceGLX->glx_pixmap,
 651         GLX_FRONT_LEFT_EXT
 652     );
 653     XSync(ctx->native_dpy, False);
 654     if (x11_untrap_errors() != 0) {
 655         va_glx_error_message("failed to release pixmap\n");
 656         return 0;
 657     }
 658
 659     glBindTexture(GL_TEXTURE_2D, 0);
 660
 661     pSurfaceGLX->is_bound = 0;
 662     return 1;
 663 }
 664
 665 // Render GLX Pixmap to texture
 666 static void render_pixmap(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 667 {
 668     const unsigned int w = pSurfaceGLX->width;
 669     const unsigned int h = pSurfaceGLX->height;
 670
 671     glColor4f(1.0f, 1.0f, 1.0f, 1.0f);
 672     glBegin(GL_QUADS);
 673     {
 674         glTexCoord2f(0.0f, 0.0f); glVertex2i(0, 0);
 675         glTexCoord2f(0.0f, 1.0f); glVertex2i(0, h);
 676         glTexCoord2f(1.0f, 1.0f); glVertex2i(w, h);
 677         glTexCoord2f(1.0f, 0.0f); glVertex2i(w, 0);
 678     }
 679     glEnd();
 680 }
 681
 682 // Create offscreen surface
 683 static int create_fbo_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 684 {
 685     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 686     GLuint fbo;
 687     GLenum status;
 688
 689     pOpenGLVTable->gl_gen_framebuffers(1, &fbo);
 690     pOpenGLVTable->gl_bind_framebuffer(GL_FRAMEBUFFER_EXT, fbo);
 691     pOpenGLVTable->gl_framebuffer_texture_2d(
 692         GL_FRAMEBUFFER_EXT,
 693         GL_COLOR_ATTACHMENT0_EXT,
 694         GL_TEXTURE_2D,
 695         pSurfaceGLX->texture,
 696         0
 697     );
 698
 699     status = pOpenGLVTable->gl_check_framebuffer_status(GL_DRAW_FRAMEBUFFER_EXT);
 700     pOpenGLVTable->gl_bind_framebuffer(GL_FRAMEBUFFER_EXT, 0);
 701     if (status != GL_FRAMEBUFFER_COMPLETE_EXT)
 702         return 0;
 703
 704     pSurfaceGLX->fbo = fbo;
 705     return 1;
 706 }
 707
 708 // Destroy offscreen surface
 709 static void destroy_fbo_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 710 {
 711     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 712
 713     if (pSurfaceGLX->fbo) {
 714         pOpenGLVTable->gl_delete_framebuffers(1, &pSurfaceGLX->fbo);
 715         pSurfaceGLX->fbo = 0;
 716     }
 717 }
 718
 719 // Setup matrices to match the FBO texture dimensions
 720 static void fbo_enter(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 721 {
 722     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 723     const unsigned int width  = pSurfaceGLX->width;
 724     const unsigned int height = pSurfaceGLX->height;
 725
 726     pOpenGLVTable->gl_bind_framebuffer(GL_FRAMEBUFFER_EXT, pSurfaceGLX->fbo);
 727     glPushAttrib(GL_VIEWPORT_BIT);
 728     glMatrixMode(GL_PROJECTION);
 729     glPushMatrix();
 730     glLoadIdentity();
 731     glMatrixMode(GL_MODELVIEW);
 732     glPushMatrix();
 733     glLoadIdentity();
 734     glViewport(0, 0, width, height);
 735     glTranslatef(-1.0f, -1.0f, 0.0f);
 736     glScalef(2.0f / width, 2.0f / height, 1.0f);
 737 }
 738
 739 // Restore original OpenGL matrices
 740 static void fbo_leave(VADriverContextP ctx)
 741 {
 742     VAOpenGLVTableP pOpenGLVTable = gl_get_vtable(ctx);
 743
 744     glPopAttrib();
 745     glMatrixMode(GL_PROJECTION);
 746     glPopMatrix();
 747     glMatrixMode(GL_MODELVIEW);
 748     glPopMatrix();
 749     pOpenGLVTable->gl_bind_framebuffer(GL_FRAMEBUFFER_EXT, 0);
 750 }
 751
 752 // Check internal texture format is supported
 753 static int is_supported_internal_format(GLenum format)
 754 {
 755     /* XXX: we don't support other textures than RGBA */
 756     switch (format) {
 757     case 4:
 758     case GL_RGBA:
 759     case GL_RGBA8:
 760         return 1;
 761     }
 762     return 0;
 763 }
 764
 765 // Destroy VA/GLX surface
 766 static void
 767 destroy_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 768 {
 769     unbind_pixmap(ctx, pSurfaceGLX);
 770     destroy_fbo_surface(ctx, pSurfaceGLX);
 771     destroy_tfp_surface(ctx, pSurfaceGLX);
 772     free(pSurfaceGLX);
 773 }
 774
 775 // Create VA/GLX surface
 776 static VASurfaceGLXP
 777 create_surface(VADriverContextP ctx, GLenum target, GLuint texture)
 778 {
 779     VASurfaceGLXP pSurfaceGLX = NULL;
 780     unsigned int internal_format, border_width, width, height;
 781     int is_error = 1;
 782
 783     pSurfaceGLX = malloc(sizeof(*pSurfaceGLX));
 784     if (!pSurfaceGLX)
 785         goto end;
 786
 787     pSurfaceGLX->magic          = VA_SURFACE_GLX_MAGIC;
 788     pSurfaceGLX->target         = target;
 789     pSurfaceGLX->texture        = texture;
 790     pSurfaceGLX->surface        = VA_INVALID_SURFACE;
 791     pSurfaceGLX->gl_context     = NULL;
 792     pSurfaceGLX->is_bound       = 0;
 793     pSurfaceGLX->pixmap         = None;
 794     pSurfaceGLX->pix_texture    = 0;
 795     pSurfaceGLX->glx_pixmap     = None;
 796     pSurfaceGLX->fbo            = 0;
 797
 798     glEnable(target);
 799     glBindTexture(target, texture);
 800     if (!gl_get_texture_param(GL_TEXTURE_INTERNAL_FORMAT, &internal_format))
 801         goto end;
 802     if (!is_supported_internal_format(internal_format))
 803         goto end;
 804
 805     /* Check texture dimensions */
 806     if (!gl_get_texture_param(GL_TEXTURE_BORDER, &border_width))
 807         goto end;
 808     if (!gl_get_texture_param(GL_TEXTURE_WIDTH, &width))
 809         goto end;
 810     if (!gl_get_texture_param(GL_TEXTURE_HEIGHT, &height))
 811         goto end;
 812
 813     width  -= 2 * border_width;
 814     height -= 2 * border_width;
 815     if (width == 0 || height == 0)
 816         goto end;
 817
 818     pSurfaceGLX->width  = width;
 819     pSurfaceGLX->height = height;
 820
 821     /* Create TFP objects */
 822     if (!create_tfp_surface(ctx, pSurfaceGLX))
 823         goto end;
 824
 825     /* Create FBO objects */
 826     if (!create_fbo_surface(ctx, pSurfaceGLX))
 827         goto end;
 828
 829     is_error = 0;
 830 end:
 831     if (is_error && pSurfaceGLX) {
 832         destroy_surface(ctx, pSurfaceGLX);
 833         pSurfaceGLX = NULL;
 834     }
 835     return pSurfaceGLX;
 836 }
 837
 838
 839 /* ========================================================================= */
 840 /* === VA/GLX implementation from the driver (fordward calls)            === */
 841 /* ========================================================================= */
 842
 843 #define INVOKE(ctx, func, args) do {                    \
 844         VADriverVTableGLXP vtable = (ctx)->vtable_glx;  \
 845         if (!vtable->va##func##GLX)                     \
 846             return VA_STATUS_ERROR_UNIMPLEMENTED;       \
 847                                                         \
 848         VAStatus status = vtable->va##func##GLX args;   \
 849         if (status != VA_STATUS_SUCCESS)                \
 850             return status;                              \
 851     } while (0)
 852
 853 static VAStatus
 854 vaCreateSurfaceGLX_impl_driver(
 855     VADriverContextP    ctx,
 856     GLenum              target,
 857     GLuint              texture,
 858     void              **gl_surface
 859 )
 860 {
 861     INVOKE(ctx, CreateSurface, (ctx, target, texture, gl_surface));
 862     return VA_STATUS_SUCCESS;
 863 }
 864
 865 static VAStatus
 866 vaDestroySurfaceGLX_impl_driver(VADriverContextP ctx, void *gl_surface)
 867 {
 868     INVOKE(ctx, DestroySurface, (ctx, gl_surface));
 869     return VA_STATUS_SUCCESS;
 870 }
 871
 872 static VAStatus
 873 vaCopySurfaceGLX_impl_driver(
 874     VADriverContextP    ctx,
 875     void               *gl_surface,
 876     VASurfaceID         surface,
 877     unsigned int        flags
 878 )
 879 {
 880     INVOKE(ctx, CopySurface, (ctx, gl_surface, surface, flags));
 881     return VA_STATUS_SUCCESS;
 882 }
 883
 884 #undef INVOKE
 885
 886
 887 /* ========================================================================= */
 888 /* === VA/GLX implementation from libVA (generic and suboptimal path)    === */
 889 /* ========================================================================= */
 890
 891 #define INIT_SURFACE(surface, surface_arg) do {         \
 892         surface = (VASurfaceGLXP)(surface_arg);         \
 893         if (!check_surface(surface))                    \
 894             return VA_STATUS_ERROR_INVALID_SURFACE;     \
 895     } while (0)
 896
 897 // Check VASurfaceGLX is valid
 898 static inline int check_surface(VASurfaceGLXP pSurfaceGLX)
 899 {
 900     return pSurfaceGLX && pSurfaceGLX->magic == VA_SURFACE_GLX_MAGIC;
 901 }
 902
 903 static VAStatus
 904 vaCreateSurfaceGLX_impl_libva(
 905     VADriverContextP    ctx,
 906     GLenum              target,
 907     GLuint              texture,
 908     void              **gl_surface
 909 )
 910 {
 911     VASurfaceGLXP pSurfaceGLX;
 912     struct OpenGLContextState old_cs, *new_cs;
 913
 914     gl_get_current_context(&old_cs);
 915     new_cs = gl_create_context(ctx, &old_cs);
 916     if (!new_cs)
 917         goto error;
 918     if (!gl_set_current_context(new_cs, NULL))
 919         goto error;
 920
 921     pSurfaceGLX = create_surface(ctx, target, texture);
 922     if (!pSurfaceGLX)
 923         goto error;
 924
 925     pSurfaceGLX->gl_context = new_cs;
 926     *gl_surface = pSurfaceGLX;
 927
 928     gl_set_current_context(&old_cs, NULL);
 929     return VA_STATUS_SUCCESS;
 930
 931 error:
 932     if (new_cs)
 933         gl_destroy_context(new_cs);
 934
 935     return VA_STATUS_ERROR_ALLOCATION_FAILED;
 936 }
 937
 938 static VAStatus
 939 vaDestroySurfaceGLX_impl_libva(VADriverContextP ctx, void *gl_surface)
 940 {
 941     VASurfaceGLXP pSurfaceGLX;
 942     struct OpenGLContextState old_cs, *new_cs;
 943
 944     INIT_SURFACE(pSurfaceGLX, gl_surface);
 945
 946     new_cs = pSurfaceGLX->gl_context;
 947     if (!gl_set_current_context(new_cs, &old_cs))
 948         return VA_STATUS_ERROR_OPERATION_FAILED;
 949
 950     destroy_surface(ctx, pSurfaceGLX);
 951
 952     gl_destroy_context(new_cs);
 953     gl_set_current_context(&old_cs, NULL);
 954     return VA_STATUS_SUCCESS;
 955 }
 956
 957 static inline VAStatus
 958 deassociate_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
 959 {
 960     if (!unbind_pixmap(ctx, pSurfaceGLX))
 961         return VA_STATUS_ERROR_OPERATION_FAILED;
 962
 963     pSurfaceGLX->surface = VA_INVALID_SURFACE;
 964     return VA_STATUS_SUCCESS;
 965 }
 966
 967 static VAStatus
 968 associate_surface(
 969     VADriverContextP    ctx,
 970     VASurfaceGLXP       pSurfaceGLX,
 971     VASurfaceID         surface,
 972     unsigned int        flags
 973 )
 974 {
 975     VAStatus status;
 976
 977     /* XXX: optimise case where we are associating the same VA surface
 978        as before an no changed occurred to it */
 979     status = deassociate_surface(ctx, pSurfaceGLX);
 980     if (status != VA_STATUS_SUCCESS)
 981         return status;
 982
 983     x11_trap_errors();
 984     status = ctx->vtable->vaPutSurface(
 985         ctx,
 986         surface,
 987         (void *)pSurfaceGLX->pixmap,
 988         0, 0, pSurfaceGLX->width, pSurfaceGLX->height,
 989         0, 0, pSurfaceGLX->width, pSurfaceGLX->height,
 990         NULL, 0,
 991         flags
 992     );
 993     XSync(ctx->native_dpy, False);
 994     if (x11_untrap_errors() != 0)
 995         return VA_STATUS_ERROR_OPERATION_FAILED;
 996     if (status != VA_STATUS_SUCCESS)
 997         return status;
 998
 999     pSurfaceGLX->surface = surface;
1000     return VA_STATUS_SUCCESS;
1001 }
1002
1003 static inline VAStatus
1004 sync_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
1005 {
1006     if (pSurfaceGLX->surface == VA_INVALID_SURFACE)
1007         return VA_STATUS_ERROR_INVALID_SURFACE;
1008
1009     return ctx->vtable->vaSyncSurface(ctx, pSurfaceGLX->surface);
1010 }
1011
1012 static inline VAStatus
1013 begin_render_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
1014 {
1015     VAStatus status;
1016
1017     status = sync_surface(ctx, pSurfaceGLX);
1018     if (status != VA_STATUS_SUCCESS)
1019         return status;
1020
1021     if (!bind_pixmap(ctx, pSurfaceGLX))
1022         return VA_STATUS_ERROR_OPERATION_FAILED;
1023
1024     return VA_STATUS_SUCCESS;
1025 }
1026
1027 static inline VAStatus
1028 end_render_surface(VADriverContextP ctx, VASurfaceGLXP pSurfaceGLX)
1029 {
1030     if (!unbind_pixmap(ctx, pSurfaceGLX))
1031         return VA_STATUS_ERROR_OPERATION_FAILED;
1032
1033     return VA_STATUS_SUCCESS;
1034 }
1035
1036 static VAStatus
1037 copy_surface(
1038     VADriverContextP    ctx,
1039     VASurfaceGLXP       pSurfaceGLX,
1040     VASurfaceID         surface,
1041     unsigned int        flags
1042 )
1043 {
1044     VAStatus status;
1045
1046     /* Associate VA surface */
1047     status = associate_surface(ctx, pSurfaceGLX, surface, flags);
1048     if (status != VA_STATUS_SUCCESS)
1049         return status;
1050
1051     /* Render to FBO */
1052     fbo_enter(ctx, pSurfaceGLX);
1053     status = begin_render_surface(ctx, pSurfaceGLX);
1054     if (status == VA_STATUS_SUCCESS) {
1055         render_pixmap(ctx, pSurfaceGLX);
1056         status = end_render_surface(ctx, pSurfaceGLX);
1057     }
1058     fbo_leave(ctx);
1059     if (status != VA_STATUS_SUCCESS)
1060         return status;
1061
1062     return deassociate_surface(ctx, pSurfaceGLX);
1063 }
1064
1065 static VAStatus
1066 vaCopySurfaceGLX_impl_libva(
1067     VADriverContextP    ctx,
1068     void               *gl_surface,
1069     VASurfaceID         surface,
1070     unsigned int        flags
1071 )
1072 {
1073     VASurfaceGLXP pSurfaceGLX;
1074     VAStatus status;
1075     struct OpenGLContextState old_cs;
1076
1077     INIT_SURFACE(pSurfaceGLX, gl_surface);
1078
1079     if (!gl_set_current_context(pSurfaceGLX->gl_context, &old_cs))
1080         return VA_STATUS_ERROR_OPERATION_FAILED;
1081
1082     status = copy_surface(ctx, pSurfaceGLX, surface, flags);
1083
1084     gl_set_current_context(&old_cs, NULL);
1085     return status;
1086 }
1087
1088 #undef INIT_SURFACE
1089
1090
1091 /* ========================================================================= */
1092 /* === Private VA/GLX vtable initialization                              === */
1093 /* ========================================================================= */
1094
1095 // Initialize GLX driver context
1096 VAStatus va_glx_init_context(VADriverContextP ctx)
1097 {
1098     VADriverContextGLXP glx_ctx = VA_DRIVER_CONTEXT_GLX(ctx);
1099     VADriverVTableGLXP  vtable  = &glx_ctx->vtable;
1100     int glx_major, glx_minor;
1101
1102     if (glx_ctx->is_initialized)
1103         return VA_STATUS_SUCCESS;
1104
1105     if (ctx->vtable_glx && ctx->vtable_glx->vaCopySurfaceGLX) {
1106         vtable->vaCreateSurfaceGLX      = vaCreateSurfaceGLX_impl_driver;
1107         vtable->vaDestroySurfaceGLX     = vaDestroySurfaceGLX_impl_driver;
1108         vtable->vaCopySurfaceGLX        = vaCopySurfaceGLX_impl_driver;
1109     }
1110     else {
1111         vtable->vaCreateSurfaceGLX      = vaCreateSurfaceGLX_impl_libva;
1112         vtable->vaDestroySurfaceGLX     = vaDestroySurfaceGLX_impl_libva;
1113         vtable->vaCopySurfaceGLX        = vaCopySurfaceGLX_impl_libva;
1114
1115         if (!glXQueryVersion(ctx->native_dpy, &glx_major, &glx_minor))
1116             return VA_STATUS_ERROR_UNIMPLEMENTED;
1117
1118         if (!check_tfp_extensions(ctx) || !load_tfp_extensions(ctx))
1119             return VA_STATUS_ERROR_UNIMPLEMENTED;
1120
1121         if (!check_fbo_extensions(ctx) || !load_fbo_extensions(ctx))
1122             return VA_STATUS_ERROR_UNIMPLEMENTED;
1123     }
1124
1125     glx_ctx->is_initialized = 1;
1126     return VA_STATUS_SUCCESS;
1127 }