gst/vaapi/gstvaapivideomemory.c

   1 /*
   2  *  gstvaapivideomemory.c - Gstreamer/VA video memory
   3  *
   4  *  Copyright (C) 2013 Intel Corporation
   5  *    Author: Gwenole Beauchesne <gwenole.beauchesne@intel.com>
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public License
   9  *  as published by the Free Software Foundation; either version 2.1
  10  *  of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free
  19  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  20  *  Boston, MA 02110-1301 USA
  21  */
  22
  23 #include "gst/vaapi/sysdeps.h"
  24 #include "gstvaapivideomemory.h"
  25
  26 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapivideomemory);
  27 #define GST_CAT_DEFAULT gst_debug_vaapivideomemory
  28
  29 #ifndef GST_VIDEO_INFO_FORMAT_STRING
  30 #define GST_VIDEO_INFO_FORMAT_STRING(vip) \
  31     gst_video_format_to_string(GST_VIDEO_INFO_FORMAT(vip))
  32 #endif
  33
  34 /* ------------------------------------------------------------------------ */
  35 /* --- GstVaapiVideoMemory                                              --- */
  36 /* ------------------------------------------------------------------------ */
  37
  38 static GstVaapiImage *
  39 new_image(GstVaapiDisplay *display, const GstVideoInfo *vip)
  40 {
  41     return gst_vaapi_image_new(display, GST_VIDEO_INFO_FORMAT(vip),
  42         GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
  43 }
  44
  45 static gboolean
  46 ensure_image(GstVaapiVideoMemory *mem)
  47 {
  48     if (!mem->image && mem->use_direct_rendering) {
  49         mem->image = gst_vaapi_surface_derive_image(mem->surface);
  50         if (!mem->image) {
  51             GST_WARNING("failed to derive image, fallbacking to copy");
  52             mem->use_direct_rendering = FALSE;
  53         }
  54     }
  55
  56     if (!mem->image) {
  57         GstVaapiDisplay * const display =
  58             gst_vaapi_video_meta_get_display(mem->meta);
  59
  60         mem->image = new_image(display, mem->image_info);
  61         if (!mem->image)
  62             return FALSE;
  63     }
  64     gst_vaapi_video_meta_set_image(mem->meta, mem->image);
  65     return TRUE;
  66 }
  67
  68 static GstVaapiSurface *
  69 new_surface(GstVaapiDisplay *display, const GstVideoInfo *vip)
  70 {
  71     GstVaapiSurface *surface;
  72     GstVaapiChromaType chroma_type;
  73
  74     /* Try with explicit format first */
  75     if (GST_VIDEO_INFO_FORMAT(vip) != GST_VIDEO_FORMAT_ENCODED) {
  76         surface = gst_vaapi_surface_new_with_format(display,
  77             GST_VIDEO_INFO_FORMAT(vip), GST_VIDEO_INFO_WIDTH(vip),
  78             GST_VIDEO_INFO_HEIGHT(vip));
  79         if (surface)
  80             return surface;
  81     }
  82
  83     /* Try to pick something compatible, i.e. with same chroma type */
  84     chroma_type = gst_vaapi_video_format_get_chroma_type(
  85         GST_VIDEO_INFO_FORMAT(vip));
  86     if (!chroma_type)
  87         return NULL;
  88     return gst_vaapi_surface_new(display, chroma_type,
  89         GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
  90 }
  91
  92 static GstVaapiSurfaceProxy *
  93 new_surface_proxy(GstVaapiVideoMemory *mem)
  94 {
  95     GstVaapiVideoAllocator * const allocator =
  96         GST_VAAPI_VIDEO_ALLOCATOR_CAST(GST_MEMORY_CAST(mem)->allocator);
  97
  98     return gst_vaapi_surface_proxy_new_from_pool(
  99         GST_VAAPI_SURFACE_POOL(allocator->surface_pool));
 100 }
 101
 102 static gboolean
 103 ensure_surface(GstVaapiVideoMemory *mem)
 104 {
 105     if (!mem->proxy) {
 106         gst_vaapi_surface_proxy_replace(&mem->proxy,
 107             gst_vaapi_video_meta_get_surface_proxy(mem->meta));
 108
 109         if (!mem->proxy) {
 110             mem->proxy = new_surface_proxy(mem);
 111             if (!mem->proxy)
 112                 return FALSE;
 113             gst_vaapi_video_meta_set_surface_proxy(mem->meta, mem->proxy);
 114         }
 115         mem->surface = GST_VAAPI_SURFACE_PROXY_SURFACE(mem->proxy);
 116     }
 117     g_return_val_if_fail(mem->surface != NULL, FALSE);
 118     return TRUE;
 119 }
 120
 121 gboolean
 122 gst_video_meta_map_vaapi_memory(GstVideoMeta *meta, guint plane,
 123     GstMapInfo *info, gpointer *data, gint *stride, GstMapFlags flags)
 124 {
 125     GstVaapiVideoMemory * const mem =
 126         GST_VAAPI_VIDEO_MEMORY_CAST(gst_buffer_peek_memory(meta->buffer, 0));
 127
 128     g_return_val_if_fail(mem, FALSE);
 129     g_return_val_if_fail(GST_VAAPI_IS_VIDEO_ALLOCATOR(mem->parent_instance.
 130                              allocator), FALSE);
 131     g_return_val_if_fail(mem->meta, FALSE);
 132
 133     if (mem->map_type &&
 134         mem->map_type != GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR)
 135         goto error_incompatible_map;
 136
 137     /* Map for writing */
 138     if (++mem->map_count == 1) {
 139         if (!ensure_surface(mem))
 140             goto error_ensure_surface;
 141         if (!ensure_image(mem))
 142             goto error_ensure_image;
 143
 144         // Check that we can actually map the surface, or image
 145         if ((flags & GST_MAP_READWRITE) != GST_MAP_WRITE &&
 146             !mem->use_direct_rendering)
 147             goto error_unsupported_map;
 148
 149         if (!gst_vaapi_image_map(mem->image))
 150             goto error_map_image;
 151         mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR;
 152     }
 153
 154     *data = gst_vaapi_image_get_plane(mem->image, plane);
 155     *stride = gst_vaapi_image_get_pitch(mem->image, plane);
 156     info->flags = flags;
 157     return TRUE;
 158
 159     /* ERRORS */
 160 error_incompatible_map:
 161     {
 162         GST_ERROR("incompatible map type (%d)", mem->map_type);
 163         return FALSE;
 164     }
 165 error_unsupported_map:
 166     {
 167         GST_ERROR("unsupported map flags (0x%x)", flags);
 168         return FALSE;
 169     }
 170 error_ensure_surface:
 171     {
 172         const GstVideoInfo * const vip = mem->surface_info;
 173         GST_ERROR("failed to create %s surface of size %ux%u",
 174                   GST_VIDEO_INFO_FORMAT_STRING(vip),
 175                   GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
 176         return FALSE;
 177     }
 178 error_ensure_image:
 179     {
 180         const GstVideoInfo * const vip = mem->image_info;
 181         GST_ERROR("failed to create %s image of size %ux%u",
 182                   GST_VIDEO_INFO_FORMAT_STRING(vip),
 183                   GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
 184         return FALSE;
 185     }
 186 error_map_image:
 187     {
 188         GST_ERROR("failed to map image %" GST_VAAPI_ID_FORMAT,
 189                   GST_VAAPI_ID_ARGS(gst_vaapi_image_get_id(mem->image)));
 190         return FALSE;
 191     }
 192 }
 193
 194 gboolean
 195 gst_video_meta_unmap_vaapi_memory(GstVideoMeta *meta, guint plane,
 196     GstMapInfo *info)
 197 {
 198     GstVaapiVideoMemory * const mem =
 199         GST_VAAPI_VIDEO_MEMORY_CAST(gst_buffer_peek_memory(meta->buffer, 0));
 200
 201     g_return_val_if_fail(mem, FALSE);
 202     g_return_val_if_fail(GST_VAAPI_IS_VIDEO_ALLOCATOR(mem->parent_instance.
 203                              allocator), FALSE);
 204     g_return_val_if_fail(mem->meta, FALSE);
 205     g_return_val_if_fail(mem->surface, FALSE);
 206     g_return_val_if_fail(mem->image, FALSE);
 207
 208     if (--mem->map_count == 0) {
 209         mem->map_type = 0;
 210
 211         /* Unmap VA image used for read/writes */
 212         if (info->flags & GST_MAP_READWRITE)
 213             gst_vaapi_image_unmap(mem->image);
 214
 215         /* Commit VA image to surface */
 216         if ((info->flags & GST_MAP_WRITE) && !mem->use_direct_rendering) {
 217             if (!gst_vaapi_surface_put_image(mem->surface, mem->image))
 218                 goto error_upload_image;
 219         }
 220     }
 221     return TRUE;
 222
 223     /* ERRORS */
 224 error_upload_image:
 225     {
 226         GST_ERROR("failed to upload image");
 227         return FALSE;
 228     }
 229 }
 230
 231 GstMemory *
 232 gst_vaapi_video_memory_new(GstAllocator *base_allocator,
 233     GstVaapiVideoMeta *meta)
 234 {
 235     GstVaapiVideoAllocator * const allocator =
 236         GST_VAAPI_VIDEO_ALLOCATOR_CAST(base_allocator);
 237     const GstVideoInfo *vip;
 238     GstVaapiVideoMemory *mem;
 239
 240     mem = g_slice_new(GstVaapiVideoMemory);
 241     if (!mem)
 242         return NULL;
 243
 244     vip = &allocator->image_info;
 245     gst_memory_init(&mem->parent_instance, 0, gst_object_ref(allocator), NULL,
 246         GST_VIDEO_INFO_SIZE(vip), 0, 0, GST_VIDEO_INFO_SIZE(vip));
 247
 248     mem->proxy = NULL;
 249     mem->surface_info = &allocator->surface_info;
 250     mem->surface = NULL;
 251     mem->image_info = &allocator->image_info;
 252     mem->image = NULL;
 253     mem->meta = gst_vaapi_video_meta_ref(meta);
 254     mem->map_type = 0;
 255     mem->map_count = 0;
 256     mem->use_direct_rendering = allocator->has_direct_rendering;
 257     return GST_MEMORY_CAST(mem);
 258 }
 259
 260 static void
 261 gst_vaapi_video_memory_free(GstVaapiVideoMemory *mem)
 262 {
 263     mem->surface = NULL;
 264     gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
 265     gst_vaapi_object_replace(&mem->image, NULL);
 266     gst_vaapi_video_meta_unref(mem->meta);
 267     gst_object_unref(GST_MEMORY_CAST(mem)->allocator);
 268     g_slice_free(GstVaapiVideoMemory, mem);
 269 }
 270
 271 void
 272 gst_vaapi_video_memory_reset_surface(GstVaapiVideoMemory *mem)
 273 {
 274     mem->surface = NULL;
 275     gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
 276     if (mem->use_direct_rendering)
 277         gst_vaapi_object_replace(&mem->image, NULL);
 278     gst_vaapi_video_meta_set_surface_proxy(mem->meta, NULL);
 279 }
 280
 281 static gpointer
 282 gst_vaapi_video_memory_map(GstVaapiVideoMemory *mem, gsize maxsize, guint flags)
 283 {
 284     if (mem->map_type &&
 285         mem->map_type != GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE)
 286         goto error_incompatible_map;
 287
 288     if (mem->map_count == 0) {
 289         gst_vaapi_surface_proxy_replace(&mem->proxy,
 290             gst_vaapi_video_meta_get_surface_proxy(mem->meta));
 291         if (!mem->proxy)
 292             goto error_no_surface_proxy;
 293         mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE;
 294     }
 295     mem->map_count++;
 296     return mem->proxy;
 297
 298     /* ERRORS */
 299 error_incompatible_map:
 300     GST_ERROR("failed to map memory to a GstVaapiSurfaceProxy");
 301     return NULL;
 302 error_no_surface_proxy:
 303     GST_ERROR("failed to extract GstVaapiSurfaceProxy from video meta");
 304     return NULL;
 305 }
 306
 307 static void
 308 gst_vaapi_video_memory_unmap(GstVaapiVideoMemory *mem)
 309 {
 310     if (mem->map_type &&
 311         mem->map_type != GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE)
 312         goto error_incompatible_map;
 313
 314     if (--mem->map_count == 0) {
 315         gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
 316         mem->map_type = 0;
 317     }
 318     return;
 319
 320     /* ERRORS */
 321 error_incompatible_map:
 322     GST_ERROR("incompatible map type (%d)", mem->map_type);
 323     return;
 324 }
 325
 326 static GstVaapiVideoMemory *
 327 gst_vaapi_video_memory_copy(GstVaapiVideoMemory *mem,
 328     gssize offset, gssize size)
 329 {
 330     GstMemory *out_mem;
 331
 332     if (offset != 0 || size != -1)
 333         goto error_unsupported;
 334
 335     out_mem = gst_vaapi_video_memory_new(mem->parent_instance.allocator,
 336         mem->meta);
 337     if (!out_mem)
 338         goto error_allocate_memory;
 339     return GST_VAAPI_VIDEO_MEMORY_CAST(out_mem);
 340
 341     /* ERRORS */
 342 error_unsupported:
 343     GST_ERROR("failed to copy partial memory (unsupported operation)");
 344     return NULL;
 345 error_allocate_memory:
 346     GST_ERROR("failed to allocate GstVaapiVideoMemory copy");
 347     return NULL;
 348 }
 349
 350 static GstVaapiVideoMemory *
 351 gst_vaapi_video_memory_share(GstVaapiVideoMemory *mem,
 352     gssize offset, gssize size)
 353 {
 354     GST_FIXME("unimplemented GstVaapiVideoAllocator::mem_share() hook");
 355     return NULL;
 356 }
 357
 358 static gboolean
 359 gst_vaapi_video_memory_is_span(GstVaapiVideoMemory *mem1,
 360     GstVaapiVideoMemory *mem2, gsize *offset_ptr)
 361 {
 362     GST_FIXME("unimplemented GstVaapiVideoAllocator::mem_is_span() hook");
 363     return FALSE;
 364 }
 365
 366 /* ------------------------------------------------------------------------ */
 367 /* --- GstVaapiVideoAllocator                                           --- */
 368 /* ------------------------------------------------------------------------ */
 369
 370 #define GST_VAAPI_VIDEO_ALLOCATOR_CLASS(klass)  \
 371     (G_TYPE_CHECK_CLASS_CAST((klass),           \
 372         GST_VAAPI_TYPE_VIDEO_ALLOCATOR,         \
 373         GstVaapiVideoAllocatorClass))
 374
 375 #define GST_VAAPI_IS_VIDEO_ALLOCATOR_CLASS(klass) \
 376     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_VIDEO_ALLOCATOR))
 377
 378 G_DEFINE_TYPE(GstVaapiVideoAllocator,
 379               gst_vaapi_video_allocator,
 380               GST_TYPE_ALLOCATOR)
 381
 382 static GstMemory *
 383 gst_vaapi_video_allocator_alloc(GstAllocator *allocator, gsize size,
 384     GstAllocationParams *params)
 385 {
 386     g_warning("use gst_vaapi_video_memory_new() to allocate from "
 387         "GstVaapiVideoMemory allocator");
 388
 389     return NULL;
 390 }
 391
 392 static void
 393 gst_vaapi_video_allocator_free(GstAllocator *allocator, GstMemory *mem)
 394 {
 395     gst_vaapi_video_memory_free(GST_VAAPI_VIDEO_MEMORY_CAST(mem));
 396 }
 397
 398 static void
 399 gst_vaapi_video_allocator_finalize(GObject *object)
 400 {
 401     GstVaapiVideoAllocator * const allocator =
 402         GST_VAAPI_VIDEO_ALLOCATOR_CAST(object);
 403
 404     gst_vaapi_video_pool_replace(&allocator->surface_pool, NULL);
 405
 406     G_OBJECT_CLASS(gst_vaapi_video_allocator_parent_class)->finalize(object);
 407 }
 408
 409 static void
 410 gst_vaapi_video_allocator_class_init(GstVaapiVideoAllocatorClass *klass)
 411 {
 412     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
 413     GstAllocatorClass * const allocator_class = GST_ALLOCATOR_CLASS(klass);
 414
 415     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapivideomemory,
 416         "vaapivideomemory", 0, "VA-API video memory allocator");
 417
 418     object_class->finalize      = gst_vaapi_video_allocator_finalize;
 419     allocator_class->alloc      = gst_vaapi_video_allocator_alloc;
 420     allocator_class->free       = gst_vaapi_video_allocator_free;
 421 }
 422
 423 static void
 424 gst_vaapi_video_allocator_init(GstVaapiVideoAllocator *allocator)
 425 {
 426     GstAllocator * const base_allocator = GST_ALLOCATOR_CAST(allocator);
 427
 428     base_allocator->mem_type = GST_VAAPI_VIDEO_MEMORY_NAME;
 429     base_allocator->mem_map = (GstMemoryMapFunction)
 430         gst_vaapi_video_memory_map;
 431     base_allocator->mem_unmap = (GstMemoryUnmapFunction)
 432         gst_vaapi_video_memory_unmap;
 433     base_allocator->mem_copy = (GstMemoryCopyFunction)
 434         gst_vaapi_video_memory_copy;
 435     base_allocator->mem_share = (GstMemoryShareFunction)
 436         gst_vaapi_video_memory_share;
 437     base_allocator->mem_is_span = (GstMemoryIsSpanFunction)
 438         gst_vaapi_video_memory_is_span;
 439 }
 440
 441 static gboolean
 442 gst_video_info_update_from_image(GstVideoInfo *vip, GstVaapiImage *image)
 443 {
 444     GstVideoFormat format;
 445     const guchar *data;
 446     guint i, num_planes, data_size, width, height;
 447
 448     /* Reset format from image */
 449     format = gst_vaapi_image_get_format(image);
 450     gst_vaapi_image_get_size(image, &width, &height);
 451     gst_video_info_set_format(vip, format, width, height);
 452
 453     num_planes = gst_vaapi_image_get_plane_count(image);
 454     g_return_val_if_fail(num_planes == GST_VIDEO_INFO_N_PLANES(vip), FALSE);
 455
 456     /* Determine the base data pointer */
 457     data = gst_vaapi_image_get_plane(image, 0);
 458     for (i = 1; i < num_planes; i++) {
 459         const guchar * const plane = gst_vaapi_image_get_plane(image, i);
 460         if (data > plane)
 461             data = plane;
 462     }
 463     data_size = gst_vaapi_image_get_data_size(image);
 464
 465     /* Check that we don't have disjoint planes */
 466     for (i = 0; i < num_planes; i++) {
 467         const guchar * const plane = gst_vaapi_image_get_plane(image, i);
 468         if (plane - data > data_size)
 469             return FALSE;
 470     }
 471
 472     /* Update GstVideoInfo structure */
 473     for (i = 0; i < num_planes; i++) {
 474         const guchar * const plane = gst_vaapi_image_get_plane(image, i);
 475         GST_VIDEO_INFO_PLANE_OFFSET(vip, i) = plane - data;
 476         GST_VIDEO_INFO_PLANE_STRIDE(vip, i) =
 477             gst_vaapi_image_get_pitch(image, i);
 478     }
 479     GST_VIDEO_INFO_SIZE(vip) = data_size;
 480     return TRUE;
 481 }
 482
 483 GstAllocator *
 484 gst_vaapi_video_allocator_new(GstVaapiDisplay *display, const GstVideoInfo *vip)
 485 {
 486     GstVaapiVideoAllocator *allocator;
 487     GstVaapiSurface *surface;
 488     GstVaapiImage *image;
 489
 490     g_return_val_if_fail(display != NULL, NULL);
 491     g_return_val_if_fail(vip != NULL, NULL);
 492
 493     allocator = g_object_new(GST_VAAPI_TYPE_VIDEO_ALLOCATOR, NULL);
 494     if (!allocator)
 495         return NULL;
 496
 497     allocator->video_info = *vip;
 498     gst_video_info_set_format(&allocator->surface_info, GST_VIDEO_FORMAT_NV12,
 499         GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
 500
 501     if (GST_VIDEO_INFO_FORMAT(vip) != GST_VIDEO_FORMAT_ENCODED) {
 502         image = NULL;
 503         do {
 504             surface = new_surface(display, vip);
 505             if (!surface)
 506                 break;
 507             image = gst_vaapi_surface_derive_image(surface);
 508             if (!image)
 509                 break;
 510             if (GST_VAAPI_IMAGE_FORMAT(image) != GST_VIDEO_INFO_FORMAT(vip))
 511                 break;
 512             if (!gst_vaapi_image_map(image))
 513                 break;
 514             allocator->has_direct_rendering = gst_video_info_update_from_image(
 515                 &allocator->surface_info, image);
 516             gst_vaapi_image_unmap(image);
 517             GST_INFO("has direct-rendering for %s surfaces: %s",
 518                      GST_VIDEO_INFO_FORMAT_STRING(&allocator->surface_info),
 519                      allocator->has_direct_rendering ? "yes" : "no");
 520         } while (0);
 521         if (surface)
 522             gst_vaapi_object_unref(surface);
 523         if (image)
 524             gst_vaapi_object_unref(image);
 525     }
 526
 527     allocator->surface_pool = gst_vaapi_surface_pool_new(display,
 528         &allocator->surface_info);
 529     if (!allocator->surface_pool)
 530         goto error_create_pool;
 531
 532     allocator->image_info = *vip;
 533     if (GST_VIDEO_INFO_FORMAT(vip) == GST_VIDEO_FORMAT_ENCODED)
 534         gst_video_info_set_format(&allocator->image_info, GST_VIDEO_FORMAT_NV12,
 535             GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
 536
 537     if (allocator->has_direct_rendering)
 538         allocator->image_info = allocator->surface_info;
 539     else {
 540         do {
 541             image = new_image(display, &allocator->image_info);
 542             if (!image)
 543                 break;
 544             if (!gst_vaapi_image_map(image))
 545                 break;
 546             gst_video_info_update_from_image(&allocator->image_info, image);
 547             gst_vaapi_image_unmap(image);
 548         } while (0);
 549         gst_vaapi_object_unref(image);
 550     }
 551     return GST_ALLOCATOR_CAST(allocator);
 552
 553     /* ERRORS */
 554 error_create_pool:
 555     {
 556         GST_ERROR("failed to allocate VA surface pool");
 557         gst_object_unref(allocator);
 558         return NULL;
 559     }
 560 }