gst/vaapi/gstvaapivideomemory.c

   1 /*
   2  *  gstvaapivideomemory.c - Gstreamer/VA video memory
   3  *
   4  *  Copyright (C) 2013 Intel Corporation
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public License
   8  *  as published by the Free Software Foundation; either version 2.1
   9  *  of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free
  18  *  Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  19  *  Boston, MA 02110-1301 USA
  20  */
  21
  22 #include "gst/vaapi/sysdeps.h"
  23 #include "gstvaapivideomemory.h"
  24
  25 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapivideomemory);
  26 #define GST_CAT_DEFAULT gst_debug_vaapivideomemory
  27
  28 #ifndef GST_VIDEO_INFO_FORMAT_STRING
  29 #define GST_VIDEO_INFO_FORMAT_STRING(vip) \
  30     gst_video_format_to_string(GST_VIDEO_INFO_FORMAT(vip))
  31 #endif
  32
  33 /* ------------------------------------------------------------------------ */
  34 /* --- GstVaapiVideoMemory                                              --- */
  35 /* ------------------------------------------------------------------------ */
  36
  37 static GstVaapiImage *
  38 new_image(GstVaapiDisplay *display, const GstVideoInfo *vip)
  39 {
  40     return gst_vaapi_image_new(display, GST_VIDEO_INFO_FORMAT(vip),
  41         GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
  42 }
  43
  44 static gboolean
  45 ensure_image(GstVaapiVideoMemory *mem)
  46 {
  47     if (!mem->image && mem->use_direct_rendering) {
  48         mem->image = gst_vaapi_surface_derive_image(mem->surface);
  49         if (!mem->image) {
  50             GST_WARNING("failed to derive image, fallbacking to copy");
  51             mem->use_direct_rendering = FALSE;
  52         }
  53     }
  54
  55     if (!mem->image) {
  56         GstVaapiDisplay * const display =
  57             gst_vaapi_video_meta_get_display(mem->meta);
  58
  59         mem->image = new_image(display, mem->image_info);
  60         if (!mem->image)
  61             return FALSE;
  62     }
  63     gst_vaapi_video_meta_set_image(mem->meta, mem->image);
  64     return TRUE;
  65 }
  66
  67 static GstVaapiSurface *
  68 new_surface(GstVaapiDisplay *display, const GstVideoInfo *vip)
  69 {
  70     GstVaapiSurface *surface;
  71     GstVaapiChromaType chroma_type;
  72
  73     /* Try with explicit format first */
  74     if (GST_VIDEO_INFO_FORMAT(vip) != GST_VIDEO_FORMAT_ENCODED) {
  75         surface = gst_vaapi_surface_new_with_format(display,
  76             GST_VIDEO_INFO_FORMAT(vip), GST_VIDEO_INFO_WIDTH(vip),
  77             GST_VIDEO_INFO_HEIGHT(vip));
  78         if (surface)
  79             return surface;
  80     }
  81
  82     /* Try to pick something compatible, i.e. with same chroma type */
  83     chroma_type = gst_vaapi_video_format_get_chroma_type(
  84         GST_VIDEO_INFO_FORMAT(vip));
  85     if (!chroma_type)
  86         return NULL;
  87     return gst_vaapi_surface_new(display, chroma_type,
  88         GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
  89 }
  90
  91 static GstVaapiSurfaceProxy *
  92 new_surface_proxy(GstVaapiVideoMemory *mem)
  93 {
  94     GstVaapiVideoAllocator * const allocator =
  95         GST_VAAPI_VIDEO_ALLOCATOR_CAST(GST_MEMORY_CAST(mem)->allocator);
  96
  97     return gst_vaapi_surface_proxy_new_from_pool(
  98         GST_VAAPI_SURFACE_POOL(allocator->surface_pool));
  99 }
 100
 101 static gboolean
 102 ensure_surface(GstVaapiVideoMemory *mem)
 103 {
 104     if (!mem->proxy) {
 105         gst_vaapi_surface_proxy_replace(&mem->proxy,
 106             gst_vaapi_video_meta_get_surface_proxy(mem->meta));
 107
 108         if (!mem->proxy) {
 109             mem->proxy = new_surface_proxy(mem);
 110             if (!mem->proxy)
 111                 return FALSE;
 112             gst_vaapi_video_meta_set_surface_proxy(mem->meta, mem->proxy);
 113         }
 114         mem->surface = GST_VAAPI_SURFACE_PROXY_SURFACE(mem->proxy);
 115     }
 116     g_return_val_if_fail(mem->surface != NULL, FALSE);
 117     return TRUE;
 118 }
 119
 120 gboolean
 121 gst_video_meta_map_vaapi_memory(GstVideoMeta *meta, guint plane,
 122     GstMapInfo *info, gpointer *data, gint *stride, GstMapFlags flags)
 123 {
 124     GstVaapiVideoMemory * const mem =
 125         GST_VAAPI_VIDEO_MEMORY_CAST(gst_buffer_get_memory(meta->buffer, 0));
 126
 127     g_return_val_if_fail(mem, FALSE);
 128     g_return_val_if_fail(GST_VAAPI_IS_VIDEO_ALLOCATOR(mem->parent_instance.
 129                              allocator), FALSE);
 130     g_return_val_if_fail(mem->meta, FALSE);
 131
 132     if (mem->map_type &&
 133         mem->map_type != GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR)
 134         goto error_incompatible_map;
 135
 136     /* Map for writing */
 137     if (++mem->map_count == 1) {
 138         if (!ensure_surface(mem))
 139             goto error_ensure_surface;
 140         if (!ensure_image(mem))
 141             goto error_ensure_image;
 142
 143         // Check that we can actually map the surface, or image
 144         if ((flags & GST_MAP_READWRITE) != GST_MAP_WRITE &&
 145             !mem->use_direct_rendering)
 146             goto error_unsupported_map;
 147
 148         if (!gst_vaapi_image_map(mem->image))
 149             goto error_map_image;
 150         mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR;
 151     }
 152
 153     *data = gst_vaapi_image_get_plane(mem->image, plane);
 154     *stride = gst_vaapi_image_get_pitch(mem->image, plane);
 155     info->flags = flags;
 156     return TRUE;
 157
 158     /* ERRORS */
 159 error_incompatible_map:
 160     {
 161         GST_ERROR("incompatible map type (%d)", mem->map_type);
 162         return FALSE;
 163     }
 164 error_unsupported_map:
 165     {
 166         GST_ERROR("unsupported map flags (0x%x)", flags);
 167         return FALSE;
 168     }
 169 error_ensure_surface:
 170     {
 171         const GstVideoInfo * const vip = mem->surface_info;
 172         GST_ERROR("failed to create %s surface of size %ux%u",
 173                   GST_VIDEO_INFO_FORMAT_STRING(vip),
 174                   GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
 175         return FALSE;
 176     }
 177 error_ensure_image:
 178     {
 179         const GstVideoInfo * const vip = mem->image_info;
 180         GST_ERROR("failed to create %s image of size %ux%u",
 181                   GST_VIDEO_INFO_FORMAT_STRING(vip),
 182                   GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
 183         return FALSE;
 184     }
 185 error_map_image:
 186     {
 187         GST_ERROR("failed to map image %" GST_VAAPI_ID_FORMAT,
 188                   GST_VAAPI_ID_ARGS(gst_vaapi_image_get_id(mem->image)));
 189         return FALSE;
 190     }
 191 }
 192
 193 gboolean
 194 gst_video_meta_unmap_vaapi_memory(GstVideoMeta *meta, guint plane,
 195     GstMapInfo *info)
 196 {
 197     GstVaapiVideoMemory * const mem =
 198         GST_VAAPI_VIDEO_MEMORY_CAST(gst_buffer_get_memory(meta->buffer, 0));
 199
 200     g_return_val_if_fail(mem, FALSE);
 201     g_return_val_if_fail(GST_VAAPI_IS_VIDEO_ALLOCATOR(mem->parent_instance.
 202                              allocator), FALSE);
 203     g_return_val_if_fail(mem->meta, FALSE);
 204     g_return_val_if_fail(mem->surface, FALSE);
 205     g_return_val_if_fail(mem->image, FALSE);
 206
 207     if (--mem->map_count == 0) {
 208         mem->map_type = 0;
 209
 210         /* Unmap VA image used for read/writes */
 211         if (info->flags & GST_MAP_READWRITE)
 212             gst_vaapi_image_unmap(mem->image);
 213
 214         /* Commit VA image to surface */
 215         if ((info->flags & GST_MAP_WRITE) && !mem->use_direct_rendering) {
 216             if (!gst_vaapi_surface_put_image(mem->surface, mem->image))
 217                 goto error_upload_image;
 218         }
 219     }
 220     return TRUE;
 221
 222     /* ERRORS */
 223 error_upload_image:
 224     {
 225         GST_ERROR("failed to upload image");
 226         return FALSE;
 227     }
 228 }
 229
 230 GstMemory *
 231 gst_vaapi_video_memory_new(GstAllocator *base_allocator,
 232     GstVaapiVideoMeta *meta)
 233 {
 234     GstVaapiVideoAllocator * const allocator =
 235         GST_VAAPI_VIDEO_ALLOCATOR_CAST(base_allocator);
 236     const GstVideoInfo *vip;
 237     GstVaapiVideoMemory *mem;
 238
 239     mem = g_slice_new(GstVaapiVideoMemory);
 240     if (!mem)
 241         return NULL;
 242
 243     vip = &allocator->image_info;
 244     gst_memory_init(&mem->parent_instance, 0, gst_object_ref(allocator), NULL,
 245         GST_VIDEO_INFO_SIZE(vip), 0, 0, GST_VIDEO_INFO_SIZE(vip));
 246
 247     mem->proxy = NULL;
 248     mem->surface_info = &allocator->surface_info;
 249     mem->surface = NULL;
 250     mem->image_info = &allocator->image_info;
 251     mem->image = NULL;
 252     mem->meta = gst_vaapi_video_meta_ref(meta);
 253     mem->map_type = 0;
 254     mem->map_count = 0;
 255     mem->use_direct_rendering = allocator->has_direct_rendering;
 256     return GST_MEMORY_CAST(mem);
 257 }
 258
 259 static void
 260 gst_vaapi_video_memory_free(GstVaapiVideoMemory *mem)
 261 {
 262     mem->surface = NULL;
 263     gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
 264     gst_vaapi_object_replace(&mem->image, NULL);
 265     gst_vaapi_video_meta_unref(mem->meta);
 266     gst_object_unref(GST_MEMORY_CAST(mem)->allocator);
 267     g_slice_free(GstVaapiVideoMemory, mem);
 268 }
 269
 270 void
 271 gst_vaapi_video_memory_reset_surface(GstVaapiVideoMemory *mem)
 272 {
 273     mem->surface = NULL;
 274     gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
 275     if (mem->use_direct_rendering)
 276         gst_vaapi_object_replace(&mem->image, NULL);
 277     gst_vaapi_video_meta_set_surface_proxy(mem->meta, NULL);
 278 }
 279
 280 static gpointer
 281 gst_vaapi_video_memory_map(GstVaapiVideoMemory *mem, gsize maxsize, guint flags)
 282 {
 283     if (mem->map_type &&
 284         mem->map_type != GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE)
 285         goto error_incompatible_map;
 286
 287     if (mem->map_count == 0) {
 288         gst_vaapi_surface_proxy_replace(&mem->proxy,
 289             gst_vaapi_video_meta_get_surface_proxy(mem->meta));
 290         if (!mem->proxy)
 291             goto error_no_surface_proxy;
 292         mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE;
 293     }
 294     mem->map_count++;
 295     return mem->proxy;
 296
 297     /* ERRORS */
 298 error_incompatible_map:
 299     GST_ERROR("failed to map memory to a GstVaapiSurfaceProxy");
 300     return NULL;
 301 error_no_surface_proxy:
 302     GST_ERROR("failed to extract GstVaapiSurfaceProxy from video meta");
 303     return NULL;
 304 }
 305
 306 static void
 307 gst_vaapi_video_memory_unmap(GstVaapiVideoMemory *mem)
 308 {
 309     if (mem->map_type &&
 310         mem->map_type != GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE)
 311         goto error_incompatible_map;
 312
 313     if (--mem->map_count == 0) {
 314         gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
 315         mem->map_type = 0;
 316     }
 317     return;
 318
 319     /* ERRORS */
 320 error_incompatible_map:
 321     GST_ERROR("incompatible map type (%d)", mem->map_type);
 322     return;
 323 }
 324
 325 static GstVaapiVideoMemory *
 326 gst_vaapi_video_memory_copy(GstVaapiVideoMemory *mem,
 327     gssize offset, gssize size)
 328 {
 329     GstMemory *out_mem;
 330
 331     if (offset != 0 || size != -1)
 332         goto error_unsupported;
 333
 334     out_mem = gst_vaapi_video_memory_new(mem->parent_instance.allocator,
 335         mem->meta);
 336     if (!out_mem)
 337         goto error_allocate_memory;
 338     return GST_VAAPI_VIDEO_MEMORY_CAST(out_mem);
 339
 340     /* ERRORS */
 341 error_unsupported:
 342     GST_ERROR("failed to copy partial memory (unsupported operation)");
 343     return NULL;
 344 error_allocate_memory:
 345     GST_ERROR("failed to allocate GstVaapiVideoMemory copy");
 346     return NULL;
 347 }
 348
 349 static GstVaapiVideoMemory *
 350 gst_vaapi_video_memory_share(GstVaapiVideoMemory *mem,
 351     gssize offset, gssize size)
 352 {
 353     GST_FIXME("unimplemented GstVaapiVideoAllocator::mem_share() hook");
 354     return NULL;
 355 }
 356
 357 static gboolean
 358 gst_vaapi_video_memory_is_span(GstVaapiVideoMemory *mem1,
 359     GstVaapiVideoMemory *mem2, gsize *offset_ptr)
 360 {
 361     GST_FIXME("unimplemented GstVaapiVideoAllocator::mem_is_span() hook");
 362     return FALSE;
 363 }
 364
 365 /* ------------------------------------------------------------------------ */
 366 /* --- GstVaapiVideoAllocator                                           --- */
 367 /* ------------------------------------------------------------------------ */
 368
 369 #define GST_VAAPI_VIDEO_ALLOCATOR_CLASS(klass)  \
 370     (G_TYPE_CHECK_CLASS_CAST((klass),           \
 371         GST_VAAPI_TYPE_VIDEO_ALLOCATOR,         \
 372         GstVaapiVideoAllocatorClass))
 373
 374 #define GST_VAAPI_IS_VIDEO_ALLOCATOR_CLASS(klass) \
 375     (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_VIDEO_ALLOCATOR))
 376
 377 G_DEFINE_TYPE(GstVaapiVideoAllocator,
 378               gst_vaapi_video_allocator,
 379               GST_TYPE_ALLOCATOR)
 380
 381 static GstMemory *
 382 gst_vaapi_video_allocator_alloc(GstAllocator *allocator, gsize size,
 383     GstAllocationParams *params)
 384 {
 385     g_warning("use gst_vaapi_video_memory_new() to allocate from "
 386         "GstVaapiVideoMemory allocator");
 387
 388     return NULL;
 389 }
 390
 391 static void
 392 gst_vaapi_video_allocator_free(GstAllocator *allocator, GstMemory *mem)
 393 {
 394     gst_vaapi_video_memory_free(GST_VAAPI_VIDEO_MEMORY_CAST(mem));
 395 }
 396
 397 static void
 398 gst_vaapi_video_allocator_finalize(GObject *object)
 399 {
 400     GstVaapiVideoAllocator * const allocator =
 401         GST_VAAPI_VIDEO_ALLOCATOR_CAST(object);
 402
 403     gst_vaapi_video_pool_replace(&allocator->surface_pool, NULL);
 404
 405     G_OBJECT_CLASS(gst_vaapi_video_allocator_parent_class)->finalize(object);
 406 }
 407
 408 static void
 409 gst_vaapi_video_allocator_class_init(GstVaapiVideoAllocatorClass *klass)
 410 {
 411     GObjectClass * const object_class = G_OBJECT_CLASS(klass);
 412     GstAllocatorClass * const allocator_class = GST_ALLOCATOR_CLASS(klass);
 413
 414     GST_DEBUG_CATEGORY_INIT(gst_debug_vaapivideomemory,
 415         "vaapivideomemory", 0, "VA-API video memory allocator");
 416
 417     object_class->finalize      = gst_vaapi_video_allocator_finalize;
 418     allocator_class->alloc      = gst_vaapi_video_allocator_alloc;
 419     allocator_class->free       = gst_vaapi_video_allocator_free;
 420 }
 421
 422 static void
 423 gst_vaapi_video_allocator_init(GstVaapiVideoAllocator *allocator)
 424 {
 425     GstAllocator * const base_allocator = GST_ALLOCATOR_CAST(allocator);
 426
 427     base_allocator->mem_type = GST_VAAPI_VIDEO_MEMORY_NAME;
 428     base_allocator->mem_map = (GstMemoryMapFunction)
 429         gst_vaapi_video_memory_map;
 430     base_allocator->mem_unmap = (GstMemoryUnmapFunction)
 431         gst_vaapi_video_memory_unmap;
 432     base_allocator->mem_copy = (GstMemoryCopyFunction)
 433         gst_vaapi_video_memory_copy;
 434     base_allocator->mem_share = (GstMemoryShareFunction)
 435         gst_vaapi_video_memory_share;
 436     base_allocator->mem_is_span = (GstMemoryIsSpanFunction)
 437         gst_vaapi_video_memory_is_span;
 438 }
 439
 440 static gboolean
 441 gst_video_info_update_from_image(GstVideoInfo *vip, GstVaapiImage *image)
 442 {
 443     GstVideoFormat format;
 444     const guchar *data;
 445     guint i, num_planes, data_size, width, height;
 446
 447     /* Reset format from image */
 448     format = gst_vaapi_image_get_format(image);
 449     gst_vaapi_image_get_size(image, &width, &height);
 450     gst_video_info_set_format(vip, format, width, height);
 451
 452     num_planes = gst_vaapi_image_get_plane_count(image);
 453     g_return_val_if_fail(num_planes == GST_VIDEO_INFO_N_PLANES(vip), FALSE);
 454
 455     /* Determine the base data pointer */
 456     data = gst_vaapi_image_get_plane(image, 0);
 457     for (i = 1; i < num_planes; i++) {
 458         const guchar * const plane = gst_vaapi_image_get_plane(image, i);
 459         if (data > plane)
 460             data = plane;
 461     }
 462     data_size = gst_vaapi_image_get_data_size(image);
 463
 464     /* Check that we don't have disjoint planes */
 465     for (i = 0; i < num_planes; i++) {
 466         const guchar * const plane = gst_vaapi_image_get_plane(image, i);
 467         if (plane - data > data_size)
 468             return FALSE;
 469     }
 470
 471     /* Update GstVideoInfo structure */
 472     for (i = 0; i < num_planes; i++) {
 473         const guchar * const plane = gst_vaapi_image_get_plane(image, i);
 474         GST_VIDEO_INFO_PLANE_OFFSET(vip, i) = plane - data;
 475         GST_VIDEO_INFO_PLANE_STRIDE(vip, i) =
 476             gst_vaapi_image_get_pitch(image, i);
 477     }
 478     GST_VIDEO_INFO_SIZE(vip) = data_size;
 479     return TRUE;
 480 }
 481
 482 GstAllocator *
 483 gst_vaapi_video_allocator_new(GstVaapiDisplay *display, GstCaps *caps)
 484 {
 485     GstVaapiVideoAllocator *allocator;
 486     GstVideoInfo *vip;
 487     GstVaapiSurface *surface;
 488     GstVaapiImage *image;
 489
 490     g_return_val_if_fail(display != NULL, NULL);
 491     g_return_val_if_fail(GST_IS_CAPS(caps), NULL);
 492
 493     allocator = g_object_new(GST_VAAPI_TYPE_VIDEO_ALLOCATOR, NULL);
 494     if (!allocator)
 495         return NULL;
 496
 497     vip = &allocator->video_info;
 498     gst_video_info_init(vip);
 499     gst_video_info_from_caps(vip, caps);
 500
 501     gst_video_info_set_format(&allocator->surface_info, GST_VIDEO_FORMAT_NV12,
 502         GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
 503
 504     if (GST_VIDEO_INFO_FORMAT(vip) != GST_VIDEO_FORMAT_ENCODED) {
 505         image = NULL;
 506         do {
 507             surface = new_surface(display, vip);
 508             if (!surface)
 509                 break;
 510             image = gst_vaapi_surface_derive_image(surface);
 511             if (!image)
 512                 break;
 513             if (GST_VAAPI_IMAGE_FORMAT(image) != GST_VIDEO_INFO_FORMAT(vip))
 514                 break;
 515             if (!gst_vaapi_image_map(image))
 516                 break;
 517             allocator->has_direct_rendering = gst_video_info_update_from_image(
 518                 &allocator->surface_info, image);
 519             gst_vaapi_image_unmap(image);
 520             GST_INFO("has direct-rendering for %s surfaces: %s",
 521                      GST_VIDEO_INFO_FORMAT_STRING(&allocator->surface_info),
 522                      allocator->has_direct_rendering ? "yes" : "no");
 523         } while (0);
 524         if (surface)
 525             gst_vaapi_object_unref(surface);
 526         if (image)
 527             gst_vaapi_object_unref(image);
 528     }
 529
 530     allocator->surface_pool = gst_vaapi_surface_pool_new(display,
 531         &allocator->surface_info);
 532     if (!allocator->surface_pool)
 533         goto error_create_pool;
 534
 535     allocator->image_info = *vip;
 536     if (GST_VIDEO_INFO_FORMAT(vip) == GST_VIDEO_FORMAT_ENCODED)
 537         gst_video_info_set_format(&allocator->image_info, GST_VIDEO_FORMAT_NV12,
 538             GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
 539
 540     if (allocator->has_direct_rendering)
 541         allocator->image_info = allocator->surface_info;
 542     else {
 543         do {
 544             image = new_image(display, &allocator->image_info);
 545             if (!image)
 546                 break;
 547             if (!gst_vaapi_image_map(image))
 548                 break;
 549             gst_video_info_update_from_image(&allocator->image_info, image);
 550             gst_vaapi_image_unmap(image);
 551         } while (0);
 552         gst_vaapi_object_unref(image);
 553     }
 554     return GST_ALLOCATOR_CAST(allocator);
 555
 556     /* ERRORS */
 557 error_create_pool:
 558     {
 559         GST_ERROR("failed to allocate VA surface pool");
 560         gst_object_unref(allocator);
 561         return NULL;
 562     }
 563 }