2 * gstvaapivideomemory.c - Gstreamer/VA video memory
4 * Copyright (C) 2013 Intel Corporation
5 * Author: Gwenole Beauchesne <gwenole.beauchesne@intel.com>
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public License
9 * as published by the Free Software Foundation; either version 2.1
10 * of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free
19 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
20 * Boston, MA 02110-1301 USA
23 #include "gst/vaapi/sysdeps.h"
24 #include "gstvaapivideomemory.h"
26 GST_DEBUG_CATEGORY_STATIC(gst_debug_vaapivideomemory);
27 #define GST_CAT_DEFAULT gst_debug_vaapivideomemory
29 #ifndef GST_VIDEO_INFO_FORMAT_STRING
30 #define GST_VIDEO_INFO_FORMAT_STRING(vip) \
31 gst_video_format_to_string(GST_VIDEO_INFO_FORMAT(vip))
34 /* ------------------------------------------------------------------------ */
35 /* --- GstVaapiVideoMemory --- */
36 /* ------------------------------------------------------------------------ */
39 get_image_data(GstVaapiImage *image)
44 data = gst_vaapi_image_get_plane(image, 0);
45 if (!data || !gst_vaapi_image_get_image(image, &va_image))
48 data -= va_image.offsets[0];
52 static GstVaapiImage *
53 new_image(GstVaapiDisplay *display, const GstVideoInfo *vip)
55 return gst_vaapi_image_new(display, GST_VIDEO_INFO_FORMAT(vip),
56 GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
60 ensure_image(GstVaapiVideoMemory *mem)
62 if (!mem->image && mem->use_direct_rendering) {
63 mem->image = gst_vaapi_surface_derive_image(mem->surface);
65 GST_WARNING("failed to derive image, fallbacking to copy");
66 mem->use_direct_rendering = FALSE;
68 else if (gst_vaapi_surface_get_format(mem->surface) !=
69 GST_VIDEO_INFO_FORMAT(mem->image_info)) {
70 gst_vaapi_object_replace(&mem->image, NULL);
71 mem->use_direct_rendering = FALSE;
76 GstVaapiDisplay * const display =
77 gst_vaapi_video_meta_get_display(mem->meta);
79 mem->image = new_image(display, mem->image_info);
83 gst_vaapi_video_meta_set_image(mem->meta, mem->image);
87 static GstVaapiSurface *
88 new_surface(GstVaapiDisplay *display, const GstVideoInfo *vip)
90 GstVaapiSurface *surface;
91 GstVaapiChromaType chroma_type;
93 /* Try with explicit format first */
94 if (GST_VIDEO_INFO_FORMAT(vip) != GST_VIDEO_FORMAT_ENCODED) {
95 surface = gst_vaapi_surface_new_with_format(display,
96 GST_VIDEO_INFO_FORMAT(vip), GST_VIDEO_INFO_WIDTH(vip),
97 GST_VIDEO_INFO_HEIGHT(vip));
102 /* Try to pick something compatible, i.e. with same chroma type */
103 chroma_type = gst_vaapi_video_format_get_chroma_type(
104 GST_VIDEO_INFO_FORMAT(vip));
107 return gst_vaapi_surface_new(display, chroma_type,
108 GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
111 static GstVaapiSurfaceProxy *
112 new_surface_proxy(GstVaapiVideoMemory *mem)
114 GstVaapiVideoAllocator * const allocator =
115 GST_VAAPI_VIDEO_ALLOCATOR_CAST(GST_MEMORY_CAST(mem)->allocator);
117 return gst_vaapi_surface_proxy_new_from_pool(
118 GST_VAAPI_SURFACE_POOL(allocator->surface_pool));
122 ensure_surface(GstVaapiVideoMemory *mem)
125 gst_vaapi_surface_proxy_replace(&mem->proxy,
126 gst_vaapi_video_meta_get_surface_proxy(mem->meta));
129 mem->proxy = new_surface_proxy(mem);
132 gst_vaapi_video_meta_set_surface_proxy(mem->meta, mem->proxy);
135 mem->surface = GST_VAAPI_SURFACE_PROXY_SURFACE(mem->proxy);
136 return mem->surface != NULL;
140 gst_video_meta_map_vaapi_memory(GstVideoMeta *meta, guint plane,
141 GstMapInfo *info, gpointer *data, gint *stride, GstMapFlags flags)
143 GstVaapiVideoMemory * const mem =
144 GST_VAAPI_VIDEO_MEMORY_CAST(gst_buffer_peek_memory(meta->buffer, 0));
146 g_return_val_if_fail(mem, FALSE);
147 g_return_val_if_fail(GST_VAAPI_IS_VIDEO_ALLOCATOR(mem->parent_instance.
149 g_return_val_if_fail(mem->meta, FALSE);
152 mem->map_type != GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR)
153 goto error_incompatible_map;
155 /* Map for writing */
156 if (++mem->map_count == 1) {
157 if (!ensure_surface(mem))
158 goto error_ensure_surface;
159 if (!ensure_image(mem))
160 goto error_ensure_image;
162 // Check that we can actually map the surface, or image
163 if ((flags & GST_MAP_READWRITE) == GST_MAP_READWRITE &&
164 !mem->use_direct_rendering)
165 goto error_unsupported_map;
167 // Load VA image from surface
168 if ((flags & GST_MAP_READ) && !mem->use_direct_rendering)
169 gst_vaapi_surface_get_image(mem->surface, mem->image);
171 if (!gst_vaapi_image_map(mem->image))
172 goto error_map_image;
173 mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_PLANAR;
176 *data = gst_vaapi_image_get_plane(mem->image, plane);
177 *stride = gst_vaapi_image_get_pitch(mem->image, plane);
182 error_incompatible_map:
184 GST_ERROR("incompatible map type (%d)", mem->map_type);
187 error_unsupported_map:
189 GST_ERROR("unsupported map flags (0x%x)", flags);
192 error_ensure_surface:
194 const GstVideoInfo * const vip = mem->surface_info;
195 GST_ERROR("failed to create %s surface of size %ux%u",
196 GST_VIDEO_INFO_FORMAT_STRING(vip),
197 GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
202 const GstVideoInfo * const vip = mem->image_info;
203 GST_ERROR("failed to create %s image of size %ux%u",
204 GST_VIDEO_INFO_FORMAT_STRING(vip),
205 GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
210 GST_ERROR("failed to map image %" GST_VAAPI_ID_FORMAT,
211 GST_VAAPI_ID_ARGS(gst_vaapi_image_get_id(mem->image)));
217 gst_video_meta_unmap_vaapi_memory(GstVideoMeta *meta, guint plane,
220 GstVaapiVideoMemory * const mem =
221 GST_VAAPI_VIDEO_MEMORY_CAST(gst_buffer_peek_memory(meta->buffer, 0));
223 g_return_val_if_fail(mem, FALSE);
224 g_return_val_if_fail(GST_VAAPI_IS_VIDEO_ALLOCATOR(mem->parent_instance.
226 g_return_val_if_fail(mem->meta, FALSE);
227 g_return_val_if_fail(mem->surface, FALSE);
228 g_return_val_if_fail(mem->image, FALSE);
230 if (--mem->map_count == 0) {
233 /* Unmap VA image used for read/writes */
234 if (info->flags & GST_MAP_READWRITE)
235 gst_vaapi_image_unmap(mem->image);
237 /* Commit VA image to surface */
238 if ((info->flags & GST_MAP_WRITE) && !mem->use_direct_rendering) {
239 if (!gst_vaapi_surface_put_image(mem->surface, mem->image))
240 goto error_upload_image;
248 GST_ERROR("failed to upload image");
254 gst_vaapi_video_memory_new(GstAllocator *base_allocator,
255 GstVaapiVideoMeta *meta)
257 GstVaapiVideoAllocator * const allocator =
258 GST_VAAPI_VIDEO_ALLOCATOR_CAST(base_allocator);
259 const GstVideoInfo *vip;
260 GstVaapiVideoMemory *mem;
262 mem = g_slice_new(GstVaapiVideoMemory);
266 vip = &allocator->image_info;
267 gst_memory_init(&mem->parent_instance, 0, gst_object_ref(allocator), NULL,
268 GST_VIDEO_INFO_SIZE(vip), 0, 0, GST_VIDEO_INFO_SIZE(vip));
271 mem->surface_info = &allocator->surface_info;
273 mem->image_info = &allocator->image_info;
275 mem->meta = gst_vaapi_video_meta_ref(meta);
278 mem->use_direct_rendering = allocator->has_direct_rendering;
279 return GST_MEMORY_CAST(mem);
283 gst_vaapi_video_memory_free(GstVaapiVideoMemory *mem)
286 gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
287 gst_vaapi_object_replace(&mem->image, NULL);
288 gst_vaapi_video_meta_unref(mem->meta);
289 gst_object_unref(GST_MEMORY_CAST(mem)->allocator);
290 g_slice_free(GstVaapiVideoMemory, mem);
294 gst_vaapi_video_memory_reset_surface(GstVaapiVideoMemory *mem)
297 gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
298 if (mem->use_direct_rendering)
299 gst_vaapi_object_replace(&mem->image, NULL);
300 gst_vaapi_video_meta_set_surface_proxy(mem->meta, NULL);
304 gst_vaapi_video_memory_map(GstVaapiVideoMemory *mem, gsize maxsize, guint flags)
308 if (mem->map_count == 0) {
309 switch (flags & GST_MAP_READWRITE) {
311 // No flags set: return a GstVaapiSurfaceProxy
312 gst_vaapi_surface_proxy_replace(&mem->proxy,
313 gst_vaapi_video_meta_get_surface_proxy(mem->meta));
315 goto error_no_surface_proxy;
316 mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE;
319 // Only read flag set: return raw pixels
320 if (!ensure_surface(mem))
321 goto error_no_surface;
322 if (!ensure_image(mem))
324 if (!mem->use_direct_rendering)
325 gst_vaapi_surface_get_image(mem->surface, mem->image);
326 if (!gst_vaapi_image_map(mem->image))
327 goto error_map_image;
328 mem->map_type = GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR;
331 goto error_unsupported_map;
335 switch (mem->map_type) {
336 case GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE:
338 goto error_no_surface_proxy;
341 case GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR:
344 data = get_image_data(mem->image);
347 goto error_unsupported_map_type;
353 error_unsupported_map:
354 GST_ERROR("unsupported map flags (0x%x)", flags);
356 error_unsupported_map_type:
357 GST_ERROR("unsupported map type (%d)", mem->map_type);
359 error_no_surface_proxy:
360 GST_ERROR("failed to extract GstVaapiSurfaceProxy from video meta");
363 GST_ERROR("failed to extract VA surface from video buffer");
366 GST_ERROR("failed to extract VA image from video buffer");
369 GST_ERROR("failed to map VA image");
374 gst_vaapi_video_memory_unmap(GstVaapiVideoMemory *mem)
376 if (mem->map_count == 1) {
377 switch (mem->map_type) {
378 case GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_SURFACE:
379 gst_vaapi_surface_proxy_replace(&mem->proxy, NULL);
381 case GST_VAAPI_VIDEO_MEMORY_MAP_TYPE_LINEAR:
382 gst_vaapi_image_unmap(mem->image);
385 goto error_incompatible_map;
393 error_incompatible_map:
394 GST_ERROR("incompatible map type (%d)", mem->map_type);
398 static GstVaapiVideoMemory *
399 gst_vaapi_video_memory_copy(GstVaapiVideoMemory *mem,
400 gssize offset, gssize size)
404 if (offset != 0 || size != -1)
405 goto error_unsupported;
407 out_mem = gst_vaapi_video_memory_new(mem->parent_instance.allocator,
410 goto error_allocate_memory;
411 return GST_VAAPI_VIDEO_MEMORY_CAST(out_mem);
415 GST_ERROR("failed to copy partial memory (unsupported operation)");
417 error_allocate_memory:
418 GST_ERROR("failed to allocate GstVaapiVideoMemory copy");
422 static GstVaapiVideoMemory *
423 gst_vaapi_video_memory_share(GstVaapiVideoMemory *mem,
424 gssize offset, gssize size)
426 GST_FIXME("unimplemented GstVaapiVideoAllocator::mem_share() hook");
431 gst_vaapi_video_memory_is_span(GstVaapiVideoMemory *mem1,
432 GstVaapiVideoMemory *mem2, gsize *offset_ptr)
434 GST_FIXME("unimplemented GstVaapiVideoAllocator::mem_is_span() hook");
438 /* ------------------------------------------------------------------------ */
439 /* --- GstVaapiVideoAllocator --- */
440 /* ------------------------------------------------------------------------ */
442 #define GST_VAAPI_VIDEO_ALLOCATOR_CLASS(klass) \
443 (G_TYPE_CHECK_CLASS_CAST((klass), \
444 GST_VAAPI_TYPE_VIDEO_ALLOCATOR, \
445 GstVaapiVideoAllocatorClass))
447 #define GST_VAAPI_IS_VIDEO_ALLOCATOR_CLASS(klass) \
448 (G_TYPE_CHECK_CLASS_TYPE((klass), GST_VAAPI_TYPE_VIDEO_ALLOCATOR))
450 G_DEFINE_TYPE(GstVaapiVideoAllocator,
451 gst_vaapi_video_allocator,
455 gst_vaapi_video_allocator_alloc(GstAllocator *allocator, gsize size,
456 GstAllocationParams *params)
458 g_warning("use gst_vaapi_video_memory_new() to allocate from "
459 "GstVaapiVideoMemory allocator");
465 gst_vaapi_video_allocator_free(GstAllocator *allocator, GstMemory *mem)
467 gst_vaapi_video_memory_free(GST_VAAPI_VIDEO_MEMORY_CAST(mem));
471 gst_vaapi_video_allocator_finalize(GObject *object)
473 GstVaapiVideoAllocator * const allocator =
474 GST_VAAPI_VIDEO_ALLOCATOR_CAST(object);
476 gst_vaapi_video_pool_replace(&allocator->surface_pool, NULL);
478 G_OBJECT_CLASS(gst_vaapi_video_allocator_parent_class)->finalize(object);
482 gst_vaapi_video_allocator_class_init(GstVaapiVideoAllocatorClass *klass)
484 GObjectClass * const object_class = G_OBJECT_CLASS(klass);
485 GstAllocatorClass * const allocator_class = GST_ALLOCATOR_CLASS(klass);
487 GST_DEBUG_CATEGORY_INIT(gst_debug_vaapivideomemory,
488 "vaapivideomemory", 0, "VA-API video memory allocator");
490 object_class->finalize = gst_vaapi_video_allocator_finalize;
491 allocator_class->alloc = gst_vaapi_video_allocator_alloc;
492 allocator_class->free = gst_vaapi_video_allocator_free;
496 gst_vaapi_video_allocator_init(GstVaapiVideoAllocator *allocator)
498 GstAllocator * const base_allocator = GST_ALLOCATOR_CAST(allocator);
500 base_allocator->mem_type = GST_VAAPI_VIDEO_MEMORY_NAME;
501 base_allocator->mem_map = (GstMemoryMapFunction)
502 gst_vaapi_video_memory_map;
503 base_allocator->mem_unmap = (GstMemoryUnmapFunction)
504 gst_vaapi_video_memory_unmap;
505 base_allocator->mem_copy = (GstMemoryCopyFunction)
506 gst_vaapi_video_memory_copy;
507 base_allocator->mem_share = (GstMemoryShareFunction)
508 gst_vaapi_video_memory_share;
509 base_allocator->mem_is_span = (GstMemoryIsSpanFunction)
510 gst_vaapi_video_memory_is_span;
514 gst_video_info_update_from_image(GstVideoInfo *vip, GstVaapiImage *image)
516 GstVideoFormat format;
518 guint i, num_planes, data_size, width, height;
520 /* Reset format from image */
521 format = gst_vaapi_image_get_format(image);
522 gst_vaapi_image_get_size(image, &width, &height);
523 gst_video_info_set_format(vip, format, width, height);
525 num_planes = gst_vaapi_image_get_plane_count(image);
526 g_return_val_if_fail(num_planes == GST_VIDEO_INFO_N_PLANES(vip), FALSE);
528 /* Determine the base data pointer */
529 data = get_image_data(image);
530 g_return_val_if_fail(data != NULL, FALSE);
531 data_size = gst_vaapi_image_get_data_size(image);
533 /* Check that we don't have disjoint planes */
534 for (i = 0; i < num_planes; i++) {
535 const guchar * const plane = gst_vaapi_image_get_plane(image, i);
536 if (plane - data > data_size)
540 /* Update GstVideoInfo structure */
541 for (i = 0; i < num_planes; i++) {
542 const guchar * const plane = gst_vaapi_image_get_plane(image, i);
543 GST_VIDEO_INFO_PLANE_OFFSET(vip, i) = plane - data;
544 GST_VIDEO_INFO_PLANE_STRIDE(vip, i) =
545 gst_vaapi_image_get_pitch(image, i);
547 GST_VIDEO_INFO_SIZE(vip) = data_size;
552 gst_vaapi_video_allocator_new(GstVaapiDisplay *display, const GstVideoInfo *vip)
554 GstVaapiVideoAllocator *allocator;
555 GstVaapiSurface *surface;
556 GstVaapiImage *image;
558 g_return_val_if_fail(display != NULL, NULL);
559 g_return_val_if_fail(vip != NULL, NULL);
561 allocator = g_object_new(GST_VAAPI_TYPE_VIDEO_ALLOCATOR, NULL);
565 allocator->video_info = *vip;
566 gst_video_info_set_format(&allocator->surface_info, GST_VIDEO_FORMAT_NV12,
567 GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
569 if (GST_VIDEO_INFO_FORMAT(vip) != GST_VIDEO_FORMAT_ENCODED) {
572 surface = new_surface(display, vip);
575 image = gst_vaapi_surface_derive_image(surface);
578 if (!gst_vaapi_image_map(image))
580 allocator->has_direct_rendering = gst_video_info_update_from_image(
581 &allocator->surface_info, image);
582 if (GST_VAAPI_IMAGE_FORMAT(image) != GST_VIDEO_INFO_FORMAT(vip))
583 allocator->has_direct_rendering = FALSE;
584 gst_vaapi_image_unmap(image);
585 GST_INFO("has direct-rendering for %s surfaces: %s",
586 GST_VIDEO_INFO_FORMAT_STRING(&allocator->surface_info),
587 allocator->has_direct_rendering ? "yes" : "no");
590 gst_vaapi_object_unref(surface);
592 gst_vaapi_object_unref(image);
595 allocator->surface_pool = gst_vaapi_surface_pool_new(display,
596 &allocator->surface_info);
597 if (!allocator->surface_pool)
598 goto error_create_pool;
600 allocator->image_info = *vip;
601 if (GST_VIDEO_INFO_FORMAT(vip) == GST_VIDEO_FORMAT_ENCODED)
602 gst_video_info_set_format(&allocator->image_info, GST_VIDEO_FORMAT_I420,
603 GST_VIDEO_INFO_WIDTH(vip), GST_VIDEO_INFO_HEIGHT(vip));
605 if (allocator->has_direct_rendering)
606 allocator->image_info = allocator->surface_info;
609 image = new_image(display, &allocator->image_info);
612 if (!gst_vaapi_image_map(image))
614 gst_video_info_update_from_image(&allocator->image_info, image);
615 gst_vaapi_image_unmap(image);
617 gst_vaapi_object_unref(image);
619 return GST_ALLOCATOR_CAST(allocator);
624 GST_ERROR("failed to allocate VA surface pool");
625 gst_object_unref(allocator);