1 /* GStreamer Intel MSDK plugin
2 * Copyright (c) 2018, Intel Corporation
3 * Copyright (c) 2018, Igalia S.L.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
9 * 1. Redistributions of source code must retain the above copyright notice,
10 * this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
16 * 3. Neither the name of the copyright holder nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
22 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
24 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
25 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
26 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
27 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
28 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGDECE
29 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
30 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include "gstmsdkvideomemory.h"
39 #include "gstmsdkallocator.h"
41 #define GST_MSDK_BUFFER_SURFACE gst_msdk_buffer_surface_quark_get ()
43 gst_msdk_buffer_surface_quark_get (void)
47 if (g_once_init_enter (&g_quark)) {
48 gsize quark = (gsize) g_quark_from_static_string ("GstMsdkBufferSurface");
49 g_once_init_leave (&g_quark, quark);
54 static mfxFrameSurface1 *
55 gst_msdk_video_allocator_get_surface (GstAllocator * allocator)
57 mfxFrameInfo frame_info = { {0,}, 0, };
58 mfxFrameSurface1 *surface;
59 GstMsdkContext *context = NULL;
60 mfxFrameAllocResponse *resp = NULL;
61 GstVideoInfo *vinfo = NULL;
63 if (GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
64 context = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->context;
65 resp = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->alloc_response;
66 vinfo = &GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->image_info;
67 } else if (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator)) {
68 context = GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->context;
69 resp = GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->alloc_response;
70 vinfo = &GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->image_info;
75 surface = gst_msdk_context_get_surface_available (context, resp);
77 GST_ERROR ("failed to get surface available");
81 gst_msdk_set_mfx_frame_info_from_video_info (&frame_info, vinfo);
82 surface->Info = frame_info;
88 gst_msdk_video_memory_get_surface_available (GstMemory * mem)
90 GstAllocator *allocator;
91 mfxFrameSurface1 *surface;
93 g_return_val_if_fail (mem, FALSE);
95 allocator = mem->allocator;
96 surface = gst_msdk_video_allocator_get_surface (allocator);
98 if (GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
99 GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface = surface;
100 } else if (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator)) {
101 gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
102 GST_MSDK_BUFFER_SURFACE, surface, NULL);
105 return surface ? TRUE : FALSE;
109 * Every time releasing a gst buffer, we need to check the status of surface's lock,
110 * so that we could manage locked surfaces separately in the context.
111 * Otherwise, we put the surface to the available list.
114 gst_msdk_video_memory_release_surface (GstMemory * mem)
116 mfxFrameSurface1 *surface = NULL;
117 GstMsdkContext *context = NULL;
118 mfxFrameAllocResponse *alloc_response = NULL;
120 g_return_if_fail (mem);
122 if (GST_IS_MSDK_VIDEO_ALLOCATOR (mem->allocator)) {
123 surface = GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface;
124 context = GST_MSDK_VIDEO_ALLOCATOR_CAST (mem->allocator)->context;
126 GST_MSDK_VIDEO_ALLOCATOR_CAST (mem->allocator)->alloc_response;
127 } else if (GST_IS_MSDK_DMABUF_ALLOCATOR (mem->allocator)) {
129 gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
130 GST_MSDK_BUFFER_SURFACE);
131 context = GST_MSDK_DMABUF_ALLOCATOR_CAST (mem->allocator)->context;
133 GST_MSDK_DMABUF_ALLOCATOR_CAST (mem->allocator)->alloc_response;
138 if (surface->Data.Locked > 0)
139 gst_msdk_context_put_surface_locked (context, alloc_response, surface);
141 gst_msdk_context_put_surface_available (context, alloc_response, surface);
143 if (GST_IS_MSDK_VIDEO_ALLOCATOR (mem->allocator))
144 GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface = NULL;
145 else if (GST_IS_MSDK_DMABUF_ALLOCATOR (mem->allocator))
146 gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
147 GST_MSDK_BUFFER_SURFACE, NULL, NULL);
153 gst_msdk_video_memory_new (GstAllocator * base_allocator)
155 GstMsdkVideoAllocator *allocator;
157 GstMsdkVideoMemory *mem;
159 g_return_val_if_fail (base_allocator, NULL);
160 g_return_val_if_fail (GST_IS_MSDK_VIDEO_ALLOCATOR (base_allocator), NULL);
162 allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (base_allocator);
164 mem = g_slice_new0 (GstMsdkVideoMemory);
168 mem->surface = gst_msdk_video_allocator_get_surface (base_allocator);
170 g_slice_free (GstMsdkVideoMemory, mem);
174 vip = &allocator->image_info;
175 gst_memory_init (&mem->parent_instance, 0,
176 base_allocator, NULL, GST_VIDEO_INFO_SIZE (vip), 0, 0,
177 GST_VIDEO_INFO_SIZE (vip));
179 return GST_MEMORY_CAST (mem);
183 gst_video_meta_map_msdk_memory (GstVideoMeta * meta, guint plane,
184 GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags)
186 gboolean ret = FALSE;
187 GstAllocator *allocator;
188 GstMsdkVideoAllocator *msdk_video_allocator;
189 GstMsdkVideoMemory *mem =
190 GST_MSDK_VIDEO_MEMORY_CAST (gst_buffer_peek_memory (meta->buffer, 0));
191 GstMsdkMemoryID *mem_id;
194 guint plane_id = plane;
196 g_return_val_if_fail (mem, FALSE);
198 allocator = GST_MEMORY_CAST (mem)->allocator;
199 msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
201 if (!GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
202 GST_WARNING ("The allocator is not MSDK video allocator");
207 GST_WARNING ("The surface is not allocated");
211 if ((flags & GST_MAP_WRITE) && mem->surface && mem->surface->Data.Locked) {
212 GST_WARNING ("The surface in memory %p is not still available", mem);
217 gst_msdk_frame_lock (msdk_video_allocator->context,
218 mem->surface->Data.MemId, &mem->surface->Data);
222 mem_id = mem->surface->Data.MemId;
224 /* msdk doesn't support I420 format and we used YV12 internally
225 * So we need to swap U/V planes for mapping */
226 if (meta->format == GST_VIDEO_FORMAT_I420)
227 plane_id = plane ? (plane == 1 ? 2 : 1) : plane;
230 offset = mem_id->image.offsets[plane_id];
231 pitch = mem_id->image.pitches[plane_id];
233 /* TODO: This is just to avoid compile errors on Windows.
234 * Implement handling Windows-specific video-memory.
236 offset = mem_id->offset;
237 pitch = mem_id->pitch;
240 switch (meta->format) {
241 case GST_VIDEO_FORMAT_BGRA:
242 case GST_VIDEO_FORMAT_BGRx:
243 *data = mem->surface->Data.B + offset;
246 /* The first channel in memory is V for GST_VIDEO_FORMAT_VUYA */
247 case GST_VIDEO_FORMAT_VUYA:
248 *data = mem->surface->Data.V + offset;
251 case GST_VIDEO_FORMAT_Y410:
252 case GST_VIDEO_FORMAT_Y412_LE:
253 *data = mem->surface->Data.U + offset;
257 *data = mem->surface->Data.Y + offset;
263 ret = (*data != NULL);
269 gst_video_meta_unmap_msdk_memory (GstVideoMeta * meta, guint plane,
272 GstAllocator *allocator;
273 GstMsdkVideoAllocator *msdk_video_allocator;
274 GstMsdkVideoMemory *mem =
275 GST_MSDK_VIDEO_MEMORY_CAST (gst_buffer_peek_memory (meta->buffer, 0));
277 g_return_val_if_fail (mem, FALSE);
279 allocator = GST_MEMORY_CAST (mem)->allocator;
280 msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
282 if (mem->mapped == 1)
283 gst_msdk_frame_unlock (msdk_video_allocator->context,
284 mem->surface->Data.MemId, &mem->surface->Data);
293 gst_msdk_video_memory_map_full (GstMemory * base_mem, GstMapInfo * info,
296 GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
297 GstAllocator *allocator = base_mem->allocator;
298 GstMsdkVideoAllocator *msdk_video_allocator =
299 GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
301 g_return_val_if_fail (mem, NULL);
304 GST_WARNING ("The surface is not allocated");
308 if ((info->flags & GST_MAP_WRITE) && mem->surface
309 && mem->surface->Data.Locked) {
310 GST_WARNING ("The surface in memory %p is not still available", mem);
314 gst_msdk_frame_lock (msdk_video_allocator->context, mem->surface->Data.MemId,
315 &mem->surface->Data);
317 switch (mem->surface->Info.FourCC) {
318 case MFX_FOURCC_RGB4:
319 return mem->surface->Data.B; /* The first channel is B */
321 /* The first channel in memory is V for MFX_FOURCC_AYUV (GST_VIDEO_FORMAT_VUYA) format */
322 case MFX_FOURCC_AYUV:
323 return mem->surface->Data.V;
325 #if (MFX_VERSION >= 1027)
326 case MFX_FOURCC_Y410:
327 return mem->surface->Data.U; /* Data.Y410 */
330 #if (MFX_VERSION >= 1031)
331 case MFX_FOURCC_Y416:
332 return mem->surface->Data.U;
335 #if (MFX_VERSION >= 2004)
336 case MFX_FOURCC_RGBP:
337 return mem->surface->Data.R;
339 case MFX_FOURCC_BGRP:
340 return mem->surface->Data.B;
344 return mem->surface->Data.Y;
349 gst_msdk_video_memory_unmap (GstMemory * base_mem)
351 GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
352 GstAllocator *allocator = base_mem->allocator;
353 GstMsdkVideoAllocator *msdk_video_allocator =
354 GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
356 gst_msdk_frame_unlock (msdk_video_allocator->context,
357 mem->surface->Data.MemId, &mem->surface->Data);
361 gst_msdk_video_memory_copy (GstMemory * base_mem, gssize offset, gssize size)
365 GstMsdkVideoAllocator *msdk_video_allocator;
367 GstMapInfo src_map, dst_map;
369 /* FIXME: can we consider offset and size here ? */
370 copy = gst_msdk_video_memory_new (base_mem->allocator);
373 GST_ERROR_OBJECT (base_mem->allocator, "Failed to create new video memory");
377 msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (base_mem->allocator);
379 info = &msdk_video_allocator->image_info;
380 mem_size = GST_VIDEO_INFO_SIZE (info);
382 gst_memory_map (base_mem, &src_map, GST_MAP_READ);
383 gst_memory_map (copy, &dst_map, GST_MAP_WRITE);
385 memcpy (dst_map.data, src_map.data, mem_size);
386 gst_memory_unmap (copy, &dst_map);
387 gst_memory_unmap (base_mem, &src_map);
392 /* GstMsdkVideoAllocator */
393 G_DEFINE_TYPE (GstMsdkVideoAllocator, gst_msdk_video_allocator,
397 gst_msdk_video_allocator_alloc (GstAllocator * allocator, gsize size,
398 GstAllocationParams * params)
400 return gst_msdk_video_memory_new (allocator);
404 gst_msdk_video_allocator_finalize (GObject * object)
406 GstMsdkVideoAllocator *allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (object);
408 gst_msdk_frame_free (allocator->context, allocator->alloc_response);
410 gst_object_unref (allocator->context);
411 G_OBJECT_CLASS (gst_msdk_video_allocator_parent_class)->finalize (object);
415 gst_msdk_video_allocator_free (GstAllocator * allocator, GstMemory * base_mem)
417 GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
419 g_slice_free (GstMsdkVideoMemory, mem);
423 gst_msdk_video_allocator_class_init (GstMsdkVideoAllocatorClass * klass)
425 GObjectClass *const object_class = G_OBJECT_CLASS (klass);
426 GstAllocatorClass *const allocator_class = GST_ALLOCATOR_CLASS (klass);
428 object_class->finalize = gst_msdk_video_allocator_finalize;
430 allocator_class->alloc = gst_msdk_video_allocator_alloc;
431 allocator_class->free = gst_msdk_video_allocator_free;
435 gst_msdk_video_allocator_init (GstMsdkVideoAllocator * allocator)
437 GstAllocator *const base_allocator = GST_ALLOCATOR_CAST (allocator);
439 base_allocator->mem_type = GST_MSDK_VIDEO_MEMORY_NAME;
440 base_allocator->mem_map_full = gst_msdk_video_memory_map_full;
441 base_allocator->mem_unmap = gst_msdk_video_memory_unmap;
442 base_allocator->mem_copy = gst_msdk_video_memory_copy;
444 GST_OBJECT_FLAG_SET (allocator, GST_ALLOCATOR_FLAG_CUSTOM_ALLOC);
448 gst_msdk_video_allocator_new (GstMsdkContext * context,
449 GstVideoInfo * image_info, mfxFrameAllocResponse * alloc_resp)
451 GstMsdkVideoAllocator *allocator;
452 GstMsdkAllocResponse *cached = NULL;
454 g_return_val_if_fail (context != NULL, NULL);
455 g_return_val_if_fail (image_info != NULL, NULL);
457 cached = gst_msdk_context_get_cached_alloc_responses (context, alloc_resp);
460 GST_ERROR ("Failed to get the cached alloc response");
464 allocator = g_object_new (GST_TYPE_MSDK_VIDEO_ALLOCATOR, NULL);
468 g_atomic_int_inc (&cached->refcount);
469 allocator->context = gst_object_ref (context);
470 allocator->image_info = *image_info;
471 allocator->mfx_response = *alloc_resp;
472 allocator->alloc_response = &allocator->mfx_response;
474 return GST_ALLOCATOR_CAST (allocator);
477 /* GstMsdkDmaBufMemory */
479 gst_msdk_dmabuf_memory_new (GstAllocator * base_allocator)
482 mfxFrameSurface1 *surface;
484 g_return_val_if_fail (base_allocator, NULL);
485 g_return_val_if_fail (GST_IS_MSDK_DMABUF_ALLOCATOR (base_allocator), NULL);
487 surface = gst_msdk_video_allocator_get_surface (base_allocator);
491 return gst_msdk_dmabuf_memory_new_with_surface (base_allocator, surface);
498 gst_msdk_dmabuf_memory_new_with_surface (GstAllocator * allocator,
499 mfxFrameSurface1 * surface)
503 GstMsdkMemoryID *mem_id;
507 g_return_val_if_fail (allocator, NULL);
508 g_return_val_if_fail (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator), NULL);
510 mem_id = surface->Data.MemId;
512 g_assert (mem_id->desc.num_objects == 1);
514 fd = mem_id->desc.objects[0].fd;
515 size = mem_id->desc.objects[0].size;
518 GST_ERROR ("Failed to get dmabuf handle");
522 mem = gst_fd_allocator_alloc (allocator, fd, size,
523 GST_FD_MEMORY_FLAG_DONT_CLOSE);
526 GST_ERROR ("failed ! dmabuf fd: %d", fd);
530 gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
531 GST_MSDK_BUFFER_SURFACE, surface, NULL);
539 /* GstMsdkDmaBufAllocator */
540 G_DEFINE_TYPE (GstMsdkDmaBufAllocator, gst_msdk_dmabuf_allocator,
541 GST_TYPE_DMABUF_ALLOCATOR);
544 gst_msdk_dmabuf_allocator_finalize (GObject * object)
546 GstMsdkDmaBufAllocator *allocator = GST_MSDK_DMABUF_ALLOCATOR_CAST (object);
548 gst_msdk_frame_free (allocator->context, allocator->alloc_response);
550 gst_object_unref (allocator->context);
551 G_OBJECT_CLASS (gst_msdk_dmabuf_allocator_parent_class)->finalize (object);
555 gst_msdk_dmabuf_allocator_class_init (GstMsdkDmaBufAllocatorClass * klass)
557 GObjectClass *const object_class = G_OBJECT_CLASS (klass);
559 object_class->finalize = gst_msdk_dmabuf_allocator_finalize;
563 gst_msdk_dmabuf_allocator_init (GstMsdkDmaBufAllocator * allocator)
565 GstAllocator *const base_allocator = GST_ALLOCATOR_CAST (allocator);
566 base_allocator->mem_type = GST_MSDK_DMABUF_MEMORY_NAME;
570 gst_msdk_dmabuf_allocator_new (GstMsdkContext * context,
571 GstVideoInfo * image_info, mfxFrameAllocResponse * alloc_resp)
573 GstMsdkDmaBufAllocator *allocator;
574 GstMsdkAllocResponse *cached = NULL;
576 g_return_val_if_fail (context != NULL, NULL);
577 g_return_val_if_fail (image_info != NULL, NULL);
579 cached = gst_msdk_context_get_cached_alloc_responses (context, alloc_resp);
582 GST_ERROR ("Failed to get the cached alloc response");
586 allocator = g_object_new (GST_TYPE_MSDK_DMABUF_ALLOCATOR, NULL);
590 g_atomic_int_inc (&cached->refcount);
591 allocator->context = gst_object_ref (context);
592 allocator->image_info = *image_info;
593 allocator->mfx_response = *alloc_resp;
594 allocator->alloc_response = &allocator->mfx_response;
596 return GST_ALLOCATOR_CAST (allocator);