2 * Copyright (C) 2021 Seungha Yang <seungha@centricular.com>
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Library General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Library General Public License for more details.
14 * You should have received a copy of the GNU Library General Public
15 * License along with this library; if not, write to the
16 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17 * Boston, MA 02110-1301, USA.
24 #include "gstqsvallocator_d3d11.h"
27 GST_DEBUG_CATEGORY_EXTERN (gst_qsv_allocator_debug);
28 #define GST_CAT_DEFAULT gst_qsv_allocator_debug
30 struct _GstQsvD3D11Allocator
32 GstQsvAllocator parent;
34 GstD3D11Device *device;
37 #define gst_qsv_d3d11_allocator_parent_class parent_class
38 G_DEFINE_TYPE (GstQsvD3D11Allocator, gst_qsv_d3d11_allocator,
39 GST_TYPE_QSV_ALLOCATOR);
41 static void gst_qsv_d3d11_allocator_dispose (GObject * object);
42 static mfxStatus gst_qsv_d3d11_allocator_alloc (GstQsvAllocator * allocator,
43 gboolean dummy_alloc, mfxFrameAllocRequest * request,
44 mfxFrameAllocResponse * response);
45 static GstBuffer *gst_qsv_d3d11_allocator_upload (GstQsvAllocator * allocator,
46 const GstVideoInfo * info, GstBuffer * buffer, GstBufferPool * pool);
47 static GstBuffer *gst_qsv_d3d11_allocator_download (GstQsvAllocator * allocator,
48 const GstVideoInfo * info, gboolean force_copy, GstQsvFrame * frame,
49 GstBufferPool * pool);
52 gst_qsv_d3d11_allocator_class_init (GstQsvD3D11AllocatorClass * klass)
54 GObjectClass *object_class = G_OBJECT_CLASS (klass);
55 GstQsvAllocatorClass *alloc_class = GST_QSV_ALLOCATOR_CLASS (klass);
57 object_class->dispose = gst_qsv_d3d11_allocator_dispose;
59 alloc_class->alloc = GST_DEBUG_FUNCPTR (gst_qsv_d3d11_allocator_alloc);
60 alloc_class->upload = GST_DEBUG_FUNCPTR (gst_qsv_d3d11_allocator_upload);
61 alloc_class->download = GST_DEBUG_FUNCPTR (gst_qsv_d3d11_allocator_download);
65 gst_qsv_d3d11_allocator_init (GstQsvD3D11Allocator * self)
70 gst_qsv_d3d11_allocator_dispose (GObject * object)
72 GstQsvD3D11Allocator *self = GST_QSV_D3D11_ALLOCATOR (object);
74 gst_clear_object (&self->device);
76 G_OBJECT_CLASS (parent_class)->dispose (object);
80 gst_qsv_d3d11_allocator_alloc (GstQsvAllocator * allocator,
81 gboolean dummy_alloc, mfxFrameAllocRequest * request,
82 mfxFrameAllocResponse * response)
84 GstQsvD3D11Allocator *self = GST_QSV_D3D11_ALLOCATOR (allocator);
85 DXGI_FORMAT dxgi_format = DXGI_FORMAT_UNKNOWN;
86 GstQsvFrame **mids = nullptr;
88 /* Something unexpected and went wrong */
89 if ((request->Type & MFX_MEMTYPE_SYSTEM_MEMORY) != 0) {
90 GST_ERROR_OBJECT (self,
91 "MFX is requesting system memory, type 0x%x", request->Type);
92 return MFX_ERR_UNSUPPORTED;
95 switch (request->Info.FourCC) {
97 dxgi_format = DXGI_FORMAT_NV12;
100 dxgi_format = DXGI_FORMAT_P010;
102 case MFX_FOURCC_AYUV:
103 dxgi_format = DXGI_FORMAT_AYUV;
105 case MFX_FOURCC_Y410:
106 dxgi_format = DXGI_FORMAT_Y410;
108 case MFX_FOURCC_RGB4:
109 dxgi_format = DXGI_FORMAT_B8G8R8A8_UNORM;
111 case MFX_FOURCC_BGR4:
112 dxgi_format = DXGI_FORMAT_R8G8B8A8_UNORM;
115 /* TODO: add more formats */
119 if (dxgi_format == DXGI_FORMAT_UNKNOWN &&
120 request->Info.FourCC != MFX_FOURCC_P8) {
121 GST_ERROR_OBJECT (self, "Failed to convert %d to DXGI format",
122 request->Info.FourCC);
124 return MFX_ERR_UNSUPPORTED;
127 if (request->Info.FourCC == MFX_FOURCC_P8) {
128 GstD3D11Allocator *d3d11_alloc = nullptr;
129 D3D11_BUFFER_DESC desc;
133 gsize offset[GST_VIDEO_MAX_PLANES] = { 0, };
134 gint stride[GST_VIDEO_MAX_PLANES] = { 0, };
138 (GstD3D11Allocator *) gst_allocator_find (GST_D3D11_MEMORY_NAME);
140 GST_ERROR_OBJECT (self, "D3D11 allocator is unavailable");
142 return MFX_ERR_MEMORY_ALLOC;
145 memset (&desc, 0, sizeof (D3D11_BUFFER_DESC));
147 desc.ByteWidth = request->Info.Width * request->Info.Height;
148 desc.Usage = D3D11_USAGE_STAGING;
149 desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
151 mem = gst_d3d11_allocator_alloc_buffer (d3d11_alloc, self->device, &desc);
152 gst_object_unref (d3d11_alloc);
155 GST_ERROR_OBJECT (self, "Failed to allocate buffer");
156 return MFX_ERR_MEMORY_ALLOC;
159 size = request->Info.Width * request->Info.Height;
162 gst_video_info_set_format (&info, GST_VIDEO_FORMAT_GRAY8, size, 1);
164 buffer = gst_buffer_new ();
165 gst_buffer_append_memory (buffer, mem);
166 gst_buffer_add_video_meta_full (buffer, GST_VIDEO_FRAME_FLAG_NONE,
167 GST_VIDEO_FORMAT_GRAY8, size, 1, 1, offset, stride);
169 mids = g_new0 (GstQsvFrame *, 1);
170 response->NumFrameActual = 1;
171 mids[0] = gst_qsv_allocator_acquire_frame (allocator,
172 GST_QSV_VIDEO_MEMORY | GST_QSV_ENCODER_IN_MEMORY, &info, buffer,
176 GstVideoFormat format;
179 GstStructure *config;
180 GstD3D11AllocationParams *params;
181 guint bind_flags = 0;
182 GstVideoAlignment align;
183 GstQsvMemoryType mem_type = GST_QSV_VIDEO_MEMORY;
185 if ((request->Type & MFX_MEMTYPE_VIDEO_MEMORY_ENCODER_TARGET) != 0) {
186 bind_flags |= D3D11_BIND_VIDEO_ENCODER;
187 mem_type |= GST_QSV_ENCODER_IN_MEMORY;
190 if ((request->Type & MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET) != 0) {
191 bind_flags |= D3D11_BIND_DECODER;
192 mem_type |= GST_QSV_DECODER_OUT_MEMORY;
195 if (mem_type == GST_QSV_VIDEO_MEMORY) {
196 GST_ERROR_OBJECT (self, "Unknown read/write access");
197 return MFX_ERR_UNSUPPORTED;
200 mids = g_new0 (GstQsvFrame *, request->NumFrameSuggested);
201 response->NumFrameActual = request->NumFrameSuggested;
203 format = gst_d3d11_dxgi_format_to_gst (dxgi_format);
204 gst_video_info_set_format (&info,
205 format, request->Info.CropW, request->Info.CropH);
208 for (guint i = 0; i < request->NumFrameSuggested; i++) {
209 mids[i] = gst_qsv_allocator_acquire_frame (allocator,
210 mem_type, &info, nullptr, nullptr);
213 response->mids = (mfxMemId *) mids;
218 caps = gst_video_info_to_caps (&info);
219 gst_video_alignment_reset (&align);
220 align.padding_right = request->Info.Width - request->Info.CropW;
221 align.padding_bottom = request->Info.Height - request->Info.CropH;
223 pool = gst_d3d11_buffer_pool_new (self->device);
224 params = gst_d3d11_allocation_params_new (self->device, &info,
225 (GstD3D11AllocationFlags) 0, bind_flags);
227 gst_d3d11_allocation_params_alignment (params, &align);
229 config = gst_buffer_pool_get_config (pool);
230 gst_buffer_pool_config_set_d3d11_allocation_params (config, params);
231 gst_d3d11_allocation_params_free (params);
232 gst_buffer_pool_config_set_params (config, caps,
233 GST_VIDEO_INFO_SIZE (&info), 0, 0);
234 gst_caps_unref (caps);
235 gst_buffer_pool_set_config (pool, config);
236 gst_buffer_pool_set_active (pool, TRUE);
238 for (guint i = 0; i < request->NumFrameSuggested; i++) {
241 if (gst_buffer_pool_acquire_buffer (pool, &buffer, nullptr) !=
243 GST_ERROR_OBJECT (self, "Failed to allocate texture buffer");
244 gst_buffer_pool_set_active (pool, FALSE);
245 gst_object_unref (pool);
249 mids[i] = gst_qsv_allocator_acquire_frame (allocator,
250 mem_type, &info, buffer, nullptr);
253 gst_buffer_pool_set_active (pool, FALSE);
254 gst_object_unref (pool);
257 response->mids = (mfxMemId *) mids;
263 for (guint i = 0; i < response->NumFrameActual; i++)
264 gst_clear_qsv_frame (&mids[i]);
269 response->NumFrameActual = 0;
271 return MFX_ERR_MEMORY_ALLOC;
275 gst_qsv_frame_copy_d3d11 (const GstVideoInfo * info, GstBuffer * src_buf,
278 D3D11_TEXTURE2D_DESC src_desc, dst_desc;
280 guint subresource_idx;
281 GstMemory *src_mem, *dst_mem;
282 GstMapInfo src_info, dst_info;
283 ID3D11Texture2D *src_tex, *dst_tex;
284 GstD3D11Device *device;
285 ID3D11DeviceContext *device_context;
287 GST_TRACE ("Copying D3D11 buffer %" GST_PTR_FORMAT, src_buf);
289 src_mem = gst_buffer_peek_memory (src_buf, 0);
290 dst_mem = gst_buffer_peek_memory (dst_buf, 0);
292 device = GST_D3D11_MEMORY_CAST (dst_mem)->device;
293 device_context = gst_d3d11_device_get_device_context_handle (device);
295 if (!gst_memory_map (src_mem,
296 &src_info, (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) {
297 GST_WARNING ("Failed to map src memory");
298 gst_buffer_unref (dst_buf);
302 if (!gst_memory_map (dst_mem,
303 &dst_info, (GstMapFlags) (GST_MAP_WRITE | GST_MAP_D3D11))) {
304 GST_WARNING ("Failed to map dst memory");
305 gst_memory_unmap (src_mem, &src_info);
306 gst_buffer_unref (dst_buf);
310 src_tex = (ID3D11Texture2D *) src_info.data;
311 dst_tex = (ID3D11Texture2D *) dst_info.data;
313 src_tex->GetDesc (&src_desc);
314 dst_tex->GetDesc (&dst_desc);
317 gst_d3d11_memory_get_subresource_index (GST_D3D11_MEMORY_CAST (src_mem));
323 src_box.right = MIN (src_desc.Width, dst_desc.Width);
324 src_box.bottom = MIN (src_desc.Height, dst_desc.Height);
326 gst_d3d11_device_lock (device);
327 device_context->CopySubresourceRegion (dst_tex, 0,
328 0, 0, 0, src_tex, subresource_idx, &src_box);
329 gst_d3d11_device_unlock (device);
331 gst_memory_unmap (dst_mem, &dst_info);
332 gst_memory_unmap (src_mem, &src_info);
338 gst_qsv_frame_upload_sysmem (const GstVideoInfo * info, GstBuffer * src_buf,
341 GstVideoFrame src_frame, dst_frame;
343 GST_TRACE ("Uploading sysmem buffer %" GST_PTR_FORMAT, src_buf);
345 if (!gst_video_frame_map (&src_frame, info, src_buf, GST_MAP_READ)) {
346 GST_WARNING ("Failed to map src frame");
347 gst_buffer_unref (dst_buf);
351 if (!gst_video_frame_map (&dst_frame, info, dst_buf, GST_MAP_WRITE)) {
352 GST_WARNING ("Failed to map src frame");
353 gst_video_frame_unmap (&src_frame);
354 gst_buffer_unref (dst_buf);
358 for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES (&src_frame); i++) {
359 guint src_width_in_bytes, src_height;
360 guint dst_width_in_bytes, dst_height;
361 guint width_in_bytes, height;
362 guint src_stride, dst_stride;
363 guint8 *src_data, *dst_data;
365 src_width_in_bytes = GST_VIDEO_FRAME_COMP_WIDTH (&src_frame, i) *
366 GST_VIDEO_FRAME_COMP_PSTRIDE (&src_frame, i);
367 src_height = GST_VIDEO_FRAME_COMP_HEIGHT (&src_frame, i);
368 src_stride = GST_VIDEO_FRAME_COMP_STRIDE (&src_frame, i);
370 dst_width_in_bytes = GST_VIDEO_FRAME_COMP_WIDTH (&dst_frame, i) *
371 GST_VIDEO_FRAME_COMP_PSTRIDE (&src_frame, i);
372 dst_height = GST_VIDEO_FRAME_COMP_HEIGHT (&src_frame, i);
373 dst_stride = GST_VIDEO_FRAME_COMP_STRIDE (&dst_frame, i);
375 width_in_bytes = MIN (src_width_in_bytes, dst_width_in_bytes);
376 height = MIN (src_height, dst_height);
378 src_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&src_frame, i);
379 dst_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&dst_frame, i);
381 for (guint j = 0; j < height; j++) {
382 memcpy (dst_data, src_data, width_in_bytes);
383 dst_data += dst_stride;
384 src_data += src_stride;
388 gst_video_frame_unmap (&dst_frame);
389 gst_video_frame_unmap (&src_frame);
395 gst_qsv_d3d11_allocator_upload (GstQsvAllocator * allocator,
396 const GstVideoInfo * info, GstBuffer * buffer, GstBufferPool * pool)
399 GstD3D11Memory *dmem, *dst_dmem;
400 D3D11_TEXTURE2D_DESC desc, dst_desc;
402 GstFlowReturn flow_ret;
404 /* 1) D3D11 buffer from the same d3d11device with ours
405 * 1-1) Same resolution
406 * -> Increase refcount and wrap with GstQsvFrame
407 * 1-2) Different resolution
409 * 2) non-D3D11 buffer or from other d3d11 device
413 if (!GST_IS_D3D11_BUFFER_POOL (pool)) {
414 GST_ERROR_OBJECT (allocator, "Not a d3d11 buffer pool");
418 flow_ret = gst_buffer_pool_acquire_buffer (pool, &dst_buf, nullptr);
419 if (flow_ret != GST_FLOW_OK) {
420 GST_WARNING ("Failed to acquire buffer from pool, return %s",
421 gst_flow_get_name (flow_ret));
425 mem = gst_buffer_peek_memory (buffer, 0);
426 if (!gst_is_d3d11_memory (mem) || gst_buffer_n_memory (buffer) > 1) {
427 /* d3d11 buffer should hold single memory object */
428 return gst_qsv_frame_upload_sysmem (info, buffer, dst_buf);
431 /* FIXME: Add support for shared texture for GPU copy or wrapping
432 * texture from different device */
433 dmem = GST_D3D11_MEMORY_CAST (mem);
434 if (dmem->device != GST_D3D11_BUFFER_POOL (pool)->device)
435 return gst_qsv_frame_upload_sysmem (info, buffer, dst_buf);
437 dst_dmem = (GstD3D11Memory *) gst_buffer_peek_memory (dst_buf, 0);
438 gst_d3d11_memory_get_texture_desc (dmem, &desc);
439 gst_d3d11_memory_get_texture_desc (dst_dmem, &dst_desc);
441 if (desc.Width == dst_desc.Width && desc.Height == dst_desc.Height &&
442 desc.Usage == D3D11_USAGE_DEFAULT) {
443 /* Identical size and non-staging texture, wrap without copying */
444 GST_TRACE ("Wrapping D3D11 buffer without copy");
445 gst_buffer_unref (dst_buf);
447 return gst_buffer_ref (buffer);
450 return gst_qsv_frame_copy_d3d11 (info, buffer, dst_buf);
454 gst_qsv_d3d11_allocator_download (GstQsvAllocator * allocator,
455 const GstVideoInfo * info, gboolean force_copy, GstQsvFrame * frame,
456 GstBufferPool * pool)
458 GstBuffer *src_buf, *dst_buf;
460 GstD3D11Memory *dmem;
463 GST_TRACE_OBJECT (allocator, "Download");
465 src_buf = gst_qsv_frame_peek_buffer (frame);
468 return gst_buffer_ref (src_buf);
470 mem = gst_buffer_peek_memory (src_buf, 0);
471 if (!gst_is_d3d11_memory (mem) || gst_buffer_n_memory (src_buf) != 1) {
472 GST_ERROR_OBJECT (allocator, "frame holds invalid d3d11 memory");
476 if (!GST_IS_D3D11_BUFFER_POOL (pool) &&
477 !GST_IS_D3D11_STAGING_BUFFER_POOL (pool)) {
478 GST_TRACE_OBJECT (allocator, "Output is not d3d11 memory");
482 dmem = GST_D3D11_MEMORY_CAST (mem);
484 /* both pool and qsvframe should hold the same d3d11 device already */
485 if (GST_IS_D3D11_BUFFER_POOL (pool)) {
486 GstD3D11BufferPool *d3d11_pool = GST_D3D11_BUFFER_POOL (pool);
488 if (d3d11_pool->device != dmem->device) {
489 GST_WARNING_OBJECT (allocator, "Pool holds different device");
493 GstD3D11StagingBufferPool *d3d11_pool =
494 GST_D3D11_STAGING_BUFFER_POOL (pool);
496 if (d3d11_pool->device != dmem->device) {
497 GST_WARNING_OBJECT (allocator, "Staging pool holds different device");
502 ret = gst_buffer_pool_acquire_buffer (pool, &dst_buf, nullptr);
503 if (ret != GST_FLOW_OK) {
504 GST_WARNING_OBJECT (allocator, "Failed to allocate output buffer");
508 return gst_qsv_frame_copy_d3d11 (info, src_buf, dst_buf);
511 GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD);
513 return GST_QSV_ALLOCATOR_CLASS (parent_class)->download (allocator,
514 info, TRUE, frame, pool);
518 gst_qsv_d3d11_allocator_new (GstD3D11Device * device)
520 GstQsvD3D11Allocator *self;
522 g_return_val_if_fail (GST_IS_D3D11_DEVICE (device), nullptr);
524 self = (GstQsvD3D11Allocator *)
525 g_object_new (GST_TYPE_QSV_D3D11_ALLOCATOR, nullptr);
526 self->device = (GstD3D11Device *) gst_object_ref (device);
528 gst_object_ref_sink (self);
530 return GST_QSV_ALLOCATOR (self);