3 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
25 #include "gstd3d11window_dummy.h"
26 #include "gstd3d11pluginutils.h"
30 using namespace Microsoft::WRL;
33 GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_window_debug);
34 #define GST_CAT_DEFAULT gst_d3d11_window_debug
36 struct _GstD3D11WindowDummy
38 GstD3D11Window parent;
40 ID3D11Texture2D *fallback_texture;
41 ID3D11VideoProcessorOutputView *fallback_pov;
42 ID3D11RenderTargetView *fallback_rtv;
45 #define gst_d3d11_window_dummy_parent_class parent_class
46 G_DEFINE_TYPE (GstD3D11WindowDummy, gst_d3d11_window_dummy,
47 GST_TYPE_D3D11_WINDOW);
49 static void gst_d3d11_window_dummy_on_resize (GstD3D11Window * window,
50 guint width, guint height);
51 static gboolean gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
52 guint display_width, guint display_height, GstCaps * caps,
53 gboolean * video_processor_available, GError ** error);
54 static void gst_d3d11_window_dummy_unprepare (GstD3D11Window * window);
56 gst_d3d11_window_dummy_open_shared_handle (GstD3D11Window * window,
57 GstD3D11WindowSharedHandleData * data);
59 gst_d3d11_window_dummy_release_shared_handle (GstD3D11Window * window,
60 GstD3D11WindowSharedHandleData * data);
63 gst_d3d11_window_dummy_class_init (GstD3D11WindowDummyClass * klass)
65 GstD3D11WindowClass *window_class = GST_D3D11_WINDOW_CLASS (klass);
67 window_class->on_resize =
68 GST_DEBUG_FUNCPTR (gst_d3d11_window_dummy_on_resize);
69 window_class->prepare = GST_DEBUG_FUNCPTR (gst_d3d11_window_dummy_prepare);
70 window_class->unprepare =
71 GST_DEBUG_FUNCPTR (gst_d3d11_window_dummy_unprepare);
72 window_class->open_shared_handle =
73 GST_DEBUG_FUNCPTR (gst_d3d11_window_dummy_open_shared_handle);
74 window_class->release_shared_handle =
75 GST_DEBUG_FUNCPTR (gst_d3d11_window_dummy_release_shared_handle);
79 gst_d3d11_window_dummy_init (GstD3D11WindowDummy * self)
84 gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
85 guint display_width, guint display_height, GstCaps * caps,
86 gboolean * video_processor_available, GError ** error)
88 g_clear_pointer (&window->processor, gst_d3d11_video_processor_free);
89 g_clear_pointer (&window->converter, gst_d3d11_converter_free);
90 g_clear_pointer (&window->compositor, gst_d3d11_overlay_compositor_free);
92 /* We are supporting only RGBA, BGRA or RGB10A2_LE formats but we don't know
93 * which format texture will be used at this moment */
95 gst_video_info_from_caps (&window->info, caps);
96 window->render_rect.left = 0;
97 window->render_rect.top = 0;
98 window->render_rect.right = display_width;
99 window->render_rect.bottom = display_height;
101 window->input_rect.left = 0;
102 window->input_rect.top = 0;
103 window->input_rect.right = GST_VIDEO_INFO_WIDTH (&window->info);
104 window->input_rect.bottom = GST_VIDEO_INFO_HEIGHT (&window->info);
106 gst_video_info_set_format (&window->render_info,
107 GST_VIDEO_FORMAT_BGRA, display_width, display_height);
109 /* TODO: not sure which colorspace should be used, let's use BT709 since
110 * it's default and most common one */
111 window->render_info.colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_BT709;
112 window->render_info.colorimetry.transfer = GST_VIDEO_TRANSFER_BT709;
113 window->render_info.colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255;
115 gst_d3d11_device_lock (window->device);
117 #if (GST_D3D11_DXGI_HEADER_VERSION >= 4)
119 const GstDxgiColorSpace *in_color_space =
120 gst_d3d11_video_info_to_dxgi_color_space (&window->info);
121 const GstD3D11Format *in_format =
122 gst_d3d11_device_format_from_gst (window->device,
123 GST_VIDEO_INFO_FORMAT (&window->info));
124 gboolean hardware = FALSE;
125 GstD3D11VideoProcessor *processor = NULL;
127 DXGI_FORMAT formats_to_check[] = {
128 DXGI_FORMAT_R8G8B8A8_UNORM,
129 DXGI_FORMAT_B8G8R8A8_UNORM,
130 DXGI_FORMAT_R10G10B10A2_UNORM
133 if (in_color_space && in_format &&
134 in_format->dxgi_format != DXGI_FORMAT_UNKNOWN) {
135 g_object_get (window->device, "hardware", &hardware, NULL);
140 gst_d3d11_video_processor_new (window->device,
141 GST_VIDEO_INFO_WIDTH (&window->info),
142 GST_VIDEO_INFO_HEIGHT (&window->info), display_width, display_height);
145 /* Check if video processor can support all possible output dxgi formats */
146 for (i = 0; i < G_N_ELEMENTS (formats_to_check) && processor; i++) {
147 DXGI_FORMAT in_dxgi_format = in_format->dxgi_format;
148 DXGI_FORMAT out_dxgi_format = formats_to_check[i];
149 DXGI_COLOR_SPACE_TYPE in_dxgi_color_space =
150 (DXGI_COLOR_SPACE_TYPE) in_color_space->dxgi_color_space_type;
152 if (!gst_d3d11_video_processor_check_format_conversion (processor,
153 in_dxgi_format, in_dxgi_color_space, out_dxgi_format,
154 DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709)) {
155 GST_DEBUG_OBJECT (window, "Conversion is not supported by device");
156 g_clear_pointer (&processor, gst_d3d11_video_processor_free);
162 gst_d3d11_video_processor_set_input_dxgi_color_space (processor,
163 (DXGI_COLOR_SPACE_TYPE) in_color_space->dxgi_color_space_type);
164 gst_d3d11_video_processor_set_output_dxgi_color_space (processor,
165 DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709);
168 window->processor = processor;
171 *video_processor_available = !!window->processor;
174 gst_d3d11_converter_new (window->device, &window->info,
175 &window->render_info, nullptr);
177 if (!window->converter) {
178 GST_ERROR_OBJECT (window, "Cannot create converter");
179 g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
180 "Cannot create converter");
185 gst_d3d11_overlay_compositor_new (window->device, &window->render_info);
186 if (!window->compositor) {
187 GST_ERROR_OBJECT (window, "Cannot create overlay compositor");
188 g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
189 "Cannot create overlay compositor");
193 gst_d3d11_device_unlock (window->device);
198 gst_d3d11_device_unlock (window->device);
204 gst_d3d11_window_dummy_clear_resources (GstD3D11WindowDummy * self)
206 GST_D3D11_CLEAR_COM (self->fallback_pov);
207 GST_D3D11_CLEAR_COM (self->fallback_rtv);
208 GST_D3D11_CLEAR_COM (self->fallback_texture);
212 gst_d3d11_window_dummy_unprepare (GstD3D11Window * window)
214 GstD3D11WindowDummy *self = GST_D3D11_WINDOW_DUMMY (window);
216 gst_d3d11_window_dummy_clear_resources (self);
220 gst_d3d11_window_dummy_on_resize (GstD3D11Window * window,
221 guint width, guint height)
223 GstVideoRectangle src_rect, dst_rect, rst_rect;
230 if (window->force_aspect_ratio) {
233 src_rect.w = GST_VIDEO_INFO_WIDTH (&window->render_info);
234 src_rect.h = GST_VIDEO_INFO_HEIGHT (&window->render_info);
236 gst_video_sink_center_rect (src_rect, dst_rect, &rst_rect, TRUE);
241 window->render_rect.left = rst_rect.x;
242 window->render_rect.top = rst_rect.y;
243 window->render_rect.right = rst_rect.x + rst_rect.w;
244 window->render_rect.bottom = rst_rect.y + rst_rect.h;
246 window->first_present = TRUE;
250 gst_d3d11_window_dummy_setup_fallback_texture (GstD3D11Window * window,
251 D3D11_TEXTURE2D_DESC * shared_desc)
253 GstD3D11WindowDummy *self = GST_D3D11_WINDOW_DUMMY (window);
254 D3D11_TEXTURE2D_DESC desc = { 0, };
255 D3D11_RENDER_TARGET_VIEW_DESC rtv_desc;
256 ID3D11Device *device_handle =
257 gst_d3d11_device_get_device_handle (window->device);
258 gboolean need_new_texture = FALSE;
261 if (!self->fallback_texture) {
262 GST_DEBUG_OBJECT (self,
263 "We have no configured fallback texture, create new one");
264 need_new_texture = TRUE;
266 self->fallback_texture->GetDesc (&desc);
267 if (shared_desc->Format != desc.Format) {
268 GST_DEBUG_OBJECT (self, "Texture formats are different, create new one");
269 need_new_texture = TRUE;
270 } else if (shared_desc->Width > desc.Width ||
271 shared_desc->Height > desc.Height) {
272 GST_DEBUG_OBJECT (self, "Needs larger size of fallback texture");
273 need_new_texture = TRUE;
277 if (!need_new_texture)
280 gst_d3d11_window_dummy_clear_resources (self);
282 desc.Width = shared_desc->Width;
283 desc.Height = shared_desc->Height;
286 desc.Format = shared_desc->Format;
287 desc.SampleDesc.Count = 1;
288 desc.SampleDesc.Quality = 0;
289 desc.Usage = D3D11_USAGE_DEFAULT;
290 desc.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET;
292 hr = device_handle->CreateTexture2D (&desc, NULL, &self->fallback_texture);
293 if (!gst_d3d11_result (hr, window->device)) {
294 GST_ERROR_OBJECT (self, "Couldn't create fallback texture");
298 rtv_desc.Format = DXGI_FORMAT_UNKNOWN;
299 rtv_desc.ViewDimension = D3D11_RTV_DIMENSION_TEXTURE2D;
300 rtv_desc.Texture2D.MipSlice = 0;
302 hr = device_handle->CreateRenderTargetView (self->fallback_texture, &rtv_desc,
303 &self->fallback_rtv);
304 if (!gst_d3d11_result (hr, window->device)) {
305 GST_ERROR_OBJECT (self,
306 "Couldn't get render target view from fallback texture");
307 gst_d3d11_window_dummy_clear_resources (self);
311 if (window->processor) {
312 D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC pov_desc;
314 pov_desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D;
315 pov_desc.Texture2D.MipSlice = 0;
317 if (!gst_d3d11_video_processor_create_output_view (window->processor,
318 &pov_desc, self->fallback_texture, &self->fallback_pov)) {
319 GST_ERROR_OBJECT (window,
320 "ID3D11VideoProcessorOutputView is unavailable");
321 gst_d3d11_window_dummy_clear_resources (self);
331 gst_d3d11_window_dummy_open_shared_handle (GstD3D11Window * window,
332 GstD3D11WindowSharedHandleData * data)
334 GstD3D11WindowDummy *self = GST_D3D11_WINDOW_DUMMY (window);
335 GstD3D11Device *device = window->device;
336 ID3D11Device *device_handle;
338 ID3D11Texture2D *texture = NULL;
339 IDXGIKeyedMutex *keyed_mutex = NULL;
340 ID3D11VideoProcessorOutputView *pov = NULL;
341 ID3D11RenderTargetView *rtv = NULL;
342 D3D11_TEXTURE2D_DESC desc;
343 gboolean use_keyed_mutex = FALSE;
344 gboolean need_fallback_texture = FALSE;
346 device_handle = gst_d3d11_device_get_device_handle (device);
348 if ((data->texture_misc_flags & D3D11_RESOURCE_MISC_SHARED_NTHANDLE) ==
349 D3D11_RESOURCE_MISC_SHARED_NTHANDLE) {
350 ComPtr<ID3D11Device1> device1_handle;
352 hr = device_handle->QueryInterface (IID_PPV_ARGS (&device1_handle));
353 if (!gst_d3d11_result (hr, device))
356 hr = device1_handle->OpenSharedResource1 (data->shared_handle,
357 IID_PPV_ARGS (&texture));
359 hr = device_handle->OpenSharedResource (data->shared_handle,
360 IID_PPV_ARGS (&texture));
363 if (!gst_d3d11_result (hr, device))
366 texture->GetDesc (&desc);
367 use_keyed_mutex = (desc.MiscFlags & D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX) ==
368 D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX;
370 if (use_keyed_mutex) {
371 hr = texture->QueryInterface (IID_PPV_ARGS (&keyed_mutex));
372 if (!gst_d3d11_result (hr, device))
376 if (window->processor) {
377 if (use_keyed_mutex) {
378 D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC pov_desc;
380 pov_desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D;
381 pov_desc.Texture2D.MipSlice = 0;
383 if (!gst_d3d11_video_processor_create_output_view (window->processor,
384 &pov_desc, texture, &pov)) {
385 GST_WARNING_OBJECT (window,
386 "ID3D11VideoProcessorOutputView is unavailable");
389 /* HACK: If external texture was created without keyed mutext
390 * and we need to used videoprocessor to convert decoder output texture
391 * to external texture, converted texture by videoprocessor seems to be broken
392 * Probably that's because of missing flush/sync API around videoprocessor.
393 * (e.g., ID3D11VideoContext and ID3D11VideoProcessor have no
394 * flushing api such as ID3D11DeviceContext::Flush).
395 * To workaround the case, we need to use fallback texture and copy back
396 * to external texture
399 need_fallback_texture = TRUE;
401 GST_TRACE_OBJECT (window,
402 "We are using video processor but keyed mutex is unavailable");
403 if (!gst_d3d11_window_dummy_setup_fallback_texture (window, &desc)) {
409 hr = device_handle->CreateRenderTargetView ((ID3D11Resource *) texture,
411 if (!gst_d3d11_result (hr, device))
415 hr = keyed_mutex->AcquireSync(data->acquire_key, INFINITE);
416 if (!gst_d3d11_result (hr, device))
420 /* Everything is prepared now */
421 gst_d3d11_window_dummy_on_resize (window, desc.Width, desc.Height);
423 /* Move owned resources */
424 data->texture = texture;
425 data->keyed_mutex = keyed_mutex;
429 if (need_fallback_texture) {
430 data->fallback_pov = self->fallback_pov;
431 data->fallback_rtv = self->fallback_rtv;
433 data->fallback_pov = nullptr;
434 data->fallback_rtv = nullptr;
440 GST_D3D11_CLEAR_COM (texture);
441 GST_D3D11_CLEAR_COM (keyed_mutex);
442 GST_D3D11_CLEAR_COM (pov);
443 GST_D3D11_CLEAR_COM (rtv);
450 gst_d3d11_window_dummy_release_shared_handle (GstD3D11Window * window,
451 GstD3D11WindowSharedHandleData * data)
453 GstD3D11WindowDummy *self = GST_D3D11_WINDOW_DUMMY (window);
454 GstD3D11Device *device = window->device;
457 /* TODO: cache owned resource for the later reuse? */
458 if (data->keyed_mutex) {
459 hr = data->keyed_mutex->ReleaseSync (data->release_key);
460 gst_d3d11_result (hr, device);
462 data->keyed_mutex->Release ();
465 ComPtr<ID3D11Query> query;
467 D3D11_QUERY_DESC query_desc;
468 ID3D11Device *device_handle = gst_d3d11_device_get_device_handle (device);
469 ID3D11DeviceContext *context_handle =
470 gst_d3d11_device_get_device_context_handle (device);
471 BOOL sync_done = FALSE;
473 /* If keyed mutex is not used, let's handle sync manually by using
474 * ID3D11Query. Issued GPU commands might not be finished yet */
475 query_desc.Query = D3D11_QUERY_EVENT;
476 query_desc.MiscFlags = 0;
478 hr = device_handle->CreateQuery (&query_desc, &query);
479 if (!gst_d3d11_result (hr, device)) {
480 GST_ERROR_OBJECT (self, "Couldn't Create event query");
484 /* Copy from fallback texture to user's texture */
485 if (data->fallback_rtv) {
487 D3D11_TEXTURE2D_DESC desc;
488 ID3D11DeviceContext *context_handle =
489 gst_d3d11_device_get_device_context_handle (device);
491 data->texture->GetDesc (&desc);
497 src_box.right = desc.Width;
498 src_box.bottom = desc.Height;
500 context_handle->CopySubresourceRegion (data->texture, 0, 0, 0, 0,
501 self->fallback_texture, 0, &src_box);
503 context_handle->End (query.Get ());
505 /* Wait until all issued GPU commands are finished */
507 context_handle->GetData (query.Get (), &sync_done, sizeof (BOOL), 0);
508 } while (!sync_done && (hr == S_OK || hr == S_FALSE));
510 if (!gst_d3d11_result (hr, device)) {
511 GST_ERROR_OBJECT (self, "Couldn't sync GPU operation");
516 GST_D3D11_CLEAR_COM (data->rtv);
517 GST_D3D11_CLEAR_COM (data->pov);
518 GST_D3D11_CLEAR_COM (data->texture);
524 gst_d3d11_window_dummy_new (GstD3D11Device * device)
526 GstD3D11Window *window;
528 g_return_val_if_fail (GST_IS_D3D11_DEVICE (device), NULL);
530 window = (GstD3D11Window *)
531 g_object_new (GST_TYPE_D3D11_WINDOW_DUMMY, "d3d11device", device, NULL);
533 window->initialized = TRUE;
534 g_object_ref_sink (window);