2 * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
3 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 * SECTION:element-d3d11videosink
23 * @title: d3d11videosink
25 * Direct3D11 based video render element
27 * ## Example launch line
29 * gst-launch-1.0 videotestsrc ! d3d11upload ! d3d11videosink
31 * This pipeline will display test video stream on screen via d3d11videosink
41 #include "gstd3d11videosink.h"
42 #include "gstd3d11pluginutils.h"
45 #if GST_D3D11_WINAPI_APP
46 #include "gstd3d11window_corewindow.h"
47 #include "gstd3d11window_swapchainpanel.h"
49 #if (!GST_D3D11_WINAPI_ONLY_APP)
50 #include "gstd3d11window_win32.h"
52 #include "gstd3d11window_dummy.h"
58 PROP_FORCE_ASPECT_RATIO,
59 PROP_ENABLE_NAVIGATION_EVENTS,
60 PROP_FULLSCREEN_TOGGLE_MODE,
62 PROP_DRAW_ON_SHARED_TEXTURE,
66 #define DEFAULT_ADAPTER -1
67 #define DEFAULT_FORCE_ASPECT_RATIO TRUE
68 #define DEFAULT_ENABLE_NAVIGATION_EVENTS TRUE
69 #define DEFAULT_FULLSCREEN_TOGGLE_MODE GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_NONE
70 #define DEFAULT_FULLSCREEN FALSE
71 #define DEFAULT_DRAW_ON_SHARED_TEXTURE FALSE
84 static guint gst_d3d11_video_sink_signals[LAST_SIGNAL] = { 0, };
86 static GstStaticCaps pad_template_caps =
87 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
88 (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_SINK_FORMATS) "; "
89 GST_VIDEO_CAPS_MAKE_WITH_FEATURES
90 (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY ","
91 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
92 GST_D3D11_SINK_FORMATS) ";"
93 GST_VIDEO_CAPS_MAKE (GST_D3D11_SINK_FORMATS) "; "
94 GST_VIDEO_CAPS_MAKE_WITH_FEATURES
95 (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY ","
96 GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
97 GST_D3D11_SINK_FORMATS));
99 GST_DEBUG_CATEGORY (d3d11_video_sink_debug);
100 #define GST_CAT_DEFAULT d3d11_video_sink_debug
102 struct _GstD3D11VideoSink
105 GstD3D11Device *device;
106 GstD3D11Window *window;
114 gboolean caps_updated;
118 gboolean force_aspect_ratio;
119 gboolean enable_navigation_events;
120 GstD3D11WindowFullscreenToggleMode fullscreen_toggle_mode;
122 gboolean draw_on_shared_texture;
124 /* saved render rectangle until we have a window */
125 GstVideoRectangle render_rect;
126 gboolean pending_render_rect;
128 /* For drawing on user texture */
130 GstBuffer *current_buffer;
135 /* method configured via property */
136 GstVideoOrientationMethod method;
137 /* method parsed from tag */
138 GstVideoOrientationMethod tag_method;
139 /* method currently selected based on "method" and "tag_method" */
140 GstVideoOrientationMethod selected_method;
143 #define GST_D3D11_VIDEO_SINK_GET_LOCK(d) (&(GST_D3D11_VIDEO_SINK_CAST(d)->lock))
144 #define GST_D3D11_VIDEO_SINK_LOCK(d) G_STMT_START { \
145 GST_TRACE_OBJECT (d, "Locking from thread %p", g_thread_self()); \
146 g_rec_mutex_lock (GST_D3D11_VIDEO_SINK_GET_LOCK (d)); \
147 GST_TRACE_OBJECT (d, "Locked from thread %p", g_thread_self()); \
150 #define GST_D3D11_VIDEO_SINK_UNLOCK(d) G_STMT_START { \
151 GST_TRACE_OBJECT (d, "Unlocking from thread %p", g_thread_self()); \
152 g_rec_mutex_unlock (GST_D3D11_VIDEO_SINK_GET_LOCK (d)); \
155 static void gst_d3d11_videosink_set_property (GObject * object, guint prop_id,
156 const GValue * value, GParamSpec * pspec);
157 static void gst_d3d11_videosink_get_property (GObject * object, guint prop_id,
158 GValue * value, GParamSpec * pspec);
159 static void gst_d3d11_video_sink_finalize (GObject * object);
161 gst_d3d11_video_sink_draw_action (GstD3D11VideoSink * self,
162 gpointer shared_handle, guint texture_misc_flags, guint64 acquire_key,
163 guint64 release_key);
166 gst_d3d11_video_sink_video_overlay_init (GstVideoOverlayInterface * iface);
168 gst_d3d11_video_sink_navigation_init (GstNavigationInterface * iface);
170 static void gst_d3d11_video_sink_set_context (GstElement * element,
171 GstContext * context);
172 static GstCaps *gst_d3d11_video_sink_get_caps (GstBaseSink * sink,
174 static gboolean gst_d3d11_video_sink_set_caps (GstBaseSink * sink,
177 static gboolean gst_d3d11_video_sink_start (GstBaseSink * sink);
178 static gboolean gst_d3d11_video_sink_stop (GstBaseSink * sink);
179 static gboolean gst_d3d11_video_sink_propose_allocation (GstBaseSink * sink,
181 static gboolean gst_d3d11_video_sink_query (GstBaseSink * sink,
183 static gboolean gst_d3d11_video_sink_unlock (GstBaseSink * sink);
184 static gboolean gst_d3d11_video_sink_unlock_stop (GstBaseSink * sink);
185 static gboolean gst_d3d11_video_sink_event (GstBaseSink * sink,
188 gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf);
189 static gboolean gst_d3d11_video_sink_prepare_window (GstD3D11VideoSink * self);
190 static void gst_d3d11_video_sink_set_orientation (GstD3D11VideoSink * self,
191 GstVideoOrientationMethod method, gboolean from_tag);
193 #define gst_d3d11_video_sink_parent_class parent_class
194 G_DEFINE_TYPE_WITH_CODE (GstD3D11VideoSink, gst_d3d11_video_sink,
196 G_IMPLEMENT_INTERFACE (GST_TYPE_VIDEO_OVERLAY,
197 gst_d3d11_video_sink_video_overlay_init);
198 G_IMPLEMENT_INTERFACE (GST_TYPE_NAVIGATION,
199 gst_d3d11_video_sink_navigation_init);
200 GST_DEBUG_CATEGORY_INIT (d3d11_video_sink_debug,
201 "d3d11videosink", 0, "Direct3D11 Video Sink"));
204 gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass)
206 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
207 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
208 GstBaseSinkClass *basesink_class = GST_BASE_SINK_CLASS (klass);
209 GstVideoSinkClass *videosink_class = GST_VIDEO_SINK_CLASS (klass);
212 gobject_class->set_property = gst_d3d11_videosink_set_property;
213 gobject_class->get_property = gst_d3d11_videosink_get_property;
214 gobject_class->finalize = gst_d3d11_video_sink_finalize;
216 g_object_class_install_property (gobject_class, PROP_ADAPTER,
217 g_param_spec_int ("adapter", "Adapter",
218 "Adapter index for creating device (-1 for default)",
219 -1, G_MAXINT32, DEFAULT_ADAPTER,
220 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
221 G_PARAM_STATIC_STRINGS)));
223 g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO,
224 g_param_spec_boolean ("force-aspect-ratio",
225 "Force aspect ratio",
226 "When enabled, scaling will respect original aspect ratio",
227 DEFAULT_FORCE_ASPECT_RATIO,
228 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
230 g_object_class_install_property (gobject_class, PROP_ENABLE_NAVIGATION_EVENTS,
231 g_param_spec_boolean ("enable-navigation-events",
232 "Enable navigation events",
233 "When enabled, navigation events are sent upstream",
234 DEFAULT_ENABLE_NAVIGATION_EVENTS,
235 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
237 g_object_class_install_property (gobject_class, PROP_FULLSCREEN_TOGGLE_MODE,
238 g_param_spec_flags ("fullscreen-toggle-mode",
239 "Full screen toggle mode",
240 "Full screen toggle mode used to trigger fullscreen mode change",
241 GST_D3D11_WINDOW_TOGGLE_MODE_GET_TYPE, DEFAULT_FULLSCREEN_TOGGLE_MODE,
242 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
244 g_object_class_install_property (gobject_class, PROP_FULLSCREEN,
245 g_param_spec_boolean ("fullscreen",
247 "Ignored when \"fullscreen-toggle-mode\" does not include \"property\"",
249 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
252 * GstD3D11VideoSink:draw-on-shared-texture:
254 * Instruct the sink to draw on a shared texture provided by user.
255 * User must watch #d3d11videosink::begin-draw signal and should call
256 * #d3d11videosink::draw method on the #d3d11videosink::begin-draw
259 * Currently supported formats for user texture are:
260 * - DXGI_FORMAT_R8G8B8A8_UNORM
261 * - DXGI_FORMAT_B8G8R8A8_UNORM
262 * - DXGI_FORMAT_R10G10B10A2_UNORM
266 g_object_class_install_property (gobject_class, PROP_DRAW_ON_SHARED_TEXTURE,
267 g_param_spec_boolean ("draw-on-shared-texture",
268 "Draw on shared texture",
269 "Draw on user provided shared texture instead of window. "
270 "When enabled, user can pass application's own texture to sink "
271 "by using \"draw\" action signal on \"begin-draw\" signal handler, "
272 "so that sink can draw video data on application's texture. "
273 "Supported texture formats for user texture are "
274 "DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_FORMAT_B8G8R8A8_UNORM, and "
275 "DXGI_FORMAT_R10G10B10A2_UNORM.",
276 DEFAULT_DRAW_ON_SHARED_TEXTURE,
277 (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
278 G_PARAM_STATIC_STRINGS)));
281 * GstD3D11VideoSink:rotate-method:
283 * Video rotation/flip method to use
287 g_object_class_install_property (gobject_class, PROP_ROTATE_METHOD,
288 g_param_spec_enum ("rotate-method", "Rotate Method",
289 "Rotate method to use",
290 GST_TYPE_VIDEO_ORIENTATION_METHOD, GST_VIDEO_ORIENTATION_IDENTITY,
291 (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
294 * GstD3D11VideoSink::begin-draw:
295 * @videosink: the #d3d11videosink
297 * Emitted when sink has a texture to draw. Application needs to invoke
298 * #d3d11videosink::draw action signal before returning from
299 * #d3d11videosink::begin-draw signal handler.
303 gst_d3d11_video_sink_signals[SIGNAL_BEGIN_DRAW] =
304 g_signal_new ("begin-draw", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST,
305 G_STRUCT_OFFSET (GstD3D11VideoSinkClass, begin_draw),
306 NULL, NULL, NULL, G_TYPE_NONE, 0, G_TYPE_NONE);
309 * GstD3D11VideoSink::draw:
310 * @videosink: the #d3d11videosink
311 * @shard_handle: a pointer to HANDLE
312 * @texture_misc_flags: a D3D11_RESOURCE_MISC_FLAG value
313 * @acquire_key: a key value used for IDXGIKeyedMutex::AcquireSync
314 * @release_key: a key value used for IDXGIKeyedMutex::ReleaseSync
316 * Draws on a shared texture. @shard_handle must be a valid pointer to
317 * a HANDLE which was obtained via IDXGIResource::GetSharedHandle or
318 * IDXGIResource1::CreateSharedHandle.
320 * If the texture was created with D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag,
321 * caller must specify valid @acquire_key and @release_key.
322 * Otherwise (i.e., created with D3D11_RESOURCE_MISC_SHARED flag),
323 * @acquire_key and @release_key will be ignored.
327 gst_d3d11_video_sink_signals[SIGNAL_DRAW] =
328 g_signal_new ("draw", G_TYPE_FROM_CLASS (klass),
329 (GSignalFlags) (G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
330 G_STRUCT_OFFSET (GstD3D11VideoSinkClass, draw), NULL, NULL, NULL,
331 G_TYPE_BOOLEAN, 4, G_TYPE_POINTER, G_TYPE_UINT, G_TYPE_UINT64,
334 element_class->set_context =
335 GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_set_context);
337 gst_element_class_set_static_metadata (element_class,
338 "Direct3D11 video sink", "Sink/Video",
339 "A Direct3D11 based videosink",
340 "Seungha Yang <seungha.yang@navercorp.com>");
342 caps = gst_d3d11_get_updated_template_caps (&pad_template_caps);
343 gst_element_class_add_pad_template (element_class,
344 gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, caps));
345 gst_caps_unref (caps);
347 basesink_class->get_caps = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_get_caps);
348 basesink_class->set_caps = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_set_caps);
349 basesink_class->start = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_start);
350 basesink_class->stop = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_stop);
351 basesink_class->propose_allocation =
352 GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_propose_allocation);
353 basesink_class->query = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_query);
354 basesink_class->unlock = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_unlock);
355 basesink_class->unlock_stop =
356 GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_unlock_stop);
357 basesink_class->event = GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_event);
359 videosink_class->show_frame =
360 GST_DEBUG_FUNCPTR (gst_d3d11_video_sink_show_frame);
362 klass->draw = gst_d3d11_video_sink_draw_action;
364 gst_type_mark_as_plugin_api (GST_D3D11_WINDOW_TOGGLE_MODE_GET_TYPE,
365 (GstPluginAPIFlags) 0);
369 gst_d3d11_video_sink_init (GstD3D11VideoSink * self)
371 self->adapter = DEFAULT_ADAPTER;
372 self->force_aspect_ratio = DEFAULT_FORCE_ASPECT_RATIO;
373 self->enable_navigation_events = DEFAULT_ENABLE_NAVIGATION_EVENTS;
374 self->fullscreen_toggle_mode = DEFAULT_FULLSCREEN_TOGGLE_MODE;
375 self->fullscreen = DEFAULT_FULLSCREEN;
376 self->draw_on_shared_texture = DEFAULT_DRAW_ON_SHARED_TEXTURE;
378 g_rec_mutex_init (&self->lock);
382 gst_d3d11_videosink_set_property (GObject * object, guint prop_id,
383 const GValue * value, GParamSpec * pspec)
385 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (object);
387 GST_D3D11_VIDEO_SINK_LOCK (self);
390 self->adapter = g_value_get_int (value);
392 case PROP_FORCE_ASPECT_RATIO:
393 self->force_aspect_ratio = g_value_get_boolean (value);
395 g_object_set (self->window,
396 "force-aspect-ratio", self->force_aspect_ratio, NULL);
398 case PROP_ENABLE_NAVIGATION_EVENTS:
399 self->enable_navigation_events = g_value_get_boolean (value);
401 g_object_set (self->window,
402 "enable-navigation-events", self->enable_navigation_events, NULL);
405 case PROP_FULLSCREEN_TOGGLE_MODE:
406 self->fullscreen_toggle_mode =
407 (GstD3D11WindowFullscreenToggleMode) g_value_get_flags (value);
409 g_object_set (self->window,
410 "fullscreen-toggle-mode", self->fullscreen_toggle_mode, NULL);
413 case PROP_FULLSCREEN:
414 self->fullscreen = g_value_get_boolean (value);
416 g_object_set (self->window, "fullscreen", self->fullscreen, NULL);
419 case PROP_DRAW_ON_SHARED_TEXTURE:
420 self->draw_on_shared_texture = g_value_get_boolean (value);
422 case PROP_ROTATE_METHOD:
423 gst_d3d11_video_sink_set_orientation (self,
424 (GstVideoOrientationMethod) g_value_get_enum (value), FALSE);
427 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
430 GST_D3D11_VIDEO_SINK_UNLOCK (self);
434 gst_d3d11_videosink_get_property (GObject * object, guint prop_id,
435 GValue * value, GParamSpec * pspec)
437 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (object);
439 GST_D3D11_VIDEO_SINK_LOCK (self);
442 g_value_set_int (value, self->adapter);
444 case PROP_FORCE_ASPECT_RATIO:
445 g_value_set_boolean (value, self->force_aspect_ratio);
447 case PROP_ENABLE_NAVIGATION_EVENTS:
448 g_value_set_boolean (value, self->enable_navigation_events);
450 case PROP_FULLSCREEN_TOGGLE_MODE:
451 g_value_set_flags (value, self->fullscreen_toggle_mode);
453 case PROP_FULLSCREEN:
455 g_object_get_property (G_OBJECT (self->window), pspec->name, value);
457 g_value_set_boolean (value, self->fullscreen);
460 case PROP_DRAW_ON_SHARED_TEXTURE:
461 g_value_set_boolean (value, self->draw_on_shared_texture);
463 case PROP_ROTATE_METHOD:
464 g_value_set_enum (value, self->method);
467 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
470 GST_D3D11_VIDEO_SINK_UNLOCK (self);
474 gst_d3d11_video_sink_finalize (GObject * object)
476 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (object);
478 g_rec_mutex_clear (&self->lock);
479 g_free (self->title);
481 G_OBJECT_CLASS (parent_class)->finalize (object);
485 gst_d3d11_video_sink_set_context (GstElement * element, GstContext * context)
487 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (element);
489 gst_d3d11_handle_set_context (element, context, self->adapter, &self->device);
491 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
495 gst_d3d11_video_sink_get_caps (GstBaseSink * sink, GstCaps * filter)
497 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
498 GstCaps *caps = NULL;
500 caps = gst_pad_get_pad_template_caps (GST_VIDEO_SINK_PAD (sink));
503 gboolean is_hardware = FALSE;
505 g_object_get (self->device, "hardware", &is_hardware, NULL);
507 /* In case of WARP device, conversion via shader would be inefficient than
508 * upstream videoconvert. Allow native formats in this case */
510 GValue format_list = G_VALUE_INIT;
511 GValue format = G_VALUE_INIT;
513 g_value_init (&format_list, GST_TYPE_LIST);
514 g_value_init (&format, G_TYPE_STRING);
516 g_value_set_string (&format, "RGBA");
517 gst_value_list_append_and_take_value (&format_list, &format);
519 format = G_VALUE_INIT;
520 g_value_init (&format, G_TYPE_STRING);
521 g_value_set_string (&format, "BGRA");
522 gst_value_list_append_and_take_value (&format_list, &format);
524 caps = gst_caps_make_writable (caps);
525 gst_caps_set_value (caps, "format", &format_list);
526 g_value_unset (&format_list);
532 isect = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
533 gst_caps_unref (caps);
541 gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
543 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
545 GST_DEBUG_OBJECT (self, "set caps %" GST_PTR_FORMAT, caps);
547 /* We will update window on show_frame() */
548 self->caps_updated = TRUE;
554 gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
556 gint video_width, video_height;
557 gint video_par_n, video_par_d; /* video's PAR */
558 gint display_par_n = 1, display_par_d = 1; /* display's PAR */
560 GError *error = NULL;
562 GST_DEBUG_OBJECT (self, "Updating window with caps %" GST_PTR_FORMAT, caps);
564 self->caps_updated = FALSE;
566 GST_D3D11_VIDEO_SINK_LOCK (self);
567 if (!gst_d3d11_video_sink_prepare_window (self)) {
568 GST_D3D11_VIDEO_SINK_UNLOCK (self);
570 GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, (nullptr),
571 ("Failed to open window."));
576 if (!gst_video_info_from_caps (&self->info, caps)) {
577 GST_DEBUG_OBJECT (self,
578 "Could not locate image format from caps %" GST_PTR_FORMAT, caps);
579 GST_D3D11_VIDEO_SINK_UNLOCK (self);
583 video_width = GST_VIDEO_INFO_WIDTH (&self->info);
584 video_height = GST_VIDEO_INFO_HEIGHT (&self->info);
585 video_par_n = GST_VIDEO_INFO_PAR_N (&self->info);
586 video_par_d = GST_VIDEO_INFO_PAR_D (&self->info);
588 /* get aspect ratio from caps if it's present, and
589 * convert video width and height to a display width and height
590 * using wd / hd = wv / hv * PARv / PARd */
592 if (!gst_video_calculate_display_ratio (&num, &den, video_width,
593 video_height, video_par_n, video_par_d, display_par_n,
595 GST_D3D11_VIDEO_SINK_UNLOCK (self);
597 GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, (nullptr),
598 ("Error calculating the output display ratio of the video."));
602 GST_DEBUG_OBJECT (self,
603 "video width/height: %dx%d, calculated display ratio: %d/%d format: %s",
604 video_width, video_height, num, den,
605 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&self->info)));
607 /* now find a width x height that respects this display ratio.
608 * prefer those that have one of w/h the same as the incoming video
609 * using wd / hd = num / den
612 /* start with same height, because of interlaced video
613 * check hd / den is an integer scale factor, and scale wd with the PAR
615 if (video_height % den == 0) {
616 GST_DEBUG_OBJECT (self, "keeping video height");
617 GST_VIDEO_SINK_WIDTH (self) = (guint)
618 gst_util_uint64_scale_int (video_height, num, den);
619 GST_VIDEO_SINK_HEIGHT (self) = video_height;
620 } else if (video_width % num == 0) {
621 GST_DEBUG_OBJECT (self, "keeping video width");
622 GST_VIDEO_SINK_WIDTH (self) = video_width;
623 GST_VIDEO_SINK_HEIGHT (self) = (guint)
624 gst_util_uint64_scale_int (video_width, den, num);
626 GST_DEBUG_OBJECT (self, "approximating while keeping video height");
627 GST_VIDEO_SINK_WIDTH (self) = (guint)
628 gst_util_uint64_scale_int (video_height, num, den);
629 GST_VIDEO_SINK_HEIGHT (self) = video_height;
632 GST_DEBUG_OBJECT (self, "scaling to %dx%d",
633 GST_VIDEO_SINK_WIDTH (self), GST_VIDEO_SINK_HEIGHT (self));
634 self->video_width = video_width;
635 self->video_height = video_height;
637 if (GST_VIDEO_SINK_WIDTH (self) <= 0 || GST_VIDEO_SINK_HEIGHT (self) <= 0) {
638 GST_D3D11_VIDEO_SINK_UNLOCK (self);
640 GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, (nullptr),
641 ("Error calculating the output display ratio of the video."));
645 if (self->pending_render_rect) {
646 GstVideoRectangle rect = self->render_rect;
648 self->pending_render_rect = FALSE;
649 gst_d3d11_window_set_render_rectangle (self->window, &rect);
652 if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self),
653 GST_VIDEO_SINK_HEIGHT (self), caps, &error)) {
654 GstMessage *error_msg;
656 GST_D3D11_VIDEO_SINK_UNLOCK (self);
658 GST_ERROR_OBJECT (self, "cannot create swapchain");
659 error_msg = gst_message_new_error (GST_OBJECT_CAST (self),
660 error, "Failed to prepare d3d11window");
661 g_clear_error (&error);
662 gst_element_post_message (GST_ELEMENT (self), error_msg);
668 gst_d3d11_window_set_title (self->window, self->title);
669 g_clear_pointer (&self->title, g_free);
672 GST_D3D11_VIDEO_SINK_UNLOCK (self);
678 gst_d3d11_video_sink_key_event (GstD3D11Window * window, const gchar * event,
679 const gchar * key, GstD3D11VideoSink * self)
681 GstEvent *key_event = NULL;
683 if (self->enable_navigation_events) {
684 GST_LOG_OBJECT (self, "send key event %s, key %s", event, key);
685 if (0 == g_strcmp0 ("key-press", event))
687 gst_navigation_event_new_key_press (key,
688 GST_NAVIGATION_MODIFIER_NONE);
689 else if (0 == g_strcmp0 ("key-release", event))
691 gst_navigation_event_new_key_release (key,
692 GST_NAVIGATION_MODIFIER_NONE);
695 gst_navigation_send_event_simple (GST_NAVIGATION (self), key_event);
700 gst_d3d11_video_mouse_key_event (GstD3D11Window * window, const gchar * event,
701 gint button, gdouble x, gdouble y, GstD3D11VideoSink * self)
703 GstEvent *mouse_event = NULL;
705 if (self->enable_navigation_events) {
706 GST_LOG_OBJECT (self,
707 "send mouse event %s, button %d (%.1f, %.1f)", event, button, x, y);
708 if (0 == g_strcmp0 ("mouse-button-press", event))
710 gst_navigation_event_new_mouse_button_press (button, x, y,
711 GST_NAVIGATION_MODIFIER_NONE);
712 else if (0 == g_strcmp0 ("mouse-button-release", event))
714 gst_navigation_event_new_mouse_button_release (button, x, y,
715 GST_NAVIGATION_MODIFIER_NONE);
716 else if (0 == g_strcmp0 ("mouse-move", event))
718 gst_navigation_event_new_mouse_move (x, y,
719 GST_NAVIGATION_MODIFIER_NONE);
722 gst_navigation_send_event_simple (GST_NAVIGATION (self), mouse_event);
727 gst_d3d11_video_sink_start (GstBaseSink * sink)
729 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
731 GST_DEBUG_OBJECT (self, "Start");
733 if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self), self->adapter,
735 GST_ERROR_OBJECT (sink, "Cannot create d3d11device");
742 /* called with lock */
744 gst_d3d11_video_sink_prepare_window (GstD3D11VideoSink * self)
746 GstD3D11WindowNativeType window_type = GST_D3D11_WINDOW_NATIVE_TYPE_HWND;
751 if (self->draw_on_shared_texture) {
752 GST_INFO_OBJECT (self,
753 "Create dummy window for rendering on shared texture");
754 self->window = gst_d3d11_window_dummy_new (self->device);
758 if (!self->window_id)
759 gst_video_overlay_prepare_window_handle (GST_VIDEO_OVERLAY (self));
761 if (self->window_id) {
763 gst_d3d11_window_get_native_type_from_handle (self->window_id);
765 if (window_type != GST_D3D11_WINDOW_NATIVE_TYPE_NONE) {
766 GST_DEBUG_OBJECT (self, "Have window handle %" G_GUINTPTR_FORMAT,
768 gst_video_overlay_got_window_handle (GST_VIDEO_OVERLAY (self),
773 GST_DEBUG_OBJECT (self, "Create window (type: %s)",
774 gst_d3d11_window_get_native_type_to_string (window_type));
776 #if GST_D3D11_WINAPI_ONLY_APP
777 if (window_type != GST_D3D11_WINDOW_NATIVE_TYPE_CORE_WINDOW &&
778 window_type != GST_D3D11_WINDOW_NATIVE_TYPE_SWAP_CHAIN_PANEL) {
779 GST_ERROR_OBJECT (self, "Overlay handle must be set before READY state");
784 switch (window_type) {
785 #if (!GST_D3D11_WINAPI_ONLY_APP)
786 case GST_D3D11_WINDOW_NATIVE_TYPE_HWND:
787 self->window = gst_d3d11_window_win32_new (self->device, self->window_id);
790 #if GST_D3D11_WINAPI_APP
791 case GST_D3D11_WINDOW_NATIVE_TYPE_CORE_WINDOW:
792 self->window = gst_d3d11_window_core_window_new (self->device,
795 case GST_D3D11_WINDOW_NATIVE_TYPE_SWAP_CHAIN_PANEL:
796 self->window = gst_d3d11_window_swap_chain_panel_new (self->device,
806 GST_ERROR_OBJECT (self, "Cannot create d3d11window");
810 g_object_set (self->window,
811 "force-aspect-ratio", self->force_aspect_ratio,
812 "fullscreen-toggle-mode", self->fullscreen_toggle_mode,
813 "fullscreen", self->fullscreen,
814 "enable-navigation-events", self->enable_navigation_events, NULL);
816 gst_d3d11_window_set_orientation (self->window, self->selected_method);
818 g_signal_connect (self->window, "key-event",
819 G_CALLBACK (gst_d3d11_video_sink_key_event), self);
820 g_signal_connect (self->window, "mouse-event",
821 G_CALLBACK (gst_d3d11_video_mouse_key_event), self);
827 gst_d3d11_video_sink_stop (GstBaseSink * sink)
829 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
831 GST_DEBUG_OBJECT (self, "Stop");
833 GST_D3D11_VIDEO_SINK_LOCK (self);
835 gst_d3d11_window_unprepare (self->window);
837 gst_clear_object (&self->window);
838 GST_D3D11_VIDEO_SINK_UNLOCK (self);
840 gst_clear_object (&self->device);
842 g_clear_pointer (&self->title, g_free);
848 gst_d3d11_video_sink_propose_allocation (GstBaseSink * sink, GstQuery * query)
850 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
852 GstBufferPool *pool = NULL;
860 gst_query_parse_allocation (query, &caps, &need_pool);
865 if (!gst_video_info_from_caps (&info, caps))
868 /* the normal size of a frame */
872 GstCapsFeatures *features;
873 GstStructure *config;
874 gboolean is_d3d11 = FALSE;
876 features = gst_caps_get_features (caps, 0);
878 && gst_caps_features_contains (features,
879 GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) {
880 GST_DEBUG_OBJECT (self, "upstream support d3d11 memory");
881 pool = gst_d3d11_buffer_pool_new (self->device);
884 pool = gst_video_buffer_pool_new ();
887 config = gst_buffer_pool_get_config (pool);
888 gst_buffer_pool_config_add_option (config,
889 GST_BUFFER_POOL_OPTION_VIDEO_META);
891 gst_buffer_pool_config_add_option (config,
892 GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
895 size = GST_VIDEO_INFO_SIZE (&info);
897 GstD3D11AllocationParams *d3d11_params;
900 gst_d3d11_allocation_params_new (self->device,
901 &info, GST_D3D11_ALLOCATION_FLAG_DEFAULT, D3D11_BIND_SHADER_RESOURCE,
904 gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
905 gst_d3d11_allocation_params_free (d3d11_params);
908 gst_buffer_pool_config_set_params (config, caps, (guint) size, 2, 0);
910 if (!gst_buffer_pool_set_config (pool, config)) {
911 GST_ERROR_OBJECT (pool, "Couldn't set config");
912 gst_object_unref (pool);
917 /* d3d11 buffer pool will update buffer size based on allocated texture,
918 * get size from config again */
919 config = gst_buffer_pool_get_config (pool);
920 gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr,
922 gst_structure_free (config);
925 /* In case of system memory, we will upload video frame to GPU memory,
926 * (which is copy in any case), so crop meta support for system memory
927 * is almost pointless */
928 gst_query_add_allocation_meta (query,
929 GST_VIDEO_CROP_META_API_TYPE, nullptr);
933 /* We need at least 2 buffers because we hold on to the last one for redrawing
934 * on window-resize event */
935 gst_query_add_allocation_pool (query, pool, size, 2, 0);
937 g_object_unref (pool);
939 gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
940 gst_query_add_allocation_meta (query,
941 GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL);
948 GST_WARNING_OBJECT (self, "no caps specified");
953 GST_WARNING_OBJECT (self, "invalid caps specified");
961 gst_d3d11_video_sink_query (GstBaseSink * sink, GstQuery * query)
963 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
965 switch (GST_QUERY_TYPE (query)) {
966 case GST_QUERY_CONTEXT:
967 if (gst_d3d11_handle_context_query (GST_ELEMENT (self), query,
976 return GST_BASE_SINK_CLASS (parent_class)->query (sink, query);
980 gst_d3d11_video_sink_unlock (GstBaseSink * sink)
982 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
984 GST_D3D11_VIDEO_SINK_LOCK (self);
986 gst_d3d11_window_unlock (self->window);
987 GST_D3D11_VIDEO_SINK_UNLOCK (self);
993 gst_d3d11_video_sink_unlock_stop (GstBaseSink * sink)
995 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
997 GST_D3D11_VIDEO_SINK_LOCK (self);
999 gst_d3d11_window_unlock_stop (self->window);
1000 GST_D3D11_VIDEO_SINK_UNLOCK (self);
1006 gst_d3d11_video_sink_event (GstBaseSink * sink, GstEvent * event)
1008 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
1010 switch (GST_EVENT_TYPE (event)) {
1011 case GST_EVENT_TAG:{
1012 GstTagList *taglist;
1013 gchar *title = nullptr;
1014 GstVideoOrientationMethod method = GST_VIDEO_ORIENTATION_IDENTITY;
1016 gst_event_parse_tag (event, &taglist);
1017 gst_tag_list_get_string (taglist, GST_TAG_TITLE, &title);
1020 const gchar *app_name = g_get_application_name ();
1021 std::string title_string;
1024 title_string = std::string (title) + " : " + std::string (app_name);
1026 title_string = std::string (title);
1029 GST_D3D11_VIDEO_SINK_LOCK (self);
1031 gst_d3d11_window_set_title (self->window, title_string.c_str ());
1033 g_free (self->title);
1034 self->title = g_strdup (title_string.c_str ());
1036 GST_D3D11_VIDEO_SINK_UNLOCK (self);
1041 if (gst_video_orientation_from_tag (taglist, &method)) {
1042 GST_D3D11_VIDEO_SINK_LOCK (self);
1043 gst_d3d11_video_sink_set_orientation (self, method, TRUE);
1044 GST_D3D11_VIDEO_SINK_UNLOCK (self);
1052 return GST_BASE_SINK_CLASS (parent_class)->event (sink, event);
1055 /* called with lock */
1057 gst_d3d11_video_sink_set_orientation (GstD3D11VideoSink * self,
1058 GstVideoOrientationMethod method, gboolean from_tag)
1060 if (method == GST_VIDEO_ORIENTATION_CUSTOM) {
1061 GST_WARNING_OBJECT (self, "Unsupported custom orientation");
1066 self->tag_method = method;
1068 self->method = method;
1070 if (self->method == GST_VIDEO_ORIENTATION_AUTO) {
1071 self->selected_method = self->tag_method;
1073 self->selected_method = self->method;
1077 gst_d3d11_window_set_orientation (self->window, self->selected_method);
1081 gst_d3d11_video_sink_check_device_update (GstD3D11VideoSink * self,
1085 GstD3D11Memory *dmem;
1086 gboolean update_device = FALSE;
1088 /* We have configured window already, cannot update device */
1092 mem = gst_buffer_peek_memory (buf, 0);
1093 if (!gst_is_d3d11_memory (mem))
1096 dmem = GST_D3D11_MEMORY_CAST (mem);
1097 /* Same device, nothing to do */
1098 if (dmem->device == self->device)
1101 if (self->adapter < 0) {
1102 update_device = TRUE;
1106 g_object_get (dmem->device, "adapter", &adapter, NULL);
1107 /* The same GPU as what user wanted, update */
1108 if (adapter == (guint) self->adapter)
1109 update_device = TRUE;
1115 GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %"
1116 GST_PTR_FORMAT, self->device, dmem->device);
1118 gst_object_unref (self->device);
1119 self->device = (GstD3D11Device *) gst_object_ref (dmem->device);
1122 static GstFlowReturn
1123 gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
1125 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
1126 GstFlowReturn ret = GST_FLOW_OK;
1128 gst_d3d11_video_sink_check_device_update (self, buf);
1130 if (self->caps_updated || !self->window) {
1131 GstCaps *caps = gst_pad_get_current_caps (GST_BASE_SINK_PAD (sink));
1132 gboolean update_ret;
1134 /* shouldn't happen */
1136 return GST_FLOW_NOT_NEGOTIATED;
1138 update_ret = gst_d3d11_video_sink_update_window (self, caps);
1139 gst_caps_unref (caps);
1142 return GST_FLOW_NOT_NEGOTIATED;
1145 gst_d3d11_window_show (self->window);
1147 if (self->draw_on_shared_texture) {
1148 GST_D3D11_VIDEO_SINK_LOCK (self);
1149 self->current_buffer = buf;
1150 self->drawing = TRUE;
1152 GST_LOG_OBJECT (self, "Begin drawing");
1154 /* Application should call draw method on this callback */
1155 g_signal_emit (self, gst_d3d11_video_sink_signals[SIGNAL_BEGIN_DRAW], 0,
1158 GST_LOG_OBJECT (self, "End drawing");
1159 self->drawing = FALSE;
1160 self->current_buffer = nullptr;
1161 GST_D3D11_VIDEO_SINK_UNLOCK (self);
1163 ret = gst_d3d11_window_render (self->window, buf);
1166 if (ret == GST_D3D11_WINDOW_FLOW_CLOSED) {
1167 GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND,
1168 ("Output window was closed"), (NULL));
1170 ret = GST_FLOW_ERROR;
1176 /* VideoOverlay interface */
1178 gst_d3d11_video_sink_set_window_handle (GstVideoOverlay * overlay,
1181 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (overlay);
1183 GST_DEBUG ("set window handle %" G_GUINTPTR_FORMAT, window_id);
1185 self->window_id = window_id;
1189 gst_d3d11_video_sink_set_render_rectangle (GstVideoOverlay * overlay, gint x,
1190 gint y, gint width, gint height)
1192 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (overlay);
1194 GST_DEBUG_OBJECT (self,
1195 "render rect x: %d, y: %d, width: %d, height %d", x, y, width, height);
1197 GST_D3D11_VIDEO_SINK_LOCK (self);
1199 GstVideoRectangle rect;
1206 self->render_rect = rect;
1208 gst_d3d11_window_set_render_rectangle (self->window, &rect);
1210 self->render_rect.x = x;
1211 self->render_rect.y = y;
1212 self->render_rect.w = width;
1213 self->render_rect.h = height;
1214 self->pending_render_rect = TRUE;
1217 GST_D3D11_VIDEO_SINK_UNLOCK (self);
1221 gst_d3d11_video_sink_expose (GstVideoOverlay * overlay)
1223 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (overlay);
1225 GST_D3D11_VIDEO_SINK_LOCK (self);
1226 if (self->window && self->window->swap_chain)
1227 gst_d3d11_window_render (self->window, nullptr);
1228 GST_D3D11_VIDEO_SINK_UNLOCK (self);
1232 gst_d3d11_video_sink_video_overlay_init (GstVideoOverlayInterface * iface)
1234 iface->set_window_handle = gst_d3d11_video_sink_set_window_handle;
1235 iface->set_render_rectangle = gst_d3d11_video_sink_set_render_rectangle;
1236 iface->expose = gst_d3d11_video_sink_expose;
1239 /* Navigation interface */
1241 gst_d3d11_video_sink_navigation_send_event (GstNavigation * navigation,
1244 GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (navigation);
1246 /* TODO: add support for translating native coordinate and video coordinate
1247 * when force-aspect-ratio is set */
1251 gst_event_ref (event);
1252 handled = gst_pad_push_event (GST_VIDEO_SINK_PAD (self), event);
1255 gst_element_post_message (GST_ELEMENT_CAST (self),
1256 gst_navigation_message_new_event (GST_OBJECT_CAST (self), event));
1258 gst_event_unref (event);
1263 gst_d3d11_video_sink_navigation_init (GstNavigationInterface * iface)
1265 iface->send_event_simple = gst_d3d11_video_sink_navigation_send_event;
1269 gst_d3d11_video_sink_draw_action (GstD3D11VideoSink * self,
1270 gpointer shared_handle, guint texture_misc_flags,
1271 guint64 acquire_key, guint64 release_key)
1274 g_return_val_if_fail (shared_handle != NULL, FALSE);
1276 if (!self->draw_on_shared_texture) {
1277 GST_ERROR_OBJECT (self, "Invalid draw call, we are drawing on window");
1281 if (!shared_handle) {
1282 GST_ERROR_OBJECT (self, "Invalid handle");
1286 GST_D3D11_VIDEO_SINK_LOCK (self);
1287 if (!self->drawing || !self->current_buffer) {
1288 GST_WARNING_OBJECT (self, "Nothing to draw");
1289 GST_D3D11_VIDEO_SINK_UNLOCK (self);
1293 GST_LOG_OBJECT (self, "Drawing on shared handle %p, MiscFlags: 0x%x"
1294 ", acquire key: %" G_GUINT64_FORMAT ", release key: %"
1295 G_GUINT64_FORMAT, shared_handle, texture_misc_flags, acquire_key,
1298 ret = gst_d3d11_window_render_on_shared_handle (self->window,
1299 self->current_buffer, shared_handle, texture_misc_flags, acquire_key,
1301 GST_D3D11_VIDEO_SINK_UNLOCK (self);
1303 return ret == GST_FLOW_OK;