From 957034d71aaa075cec327effcaa0696897d02120 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Tue, 27 Apr 2021 21:52:31 +0900 Subject: [PATCH] d3d11: Handle device change If incoming buffer holds other d3d11 device, and user wants any device (i.e., adapter index wasn't specified explicitly) update our device with that of buffer. Part-of: --- sys/d3d11/gstd3d11basefilter.cpp | 73 +++++++++++++++++++++ sys/d3d11/gstd3d11compositor.cpp | 135 +++++++++++++++++++++++++++++++++++--- sys/d3d11/gstd3d11deinterlace.cpp | 66 +++++++++++++++++++ sys/d3d11/gstd3d11videosink.cpp | 44 +++++++++++++ 4 files changed, 310 insertions(+), 8 deletions(-) diff --git a/sys/d3d11/gstd3d11basefilter.cpp b/sys/d3d11/gstd3d11basefilter.cpp index b9015ed..b80ad6c 100644 --- a/sys/d3d11/gstd3d11basefilter.cpp +++ b/sys/d3d11/gstd3d11basefilter.cpp @@ -55,6 +55,8 @@ static gboolean gst_d3d11_base_filter_get_unit_size (GstBaseTransform * trans, static gboolean gst_d3d11_base_filter_query (GstBaseTransform * trans, GstPadDirection direction, GstQuery * query); +static void gst_d3d11_base_filter_before_transform (GstBaseTransform * trans, + GstBuffer * buffer); static void gst_d3d11_base_filter_class_init (GstD3D11BaseFilterClass * klass) @@ -92,6 +94,8 @@ gst_d3d11_base_filter_class_init (GstD3D11BaseFilterClass * klass) trans_class->get_unit_size = GST_DEBUG_FUNCPTR (gst_d3d11_base_filter_get_unit_size); trans_class->query = GST_DEBUG_FUNCPTR (gst_d3d11_base_filter_query); + trans_class->before_transform = + GST_DEBUG_FUNCPTR (gst_d3d11_base_filter_before_transform); gst_type_mark_as_plugin_api (GST_TYPE_D3D11_BASE_FILTER, (GstPluginAPIFlags) 0); @@ -260,3 +264,72 @@ gst_d3d11_base_filter_query (GstBaseTransform * trans, return GST_BASE_TRANSFORM_CLASS (parent_class)->query (trans, direction, query); } + +static void +gst_d3d11_base_filter_before_transform (GstBaseTransform * trans, + GstBuffer * buffer) +{ + GstD3D11BaseFilter *self = GST_D3D11_BASE_FILTER (trans); + GstD3D11Memory *dmem; + GstMemory *mem; + gboolean update_device = FALSE; + GstCaps *in_caps = NULL; + GstCaps *out_caps = NULL; + + mem = gst_buffer_peek_memory (buffer, 0); + /* Can happens (e.g., d3d11upload) */ + if (!gst_is_d3d11_memory (mem)) + return; + + dmem = GST_D3D11_MEMORY_CAST (mem); + /* Same device, nothing to do */ + if (dmem->device == self->device) + return; + + /* Can accept any device, update */ + if (self->adapter < 0) { + update_device = TRUE; + } else { + guint adapter = 0; + + g_object_get (dmem->device, "adapter", &adapter, NULL); + /* The same GPU as what user wanted, update */ + if (adapter == (guint) self->adapter) + update_device = TRUE; + } + + if (!update_device) + return; + + GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %" + GST_PTR_FORMAT, self->device, dmem->device); + + gst_object_unref (self->device); + self->device = (GstD3D11Device *) gst_object_ref (dmem->device); + + in_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM_SINK_PAD (trans)); + if (!in_caps) { + GST_WARNING_OBJECT (self, "sinkpad has null caps"); + goto out; + } + + out_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM_SRC_PAD (trans)); + if (!out_caps) { + GST_WARNING_OBJECT (self, "Has no configured output caps"); + goto out; + } + + /* subclass will update internal object. + * Note that gst_base_transform_reconfigure() might not trigger this + * unless caps was changed meanwhile */ + gst_d3d11_base_filter_set_caps (trans, in_caps, out_caps); + + /* Mark reconfigure so that we can update pool */ + gst_base_transform_reconfigure_src (trans); + +out: + gst_clear_caps (&in_caps); + gst_clear_caps (&out_caps); + + return; +} diff --git a/sys/d3d11/gstd3d11compositor.cpp b/sys/d3d11/gstd3d11compositor.cpp index 7852598..717c10d 100644 --- a/sys/d3d11/gstd3d11compositor.cpp +++ b/sys/d3d11/gstd3d11compositor.cpp @@ -1312,10 +1312,12 @@ static gboolean gst_d3d11_compositor_decide_allocation (GstAggregator * aggregator, GstQuery * query); static gboolean gst_d3d11_compositor_sink_event (GstAggregator * agg, GstAggregatorPad * pad, GstEvent * event); - static GstFlowReturn gst_d3d11_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf); +static GstFlowReturn +gst_d3d11_compositor_create_output_buffer (GstVideoAggregator * vagg, + GstBuffer ** outbuffer); #define gst_d3d11_compositor_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstD3D11Compositor, gst_d3d11_compositor, @@ -1372,6 +1374,8 @@ gst_d3d11_compositor_class_init (GstD3D11CompositorClass * klass) vagg_class->aggregate_frames = GST_DEBUG_FUNCPTR (gst_d3d11_compositor_aggregate_frames); + vagg_class->create_output_buffer = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_create_output_buffer); caps = gst_d3d11_get_updated_template_caps (&pad_template_caps); gst_element_class_add_pad_template (element_class, @@ -1519,6 +1523,21 @@ could_not_create: } } +static gboolean +gst_d3d11_compositor_pad_clear_resource (GstD3D11Compositor * self, + GstD3D11CompositorPad * cpad, gpointer user_data) +{ + gst_clear_buffer (&cpad->fallback_buf); + if (cpad->fallback_pool) { + gst_buffer_pool_set_active (cpad->fallback_pool, FALSE); + gst_clear_object (&cpad->fallback_pool); + } + g_clear_pointer (&cpad->convert, gst_d3d11_converter_free); + GST_D3D11_CLEAR_COM (cpad->blend); + + return TRUE; +} + static void gst_d3d11_compositor_release_pad (GstElement * element, GstPad * pad) { @@ -1530,13 +1549,7 @@ gst_d3d11_compositor_release_pad (GstElement * element, GstPad * pad) gst_child_proxy_child_removed (GST_CHILD_PROXY (self), G_OBJECT (pad), GST_OBJECT_NAME (pad)); - gst_clear_buffer (&cpad->fallback_buf); - if (cpad->fallback_pool) { - gst_buffer_pool_set_active (cpad->fallback_pool, FALSE); - gst_clear_object (&cpad->fallback_pool); - } - g_clear_pointer (&cpad->convert, gst_d3d11_converter_free); - GST_D3D11_CLEAR_COM (cpad->blend); + gst_d3d11_compositor_pad_clear_resource (self, cpad, NULL); GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad); } @@ -2255,3 +2268,109 @@ done: return ret; } + +typedef struct +{ + /* without holding ref */ + GstD3D11Device *other_device; + gboolean have_same_device; +} DeviceCheckData; + +static gboolean +gst_d3d11_compositor_check_device_update (GstElement * agg, + GstVideoAggregatorPad * vpad, DeviceCheckData * data) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (agg); + GstBuffer *buf; + GstMemory *mem; + GstD3D11Memory *dmem; + gboolean update_device = FALSE; + + buf = gst_video_aggregator_pad_get_current_buffer (vpad); + if (!buf) + return TRUE; + + mem = gst_buffer_peek_memory (buf, 0); + /* FIXME: we should be able to accept non-d3d11 memory later once + * we remove intermediate elements (d3d11upload and d3d11colorconvert) + */ + if (!gst_is_d3d11_memory (mem)) { + GST_ELEMENT_ERROR (agg, CORE, FAILED, (NULL), ("Invalid memory")); + return FALSE; + } + + dmem = GST_D3D11_MEMORY_CAST (mem); + + /* We can use existing device */ + if (dmem->device == self->device) { + data->have_same_device = TRUE; + return FALSE; + } + + if (self->adapter < 0) { + update_device = TRUE; + } else { + guint adapter = 0; + + g_object_get (dmem->device, "adapter", &adapter, NULL); + /* The same GPU as what user wanted, update */ + if (adapter == (guint) self->adapter) + update_device = TRUE; + } + + if (!update_device) + return TRUE; + + data->other_device = dmem->device; + + /* Keep iterate since there might be one buffer which holds the same device + * as ours */ + return TRUE; +} + +static GstFlowReturn +gst_d3d11_compositor_create_output_buffer (GstVideoAggregator * vagg, + GstBuffer ** outbuffer) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg); + DeviceCheckData data; + + /* Check whether there is at least one sinkpad which holds d3d11 buffer + * with compatible device, and if not, update our device */ + data.other_device = NULL; + data.have_same_device = FALSE; + + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), + (GstElementForeachPadFunc) gst_d3d11_compositor_check_device_update, + &data); + + if (data.have_same_device || !data.other_device) + goto done; + + /* Clear all device dependent resources */ + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), + (GstElementForeachPadFunc) gst_d3d11_compositor_pad_clear_resource, NULL); + + gst_clear_buffer (&self->fallback_buf); + if (self->fallback_pool) { + gst_buffer_pool_set_active (self->fallback_pool, FALSE); + gst_clear_object (&self->fallback_pool); + } + g_clear_pointer (&self->checker_background, gst_d3d11_quad_free); + + GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %" + GST_PTR_FORMAT, self->device, data.other_device); + gst_object_unref (self->device); + self->device = (GstD3D11Device *) gst_object_ref (data.other_device); + + /* We cannot call gst_aggregator_negotiate() here, since GstVideoAggregator + * is holding GST_VIDEO_AGGREGATOR_LOCK() already. + * Mark reconfigure and do reconfigure later */ + gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); + + return GST_AGGREGATOR_FLOW_NEED_DATA; + +done: + return GST_VIDEO_AGGREGATOR_CLASS (parent_class)->create_output_buffer (vagg, + outbuffer); +} diff --git a/sys/d3d11/gstd3d11deinterlace.cpp b/sys/d3d11/gstd3d11deinterlace.cpp index f1315c3..8d49cb0 100644 --- a/sys/d3d11/gstd3d11deinterlace.cpp +++ b/sys/d3d11/gstd3d11deinterlace.cpp @@ -328,6 +328,8 @@ gst_d3d11_deinterlace_transform (GstBaseTransform * trans, GstBuffer * inbuf, GstBuffer * outbuf); static gboolean gst_d3d11_deinterlace_sink_event (GstBaseTransform * trans, GstEvent * event); +static void gst_d3d11_deinterlace_before_transform (GstBaseTransform * trans, + GstBuffer * buffer); static void gst_d3d11_deinterlace_class_init (GstD3D11DeinterlaceClass * klass, @@ -413,6 +415,8 @@ gst_d3d11_deinterlace_class_init (GstD3D11DeinterlaceClass * klass, trans_class->transform = GST_DEBUG_FUNCPTR (gst_d3d11_deinterlace_transform); trans_class->sink_event = GST_DEBUG_FUNCPTR (gst_d3d11_deinterlace_sink_event); + trans_class->before_transform = + GST_DEBUG_FUNCPTR (gst_d3d11_deinterlace_before_transform); klass->adapter = cdata->adapter; klass->device_id = cdata->device_id; @@ -1829,6 +1833,68 @@ gst_d3d11_deinterlace_sink_event (GstBaseTransform * trans, GstEvent * event) return GST_BASE_TRANSFORM_CLASS (parent_class)->sink_event (trans, event); } +static void +gst_d3d11_deinterlace_before_transform (GstBaseTransform * trans, + GstBuffer * buffer) +{ + GstD3D11Deinterlace *self = GST_D3D11_DEINTERLACE (trans); + GstD3D11DeinterlaceClass *klass = GST_D3D11_DEINTERLACE_GET_CLASS (self); + GstD3D11Memory *dmem; + GstMemory *mem; + GstCaps *in_caps = NULL; + GstCaps *out_caps = NULL; + guint adapter = 0; + + mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_d3d11_memory (mem)) { + GST_ELEMENT_ERROR (self, CORE, FAILED, (NULL), ("Invalid memory")); + return; + } + + dmem = GST_D3D11_MEMORY_CAST (mem); + /* Same device, nothing to do */ + if (dmem->device == self->device) + return; + + g_object_get (dmem->device, "adapter", &adapter, NULL); + /* We have per-GPU deinterlace elements because of different capability + * per GPU. so, cannot accept other GPU at the moment */ + if (adapter != klass->adapter) + return; + + GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %" + GST_PTR_FORMAT, self->device, dmem->device); + + /* Drain buffers before updating device */ + gst_d3d11_deinterlace_drain (self); + + gst_object_unref (self->device); + self->device = (GstD3D11Device *) gst_object_ref (dmem->device); + + in_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM_SINK_PAD (trans)); + if (!in_caps) { + GST_WARNING_OBJECT (self, "sinkpad has null caps"); + goto out; + } + + out_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM_SRC_PAD (trans)); + if (!out_caps) { + GST_WARNING_OBJECT (self, "Has no configured output caps"); + goto out; + } + + gst_d3d11_deinterlace_set_caps (trans, in_caps, out_caps); + + /* Mark reconfigure so that we can update pool */ + gst_base_transform_reconfigure_src (trans); + +out: + gst_clear_caps (&in_caps); + gst_clear_caps (&out_caps); + + return; +} + /* FIXME: might be job of basetransform */ static GstFlowReturn gst_d3d11_deinterlace_drain (GstD3D11Deinterlace * self) diff --git a/sys/d3d11/gstd3d11videosink.cpp b/sys/d3d11/gstd3d11videosink.cpp index 005fa1b..07da3ee 100644 --- a/sys/d3d11/gstd3d11videosink.cpp +++ b/sys/d3d11/gstd3d11videosink.cpp @@ -1049,6 +1049,48 @@ error: return FALSE; } +static void +gst_d3d11_video_sink_check_device_update (GstD3D11VideoSink * self, + GstBuffer * buf) +{ + GstMemory *mem; + GstD3D11Memory *dmem; + gboolean update_device = FALSE; + + /* We have configured window already, cannot update device */ + if (self->window) + return; + + mem = gst_buffer_peek_memory (buf, 0); + if (!gst_is_d3d11_memory (mem)) + return; + + dmem = GST_D3D11_MEMORY_CAST (mem); + /* Same device, nothing to do */ + if (dmem->device == self->device) + return; + + if (self->adapter < 0) { + update_device = TRUE; + } else { + guint adapter = 0; + + g_object_get (dmem->device, "adapter", &adapter, NULL); + /* The same GPU as what user wanted, update */ + if (adapter == (guint) self->adapter) + update_device = TRUE; + } + + if (!update_device) + return; + + GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %" + GST_PTR_FORMAT, self->device, dmem->device); + + gst_object_unref (self->device); + self->device = (GstD3D11Device *) gst_object_ref (dmem->device); +} + static GstFlowReturn gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf) { @@ -1060,6 +1102,8 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf) gst_d3d11_device_get_device_handle (self->device); ID3D11ShaderResourceView *view[GST_VIDEO_MAX_PLANES]; + gst_d3d11_video_sink_check_device_update (self, buf); + if (self->caps_updated || !self->window) { GstCaps *caps = gst_pad_get_current_caps (GST_BASE_SINK_PAD (sink)); gboolean update_ret; -- 2.7.4