From 487a41d3124efa8522e92791c7f5e8048b322d22 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Tue, 24 Dec 2019 15:54:57 +0900 Subject: [PATCH] d3d11videosink: Add support for overlay composition Add d3d11overlaycompositor object to draw overlay image on render target using Blend method. --- sys/d3d11/gstd3d11colorconvert.c | 14 +- sys/d3d11/gstd3d11colorconverter.c | 27 +- sys/d3d11/gstd3d11colorconverter.h | 4 + sys/d3d11/gstd3d11download.c | 16 +- sys/d3d11/gstd3d11overlaycompositor.c | 666 ++++++++++++++++++++++++++++++++++ sys/d3d11/gstd3d11overlaycompositor.h | 52 +++ sys/d3d11/gstd3d11shader.c | 32 +- sys/d3d11/gstd3d11shader.h | 8 +- sys/d3d11/gstd3d11upload.c | 77 +++- sys/d3d11/gstd3d11videosink.c | 29 +- sys/d3d11/gstd3d11videosinkbin.c | 12 +- sys/d3d11/gstd3d11window.c | 48 ++- sys/d3d11/gstd3d11window.h | 3 + sys/d3d11/meson.build | 1 + sys/d3d11/plugin.c | 4 + 15 files changed, 952 insertions(+), 41 deletions(-) create mode 100644 sys/d3d11/gstd3d11overlaycompositor.c create mode 100644 sys/d3d11/gstd3d11overlaycompositor.h diff --git a/sys/d3d11/gstd3d11colorconvert.c b/sys/d3d11/gstd3d11colorconvert.c index f163d31..92c4ba5 100644 --- a/sys/d3d11/gstd3d11colorconvert.c +++ b/sys/d3d11/gstd3d11colorconvert.c @@ -55,14 +55,22 @@ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS)) + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS)) ); static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS)) + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS)) ); #define gst_d3d11_color_convert_parent_class parent_class @@ -361,6 +369,8 @@ gst_d3d11_color_convert_propose_allocation (GstBaseTransform * trans, goto config_failed; gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + gst_query_add_allocation_meta (query, + GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL); size = GST_D3D11_BUFFER_POOL (pool)->buffer_size; gst_query_add_allocation_pool (query, pool, size, 0, 0); diff --git a/sys/d3d11/gstd3d11colorconverter.c b/sys/d3d11/gstd3d11colorconverter.c index 3d330ed..a7ff96e 100644 --- a/sys/d3d11/gstd3d11colorconverter.c +++ b/sys/d3d11/gstd3d11colorconverter.c @@ -752,8 +752,8 @@ gst_d3d11_color_convert_setup_shader (GstD3D11Device * device, gst_d3d11_device_unlock (device); self->quad = gst_d3d11_quad_new (device, - ps, vs, layout, sampler, const_buffer, vertex_buffer, sizeof (VertexData), - index_buffer, DXGI_FORMAT_R16_UINT, index_count); + ps, vs, layout, sampler, NULL, NULL, const_buffer, vertex_buffer, + sizeof (VertexData), index_buffer, DXGI_FORMAT_R16_UINT, index_count); self->num_input_view = GST_VIDEO_INFO_N_PLANES (data->in_info); self->num_output_view = GST_VIDEO_INFO_N_PLANES (data->out_info); @@ -908,12 +908,30 @@ gst_d3d11_color_converter_convert (GstD3D11ColorConverter * converter, ID3D11ShaderResourceView * srv[GST_VIDEO_MAX_PLANES], ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES]) { + gboolean ret; + + g_return_val_if_fail (converter != NULL, FALSE); + g_return_val_if_fail (srv != NULL, FALSE); + g_return_val_if_fail (rtv != NULL, FALSE); + + gst_d3d11_device_lock (converter->device); + ret = gst_d3d11_color_converter_convert_unlocked (converter, srv, rtv); + gst_d3d11_device_lock (converter->device); + + return ret; +} + +gboolean +gst_d3d11_color_converter_convert_unlocked (GstD3D11ColorConverter * converter, + ID3D11ShaderResourceView * srv[GST_VIDEO_MAX_PLANES], + ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES]) +{ g_return_val_if_fail (converter != NULL, FALSE); g_return_val_if_fail (srv != NULL, FALSE); g_return_val_if_fail (rtv != NULL, FALSE); - return gst_d3d11_draw_quad (converter->quad, &converter->viewport, 1, - srv, converter->num_input_view, rtv, converter->num_output_view); + return gst_d3d11_draw_quad_unlocked (converter->quad, &converter->viewport, 1, + srv, converter->num_input_view, rtv, converter->num_output_view, NULL); } gboolean @@ -921,6 +939,7 @@ gst_d3d11_color_converter_update_rect (GstD3D11ColorConverter * converter, RECT * rect) { g_return_val_if_fail (converter != NULL, FALSE); + g_return_val_if_fail (rect != NULL, FALSE); converter->viewport.TopLeftX = rect->left; converter->viewport.TopLeftY = rect->top; diff --git a/sys/d3d11/gstd3d11colorconverter.h b/sys/d3d11/gstd3d11colorconverter.h index ad9fde9..544146e 100644 --- a/sys/d3d11/gstd3d11colorconverter.h +++ b/sys/d3d11/gstd3d11colorconverter.h @@ -38,6 +38,10 @@ gboolean gst_d3d11_color_converter_convert (GstD3D11ColorConvert ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES], ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES]); +gboolean gst_d3d11_color_converter_convert_unlocked (GstD3D11ColorConverter * converter, + ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES], + ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES]); + gboolean gst_d3d11_color_converter_update_rect (GstD3D11ColorConverter * converter, RECT *rect); diff --git a/sys/d3d11/gstd3d11download.c b/sys/d3d11/gstd3d11download.c index 1f612cb..05c0a62 100644 --- a/sys/d3d11/gstd3d11download.c +++ b/sys/d3d11/gstd3d11download.c @@ -34,13 +34,25 @@ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS) "; " - GST_VIDEO_CAPS_MAKE (GST_D3D11_FORMATS) + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE (GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS) )); static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_D3D11_FORMATS) + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_D3D11_FORMATS) ";" + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS) )); #define gst_d3d11_download_parent_class parent_class diff --git a/sys/d3d11/gstd3d11overlaycompositor.c b/sys/d3d11/gstd3d11overlaycompositor.c new file mode 100644 index 0000000..a90e319 --- /dev/null +++ b/sys/d3d11/gstd3d11overlaycompositor.c @@ -0,0 +1,666 @@ +/* GStreamer + * Copyright (C) <2019> Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#include "gstd3d11overlaycompositor.h" +#include "gstd3d11utils.h" +#include "gstd3d11device.h" +#include "gstd3d11shader.h" +#include "gstd3d11format.h" + +GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_overlay_compositor_debug); +#define GST_CAT_DEFAULT gst_d3d11_overlay_compositor_debug + +/* *INDENT-OFF* */ +typedef struct +{ + struct { + FLOAT x; + FLOAT y; + FLOAT z; + } position; + struct { + FLOAT x; + FLOAT y; + } texture; +} VertexData; + +static const gchar templ_pixel_shader[] = + "Texture2D shaderTexture;\n" + "SamplerState samplerState;\n" + "\n" + "struct PS_INPUT\n" + "{\n" + " float4 Position: SV_POSITION;\n" + " float3 Texture: TEXCOORD0;\n" + "};\n" + "\n" + "float4 main(PS_INPUT input): SV_TARGET\n" + "{\n" + " return shaderTexture.Sample(samplerState, input.Texture);\n" + "}\n"; + +static const gchar templ_vertex_shader[] = + "struct VS_INPUT\n" + "{\n" + " float4 Position : POSITION;\n" + " float4 Texture : TEXCOORD0;\n" + "};\n" + "\n" + "struct VS_OUTPUT\n" + "{\n" + " float4 Position: SV_POSITION;\n" + " float4 Texture: TEXCOORD0;\n" + "};\n" + "\n" + "VS_OUTPUT main(VS_INPUT input)\n" + "{\n" + " return input;\n" + "}\n"; +/* *INDENT-ON* */ + +struct _GstD3D11OverlayCompositor +{ + GstD3D11Device *device; + GstVideoInfo out_info; + + D3D11_VIEWPORT viewport; + + ID3D11PixelShader *ps; + ID3D11VertexShader *vs; + ID3D11InputLayout *layout; + ID3D11SamplerState *sampler; + ID3D11BlendState *blend; + ID3D11Buffer *index_buffer; + + /* GstD3D11CompositionOverlay */ + GList *overlays; +}; + +typedef struct +{ + GstVideoOverlayRectangle *overlay_rect; + ID3D11Texture2D *texture; + ID3D11ShaderResourceView *srv; + GstD3D11Quad *quad; +} GstD3D11CompositionOverlay; + +static GstD3D11CompositionOverlay * +gst_d3d11_composition_overlay_new (GstD3D11OverlayCompositor * self, + GstVideoOverlayRectangle * overlay_rect) +{ + GstD3D11CompositionOverlay *overlay = NULL; + gint x, y; + guint width, height; + D3D11_SUBRESOURCE_DATA subresource_data = { 0, }; + D3D11_TEXTURE2D_DESC texture_desc = { 0, }; + D3D11_SHADER_RESOURCE_VIEW_DESC srv_desc = { 0, }; + D3D11_BUFFER_DESC buffer_desc = { 0, }; + ID3D11Buffer *vertex_buffer = NULL; + D3D11_MAPPED_SUBRESOURCE map; + VertexData *vertex_data; + GstBuffer *buf; + GstVideoMeta *vmeta; + GstMapInfo info; + guint8 *data; + gint stride; + ID3D11Texture2D *texture = NULL; + ID3D11ShaderResourceView *srv = NULL; + HRESULT hr; + ID3D11Device *device_handle; + ID3D11DeviceContext *context_handle; + GstD3D11Device *device = self->device; + const guint index_count = 2 * 3; + FLOAT x1, y1, x2, y2; + + g_return_val_if_fail (overlay_rect != NULL, NULL); + + device_handle = gst_d3d11_device_get_device_handle (device); + context_handle = gst_d3d11_device_get_device_context_handle (device); + + if (!gst_video_overlay_rectangle_get_render_rectangle (overlay_rect, &x, &y, + &width, &height)) { + GST_ERROR ("Failed to get render rectangle"); + return NULL; + } + + buf = gst_video_overlay_rectangle_get_pixels_unscaled_argb (overlay_rect, + GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE); + if (!buf) { + GST_ERROR ("Failed to get overlay buffer"); + return NULL; + } + + vmeta = gst_buffer_get_video_meta (buf); + if (!vmeta) { + GST_ERROR ("Failed to get video meta"); + return NULL; + } + + if (!gst_video_meta_map (vmeta, + 0, &info, (gpointer *) & data, &stride, GST_MAP_READ)) { + GST_ERROR ("Failed to map"); + return NULL; + } + + /* Do create texture and upload data at once, for create immutable texture */ + subresource_data.pSysMem = data; + subresource_data.SysMemPitch = stride; + subresource_data.SysMemSlicePitch = 0; + + texture_desc.Width = width; + texture_desc.Height = height; + texture_desc.MipLevels = 1; + texture_desc.ArraySize = 1; + /* FIXME: need to consider non-BGRA ? */ + texture_desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + texture_desc.SampleDesc.Count = 1; + texture_desc.SampleDesc.Quality = 0; + texture_desc.Usage = D3D11_USAGE_IMMUTABLE; + texture_desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + texture_desc.CPUAccessFlags = 0; + + texture = gst_d3d11_device_create_texture (device, + &texture_desc, &subresource_data); + gst_video_meta_unmap (vmeta, 0, &info); + + if (!texture) { + GST_ERROR ("Failed to create texture"); + return NULL; + } + + srv_desc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + srv_desc.Texture2D.MipLevels = 1; + + hr = ID3D11Device_CreateShaderResourceView (device_handle, + (ID3D11Resource *) texture, &srv_desc, &srv); + if (!gst_d3d11_result (hr, device) || !srv) { + GST_ERROR ("Failed to create shader resource view"); + goto clear; + } + + buffer_desc.Usage = D3D11_USAGE_DYNAMIC; + buffer_desc.ByteWidth = sizeof (VertexData) * 4; + buffer_desc.BindFlags = D3D11_BIND_VERTEX_BUFFER; + buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + + hr = ID3D11Device_CreateBuffer (device_handle, &buffer_desc, NULL, + &vertex_buffer); + if (!gst_d3d11_result (hr, device)) { + GST_ERROR ("Couldn't create vertex buffer, hr: 0x%x", (guint) hr); + goto clear; + } + + gst_d3d11_device_lock (device); + hr = ID3D11DeviceContext_Map (context_handle, + (ID3D11Resource *) vertex_buffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &map); + + if (!gst_d3d11_result (hr, device)) { + GST_ERROR ("Couldn't map vertex buffer, hr: 0x%x", (guint) hr); + gst_d3d11_device_unlock (device); + goto clear; + } + + vertex_data = (VertexData *) map.pData; + x1 = (x / (gfloat) GST_VIDEO_INFO_WIDTH (&self->out_info)) * 2.0f - 1.0f; + y1 = (y / (gfloat) GST_VIDEO_INFO_HEIGHT (&self->out_info)) * 2.0f - 1.0f; + + x2 = ((x + width) / + (gfloat) GST_VIDEO_INFO_WIDTH (&self->out_info)) * 2.0f - 1.0f; + y2 = ((y + height) / + (gfloat) GST_VIDEO_INFO_HEIGHT (&self->out_info)) * 2.0f - 1.0f; + + vertex_data[0].position.x = x1; + vertex_data[0].position.y = y1; + vertex_data[0].position.z = 0.0f; + vertex_data[0].texture.x = 0.0f; + vertex_data[0].texture.y = 1.0f; + + vertex_data[1].position.x = x1; + vertex_data[1].position.y = y2; + vertex_data[1].position.z = 0.0f; + vertex_data[1].texture.x = 0.0f; + vertex_data[1].texture.y = 0.0f; + + vertex_data[2].position.x = x2; + vertex_data[2].position.y = y2; + vertex_data[2].position.z = 0.0f; + vertex_data[2].texture.x = 1.0f; + vertex_data[2].texture.y = 0.0f; + + vertex_data[3].position.x = x2; + vertex_data[3].position.y = y1; + vertex_data[3].position.z = 0.0f; + vertex_data[3].texture.x = 1.0f; + vertex_data[3].texture.y = 1.0f; + + ID3D11DeviceContext_Unmap (context_handle, + (ID3D11Resource *) vertex_buffer, 0); + gst_d3d11_device_unlock (device); + + overlay = g_new0 (GstD3D11CompositionOverlay, 1); + overlay->overlay_rect = gst_video_overlay_rectangle_ref (overlay_rect); + overlay->texture = texture; + overlay->srv = srv; + overlay->quad = gst_d3d11_quad_new (device, + self->ps, self->vs, self->layout, self->sampler, self->blend, NULL, NULL, + vertex_buffer, sizeof (VertexData), + self->index_buffer, DXGI_FORMAT_R16_UINT, index_count); + +clear: + if (!overlay) { + if (srv) + ID3D11ShaderResourceView_Release (srv); + if (texture) + ID3D11Texture2D_Release (texture); + } + + if (vertex_buffer) + ID3D11Buffer_Release (vertex_buffer); + + return overlay; +} + +static void +gst_d3d11_composition_overlay_free (GstD3D11CompositionOverlay * overlay) +{ + if (!overlay) + return; + + if (overlay->overlay_rect) + gst_video_overlay_rectangle_unref (overlay->overlay_rect); + + if (overlay->srv) + ID3D11ShaderResourceView_Release (overlay->srv); + + if (overlay->texture) + ID3D11Texture2D_Release (overlay->texture); + + if (overlay->quad) + gst_d3d11_quad_free (overlay->quad); + + g_free (overlay); +} + +static gboolean +gst_d3d11_overlay_compositor_setup_shader (GstD3D11OverlayCompositor * self, + GstD3D11Device * device) +{ + HRESULT hr; + D3D11_SAMPLER_DESC sampler_desc = { 0, }; + D3D11_INPUT_ELEMENT_DESC input_desc[2] = { 0, }; + D3D11_BUFFER_DESC buffer_desc = { 0, }; + D3D11_BLEND_DESC blend_desc = { 0, }; + D3D11_MAPPED_SUBRESOURCE map; + WORD *indices; + ID3D11Device *device_handle; + ID3D11DeviceContext *context_handle; + ID3D11PixelShader *ps = NULL; + ID3D11VertexShader *vs = NULL; + ID3D11InputLayout *layout = NULL; + ID3D11SamplerState *sampler = NULL; + ID3D11BlendState *blend = NULL; + ID3D11Buffer *index_buffer = NULL; + const guint index_count = 2 * 3; + gboolean ret = TRUE; + + device_handle = gst_d3d11_device_get_device_handle (device); + context_handle = gst_d3d11_device_get_device_context_handle (device); + + /* bilinear filtering */ + sampler_desc.Filter = D3D11_FILTER_MIN_MAG_LINEAR_MIP_POINT; + sampler_desc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.ComparisonFunc = D3D11_COMPARISON_ALWAYS; + sampler_desc.MinLOD = 0; + sampler_desc.MaxLOD = D3D11_FLOAT32_MAX; + + hr = ID3D11Device_CreateSamplerState (device_handle, &sampler_desc, &sampler); + if (!gst_d3d11_result (hr, device)) { + GST_ERROR ("Couldn't create sampler state, hr: 0x%x", (guint) hr); + ret = FALSE; + goto clear; + } + + GST_LOG ("Create Pixel Shader \n%s", templ_pixel_shader); + + if (!gst_d3d11_create_pixel_shader (device, templ_pixel_shader, &ps)) { + GST_ERROR ("Couldn't create pixel shader"); + ret = FALSE; + goto clear; + } + + input_desc[0].SemanticName = "POSITION"; + input_desc[0].SemanticIndex = 0; + input_desc[0].Format = DXGI_FORMAT_R32G32B32_FLOAT; + input_desc[0].InputSlot = 0; + input_desc[0].AlignedByteOffset = D3D11_APPEND_ALIGNED_ELEMENT; + input_desc[0].InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA; + input_desc[0].InstanceDataStepRate = 0; + + input_desc[1].SemanticName = "TEXCOORD"; + input_desc[1].SemanticIndex = 0; + input_desc[1].Format = DXGI_FORMAT_R32G32_FLOAT; + input_desc[1].InputSlot = 0; + input_desc[1].AlignedByteOffset = D3D11_APPEND_ALIGNED_ELEMENT; + input_desc[1].InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA; + input_desc[1].InstanceDataStepRate = 0; + + if (!gst_d3d11_create_vertex_shader (device, templ_vertex_shader, + input_desc, G_N_ELEMENTS (input_desc), &vs, &layout)) { + GST_ERROR ("Couldn't vertex pixel shader"); + ret = FALSE; + goto clear; + } + + blend_desc.AlphaToCoverageEnable = FALSE; + blend_desc.IndependentBlendEnable = FALSE; + blend_desc.RenderTarget[0].BlendEnable = TRUE; + blend_desc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA; + blend_desc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA; + blend_desc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD; + blend_desc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE; + blend_desc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_ZERO; + blend_desc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD; + blend_desc.RenderTarget[0].RenderTargetWriteMask = + D3D11_COLOR_WRITE_ENABLE_ALL; + + hr = ID3D11Device_CreateBlendState (device_handle, &blend_desc, &blend); + if (!gst_d3d11_result (hr, device)) { + GST_ERROR ("Couldn't create blend staten, hr: 0x%x", (guint) hr); + ret = FALSE; + goto clear; + } + + buffer_desc.Usage = D3D11_USAGE_DYNAMIC; + buffer_desc.ByteWidth = sizeof (WORD) * index_count; + buffer_desc.BindFlags = D3D11_BIND_INDEX_BUFFER; + buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + + hr = ID3D11Device_CreateBuffer (device_handle, &buffer_desc, NULL, + &index_buffer); + if (!gst_d3d11_result (hr, device)) { + GST_ERROR ("Couldn't create index buffer, hr: 0x%x", (guint) hr); + ret = FALSE; + goto clear; + } + + gst_d3d11_device_lock (device); + hr = ID3D11DeviceContext_Map (context_handle, + (ID3D11Resource *) index_buffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &map); + + if (!gst_d3d11_result (hr, device)) { + GST_ERROR ("Couldn't map index buffer, hr: 0x%x", (guint) hr); + gst_d3d11_device_unlock (device); + ret = FALSE; + goto clear; + } + + indices = (WORD *) map.pData; + + /* clockwise indexing */ + indices[0] = 0; /* bottom left */ + indices[1] = 1; /* top left */ + indices[2] = 2; /* top right */ + + indices[3] = 3; /* bottom right */ + indices[4] = 0; /* bottom left */ + indices[5] = 2; /* top right */ + + ID3D11DeviceContext_Unmap (context_handle, + (ID3D11Resource *) index_buffer, 0); + gst_d3d11_device_unlock (device); + + self->ps = ps; + self->vs = vs; + self->layout = layout; + self->sampler = sampler; + self->blend = blend; + self->index_buffer = index_buffer; + +clear: + if (ret) + return TRUE; + + if (ps) + ID3D11PixelShader_Release (ps); + if (vs) + ID3D11VertexShader_Release (vs); + if (layout) + ID3D11InputLayout_Release (layout); + if (sampler) + ID3D11SamplerState_Release (sampler); + if (blend) + ID3D11BlendState_Release (blend); + if (index_buffer) + ID3D11Buffer_Release (index_buffer); + + return FALSE; +} + + +GstD3D11OverlayCompositor * +gst_d3d11_overlay_compositor_new (GstD3D11Device * device, + GstVideoInfo * out_info) +{ + GstD3D11OverlayCompositor *compositor = NULL; + + g_return_val_if_fail (GST_IS_D3D11_DEVICE (device), NULL); + g_return_val_if_fail (out_info != NULL, NULL); + + compositor = g_new0 (GstD3D11OverlayCompositor, 1); + + if (!gst_d3d11_overlay_compositor_setup_shader (compositor, device)) { + gst_d3d11_overlay_compositor_free (compositor); + return NULL; + } + + compositor->device = gst_object_ref (device); + compositor->out_info = *out_info; + + compositor->viewport.TopLeftX = 0; + compositor->viewport.TopLeftY = 0; + compositor->viewport.Width = GST_VIDEO_INFO_WIDTH (out_info); + compositor->viewport.Height = GST_VIDEO_INFO_HEIGHT (out_info); + compositor->viewport.MinDepth = 0.0f; + compositor->viewport.MaxDepth = 1.0f; + + return compositor; +} + +void +gst_d3d11_overlay_compositor_free (GstD3D11OverlayCompositor * compositor) +{ + g_return_if_fail (compositor != NULL); + + gst_d3d11_overlay_compositor_free_overlays (compositor); + + if (compositor->ps) + ID3D11PixelShader_Release (compositor->ps); + if (compositor->vs) + ID3D11VertexShader_Release (compositor->vs); + if (compositor->layout) + ID3D11InputLayout_Release (compositor->layout); + if (compositor->sampler) + ID3D11SamplerState_Release (compositor->sampler); + if (compositor->blend) + ID3D11BlendState_Release (compositor->blend); + if (compositor->index_buffer) + ID3D11Buffer_Release (compositor->index_buffer); + + gst_clear_object (&compositor->device); + g_free (compositor); +} + +static gint +find_in_compositor (const GstD3D11CompositionOverlay * overlay, + const GstVideoOverlayRectangle * rect) +{ + return !(overlay->overlay_rect == rect); +} + +static gboolean +is_in_video_overlay_composition (GstVideoOverlayComposition * voc, + GstD3D11CompositionOverlay * overlay) +{ + guint i; + + for (i = 0; i < gst_video_overlay_composition_n_rectangles (voc); i++) { + GstVideoOverlayRectangle *rectangle = + gst_video_overlay_composition_get_rectangle (voc, i); + if (overlay->overlay_rect == rectangle) + return TRUE; + } + return FALSE; +} + +gboolean +gst_d3d11_overlay_compositor_upload (GstD3D11OverlayCompositor * compositor, + GstBuffer * buf) +{ + GstVideoOverlayCompositionMeta *meta; + gint i, num_overlays; + GList *iter; + + g_return_val_if_fail (compositor != NULL, FALSE); + g_return_val_if_fail (GST_IS_BUFFER (buf), FALSE); + + meta = gst_buffer_get_video_overlay_composition_meta (buf); + + if (!meta) { + gst_d3d11_overlay_compositor_free_overlays (compositor); + return TRUE; + } + + num_overlays = gst_video_overlay_composition_n_rectangles (meta->overlay); + if (!num_overlays) { + gst_d3d11_overlay_compositor_free_overlays (compositor); + return TRUE; + } + + GST_LOG ("Upload %d overlay rectangles", num_overlays); + + /* Upload new overlay */ + for (i = 0; i < num_overlays; i++) { + GstVideoOverlayRectangle *rectangle = + gst_video_overlay_composition_get_rectangle (meta->overlay, i); + + if (!g_list_find_custom (compositor->overlays, + rectangle, (GCompareFunc) find_in_compositor)) { + GstD3D11CompositionOverlay *overlay = NULL; + + overlay = gst_d3d11_composition_overlay_new (compositor, rectangle); + + if (!overlay) + return FALSE; + + compositor->overlays = g_list_append (compositor->overlays, overlay); + } + } + + /* Remove old overlay */ + iter = compositor->overlays; + while (iter) { + GstD3D11CompositionOverlay *overlay = + (GstD3D11CompositionOverlay *) iter->data; + GList *next = iter->next; + + if (!is_in_video_overlay_composition (meta->overlay, overlay)) { + compositor->overlays = g_list_delete_link (compositor->overlays, iter); + gst_d3d11_composition_overlay_free (overlay); + } + + iter = next; + } + + return TRUE; +} + +void +gst_d3d11_overlay_compositor_free_overlays (GstD3D11OverlayCompositor * + compositor) +{ + g_return_if_fail (compositor != NULL); + + if (compositor->overlays) { + g_list_free_full (compositor->overlays, + (GDestroyNotify) gst_d3d11_composition_overlay_free); + + compositor->overlays = NULL; + } +} + +gboolean +gst_d3d11_overlay_compositor_update_rect (GstD3D11OverlayCompositor * + compositor, RECT * rect) +{ + g_return_val_if_fail (compositor != NULL, FALSE); + g_return_val_if_fail (rect != NULL, FALSE); + + compositor->viewport.TopLeftX = rect->left; + compositor->viewport.TopLeftY = rect->top; + compositor->viewport.Width = rect->right - rect->left; + compositor->viewport.Height = rect->bottom - rect->top; + + return TRUE; +} + +gboolean +gst_d3d11_overlay_compositor_draw (GstD3D11OverlayCompositor * compositor, + ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES]) +{ + gboolean ret = TRUE; + + g_return_val_if_fail (compositor != NULL, FALSE); + g_return_val_if_fail (rtv != NULL, FALSE); + + gst_d3d11_device_lock (compositor->device); + ret = gst_d3d11_overlay_compositor_draw_unlocked (compositor, rtv); + gst_d3d11_device_unlock (compositor->device); + + return ret; +} + +gboolean +gst_d3d11_overlay_compositor_draw_unlocked (GstD3D11OverlayCompositor * + compositor, ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES]) +{ + gboolean ret = TRUE; + GList *iter; + + g_return_val_if_fail (compositor != NULL, FALSE); + g_return_val_if_fail (rtv != NULL, FALSE); + + for (iter = compositor->overlays; iter; iter = g_list_next (iter)) { + GstD3D11CompositionOverlay *overlay = + (GstD3D11CompositionOverlay *) iter->data; + + ret = gst_d3d11_draw_quad_unlocked (overlay->quad, + &compositor->viewport, 1, &overlay->srv, 1, rtv, 1, NULL); + + if (!ret) + break; + } + + return ret; +} diff --git a/sys/d3d11/gstd3d11overlaycompositor.h b/sys/d3d11/gstd3d11overlaycompositor.h new file mode 100644 index 0000000..1640546 --- /dev/null +++ b/sys/d3d11/gstd3d11overlaycompositor.h @@ -0,0 +1,52 @@ +/* GStreamer + * Copyright (C) <2019> Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_D3D11_OVERLAY_COMPOSITOR_H__ +#define __GST_D3D11_OVERLAY_COMPOSITOR_H__ + +#include +#include +#include "gstd3d11_fwd.h" + +G_BEGIN_DECLS + +typedef struct _GstD3D11OverlayCompositor GstD3D11OverlayCompositor; + +GstD3D11OverlayCompositor * gst_d3d11_overlay_compositor_new (GstD3D11Device * device, + GstVideoInfo * out_info); + +void gst_d3d11_overlay_compositor_free (GstD3D11OverlayCompositor * compositor); + +gboolean gst_d3d11_overlay_compositor_upload (GstD3D11OverlayCompositor * compositor, + GstBuffer * buf); + +void gst_d3d11_overlay_compositor_free_overlays (GstD3D11OverlayCompositor * compositor); + +gboolean gst_d3d11_overlay_compositor_update_rect (GstD3D11OverlayCompositor * compositor, + RECT *rect); + +gboolean gst_d3d11_overlay_compositor_draw (GstD3D11OverlayCompositor * compositor, + ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES]); + +gboolean gst_d3d11_overlay_compositor_draw_unlocked (GstD3D11OverlayCompositor * compositor, + ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES]); + +G_END_DECLS + +#endif /* __GST_D3D11_OVERLAY_COMPOSITOR_H__ */ diff --git a/sys/d3d11/gstd3d11shader.c b/sys/d3d11/gstd3d11shader.c index 73ab40b..1b2fb79 100644 --- a/sys/d3d11/gstd3d11shader.c +++ b/sys/d3d11/gstd3d11shader.c @@ -180,6 +180,8 @@ struct _GstD3D11Quad ID3D11VertexShader *vs; ID3D11InputLayout *layout; ID3D11SamplerState *sampler; + ID3D11BlendState *blend; + ID3D11DepthStencilState *depth_stencil; ID3D11Buffer *const_buffer; ID3D11Buffer *vertex_buffer; guint vertex_stride; @@ -196,7 +198,9 @@ struct _GstD3D11Quad GstD3D11Quad * gst_d3d11_quad_new (GstD3D11Device * device, ID3D11PixelShader * pixel_shader, ID3D11VertexShader * vertex_shader, ID3D11InputLayout * layout, - ID3D11SamplerState * sampler, ID3D11Buffer * const_buffer, + ID3D11SamplerState * sampler, ID3D11BlendState * blend, + ID3D11DepthStencilState * depth_stencil, + ID3D11Buffer * const_buffer, ID3D11Buffer * vertex_buffer, guint vertex_stride, ID3D11Buffer * index_buffer, DXGI_FORMAT index_format, guint index_count) { @@ -219,6 +223,8 @@ gst_d3d11_quad_new (GstD3D11Device * device, ID3D11PixelShader * pixel_shader, quad->vs = vertex_shader; quad->layout = layout; quad->sampler = sampler; + quad->blend = blend; + quad->depth_stencil = depth_stencil; quad->vertex_buffer = vertex_buffer; quad->vertex_stride = vertex_stride; quad->index_buffer = index_buffer; @@ -230,6 +236,12 @@ gst_d3d11_quad_new (GstD3D11Device * device, ID3D11PixelShader * pixel_shader, ID3D11InputLayout_AddRef (layout); ID3D11SamplerState_AddRef (sampler); + if (blend) + ID3D11BlendState_AddRef (blend); + + if (depth_stencil) + ID3D11DepthStencilState_AddRef (depth_stencil); + if (const_buffer) { quad->const_buffer = const_buffer; ID3D11Buffer_AddRef (const_buffer); @@ -253,6 +265,10 @@ gst_d3d11_quad_free (GstD3D11Quad * quad) ID3D11InputLayout_Release (quad->layout); if (quad->sampler) ID3D11SamplerState_Release (quad->sampler); + if (quad->blend) + ID3D11BlendState_Release (quad->blend); + if (quad->depth_stencil) + ID3D11DepthStencilState_Release (quad->depth_stencil); if (quad->const_buffer) ID3D11Buffer_Release (quad->const_buffer); if (quad->vertex_buffer) @@ -268,7 +284,8 @@ gboolean gst_d3d11_draw_quad (GstD3D11Quad * quad, D3D11_VIEWPORT viewport[GST_VIDEO_MAX_PLANES], guint num_viewport, ID3D11ShaderResourceView * srv[GST_VIDEO_MAX_PLANES], guint num_srv, - ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES], guint num_rtv) + ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES], guint num_rtv, + ID3D11DepthStencilView * dsv) { gboolean ret; @@ -276,7 +293,7 @@ gst_d3d11_draw_quad (GstD3D11Quad * quad, gst_d3d11_device_lock (quad->device); ret = gst_d3d11_draw_quad_unlocked (quad, viewport, num_viewport, - srv, num_srv, rtv, num_viewport); + srv, num_srv, rtv, num_viewport, dsv); gst_d3d11_device_unlock (quad->device); return ret; @@ -286,7 +303,8 @@ gboolean gst_d3d11_draw_quad_unlocked (GstD3D11Quad * quad, D3D11_VIEWPORT viewport[GST_VIDEO_MAX_PLANES], guint num_viewport, ID3D11ShaderResourceView * srv[GST_VIDEO_MAX_PLANES], guint num_srv, - ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES], guint num_rtv) + ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES], guint num_rtv, + ID3D11DepthStencilView * dsv) { ID3D11DeviceContext *context_handle; UINT offsets = 0; @@ -320,7 +338,11 @@ gst_d3d11_draw_quad_unlocked (GstD3D11Quad * quad, 0, 1, &quad->const_buffer); ID3D11DeviceContext_PSSetShaderResources (context_handle, 0, num_srv, srv); - ID3D11DeviceContext_OMSetRenderTargets (context_handle, num_rtv, rtv, NULL); + ID3D11DeviceContext_OMSetRenderTargets (context_handle, num_rtv, rtv, dsv); + ID3D11DeviceContext_OMSetBlendState (context_handle, + quad->blend, NULL, 0xffffffff); + ID3D11DeviceContext_OMSetDepthStencilState (context_handle, + quad->depth_stencil, 1); ID3D11DeviceContext_DrawIndexed (context_handle, quad->index_count, 0, 0); diff --git a/sys/d3d11/gstd3d11shader.h b/sys/d3d11/gstd3d11shader.h index 0ef064d..4d9e166 100644 --- a/sys/d3d11/gstd3d11shader.h +++ b/sys/d3d11/gstd3d11shader.h @@ -46,6 +46,8 @@ GstD3D11Quad * gst_d3d11_quad_new (GstD3D11Device * device, ID3D11VertexShader * vertex_shader, ID3D11InputLayout * layout, ID3D11SamplerState * sampler, + ID3D11BlendState * blend, + ID3D11DepthStencilState *depth_stencil, ID3D11Buffer * const_buffer, ID3D11Buffer * vertex_buffer, guint vertex_stride, @@ -61,7 +63,8 @@ gboolean gst_d3d11_draw_quad (GstD3D11Quad * quad, ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES], guint num_srv, ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES], - guint num_rtv); + guint num_rtv, + ID3D11DepthStencilView *dsv); gboolean gst_d3d11_draw_quad_unlocked (GstD3D11Quad * quad, D3D11_VIEWPORT viewport[GST_VIDEO_MAX_PLANES], @@ -69,7 +72,8 @@ gboolean gst_d3d11_draw_quad_unlocked (GstD3D11Quad * quad, ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES], guint num_srv, ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES], - guint num_rtv); + guint num_rtv, + ID3D11DepthStencilView *dsv); G_END_DECLS diff --git a/sys/d3d11/gstd3d11upload.c b/sys/d3d11/gstd3d11upload.c index 4141b3a..08ae56f 100644 --- a/sys/d3d11/gstd3d11upload.c +++ b/sys/d3d11/gstd3d11upload.c @@ -36,7 +36,14 @@ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS) "; " GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, + GST_D3D11_FORMATS) ";" + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY + "," GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, GST_D3D11_FORMATS)) ); @@ -44,7 +51,11 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS))); + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS))); #define gst_d3d11_upload_parent_class parent_class G_DEFINE_TYPE (GstD3D11Upload, gst_d3d11_upload, GST_TYPE_D3D11_BASE_FILTER); @@ -94,13 +105,52 @@ gst_d3d11_upload_init (GstD3D11Upload * upload) static GstCaps * _set_caps_features (const GstCaps * caps, const gchar * feature_name) { - GstCaps *tmp = gst_caps_copy (caps); - guint n = gst_caps_get_size (tmp); - guint i = 0; + guint i, j, m, n; + GstCaps *tmp; + GstCapsFeatures *overlay_feature = + gst_caps_features_from_string + (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + + tmp = gst_caps_new_empty (); + + n = gst_caps_get_size (caps); + for (i = 0; i < n; i++) { + GstCapsFeatures *features, *orig_features; + GstStructure *s = gst_caps_get_structure (caps, i); + + orig_features = gst_caps_get_features (caps, i); + features = gst_caps_features_new (feature_name, NULL); + + if (gst_caps_features_is_any (orig_features)) { + gst_caps_append_structure_full (tmp, gst_structure_copy (s), + gst_caps_features_copy (features)); + + if (!gst_caps_features_contains (features, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) + gst_caps_features_add (features, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + } else { + m = gst_caps_features_get_size (orig_features); + for (j = 0; j < m; j++) { + const gchar *feature = gst_caps_features_get_nth (orig_features, j); + + /* if we already have the features */ + if (gst_caps_features_contains (features, feature)) + continue; + + if (g_strcmp0 (feature, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY) == 0) + continue; + + if (gst_caps_features_contains (overlay_feature, feature)) { + gst_caps_features_add (features, feature); + } + } + } + + gst_caps_append_structure_full (tmp, gst_structure_copy (s), features); + } - for (i = 0; i < n; i++) - gst_caps_set_features (tmp, i, - gst_caps_features_from_string (feature_name)); + gst_caps_features_free (overlay_feature); return tmp; } @@ -117,13 +167,8 @@ gst_d3d11_upload_transform_caps (GstBaseTransform * trans, if (direction == GST_PAD_SINK) { tmp = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY); - tmp = gst_caps_merge (gst_caps_ref (caps), tmp); } else { - GstCaps *newcaps; - tmp = gst_caps_ref (caps); - - newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); - tmp = gst_caps_merge (tmp, newcaps); + tmp = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); } if (filter) { @@ -197,8 +242,6 @@ gst_d3d11_upload_propose_allocation (GstBaseTransform * trans, if (!gst_buffer_pool_set_config (pool, config)) goto config_failed; - gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); - /* d3d11 buffer pool might update buffer size by self */ if (is_d3d11) size = GST_D3D11_BUFFER_POOL (pool)->buffer_size; @@ -207,6 +250,10 @@ gst_d3d11_upload_propose_allocation (GstBaseTransform * trans, gst_object_unref (pool); } + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + gst_query_add_allocation_meta (query, + GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL); + return TRUE; /* ERRORS */ diff --git a/sys/d3d11/gstd3d11videosink.c b/sys/d3d11/gstd3d11videosink.c index 67c6716..626b62c 100644 --- a/sys/d3d11/gstd3d11videosink.c +++ b/sys/d3d11/gstd3d11videosink.c @@ -48,7 +48,11 @@ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS) + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS) )); GST_DEBUG_CATEGORY (d3d11_video_sink_debug); @@ -262,9 +266,18 @@ gst_d3d11_video_sink_get_caps (GstBaseSink * sink, GstCaps * filter) GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink); GstCaps *caps = NULL; - if (self->device && !self->can_convert) + if (self->device && !self->can_convert) { + GstCaps *overlaycaps; + GstCapsFeatures *features; + caps = gst_d3d11_device_get_supported_caps (self->device, D3D11_FORMAT_SUPPORT_TEXTURE2D | D3D11_FORMAT_SUPPORT_DISPLAY); + overlaycaps = gst_caps_copy (caps); + features = gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + gst_caps_set_features_simple (overlaycaps, features); + gst_caps_append (caps, overlaycaps); + } if (!caps) caps = gst_pad_get_pad_template_caps (GST_VIDEO_SINK_PAD (sink)); @@ -607,6 +620,8 @@ gst_d3d11_video_sink_propose_allocation (GstBaseSink * sink, GstQuery * query) g_object_unref (pool); gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + gst_query_add_allocation_meta (query, + GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL); return TRUE; @@ -744,6 +759,16 @@ gst_d3d11_video_sink_upload_frame (GstD3D11VideoSink * self, GstBuffer * inbuf, done: gst_video_frame_unmap (&in_frame); + if (ret) { + GstVideoOverlayCompositionMeta *overlay_meta; + + overlay_meta = gst_buffer_get_video_overlay_composition_meta (inbuf); + if (overlay_meta) { + gst_buffer_add_video_overlay_composition_meta (outbuf, + overlay_meta->overlay); + } + } + return ret; /* ERRORS */ diff --git a/sys/d3d11/gstd3d11videosinkbin.c b/sys/d3d11/gstd3d11videosinkbin.c index a2e507e..1fd8052 100644 --- a/sys/d3d11/gstd3d11videosinkbin.c +++ b/sys/d3d11/gstd3d11videosinkbin.c @@ -85,8 +85,16 @@ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS) - "; " GST_VIDEO_CAPS_MAKE (GST_D3D11_FORMATS) + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS) ";" + GST_VIDEO_CAPS_MAKE (GST_D3D11_FORMATS) "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + GST_D3D11_FORMATS) )); GST_DEBUG_CATEGORY (d3d11_video_sink_bin_debug); diff --git a/sys/d3d11/gstd3d11window.c b/sys/d3d11/gstd3d11window.c index e051078..1c96144 100644 --- a/sys/d3d11/gstd3d11window.c +++ b/sys/d3d11/gstd3d11window.c @@ -329,6 +329,11 @@ gst_d3d11_window_dispose (GObject * object) self->converter = NULL; } + if (self->compositor) { + gst_d3d11_overlay_compositor_free (self->compositor); + self->compositor = NULL; + } + gst_clear_buffer (&self->cached_buffer); gst_clear_object (&self->device); @@ -650,6 +655,8 @@ gst_d3d11_window_on_resize (GstD3D11Window * window, gboolean redraw) goto done; } + window->first_present = TRUE; + if (redraw) gst_d3d111_window_present (window, NULL); @@ -1173,7 +1180,14 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, gst_d3d11_color_converter_free (window->converter); window->converter = NULL; + if (window->compositor) + gst_d3d11_overlay_compositor_free (window->compositor); + window->compositor = NULL; + /* preserve upstream colorimetry */ + window->render_info.width = width; + window->render_info.height = height; + window->render_info.colorimetry.primaries = window->info.colorimetry.primaries; window->render_info.colorimetry.transfer = window->info.colorimetry.transfer; @@ -1190,6 +1204,16 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, return FALSE; } + window->compositor = + gst_d3d11_overlay_compositor_new (window->device, &window->render_info); + if (!window->compositor) { + GST_ERROR_OBJECT (window, "Cannot create overlay compositor"); + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED, + "Cannot create overlay compositor"); + + return FALSE; + } + window->allow_tearing = FALSE; #if (DXGI_HEADER_VERSION >= 5) if (!gst_video_content_light_level_from_caps (&window->content_light_level, @@ -1468,7 +1492,6 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer) if (self->cached_buffer) { ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES]; gint i, j, k; - RECT rect; for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) { GstD3D11Memory *mem = @@ -1479,13 +1502,23 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer) } } - rect.left = self->render_rect.x; - rect.right = self->render_rect.x + self->render_rect.w; - rect.top = self->render_rect.y; - rect.bottom = self->render_rect.y + self->render_rect.h; + if (self->first_present) { + RECT rect; + + rect.left = self->render_rect.x; + rect.right = self->render_rect.x + self->render_rect.w; + rect.top = self->render_rect.y; + rect.bottom = self->render_rect.y + self->render_rect.h; + + gst_d3d11_color_converter_update_rect (self->converter, &rect); + gst_d3d11_overlay_compositor_update_rect (self->compositor, &rect); + } + + gst_d3d11_color_converter_convert_unlocked (self->converter, + srv, &self->rtv); - gst_d3d11_color_converter_update_rect (self->converter, &rect); - gst_d3d11_color_converter_convert (self->converter, srv, &self->rtv); + gst_d3d11_overlay_compositor_upload (self->compositor, self->cached_buffer); + gst_d3d11_overlay_compositor_draw_unlocked (self->compositor, &self->rtv); #if (DXGI_HEADER_VERSION >= 5) if (self->allow_tearing) { @@ -1494,6 +1527,7 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer) #endif hr = IDXGISwapChain_Present (self->swap_chain, 0, present_flags); + self->first_present = FALSE; if (!gst_d3d11_result (hr, self->device)) { GST_WARNING_OBJECT (self, "Direct3D cannot present texture, hr: 0x%x", diff --git a/sys/d3d11/gstd3d11window.h b/sys/d3d11/gstd3d11window.h index b70aceb..6934ebf 100644 --- a/sys/d3d11/gstd3d11window.h +++ b/sys/d3d11/gstd3d11window.h @@ -26,6 +26,7 @@ #include #include "gstd3d11_fwd.h" #include "gstd3d11colorconverter.h" +#include "gstd3d11overlaycompositor.h" G_BEGIN_DECLS @@ -66,6 +67,7 @@ struct _GstD3D11Window GstVideoInfo render_info; const GstD3D11Format *render_format; GstD3D11ColorConverter *converter; + GstD3D11OverlayCompositor *compositor; GstVideoMasteringDisplayInfo mastering_display_info; GstVideoContentLightLevel content_light_level; @@ -108,6 +110,7 @@ struct _GstD3D11Window IDXGISwapChain *swap_chain; ID3D11RenderTargetView *rtv; DXGI_FORMAT format; + gboolean first_present; GstD3D11Device *device; diff --git a/sys/d3d11/meson.build b/sys/d3d11/meson.build index 71ca923..1380ebe 100644 --- a/sys/d3d11/meson.build +++ b/sys/d3d11/meson.build @@ -14,6 +14,7 @@ d3d11_sources = [ 'gstd3d11videosinkbin.c', 'gstd3d11shader.c', 'gstd3d11colorconverter.c', + 'gstd3d11overlaycompositor.c', ] dxgi_headers = [ diff --git a/sys/d3d11/plugin.c b/sys/d3d11/plugin.c index ea822f1..8152209 100644 --- a/sys/d3d11/plugin.c +++ b/sys/d3d11/plugin.c @@ -35,6 +35,7 @@ GST_DEBUG_CATEGORY (gst_d3d11_colorconverter_debug); GST_DEBUG_CATEGORY (gst_d3d11_utils_debug); GST_DEBUG_CATEGORY (gst_d3d11_format_debug); GST_DEBUG_CATEGORY (gst_d3d11_device_debug); +GST_DEBUG_CATEGORY (gst_d3d11_overlay_compositor_debug); #if (HAVE_D3D11SDKLAYERS_H || HAVE_DXGIDEBUG_H) GST_DEBUG_CATEGORY (gst_d3d11_debug_layer_debug); @@ -53,6 +54,9 @@ plugin_init (GstPlugin * plugin) "d3d11format", 0, "d3d11 specific formats"); GST_DEBUG_CATEGORY_INIT (gst_d3d11_device_debug, "d3d11device", 0, "d3d11 device object"); + GST_DEBUG_CATEGORY_INIT (gst_d3d11_overlay_compositor_debug, + "d3d11overlaycompositor", 0, "d3d11overlaycompositor"); + #if (HAVE_D3D11SDKLAYERS_H || HAVE_DXGIDEBUG_H) /* NOTE: enabled only for debug build */ GST_DEBUG_CATEGORY_INIT (gst_d3d11_debug_layer_debug, -- 2.7.4