From: Seungha Yang Date: Tue, 31 Dec 2024 15:43:41 +0000 (+0900) Subject: d3d12screencapturesrc: Add support for HDR capture in DDA mode X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=126d6f0815f231fc2eb6e92bb1a1bd0e1ab1f159;p=platform%2Fupstream%2Fgstreamer.git d3d12screencapturesrc: Add support for HDR capture in DDA mode Use IDXGIOutput5::DuplicateOutput1() if HDR is enabled. Note that scRGB color space is not defined in GStreamer, this element will output SDR tonemapped frame with linear or reinhard filtering. Fixes: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3834 Part-of: --- diff --git a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json index 85130342b9..ddea6c7630 100644 --- a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json +++ b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json @@ -14932,7 +14932,7 @@ "klass": "Source/Video", "pad-templates": { "src": { - "caps": "video/x-raw(memory:D3D12Memory):\n format: BGRA\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\npixel-aspect-ratio: 1/1\n colorimetry: sRGB\nvideo/x-raw:\n format: BGRA\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\npixel-aspect-ratio: 1/1\n colorimetry: sRGB\n", + "caps": "video/x-raw(memory:D3D12Memory):\n format: { BGRA, RGBA64_LE }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\npixel-aspect-ratio: 1/1\n colorimetry: sRGB\nvideo/x-raw:\n format: { BGRA, RGBA64_LE }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\npixel-aspect-ratio: 1/1\n colorimetry: sRGB\n", "direction": "src", "presence": "always" } @@ -15072,6 +15072,18 @@ "type": "gboolean", "writable": true }, + "tonemap": { + "blurb": "Tonemapping method to use when HDR capturing is enabled", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "linear (0)", + "mutable": "ready", + "readable": true, + "type": "GstD3D12ScreenCaptureTonemap", + "writable": true + }, "window-capture-mode": { "blurb": "Window capture mode to use if \"window-handle\" is set", "conditionally-available": true, @@ -16540,6 +16552,21 @@ } ] }, + "GstD3D12ScreenCaptureTonemap": { + "kind": "enum", + "values": [ + { + "desc": "Linear scaling", + "name": "linear", + "value": "0" + }, + { + "desc": "Reinhard tonemap", + "name": "reinhard", + "value": "1" + } + ] + }, "GstD3D12TestSrcPattern": { "kind": "enum", "values": [ diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12dxgicapture.cpp b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12dxgicapture.cpp index 6bd8ccda25..d9e88ad261 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12dxgicapture.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12dxgicapture.cpp @@ -56,6 +56,7 @@ #include #include #include +#include #define _XM_NO_INTRINSICS_ #include @@ -136,6 +137,110 @@ flow_return_from_hr (ID3D11Device * device, return GST_FLOW_ERROR; } +static guint +get_sdr_white_level (PCWSTR name) +{ + LONG ret = ERROR_SUCCESS; + std::vector < DISPLAYCONFIG_PATH_INFO > path_info; + std::vector < DISPLAYCONFIG_MODE_INFO > mode_info; + gint retry_count = 0; + guint nits = 80; + + /* QueryDisplayConfig() may return ERROR_INSUFFICIENT_BUFFER if there was + * configuration update between GetDisplayConfigBufferSizes() and + * QueryDisplayConfig() call. */ + while (1) { + UINT32 n_path = 0; + UINT32 n_mode = 0; + + ret = GetDisplayConfigBufferSizes (QDC_ONLY_ACTIVE_PATHS, &n_path, &n_mode); + if (ret != ERROR_SUCCESS) { + GST_WARNING ("GetDisplayConfigBufferSizes failed %d", (gint) ret); + return nits; + } + + path_info.resize (n_path); + mode_info.resize (n_mode); + + ret = QueryDisplayConfig (QDC_ONLY_ACTIVE_PATHS, &n_path, path_info.data (), + &n_mode, mode_info.data (), nullptr); + if (ret == ERROR_INSUFFICIENT_BUFFER) { + /* XXX: avoid infinite loop */ + retry_count++; + if (retry_count > 100) { + GST_WARNING ("Too many retry, give up"); + return nits; + } + + GST_DEBUG ("Insufficient buffer, retrying"); + continue; + } else if (ret != ERROR_SUCCESS) { + GST_WARNING ("QueryDisplayConfig failed %d", (gint) ret); + return nits; + } + + path_info.resize (n_path); + mode_info.resize (n_mode); + break; + } + + for (size_t i = 0; i < path_info.size (); i++) { + DISPLAYCONFIG_SOURCE_DEVICE_NAME src_name = { }; + src_name.header.type = DISPLAYCONFIG_DEVICE_INFO_GET_SOURCE_NAME; + src_name.header.size = sizeof (DISPLAYCONFIG_SOURCE_DEVICE_NAME); + src_name.header.adapterId = path_info[i].sourceInfo.adapterId; + src_name.header.id = path_info[i].sourceInfo.id; + + ret = DisplayConfigGetDeviceInfo (&src_name.header); + if (ret == ERROR_SUCCESS && wcscmp (name, src_name.viewGdiDeviceName) == 0) { + DISPLAYCONFIG_SDR_WHITE_LEVEL level; + level.header.type = DISPLAYCONFIG_DEVICE_INFO_GET_SDR_WHITE_LEVEL; + level.header.size = sizeof (level); + level.header.adapterId = path_info[i].targetInfo.adapterId; + level.header.id = path_info[i].targetInfo.id; + ret = DisplayConfigGetDeviceInfo (&level.header); + if (ret != ERROR_SUCCESS) { + GST_WARNING ("Couldn't get SDR white level info"); + return nits; + } + + return (level.SDRWhiteLevel * 80) / 1000; + } + } + + return nits; +} + +struct DxgiCaptureVTable +{ + gboolean loaded; + DPI_AWARENESS_CONTEXT (WINAPI * SetThreadDpiAwarenessContext) (DPI_AWARENESS_CONTEXT context); +}; + +static DxgiCaptureVTable g_vtable = { }; + +static gboolean +gst_d3d12_dxgi_capture_load_library (void) +{ + static GModule *user32_module = nullptr; + + GST_D3D12_CALL_ONCE_BEGIN { + g_vtable.loaded = FALSE; + user32_module = g_module_open ("user32.dll", G_MODULE_BIND_LAZY); + if (!user32_module) + return; + + if (!g_module_symbol (user32_module, "SetThreadDpiAwarenessContext", + (gpointer *) &g_vtable.SetThreadDpiAwarenessContext)) { + return; + } + + g_vtable.loaded = TRUE; + } GST_D3D12_CALL_ONCE_END; + + return g_vtable.loaded; +} + struct PtrInfo { PtrInfo () @@ -259,6 +364,12 @@ struct VERTEX XMFLOAT2 TexCoord; }; +struct PSConstBuffer +{ + float sdr_white_level; + float padding[3]; +}; + class DesktopDupCtx { public: @@ -275,11 +386,14 @@ public: GstFlowReturn Init (HMONITOR monitor, ID3D11Device5 * device, ID3D11DeviceContext4 * context, ID3D11Fence * fence, ID3D11SamplerState * sampler, ID3D11PixelShader * ps, - ID3D11VertexShader * vs, ID3D11InputLayout * layout) + ID3D11PixelShader * ps_scrgb, ID3D11PixelShader * ps_scrgb_tonemap, + ID3D11Buffer * ps_cbuf, ID3D11VertexShader * vs, + ID3D11InputLayout * layout, gboolean use_reinhard) { ComPtr adapter; ComPtr output; ComPtr output1; + ComPtr output6; HRESULT hr = gst_d3d12_screen_capture_find_output_for_monitor (monitor, &adapter, &output); @@ -294,6 +408,27 @@ public: return GST_FLOW_ERROR; } + PSConstBuffer cbuf; + cbuf.sdr_white_level = 80.0; + gboolean is_hdr = FALSE; + + if (gst_d3d12_dxgi_capture_load_library ()) { + hr = output.As (&output6); + if (SUCCEEDED (hr)) { + DXGI_OUTPUT_DESC1 desc1; + hr = output6->GetDesc1 (&desc1); + if (SUCCEEDED (hr) && + desc1.ColorSpace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020) { + is_hdr = TRUE; + + MONITORINFOEXW monitor_info = { }; + monitor_info.cbSize = sizeof (MONITORINFOEXW); + if (GetMonitorInfoW (desc1.Monitor, (LPMONITORINFO) & monitor_info)) + cbuf.sdr_white_level = get_sdr_white_level (monitor_info.szDevice); + } + } + } + HDESK hdesk = OpenInputDesktop (0, FALSE, GENERIC_ALL); if (hdesk) { if (!SetThreadDesktop (hdesk)) { @@ -305,8 +440,35 @@ public: GST_WARNING ("OpenInputDesktop() failed, error %lu", GetLastError()); } - /* FIXME: Use DuplicateOutput1 to avoid potentail color conversion */ - hr = output1->DuplicateOutput(device, &dupl_); + hr = E_FAIL; + output_format_ = DXGI_FORMAT_B8G8R8A8_UNORM; + if (is_hdr) { + DXGI_FORMAT formats[] = { + DXGI_FORMAT_R16G16B16A16_FLOAT, + DXGI_FORMAT_B8G8R8A8_UNORM, + }; + + /* XXX: DuplicateOutput1() would fail if dpi awareness is not configured */ + auto prev_ctx = g_vtable.SetThreadDpiAwarenessContext + (DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); + hr = output6->DuplicateOutput1(device, 0, 2, formats, &dupl_); + + /* And restore dpi context for the current thread */ + if (prev_ctx != nullptr) + g_vtable.SetThreadDpiAwarenessContext (prev_ctx); + + if (FAILED (hr)) { + GST_WARNING ("IDXGIOutput5::DuplicateOutput1 returned 0x%x", + (guint) hr); + is_hdr = FALSE; + } else { + output_format_ = DXGI_FORMAT_R16G16B16A16_UNORM; + } + } + + if (FAILED (hr)) + hr = output1->DuplicateOutput(device, &dupl_); + if (FAILED (hr)) { if (hr == DXGI_ERROR_NOT_CURRENTLY_AVAILABLE) { GST_ERROR ("Hit the max allowed number of Desktop Duplication session"); @@ -328,14 +490,6 @@ public: CreateDuplicationExpectedErrors); } - device_ = device; - context_ = context; - shared_fence_ = fence; - sampler_ = sampler; - ps_ = ps; - vs_ = vs; - layout_ = layout; - dupl_->GetDesc (&output_desc_); D3D11_TEXTURE2D_DESC desc = { }; @@ -343,23 +497,55 @@ public: desc.Height = output_desc_.ModeDesc.Height; desc.MipLevels = 1; desc.ArraySize = 1; - desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.Format = output_format_; desc.SampleDesc.Count = 1; desc.Usage = D3D11_USAGE_DEFAULT; desc.BindFlags = D3D11_BIND_RENDER_TARGET; - hr = device_->CreateTexture2D (&desc, nullptr, &texture_); + device_ = device; + context_ = context; + shared_fence_ = fence; + sampler_ = sampler; + ps_cbuf_ = ps_cbuf; + vs_ = vs; + layout_ = layout; + + if (is_hdr) { + GST_INFO ("HDR with SDR white level %d nits", + (guint) cbuf.sdr_white_level); + if (!use_reinhard) { + GST_INFO ("Use scRGB sampling"); + ps_ = ps_scrgb; + } else { + GST_INFO ("use scRGB sampling with reinhard tonemapping"); + ps_ = ps_scrgb_tonemap; + } + } else { + GST_INFO ("Monitor is SDR mode"); + ps_ = ps; + } + + hr = device->CreateTexture2D (&desc, nullptr, &texture_); if (FAILED (hr)) { GST_ERROR ("Couldn't create texture"); return GST_FLOW_ERROR; } - hr = device_->CreateRenderTargetView (texture_.Get (), nullptr, &rtv_); + hr = device->CreateRenderTargetView (texture_.Get (), nullptr, &rtv_); if (FAILED (hr)) { GST_ERROR ("Couldn't create render target view"); return GST_FLOW_ERROR; } + D3D11_MAPPED_SUBRESOURCE map; + hr = context->Map (ps_cbuf_.Get (), 0, D3D11_MAP_WRITE_DISCARD, 0, &map); if (FAILED (hr)) { + GST_ERROR ("Couldn't map constant buffer"); + return GST_FLOW_ERROR; + } + + memcpy (map.pData, &cbuf, sizeof (PSConstBuffer)); + context->Unmap (ps_cbuf_.Get (), 0); + viewport_.TopLeftX = 0; viewport_.TopLeftY = 0; viewport_.MinDepth = 0; @@ -647,6 +833,8 @@ public: context_->IASetInputLayout(layout_.Get()); context_->VSSetShader(vs_.Get(), nullptr, 0); context_->PSSetShader(ps_.Get(), nullptr, 0); + ID3D11Buffer *ps_cbuf[] = { ps_cbuf_.Get () }; + context_->PSSetConstantBuffers (0, 1, ps_cbuf); ID3D11ShaderResourceView *srv[] = { cur_srv.Get () }; context_->PSSetShaderResources(0, 1, srv); @@ -751,6 +939,18 @@ public: if (hr != DXGI_ERROR_WAIT_TIMEOUT) { if (FAILED (hr)) { GST_WARNING ("AcquireNextFrame failed with 0x%x", (guint) hr); + /* XXX: HDR <-> SDR mode switching seems to be racy, + * and AcquireNextFrame() seems to return DXGI_ERROR_INVALID_CALL + * sometimes on HDR <-> SDR mode switching. + * Do return GST_D3D12_SCREEN_CAPTURE_FLOW_UNSUPPORTED here + * if AcquireNextFrame() returns DXGI_ERROR_INVALID_CALL, then + * source element will do retry a bit more */ + if (hr == DXGI_ERROR_INVALID_CALL) { + GST_WARNING ("DXGI_ERROR_INVALID_CALL, trying again"); + dupl_->ReleaseFrame (); + return GST_D3D12_SCREEN_CAPTURE_FLOW_UNSUPPORTED; + } + dupl_->ReleaseFrame (); return flow_return_from_hr (device_.Get (), hr, FrameInfoExpectedErrors); } @@ -777,6 +977,11 @@ public: *height = output_desc_.ModeDesc.Height; } + DXGI_FORMAT GetFormat () + { + return output_format_; + } + DXGI_OUTDUPL_DESC GetDesc () { return output_desc_; @@ -810,12 +1015,16 @@ private: ComPtr rtv_; ComPtr sampler_; ComPtr ps_; + ComPtr ps_scrgb_; + ComPtr ps_scrgb_tonemap_; + ComPtr ps_cbuf_; ComPtr vs_; ComPtr layout_; ComPtr vertex_buf_; UINT vertext_buf_size_ = 0; D3D11_VIEWPORT viewport_ = { }; std::vector dirty_vertex_; + DXGI_FORMAT output_format_ = DXGI_FORMAT_B8G8R8A8_UNORM; /* frame metadata */ std::vector metadata_buffer_; @@ -838,6 +1047,8 @@ struct GstD3D12DxgiCapturePrivate gst_clear_object (&fence_data_pool); gst_clear_object (&mouse_blend); gst_clear_object (&mouse_xor_blend); + gst_clear_object (&mouse_blend_scrgb); + gst_clear_object (&mouse_xor_blend_scrgb); } void WaitGPU () @@ -860,17 +1071,24 @@ struct GstD3D12DxgiCapturePrivate ComPtr shared_fence11; ComPtr sampler; ComPtr ps; + ComPtr ps_scrgb; + ComPtr ps_scrgb_tonemap; ComPtr vs; ComPtr layout; + ComPtr const_buf; GstBuffer *mouse_buf = nullptr; GstBuffer *mouse_xor_buf = nullptr; GstD3D12Converter *mouse_blend = nullptr; GstD3D12Converter *mouse_xor_blend = nullptr; + GstD3D12Converter *mouse_blend_scrgb = nullptr; + GstD3D12Converter *mouse_xor_blend_scrgb = nullptr; HMONITOR monitor_handle = nullptr; RECT desktop_coordinates = { }; + guint sdr_white_level = 80; + guint prepare_flags = 0; guint cached_width = 0; guint cached_height = 0; @@ -894,10 +1112,12 @@ struct _GstD3D12DxgiCapture static void gst_d3d12_dxgi_capture_finalize (GObject * object); static GstFlowReturn -gst_d3d12_dxgi_capture_prepare (GstD3D12ScreenCapture * capture); +gst_d3d12_dxgi_capture_prepare (GstD3D12ScreenCapture * capture, guint flags); static gboolean gst_d3d12_dxgi_capture_get_size (GstD3D12ScreenCapture * capture, guint * width, guint * height); +static GstVideoFormat +gst_d3d12_dxgi_capture_get_format (GstD3D12ScreenCapture * capture); #define gst_d3d12_dxgi_capture_parent_class parent_class G_DEFINE_TYPE (GstD3D12DxgiCapture, gst_d3d12_dxgi_capture, @@ -913,6 +1133,8 @@ gst_d3d12_dxgi_capture_class_init (GstD3D12DxgiCaptureClass * klass) capture_class->prepare = GST_DEBUG_FUNCPTR (gst_d3d12_dxgi_capture_prepare); capture_class->get_size = GST_DEBUG_FUNCPTR (gst_d3d12_dxgi_capture_get_size); + capture_class->get_format = + GST_DEBUG_FUNCPTR (gst_d3d12_dxgi_capture_get_format); } static void @@ -948,6 +1170,7 @@ gst_d3d12_dxgi_capture_open (GstD3D12DxgiCapture * self, priv->monitor_handle = monitor_handle; ComPtr < IDXGIOutput > output; + ComPtr < IDXGIOutput6 > output6; ComPtr < IDXGIAdapter1 > adapter; auto hr = gst_d3d12_screen_capture_find_output_for_monitor (monitor_handle, &adapter, &output); @@ -991,6 +1214,19 @@ gst_d3d12_dxgi_capture_open (GstD3D12DxgiCapture * self, return FALSE; } + priv->sdr_white_level = 80; + hr = output.As (&output6); + if (SUCCEEDED (hr)) { + DXGI_OUTPUT_DESC1 desc1; + hr = output6->GetDesc1 (&desc1); + if (SUCCEEDED (hr) && + desc1.ColorSpace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020) { + priv->sdr_white_level = get_sdr_white_level (monitor_info.szDevice); + GST_INFO_OBJECT (self, "HDR mode detected, SDR white level in nits: %d", + priv->sdr_white_level); + } + } + priv->desktop_coordinates.left = dev_mode.dmPosition.x; priv->desktop_coordinates.top = dev_mode.dmPosition.y; priv->desktop_coordinates.right = @@ -1013,8 +1249,11 @@ gst_d3d12_dxgi_capture_open (GstD3D12DxgiCapture * self, /* size will be updated later */ GstVideoInfo info; + GstVideoInfo scrgb_info; gst_video_info_set_format (&info, GST_VIDEO_FORMAT_BGRA, priv->cached_width, priv->cached_height); + gst_video_info_set_format (&scrgb_info, GST_VIDEO_FORMAT_RGBA64_LE, + priv->cached_width, priv->cached_height); D3D12_BLEND_DESC blend_desc = CD3DX12_BLEND_DESC (D3D12_DEFAULT); blend_desc.RenderTarget[0].BlendEnable = TRUE; @@ -1031,11 +1270,15 @@ gst_d3d12_dxgi_capture_open (GstD3D12DxgiCapture * self, priv->mouse_blend = gst_d3d12_converter_new (self->device, nullptr, &info, &info, &blend_desc, nullptr, nullptr); + priv->mouse_blend_scrgb = gst_d3d12_converter_new (self->device, nullptr, + &info, &scrgb_info, &blend_desc, nullptr, nullptr); blend_desc.RenderTarget[0].SrcBlend = D3D12_BLEND_INV_DEST_COLOR; blend_desc.RenderTarget[0].DestBlend = D3D12_BLEND_INV_SRC_COLOR; priv->mouse_xor_blend = gst_d3d12_converter_new (self->device, nullptr, &info, &info, &blend_desc, nullptr, nullptr); + priv->mouse_xor_blend_scrgb = gst_d3d12_converter_new (self->device, nullptr, + &info, &scrgb_info, &blend_desc, nullptr, nullptr); hr = device->CreateFence (0, D3D12_FENCE_FLAG_SHARED, IID_PPV_ARGS (&priv->shared_fence)); @@ -1083,19 +1326,12 @@ gst_d3d12_dxgi_capture_open (GstD3D12DxgiCapture * self, } GstD3DShaderByteCode vs_code; - GstD3DShaderByteCode ps_code; if (!gst_d3d_plugin_shader_get_vs_blob (GST_D3D_PLUGIN_VS_COORD, GST_D3D_SM_5_0, &vs_code)) { GST_ERROR_OBJECT (self, "Couldn't get vs bytecode"); return FALSE; } - if (!gst_d3d_plugin_shader_get_ps_blob (GST_D3D_PLUGIN_PS_SAMPLE, - GST_D3D_SM_5_0, &ps_code)) { - GST_ERROR_OBJECT (self, "Couldn't get ps bytecode"); - return FALSE; - } - D3D11_INPUT_ELEMENT_DESC input_desc[2] = { }; input_desc[0].SemanticName = "POSITION"; input_desc[0].SemanticIndex = 0; @@ -1126,6 +1362,12 @@ gst_d3d12_dxgi_capture_open (GstD3D12DxgiCapture * self, return FALSE; } + GstD3DShaderByteCode ps_code; + if (!gst_d3d_plugin_shader_get_ps_blob (GST_D3D_PLUGIN_PS_SAMPLE, + GST_D3D_SM_5_0, &ps_code)) { + GST_ERROR_OBJECT (self, "Couldn't get ps bytecode"); + return FALSE; + } hr = priv->device11->CreatePixelShader (ps_code.byte_code, ps_code.byte_code_len, nullptr, &priv->ps); if (FAILED (hr)) { @@ -1133,6 +1375,48 @@ gst_d3d12_dxgi_capture_open (GstD3D12DxgiCapture * self, return FALSE; } + if (!gst_d3d_plugin_shader_get_ps_blob (GST_D3D_PLUGIN_PS_SAMPLE_SCRGB, + GST_D3D_SM_5_0, &ps_code)) { + GST_ERROR_OBJECT (self, "Couldn't get ps bytecode"); + return FALSE; + } + hr = priv->device11->CreatePixelShader (ps_code.byte_code, + ps_code.byte_code_len, nullptr, &priv->ps_scrgb); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Couldn't create pixel shader"); + return FALSE; + } + + if (!gst_d3d_plugin_shader_get_ps_blob + (GST_D3D_PLUGIN_PS_SAMPLE_SCRGB_TONEMAP, GST_D3D_SM_5_0, &ps_code)) { + GST_ERROR_OBJECT (self, "Couldn't get ps bytecode"); + return FALSE; + } + hr = priv->device11->CreatePixelShader (ps_code.byte_code, + ps_code.byte_code_len, nullptr, &priv->ps_scrgb_tonemap); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Couldn't create pixel shader"); + return FALSE; + } + + PSConstBuffer cbuf; + cbuf.sdr_white_level = (float) priv->sdr_white_level; + + D3D11_BUFFER_DESC buffer_desc = { }; + D3D11_SUBRESOURCE_DATA subresource = { }; + buffer_desc.Usage = D3D11_USAGE_DYNAMIC; + buffer_desc.ByteWidth = sizeof (PSConstBuffer); + buffer_desc.BindFlags = D3D11_BIND_CONSTANT_BUFFER; + buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + subresource.pSysMem = &cbuf; + subresource.SysMemPitch = sizeof (PSConstBuffer); + hr = priv->device11->CreateBuffer (&buffer_desc, &subresource, + &priv->const_buf); + if (FAILED (hr)) { + GST_ERROR_OBJECT (self, "Couldn't create constant buffer"); + return FALSE; + } + D3D11_SAMPLER_DESC sampler_desc = { }; sampler_desc.Filter = D3D11_FILTER_MIN_MAG_MIP_POINT; sampler_desc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP; @@ -1156,6 +1440,8 @@ gst_d3d12_dxgi_capture_new (GstD3D12Device * device, HMONITOR monitor_handle) g_return_val_if_fail (GST_IS_D3D12_DEVICE (device), nullptr); + gst_d3d12_dxgi_capture_load_library (); + /* Check if we have dup object corresponding to monitor_handle, * and if there is already configured capture object, reuse it. * This is because of the limitation of desktop duplication API @@ -1206,8 +1492,9 @@ gst_d3d12_dxgi_capture_prepare_unlocked (GstD3D12DxgiCapture * self) auto ctx = std::make_unique < DesktopDupCtx > (); auto ret = ctx->Init (priv->monitor_handle, priv->device11.Get (), priv->context11.Get (), priv->shared_fence11.Get (), - priv->sampler.Get (), priv->ps.Get (), priv->vs.Get (), - priv->layout.Get ()); + priv->sampler.Get (), priv->ps.Get (), priv->ps_scrgb.Get (), + priv->ps_scrgb_tonemap.Get (), priv->const_buf.Get (), + priv->vs.Get (), priv->layout.Get (), priv->prepare_flags ? TRUE : FALSE); if (ret != GST_FLOW_OK) { GST_WARNING_OBJECT (self, "Couldn't prepare capturing, %sexpected failure", @@ -1223,12 +1510,13 @@ gst_d3d12_dxgi_capture_prepare_unlocked (GstD3D12DxgiCapture * self) } static GstFlowReturn -gst_d3d12_dxgi_capture_prepare (GstD3D12ScreenCapture * capture) +gst_d3d12_dxgi_capture_prepare (GstD3D12ScreenCapture * capture, guint flags) { auto self = GST_D3D12_DXGI_CAPTURE (capture); auto priv = self->priv; std::lock_guard < std::mutex > lk (priv->lock); + priv->prepare_flags = flags; return gst_d3d12_dxgi_capture_prepare_unlocked (self); } @@ -1262,9 +1550,26 @@ gst_d3d12_dxgi_capture_get_size (GstD3D12ScreenCapture * capture, return gst_d3d12_dxgi_capture_get_size_unlocked (self, width, height); } +static GstVideoFormat +gst_d3d12_dxgi_capture_get_format (GstD3D12ScreenCapture * capture) +{ + auto self = GST_D3D12_DXGI_CAPTURE (capture); + auto priv = self->priv; + + std::lock_guard < std::mutex > lk (priv->lock); + if (!priv->ctx) + return GST_VIDEO_FORMAT_BGRA; + + auto format = priv->ctx->GetFormat (); + if (format == DXGI_FORMAT_R16G16B16A16_UNORM) + return GST_VIDEO_FORMAT_RGBA64_LE; + + return GST_VIDEO_FORMAT_BGRA; +} + static gboolean gst_d3d12_dxgi_capture_draw_mouse (GstD3D12DxgiCapture * self, - GstBuffer * buffer, const D3D12_BOX * crop_box) + GstBuffer * buffer, const D3D12_BOX * crop_box, gboolean is_hdr) { auto priv = self->priv; const auto & info = priv->ctx->GetPointerInfo (); @@ -1404,13 +1709,15 @@ gst_d3d12_dxgi_capture_draw_mouse (GstD3D12DxgiCapture * self, gint ptr_w = info.width_; gint ptr_h = info.height_; - g_object_set (priv->mouse_blend, "src-x", 0, "src-y", 0, "src-width", + auto blend_conv = is_hdr ? priv->mouse_blend_scrgb : priv->mouse_blend; + + g_object_set (blend_conv, "src-x", 0, "src-y", 0, "src-width", ptr_w, "src-height", ptr_h, "dest-x", ptr_x, "dest-y", ptr_y, "dest-width", ptr_w, "dest-height", ptr_h, nullptr); auto cq = gst_d3d12_device_get_cmd_queue (self->device, D3D12_COMMAND_LIST_TYPE_DIRECT); - if (!gst_d3d12_converter_convert_buffer (priv->mouse_blend, + if (!gst_d3d12_converter_convert_buffer (blend_conv, priv->mouse_buf, buffer, fence_data, cl.Get (), TRUE)) { GST_ERROR_OBJECT (self, "Couldn't build mouse blend command"); gst_d3d12_fence_data_unref (fence_data); @@ -1418,11 +1725,12 @@ gst_d3d12_dxgi_capture_draw_mouse (GstD3D12DxgiCapture * self, } if (priv->mouse_xor_buf) { - g_object_set (priv->mouse_xor_blend, "src-x", 0, "src-y", 0, "src-width", + blend_conv = is_hdr ? priv->mouse_xor_blend_scrgb : priv->mouse_xor_blend; + g_object_set (blend_conv, "src-x", 0, "src-y", 0, "src-width", ptr_w, "src-height", ptr_h, "dest-x", ptr_x, "dest-y", ptr_y, "dest-width", ptr_w, "dest-height", ptr_h, nullptr); - if (!gst_d3d12_converter_convert_buffer (priv->mouse_xor_blend, + if (!gst_d3d12_converter_convert_buffer (blend_conv, priv->mouse_xor_buf, buffer, fence_data, cl.Get (), FALSE)) { GST_ERROR_OBJECT (self, "Couldn't build mouse blend command"); gst_d3d12_fence_data_unref (fence_data); @@ -1492,6 +1800,13 @@ gst_d3d12_dxgi_capture_do_capture (GstD3D12DxgiCapture * capture, return GST_FLOW_ERROR; } + D3D11_TEXTURE2D_DESC tex_desc; + texture->GetDesc (&tex_desc); + if (tex_desc.Format != priv->ctx->GetFormat ()) { + GST_INFO_OBJECT (self, "Format mismatch"); + return GST_D3D12_SCREEN_CAPTURE_FLOW_SIZE_CHANGED; + } + priv->fence_val++; ret = priv->ctx->Execute (texture, (D3D11_BOX *) crop_box, priv->fence_val); if (ret != GST_FLOW_OK) { @@ -1513,7 +1828,8 @@ gst_d3d12_dxgi_capture_do_capture (GstD3D12DxgiCapture * capture, GST_MINI_OBJECT_FLAG_SET (dmem, GST_D3D12_MEMORY_TRANSFER_NEED_DOWNLOAD); GST_MINI_OBJECT_FLAG_UNSET (dmem, GST_D3D12_MEMORY_TRANSFER_NEED_UPLOAD); - if (draw_mouse && !gst_d3d12_dxgi_capture_draw_mouse (self, buffer, crop_box)) { + if (draw_mouse && !gst_d3d12_dxgi_capture_draw_mouse (self, buffer, crop_box, + tex_desc.Format == DXGI_FORMAT_R16G16B16A16_UNORM)) { priv->WaitGPU (); priv->ctx = nullptr; return GST_FLOW_ERROR; diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12graphicscapture.cpp b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12graphicscapture.cpp index 81426a085a..b3f37dd6f8 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12graphicscapture.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12graphicscapture.cpp @@ -890,7 +890,8 @@ struct _GstD3D12GraphicsCapture static void gst_d3d12_graphics_capture_finalize (GObject * object); static GstFlowReturn -gst_d3d12_graphics_capture_prepare (GstD3D12ScreenCapture * capture); +gst_d3d12_graphics_capture_prepare (GstD3D12ScreenCapture * capture, + guint flags); static gboolean gst_d3d12_graphics_capture_get_size (GstD3D12ScreenCapture * capture, guint * width, guint * height); @@ -1162,7 +1163,8 @@ out: } static GstFlowReturn -gst_d3d12_graphics_capture_prepare (GstD3D12ScreenCapture * capture) +gst_d3d12_graphics_capture_prepare (GstD3D12ScreenCapture * capture, + guint flags) { return GST_FLOW_OK; } diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapture.cpp b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapture.cpp index 569239c6da..bacf189485 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapture.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapture.cpp @@ -49,14 +49,14 @@ gst_d3d12_screen_capture_init (GstD3D12ScreenCapture * self) } GstFlowReturn -gst_d3d12_screen_capture_prepare (GstD3D12ScreenCapture * capture) +gst_d3d12_screen_capture_prepare (GstD3D12ScreenCapture * capture, guint flags) { g_return_val_if_fail (GST_IS_D3D12_SCREEN_CAPTURE (capture), GST_FLOW_ERROR); auto klass = GST_D3D12_SCREEN_CAPTURE_GET_CLASS (capture); g_assert (klass->prepare); - return klass->prepare (capture); + return klass->prepare (capture, flags); } gboolean @@ -73,6 +73,19 @@ gst_d3d12_screen_capture_get_size (GstD3D12ScreenCapture * capture, return klass->get_size (capture, width, height); } +GstVideoFormat +gst_d3d12_screen_capture_get_format (GstD3D12ScreenCapture * capture) +{ + g_return_val_if_fail (GST_IS_D3D12_SCREEN_CAPTURE (capture), + GST_VIDEO_FORMAT_BGRA); + + auto klass = GST_D3D12_SCREEN_CAPTURE_GET_CLASS (capture); + if (klass->get_format) + return klass->get_format (capture); + + return GST_VIDEO_FORMAT_BGRA; +} + gboolean gst_d3d12_screen_capture_unlock (GstD3D12ScreenCapture * capture) { diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapture.h b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapture.h index 9bfc6981aa..c870870921 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapture.h +++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapture.h @@ -57,12 +57,15 @@ struct _GstD3D12ScreenCaptureClass { GstObjectClass parent_class; - GstFlowReturn (*prepare) (GstD3D12ScreenCapture * capture); + GstFlowReturn (*prepare) (GstD3D12ScreenCapture * capture, + guint flags); gboolean (*get_size) (GstD3D12ScreenCapture * capture, guint * width, guint * height); + GstVideoFormat (*get_format) (GstD3D12ScreenCapture * capture); + gboolean (*unlock) (GstD3D12ScreenCapture * capture); gboolean (*unlock_stop) (GstD3D12ScreenCapture * capture); @@ -70,12 +73,15 @@ struct _GstD3D12ScreenCaptureClass GType gst_d3d12_screen_capture_get_type (void); -GstFlowReturn gst_d3d12_screen_capture_prepare (GstD3D12ScreenCapture * capture); +GstFlowReturn gst_d3d12_screen_capture_prepare (GstD3D12ScreenCapture * capture, + guint flags); gboolean gst_d3d12_screen_capture_get_size (GstD3D12ScreenCapture * capture, guint * width, guint * height); +GstVideoFormat gst_d3d12_screen_capture_get_format (GstD3D12ScreenCapture * capture); + gboolean gst_d3d12_screen_capture_unlock (GstD3D12ScreenCapture * capture); gboolean gst_d3d12_screen_capture_unlock_stop (GstD3D12ScreenCapture * capture); diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapturedevice.cpp b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapturedevice.cpp index 5751049676..9e57dc1cb2 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapturedevice.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapturedevice.cpp @@ -40,8 +40,9 @@ GST_DEBUG_CATEGORY_EXTERN (gst_d3d12_screen_capture_debug); static GstStaticCaps template_caps = GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, - "BGRA") ", pixel-aspect-ratio = 1/1, colorimetry = (string) sRGB; " - GST_VIDEO_CAPS_MAKE ("BGRA") ", pixel-aspect-ratio = 1/1, " + "{ BGRA, RGBA64_LE }") + ", pixel-aspect-ratio = 1/1, colorimetry = (string) sRGB; " + GST_VIDEO_CAPS_MAKE ("{ BGRA, RGBA64_LE }") ", pixel-aspect-ratio = 1/1, " "colorimetry = (string) sRGB"); enum diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapturesrc.cpp b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapturesrc.cpp index 50c1160b4e..511ffcd113 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapturesrc.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12screencapturesrc.cpp @@ -68,6 +68,7 @@ enum PROP_CAPTURE_API, PROP_ADAPTER, PROP_WINDOW_CAPTURE_MODE, + PROP_TONEMAP, }; enum GstD3D12ScreenCaptureAPI @@ -82,6 +83,12 @@ enum GstD3D12WindowCaptureMode GST_D3D12_WINDOW_CAPTURE_CLIENT, }; +enum GstD3D12ScreenCaptureTonemap +{ + GST_D3D12_SCREEN_CAPTURE_TONEMAP_LINEAR, + GST_D3D12_SCREEN_CAPTURE_TONEMAP_REINHARD, +}; + #ifdef HAVE_WGC /** * GstD3D12ScreenCaptureAPI: @@ -155,20 +162,58 @@ gst_d3d12_window_capture_mode_get_type (void) } #endif +/** + * GstD3D12ScreenCaptureTonemap: + * + * Since: 1.26 + */ +#define GST_TYPE_D3D12_SCREEN_CAPTURE_TONEMAP (gst_d3d12_screen_capture_tonemap_get_type()) +static GType +gst_d3d12_screen_capture_tonemap_get_type (void) +{ + static GType type = 0; + + GST_D3D12_CALL_ONCE_BEGIN { + static const GEnumValue modes[] = { + /** + * GstD3D12ScreenCaptureTonemap::linear: + * + * Since: 1.26 + */ + {GST_D3D12_SCREEN_CAPTURE_TONEMAP_LINEAR, + "Linear scaling", "linear"}, + + /** + * GstD3D12ScreenCaptureTonemap::reinhard: + * + * Since: 1.26 + */ + {GST_D3D12_SCREEN_CAPTURE_TONEMAP_REINHARD, "Reinhard tonemap", + "reinhard"}, + {0, nullptr, nullptr}, + }; + + type = g_enum_register_static ("GstD3D12ScreenCaptureTonemap", modes); + } GST_D3D12_CALL_ONCE_END; + + return type; +} + #define DEFAULT_MONITOR_INDEX -1 #define DEFAULT_SHOW_CURSOR FALSE #define DEFAULT_SHOW_BORDER FALSE #define DEFAULT_CAPTURE_API GST_D3D12_SCREEN_CAPTURE_API_DXGI #define DEFAULT_ADAPTER -1 #define DEFAULT_WINDOW_CAPTURE_MODE GST_D3D12_WINDOW_CAPTURE_DEFAULT +#define DEFAULT_TONEMAP GST_D3D12_SCREEN_CAPTURE_TONEMAP_LINEAR static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, "BGRA") + (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, "{ BGRA, RGBA64_LE }") ", pixel-aspect-ratio = 1/1, colorimetry = (string) sRGB; " - GST_VIDEO_CAPS_MAKE ("BGRA") ", pixel-aspect-ratio = 1/1, " - "colorimetry = (string) sRGB")); + GST_VIDEO_CAPS_MAKE ("{ BGRA, RGBA64_LE }") + ", pixel-aspect-ratio = 1/1, " "colorimetry = (string) sRGB")); struct GstD3D12ScreenCaptureSrcPrivate { @@ -203,6 +248,7 @@ struct GstD3D12ScreenCaptureSrcPrivate GstD3D12ScreenCaptureAPI capture_api = DEFAULT_CAPTURE_API; GstD3D12ScreenCaptureAPI selected_capture_api = DEFAULT_CAPTURE_API; GstD3D12WindowCaptureMode hwnd_capture_mode = DEFAULT_WINDOW_CAPTURE_MODE; + GstD3D12ScreenCaptureTonemap tonemap = DEFAULT_TONEMAP; gboolean flushing = FALSE; GstClockTime latency = GST_CLOCK_TIME_NONE; @@ -388,6 +434,23 @@ gst_d3d12_screen_capture_src_class_init (GstD3D12ScreenCaptureSrcClass * klass) } #endif + /** + * GstD3D12ScreenCaptureSrc:tonemap: + * + * Tonemapping method in case of HDR capture + * + * Since: 1.26 + */ + g_object_class_install_property (object_class, PROP_TONEMAP, + g_param_spec_enum ("tonemap", "Tonemap", + "Tonemapping method to use when HDR capturing is enabled", + GST_TYPE_D3D12_SCREEN_CAPTURE_TONEMAP, DEFAULT_TONEMAP, + (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS))); + + gst_type_mark_as_plugin_api (GST_TYPE_D3D12_SCREEN_CAPTURE_TONEMAP, + (GstPluginAPIFlags) 0); + element_class->provide_clock = GST_DEBUG_FUNCPTR (gst_d3d12_screen_capture_src_provide_clock); element_class->set_context = @@ -506,6 +569,9 @@ gst_d3d12_screen_capture_src_set_property (GObject * object, guint prop_id, } #endif break; + case PROP_TONEMAP: + priv->tonemap = (GstD3D12ScreenCaptureTonemap) g_value_get_enum (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -557,6 +623,9 @@ gst_d3d12_screen_capture_src_get_property (GObject * object, guint prop_id, case PROP_WINDOW_CAPTURE_MODE: g_value_set_enum (value, priv->hwnd_capture_mode); break; + case PROP_TONEMAP: + g_value_set_enum (value, priv->tonemap); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -622,6 +691,7 @@ gst_d3d12_screen_capture_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter) auto self = GST_D3D12_SCREEN_CAPTURE_SRC (bsrc); auto priv = self->priv; guint width, height; + GstVideoFormat format = GST_VIDEO_FORMAT_BGRA; std::lock_guard < std::recursive_mutex > lk (priv->lock); if (!priv->capture) { @@ -637,10 +707,13 @@ gst_d3d12_screen_capture_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter) height = priv->crop_box.bottom - priv->crop_box.top; } + format = gst_d3d12_screen_capture_get_format (priv->capture); + auto caps = gst_pad_get_pad_template_caps (GST_BASE_SRC_PAD (bsrc)); caps = gst_caps_make_writable (caps); - gst_caps_set_simple (caps, "width", G_TYPE_INT, width, "height", + gst_caps_set_simple (caps, "format", G_TYPE_STRING, + gst_video_format_to_string (format), "width", G_TYPE_INT, width, "height", G_TYPE_INT, height, nullptr); if (filter) { @@ -920,7 +993,8 @@ gst_d3d12_screen_capture_src_start (GstBaseSrc * bsrc) } /* Check if we can open device */ - ret = gst_d3d12_screen_capture_prepare (capture); + ret = gst_d3d12_screen_capture_prepare (capture, + priv->tonemap == GST_D3D12_SCREEN_CAPTURE_TONEMAP_REINHARD); switch (ret) { case GST_D3D12_SCREEN_CAPTURE_FLOW_EXPECTED_ERROR: case GST_FLOW_OK: