d3d11desktopdupsrc: Add support for desktop size/rotation mode change
authorSeungha Yang <seungha@centricular.com>
Thu, 20 May 2021 08:03:15 +0000 (17:03 +0900)
committerGStreamer Marge Bot <gitlab-merge-bot@gstreamer-foundation.org>
Thu, 20 May 2021 17:05:24 +0000 (17:05 +0000)
Re-negotiates with updated size on desktop size
(i.e., resolution, scaling factor), and rotation mode change

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2268>

sys/d3d11/gstd3d11desktopdup.cpp
sys/d3d11/gstd3d11desktopdup.h
sys/d3d11/gstd3d11desktopdupsrc.cpp
sys/d3d11/gstd3d11desktopdupsrc.h

index 2d8bce7..8be33c1 100644 (file)
@@ -264,9 +264,12 @@ public:
   }
 
   GstFlowReturn
-  Init (GstD3D11Device * device, ID3D11Texture2D * texture, UINT monitor_index)
+  Init (GstD3D11Device * device, UINT monitor_index)
   {
     GstFlowReturn ret;
+    ID3D11Device *device_handle;
+    HRESULT hr;
+    D3D11_TEXTURE2D_DESC texture_desc = { 0, };
 
     if (!InitShader (device))
       return GST_FLOW_ERROR;
@@ -277,7 +280,28 @@ public:
 
     GST_INFO ("Init done");
 
-    shared_texture_ = texture;
+    device_handle = gst_d3d11_device_get_device_handle (device);
+
+    texture_desc.Width = output_desc_.ModeDesc.Width;
+    texture_desc.Height = output_desc_.ModeDesc.Height;
+    texture_desc.MipLevels = 1;
+    texture_desc.ArraySize = 1;
+    /* FIXME: we can support DXGI_FORMAT_R10G10B10A2_UNORM */
+    texture_desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
+    texture_desc.SampleDesc.Count = 1;
+    texture_desc.Usage = D3D11_USAGE_DEFAULT;
+    texture_desc.BindFlags =
+        D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
+    texture_desc.CPUAccessFlags = 0;
+    texture_desc.MiscFlags = 0;
+
+    hr = device_handle->CreateTexture2D (&texture_desc,
+        nullptr, &shared_texture_);
+    if (!gst_d3d11_result (hr, device)) {
+      GST_ERROR_OBJECT (device, "Couldn't create texture, hr 0x%x", (guint) hr);
+      return GST_FLOW_ERROR;
+    }
+
     device_ = (GstD3D11Device *) gst_object_ref (device);
 
     return GST_FLOW_OK;
@@ -511,6 +535,23 @@ public:
     return true;
   }
 
+  void
+  CopyToTexture (ID3D11Texture2D * texture)
+  {
+    ID3D11DeviceContext *context_handle =
+        gst_d3d11_device_get_device_context_handle (device_);
+
+    context_handle->CopySubresourceRegion (texture, 0, 0, 0, 0,
+      shared_texture_.Get(), 0, nullptr);
+  }
+
+  void
+  GetSize (guint * width, guint * height)
+  {
+    *width = output_desc_.ModeDesc.Width;
+    *height = output_desc_.ModeDesc.Height;
+  }
+
 private:
   /* This method is not expected to be failed unless un-recoverable error case */
   bool
@@ -652,8 +693,6 @@ private:
           hr, EnumOutputsExpectedErrors);
     }
 
-    output->GetDesc(&output_desc_);
-
     hr = output.As (&output1);
     if (!gst_d3d11_result (hr, device)) {
       GST_ERROR ("Couldn't get IDXGIOutput1 interface, hr 0x%x", (guint) hr);
@@ -694,6 +733,8 @@ private:
           CreateDuplicationExpectedErrors);
     }
 
+    dupl_->GetDesc (&output_desc_);
+
     return GST_FLOW_OK;
   }
 
@@ -870,7 +911,7 @@ private:
   }
 
   void
-  SetMoveRect (RECT* SrcRect, RECT* DestRect, DXGI_OUTPUT_DESC* DeskDesc,
+  SetMoveRect (RECT* SrcRect, RECT* DestRect, DXGI_OUTDUPL_DESC* DeskDesc,
       DXGI_OUTDUPL_MOVE_RECT* MoveRect, INT TexWidth, INT TexHeight)
   {
     switch (DeskDesc->Rotation)
@@ -934,7 +975,7 @@ private:
 
   GstFlowReturn
   CopyMove (ID3D11Texture2D* SharedSurf, DXGI_OUTDUPL_MOVE_RECT* MoveBuffer,
-      UINT MoveCount,DXGI_OUTPUT_DESC* DeskDesc)
+      UINT MoveCount, DXGI_OUTDUPL_DESC* DeskDesc)
   {
     ID3D11Device *device_handle = gst_d3d11_device_get_device_handle (device_);
     ID3D11DeviceContext *device_context =
@@ -986,7 +1027,7 @@ private:
 
   void
   SetDirtyVert (VERTEX* Vertices, RECT* Dirty,
-      DXGI_OUTPUT_DESC* DeskDesc, D3D11_TEXTURE2D_DESC* FullDesc,
+      DXGI_OUTDUPL_DESC* DeskDesc, D3D11_TEXTURE2D_DESC* FullDesc,
       D3D11_TEXTURE2D_DESC* ThisDesc)
   {
     INT CenterX = FullDesc->Width / 2;
@@ -1119,7 +1160,7 @@ private:
 
   GstFlowReturn
   CopyDirty (ID3D11Texture2D* SrcSurface, ID3D11Texture2D* SharedSurf,
-      RECT* DirtyBuffer, UINT DirtyCount, DXGI_OUTPUT_DESC* DeskDesc)
+      RECT* DirtyBuffer, UINT DirtyCount, DXGI_OUTDUPL_DESC* DeskDesc)
   {
     D3D11_TEXTURE2D_DESC FullDesc;
     D3D11_TEXTURE2D_DESC ThisDesc;
@@ -1224,7 +1265,7 @@ private:
 
   GstFlowReturn
   ProcessFrame(ID3D11Texture2D * acquired_texture, ID3D11Texture2D* SharedSurf,
-      DXGI_OUTPUT_DESC* DeskDesc, UINT move_count, UINT dirty_count,
+      DXGI_OUTDUPL_DESC* DeskDesc, UINT move_count, UINT dirty_count,
       DXGI_OUTDUPL_FRAME_INFO * frame_info)
   {
     GstFlowReturn ret = GST_FLOW_OK;
@@ -1409,7 +1450,7 @@ private:
 
 private:
   PTR_INFO ptr_info_;
-  DXGI_OUTPUT_DESC output_desc_;
+  DXGI_OUTDUPL_DESC output_desc_;
   GstD3D11Device * device_;
 
   ComPtr<ID3D11Texture2D> shared_texture_;
@@ -1437,10 +1478,9 @@ struct _GstD3D11DesktopDup
   GstObject parent;
 
   GstD3D11Device *device;
-  guint width;
-  guint height;
+  guint cached_width;
+  guint cached_height;
 
-  ID3D11Texture2D *texture;
   D3D11DesktopDupObject *dupl_obj;
 
   gboolean primary;
@@ -1603,16 +1643,16 @@ gst_d3d11_desktop_dup_constructed (GObject * object)
     goto out;
   }
 
-  self->width =
+  self->cached_width =
       self->desktop_coordinates.right - self->desktop_coordinates.left;
-  self->height =
+  self->cached_height =
       self->desktop_coordinates.bottom - self->desktop_coordinates.top;
 
   GST_DEBUG_OBJECT (self,
       "Desktop coordinates left:top:right:bottom = %ld:%ld:%ld:%ld (%dx%d)",
       self->desktop_coordinates.left, self->desktop_coordinates.top,
       self->desktop_coordinates.right, self->desktop_coordinates.bottom,
-      self->width, self->height);
+      self->cached_width, self->cached_height);
 
   ret = TRUE;
 
@@ -1645,8 +1685,6 @@ gst_d3d11_desktop_dup_dispose (GObject * object)
 {
   GstD3D11DesktopDup *self = GST_D3D11_DESKTOP_DUP (object);
 
-  GST_D3D11_CLEAR_COM (self->texture);
-
   if (self->dupl_obj) {
     delete self->dupl_obj;
     self->dupl_obj = nullptr;
@@ -1667,44 +1705,6 @@ gst_d3d11_desktop_dup_finalize (GObject * object)
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
-static gboolean
-gst_d3d11_desktop_dup_setup_texture (GstD3D11DesktopDup * self)
-{
-  D3D11_TEXTURE2D_DESC texture_desc = { 0, };
-  ID3D11Device *device_handle;
-  HRESULT hr;
-  /* *INDENT-OFF* */
-  ComPtr<ID3D11Texture2D> texture;
-  /* *INDENT-ON* */
-
-  /* This texture is for copying/updating only updated region from previously
-   * captured frame (like a reference frame) */
-  device_handle = gst_d3d11_device_get_device_handle (self->device);
-
-  texture_desc.Width = self->width;
-  texture_desc.Height = self->height;
-  texture_desc.MipLevels = 1;
-  texture_desc.ArraySize = 1;
-  /* FIXME: we can support DXGI_FORMAT_R10G10B10A2_UNORM */
-  texture_desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
-  texture_desc.SampleDesc.Count = 1;
-  texture_desc.Usage = D3D11_USAGE_DEFAULT;
-  texture_desc.BindFlags =
-      D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
-  texture_desc.CPUAccessFlags = 0;
-  texture_desc.MiscFlags = 0;
-
-  hr = device_handle->CreateTexture2D (&texture_desc, nullptr, &texture);
-  if (!gst_d3d11_result (hr, self->device)) {
-    GST_ERROR_OBJECT (self, "Couldn't create texture, hr 0x%x", (guint) hr);
-    return FALSE;
-  }
-
-  self->texture = texture.Detach ();
-
-  return TRUE;
-}
-
 static void
 gst_d3d11_desktop_dup_weak_ref_notify (gpointer data, GstD3D11DesktopDup * dupl)
 {
@@ -1781,15 +1781,8 @@ gst_d3d11_desktop_dup_prepare (GstD3D11DesktopDup * desktop)
     return GST_FLOW_OK;
   }
 
-  if (!desktop->texture && !gst_d3d11_desktop_dup_setup_texture (desktop)) {
-    GST_ERROR_OBJECT (desktop, "Couldn't setup internal texture");
-    g_rec_mutex_unlock (&desktop->lock);
-    return GST_FLOW_ERROR;
-  }
-
   desktop->dupl_obj = new D3D11DesktopDupObject ();
-  ret = desktop->dupl_obj->Init (desktop->device, desktop->texture,
-      desktop->monitor_index);
+  ret = desktop->dupl_obj->Init (desktop->device, desktop->monitor_index);
   if (ret != GST_FLOW_OK) {
     GST_WARNING_OBJECT (desktop,
         "Couldn't prepare capturing, %sexpected failure",
@@ -1809,13 +1802,25 @@ gst_d3d11_desktop_dup_prepare (GstD3D11DesktopDup * desktop)
 }
 
 gboolean
-gst_d3d11_desktop_dup_get_coordinates (GstD3D11DesktopDup * desktop,
-    RECT * desktop_coordinates)
+gst_d3d11_desktop_dup_get_size (GstD3D11DesktopDup * desktop, guint * width,
+    guint * height)
 {
   g_return_val_if_fail (GST_IS_D3D11_DESKTOP_DUP (desktop), FALSE);
-  g_return_val_if_fail (desktop_coordinates != nullptr, FALSE);
+  g_return_val_if_fail (width != nullptr, FALSE);
+  g_return_val_if_fail (height != nullptr, FALSE);
+
+  g_rec_mutex_lock (&desktop->lock);
+  *width = 0;
+  *height = 0;
+
+  if (desktop->dupl_obj) {
+    desktop->dupl_obj->GetSize (&desktop->cached_width,
+        &desktop->cached_height);
+  }
 
-  *desktop_coordinates = desktop->desktop_coordinates;
+  *width = desktop->cached_width;
+  *height = desktop->cached_height;
+  g_rec_mutex_unlock (&desktop->lock);
 
   return TRUE;
 }
@@ -1826,7 +1831,8 @@ gst_d3d11_desktop_dup_capture (GstD3D11DesktopDup * desktop,
     gboolean draw_mouse)
 {
   GstFlowReturn ret = GST_FLOW_OK;
-  ID3D11DeviceContext *device_context_handle;
+  D3D11_TEXTURE2D_DESC desc;
+  guint width, height;
 
   g_return_val_if_fail (GST_IS_D3D11_DESKTOP_DUP (desktop), GST_FLOW_ERROR);
   g_return_val_if_fail (texture != nullptr, GST_FLOW_ERROR);
@@ -1841,6 +1847,18 @@ gst_d3d11_desktop_dup_capture (GstD3D11DesktopDup * desktop,
     return ret;
   }
 
+  gst_d3d11_desktop_dup_get_size (desktop, &width, &height);
+
+  texture->GetDesc (&desc);
+  if (desc.Width != width || desc.Height != height) {
+    GST_INFO_OBJECT (desktop,
+        "Different texture size, ours: %dx%d, external: %dx%d",
+        width, height, desc.Width, desc.Height);
+    g_rec_mutex_unlock (&desktop->lock);
+
+    return GST_D3D11_DESKTOP_DUP_FLOW_SIZE_CHANGED;
+  }
+
   gst_d3d11_device_lock (desktop->device);
   ret = desktop->dupl_obj->Capture (draw_mouse);
   if (ret != GST_FLOW_OK) {
@@ -1863,10 +1881,7 @@ gst_d3d11_desktop_dup_capture (GstD3D11DesktopDup * desktop,
 
   GST_LOG_OBJECT (desktop, "Capture done");
 
-  device_context_handle =
-      gst_d3d11_device_get_device_context_handle (desktop->device);
-  device_context_handle->CopySubresourceRegion (texture, 0, 0, 0, 0,
-      desktop->texture, 0, nullptr);
+  desktop->dupl_obj->CopyToTexture (texture);
   if (draw_mouse)
     desktop->dupl_obj->DrawMouse (rtv);
   gst_d3d11_device_unlock (desktop->device);
index b56ee91..5741c19 100644 (file)
@@ -28,6 +28,7 @@
 G_BEGIN_DECLS
 
 #define GST_D3D11_DESKTOP_DUP_FLOW_EXPECTED_ERROR GST_FLOW_CUSTOM_SUCCESS
+#define GST_D3D11_DESKTOP_DUP_FLOW_SIZE_CHANGED GST_FLOW_CUSTOM_SUCCESS_1
 #define GST_D3D11_DESKTOP_DUP_FLOW_UNSUPPORTED GST_FLOW_CUSTOM_ERROR
 
 #define GST_TYPE_D3D11_DESKTOP_DUP (gst_d3d11_desktop_dup_get_type())
@@ -39,8 +40,9 @@ GstD3D11DesktopDup *  gst_d3d11_desktop_dup_new (GstD3D11Device * device,
 
 GstFlowReturn   gst_d3d11_desktop_dup_prepare (GstD3D11DesktopDup * desktop);
 
-gboolean        gst_d3d11_desktop_dup_get_coordinates (GstD3D11DesktopDup * desktop,
-                                                       RECT * desktop_coordinates);
+gboolean        gst_d3d11_desktop_dup_get_size (GstD3D11DesktopDup * desktop,
+                                                guint * width,
+                                                guint * height);
 
 GstFlowReturn   gst_d3d11_desktop_dup_capture (GstD3D11DesktopDup * desktop,
                                                ID3D11Texture2D * texture,
index eb098cf..a01ef16 100644 (file)
@@ -71,7 +71,7 @@ GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
 
 struct _GstD3D11DesktopDupSrc
 {
-  GstPushSrc src;
+  GstBaseSrc src;
 
   guint64 last_frame_no;
   GstClockID clock_id;
@@ -113,12 +113,12 @@ static gboolean gst_d3d11_desktop_dup_src_unlock_stop (GstBaseSrc * bsrc);
 static gboolean
 gst_d3d11_desktop_dup_src_src_query (GstBaseSrc * bsrc, GstQuery * query);
 
-static GstFlowReturn gst_d3d11_desktop_dup_src_fill (GstPushSrc * pushsrc,
-    GstBuffer * buffer);
+static GstFlowReturn gst_d3d11_desktop_dup_src_create (GstBaseSrc * bsrc,
+    guint64 offset, guint size, GstBuffer ** buf);
 
 #define gst_d3d11_desktop_dup_src_parent_class parent_class
 G_DEFINE_TYPE (GstD3D11DesktopDupSrc, gst_d3d11_desktop_dup_src,
-    GST_TYPE_PUSH_SRC);
+    GST_TYPE_BASE_SRC);
 
 static void
 gst_d3d11_desktop_dup_src_class_init (GstD3D11DesktopDupSrcClass * klass)
@@ -126,7 +126,6 @@ gst_d3d11_desktop_dup_src_class_init (GstD3D11DesktopDupSrcClass * klass)
   GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
   GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
   GstBaseSrcClass *basesrc_class = GST_BASE_SRC_CLASS (klass);
-  GstPushSrcClass *pushsrc_class = GST_PUSH_SRC_CLASS (klass);
   GstCaps *caps;
 
   gobject_class->dispose = gst_d3d11_desktop_dup_src_dispose;
@@ -176,7 +175,7 @@ gst_d3d11_desktop_dup_src_class_init (GstD3D11DesktopDupSrcClass * klass)
   basesrc_class->query =
       GST_DEBUG_FUNCPTR (gst_d3d11_desktop_dup_src_src_query);
 
-  pushsrc_class->fill = GST_DEBUG_FUNCPTR (gst_d3d11_desktop_dup_src_fill);
+  basesrc_class->create = GST_DEBUG_FUNCPTR (gst_d3d11_desktop_dup_src_create);
 }
 
 static void
@@ -259,22 +258,18 @@ gst_d3d11_desktop_dup_src_get_caps (GstBaseSrc * bsrc, GstCaps * filter)
   GstD3D11DesktopDupSrc *self = GST_D3D11_DESKTOP_DUP_SRC (bsrc);
   GstCaps *caps = NULL;
   guint width, height;
-  RECT desktop_coordinates;
 
   if (!self->dupl) {
     GST_DEBUG_OBJECT (self, "Duplication object is not configured yet");
     return gst_pad_get_pad_template_caps (GST_BASE_SRC_PAD (bsrc));
   }
 
-  if (!gst_d3d11_desktop_dup_get_coordinates (self->dupl, &desktop_coordinates)) {
+  if (!gst_d3d11_desktop_dup_get_size (self->dupl, &width, &height)) {
     GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ,
         ("Cannot query supported resolution"), (NULL));
     return NULL;
   }
 
-  width = desktop_coordinates.right - desktop_coordinates.left;
-  height = desktop_coordinates.bottom - desktop_coordinates.top;
-
   caps =
       gst_caps_new_simple ("video/x-raw", "format", G_TYPE_STRING, "BGRA",
       "width", G_TYPE_INT, width,
@@ -520,9 +515,10 @@ gst_d3d11_desktop_dup_src_src_query (GstBaseSrc * bsrc, GstQuery * query)
 }
 
 static GstFlowReturn
-gst_d3d11_desktop_dup_src_fill (GstPushSrc * pushsrc, GstBuffer * buffer)
+gst_d3d11_desktop_dup_src_create (GstBaseSrc * bsrc, guint64 offset, guint size,
+    GstBuffer ** buf)
 {
-  GstD3D11DesktopDupSrc *self = GST_D3D11_DESKTOP_DUP_SRC (pushsrc);
+  GstD3D11DesktopDupSrc *self = GST_D3D11_DESKTOP_DUP_SRC (bsrc);
   ID3D11Texture2D *texture;
   ID3D11RenderTargetView *rtv = NULL;
   gint fps_n, fps_d;
@@ -542,6 +538,7 @@ gst_d3d11_desktop_dup_src_fill (GstPushSrc * pushsrc, GstBuffer * buffer)
   gboolean draw_mouse;
   /* Just magic number... */
   gint unsupported_retry_count = 100;
+  GstBuffer *buffer = NULL;
 
   if (!self->dupl) {
     GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ,
@@ -630,6 +627,13 @@ again:
   self->last_frame_no = next_frame_no;
   GST_OBJECT_UNLOCK (self);
 
+  if (!buffer) {
+    ret =
+        GST_BASE_SRC_CLASS (parent_class)->alloc (bsrc, offset, size, &buffer);
+    if (ret != GST_FLOW_OK)
+      goto out;
+  }
+
   /* FIXME: handle fallback case
    * (e.g., texture belongs to other device, RTV is unavailable) */
   mem = gst_buffer_peek_memory (buffer, 0);
@@ -658,21 +662,34 @@ again:
   GST_BUFFER_PTS (buffer) = next_capture_ts;
   GST_BUFFER_DURATION (buffer) = dur;
 
-  if (ret == GST_D3D11_DESKTOP_DUP_FLOW_EXPECTED_ERROR) {
-    GST_WARNING_OBJECT (self, "Got expected error, try again");
-    gst_clear_object (&clock);
-    goto again;
-  } else if (ret == GST_D3D11_DESKTOP_DUP_FLOW_UNSUPPORTED) {
-    GST_WARNING_OBJECT (self, "Got DXGI_ERROR_UNSUPPORTED error");
-    unsupported_retry_count--;
-
-    if (unsupported_retry_count < 0) {
-      ret = GST_FLOW_ERROR;
-      goto out;
-    }
+  switch (ret) {
+    case GST_D3D11_DESKTOP_DUP_FLOW_EXPECTED_ERROR:
+      GST_WARNING_OBJECT (self, "Got expected error, try again");
+      gst_clear_object (&clock);
+      goto again;
+    case GST_D3D11_DESKTOP_DUP_FLOW_UNSUPPORTED:
+      GST_WARNING_OBJECT (self, "Got DXGI_ERROR_UNSUPPORTED error");
+      unsupported_retry_count--;
 
-    gst_clear_object (&clock);
-    goto again;
+      if (unsupported_retry_count < 0) {
+        ret = GST_FLOW_ERROR;
+        goto out;
+      }
+      gst_clear_object (&clock);
+      goto again;
+    case GST_D3D11_DESKTOP_DUP_FLOW_SIZE_CHANGED:
+      GST_INFO_OBJECT (self, "Size was changed, need negotiation");
+      gst_clear_buffer (&buffer);
+      gst_clear_object (&clock);
+
+      if (!gst_base_src_negotiate (bsrc)) {
+        GST_ERROR_OBJECT (self, "Failed to negotiate with new size");
+        ret = GST_FLOW_NOT_NEGOTIATED;
+        goto out;
+      }
+      goto again;
+    default:
+      break;
   }
 
   after_capture = gst_clock_get_time (clock);
@@ -698,6 +715,7 @@ again:
 
 out:
   gst_clear_object (&clock);
+  *buf = buffer;
 
   return ret;
 }
index 8305522..2149d98 100644 (file)
 
 #include <gst/gst.h>
 #include <gst/video/video.h>
-#include <gst/base/gstpushsrc.h>
+#include <gst/base/gstbasesrc.h>
 #include <gst/d3d11/gstd3d11.h>
 
 G_BEGIN_DECLS
 
 #define GST_TYPE_D3D11_DESKTOP_DUP_SRC (gst_d3d11_desktop_dup_src_get_type())
 G_DECLARE_FINAL_TYPE (GstD3D11DesktopDupSrc, gst_d3d11_desktop_dup_src,
-    GST, D3D11_DESKTOP_DUP_SRC, GstPushSrc);
+    GST, D3D11_DESKTOP_DUP_SRC, GstBaseSrc);
 
 G_END_DECLS