d3d11: Don't hold staging texture

Staging texture is used for memory transfer between system and
gpu memory. Apart from d3d11{upload,download} elements, however,
it should happen very rarely.

Before this commit, d3d11bufferpool was allocating at least one
staging texture in order to calculate cpu accessible memory size,
and it wasn't freed for later use of the texture unconditionally.
But it will increase system memory usage. Although GstD3D11memory
object is implemented so that support CPU access, most memory
transfer will happen in d3d11{upload,download} elements.

By this commit, the initial staging texture will be freed immediately
once cpu accessible memory size is calculated.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1627>

diff --git a/sys/d3d11/gstd3d11bufferpool.c b/sys/d3d11/gstd3d11bufferpool.c
index 671f796..80cf647 100644 (file)
--- a/sys/d3d11/gstd3d11bufferpool.c
+++ b/sys/d3d11/gstd3d11bufferpool.c
@@ -24,6 +24,9 @@
 #include "gstd3d11bufferpool.h"
 #include "gstd3d11memory.h"
 #include "gstd3d11device.h"
+#include "gstd3d11utils.h"
+
+#include <string.h>
 
 GST_DEBUG_CATEGORY_STATIC (gst_d3d11_buffer_pool_debug);
 #define GST_CAT_DEFAULT gst_d3d11_buffer_pool_debug
@@ -33,11 +36,12 @@ struct _GstD3D11BufferPoolPrivate
   GstD3D11Device *device;
   GstD3D11Allocator *allocator;
 
-  /* initial buffer used for calculating buffer size */
-  GstBuffer *initial_buffer;
-
   gboolean add_videometa;
   GstD3D11AllocationParams *d3d11_params;
+
+  gint stride[GST_VIDEO_MAX_PLANES];
+  gsize size[GST_VIDEO_MAX_PLANES];
+  gsize offset[GST_VIDEO_MAX_PLANES];
 };
 
 #define gst_d3d11_buffer_pool_parent_class parent_class
@@ -87,7 +91,6 @@ gst_d3d11_buffer_pool_dispose (GObject * object)
     gst_d3d11_allocation_params_free (priv->d3d11_params);
   priv->d3d11_params = NULL;
 
-  gst_clear_buffer (&priv->initial_buffer);
   gst_clear_object (&priv->device);
   gst_clear_object (&priv->allocator);
 
@@ -114,6 +117,7 @@ gst_d3d11_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
   GstAllocator *allocator = NULL;
   gboolean ret = TRUE;
   D3D11_TEXTURE2D_DESC *desc;
+  GstBuffer *staging_buffer;
   gint i;
 
   if (!gst_buffer_pool_config_get_params (config, &caps, NULL, &min_buffers,
@@ -133,7 +137,6 @@ gst_d3d11_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
   if (!gst_buffer_pool_config_get_allocator (config, &allocator, NULL))
     goto wrong_config;
 
-  gst_clear_buffer (&priv->initial_buffer);
   gst_clear_object (&priv->allocator);
 
   if (allocator) {
@@ -227,14 +230,33 @@ gst_d3d11_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
     }
   }
 
-  gst_d3d11_buffer_pool_alloc (pool, &priv->initial_buffer, NULL);
+  staging_buffer = gst_d3d11_allocate_staging_buffer (priv->allocator,
+      &info, priv->d3d11_params->d3d11_format, priv->d3d11_params->desc, TRUE);
 
-  if (!priv->initial_buffer) {
-    GST_ERROR_OBJECT (pool, "Could not create initial buffer");
+  if (!staging_buffer) {
+    GST_ERROR_OBJECT (pool, "Couldn't allocated staging buffer");
     return FALSE;
+  } else {
+    GstVideoMeta *meta = gst_buffer_get_video_meta (staging_buffer);
+
+    if (!meta) {
+      GST_ERROR_OBJECT (pool, "Buffer doesn't have video meta");
+      gst_buffer_unref (staging_buffer);
+      return FALSE;
+    }
+
+    for (i = 0; i < gst_buffer_n_memory (staging_buffer); i++) {
+      GstMemory *mem = gst_buffer_peek_memory (staging_buffer, i);
+
+      priv->size[i] = gst_memory_get_sizes (mem, NULL, NULL);
+    }
+
+    memcpy (priv->offset, meta->offset, sizeof (priv->offset));
+    memcpy (priv->stride, meta->stride, sizeof (priv->stride));
   }
 
-  self->buffer_size = gst_buffer_get_size (priv->initial_buffer);
+  self->buffer_size = gst_buffer_get_size (staging_buffer);
+  gst_buffer_unref (staging_buffer);
 
   gst_buffer_pool_config_set_params (config,
       caps, self->buffer_size, min_buffers, max_buffers);
@@ -275,34 +297,22 @@ gst_d3d11_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer,
   GstBuffer *buf;
   GstD3D11AllocationParams *d3d11_params = priv->d3d11_params;
   GstVideoInfo *info = &d3d11_params->info;
-  GstVideoInfo *aligned_info = &d3d11_params->aligned_info;
-  gint n_texture = 0;
   gint i;
-  gsize offset[GST_VIDEO_MAX_PLANES] = { 0, };
-
-  /* consume pre-allocated buffer if any */
-  if (G_UNLIKELY (priv->initial_buffer)) {
-    *buffer = priv->initial_buffer;
-    priv->initial_buffer = NULL;
-
-    return GST_FLOW_OK;
-  }
 
   buf = gst_buffer_new ();
 
   if (d3d11_params->d3d11_format->dxgi_format == DXGI_FORMAT_UNKNOWN) {
-    for (n_texture = 0; n_texture < GST_VIDEO_INFO_N_PLANES (info); n_texture++) {
-      d3d11_params->plane = n_texture;
-      mem = gst_d3d11_allocator_alloc (priv->allocator, d3d11_params);
+    for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) {
+      mem = gst_d3d11_allocator_alloc (priv->allocator, &d3d11_params->desc[i],
+          d3d11_params->flags, priv->size[i]);
       if (!mem)
         goto error;
 
       gst_buffer_append_memory (buf, mem);
     }
   } else {
-    d3d11_params->plane = 0;
-    mem = gst_d3d11_allocator_alloc (priv->allocator, priv->d3d11_params);
-    n_texture++;
+    mem = gst_d3d11_allocator_alloc (priv->allocator, &d3d11_params->desc[0],
+        d3d11_params->flags, priv->size[0]);
 
     if (!mem)
       goto error;
@@ -310,18 +320,12 @@ gst_d3d11_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer,
     gst_buffer_append_memory (buf, mem);
   }
 
-  /* calculate offset */
-  for (i = 0; i < n_texture && i < GST_VIDEO_MAX_PLANES - 1; i++) {
-    offset[i + 1] = offset[i] +
-        d3d11_params->stride[i] * GST_VIDEO_INFO_COMP_HEIGHT (aligned_info, i);
-  }
-
   if (priv->add_videometa) {
     GST_DEBUG_OBJECT (self, "adding GstVideoMeta");
     gst_buffer_add_video_meta_full (buf, GST_VIDEO_FRAME_FLAG_NONE,
         GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
         GST_VIDEO_INFO_HEIGHT (info), GST_VIDEO_INFO_N_PLANES (info),
-        offset, d3d11_params->stride);
+        priv->offset, priv->stride);
   }
 
   *buffer = buf;

diff --git a/sys/d3d11/gstd3d11memory.c b/sys/d3d11/gstd3d11memory.c
index d1a43e5..15257cc 100644 (file)
--- a/sys/d3d11/gstd3d11memory.c
+++ b/sys/d3d11/gstd3d11memory.c
@@ -262,10 +262,54 @@ map_cpu_access_data (GstD3D11Memory * dmem, D3D11_MAP map_type)
 }
 
 static gpointer
+gst_d3d11_memory_map_staging (GstMemory * mem, GstMapFlags flags)
+{
+  GstD3D11Memory *dmem = (GstD3D11Memory *) mem;
+
+  g_mutex_lock (&dmem->lock);
+  if (dmem->cpu_map_count == 0) {
+    ID3D11DeviceContext *device_context =
+        gst_d3d11_device_get_device_context_handle (dmem->device);
+    D3D11_MAP map_type;
+    HRESULT hr;
+    gboolean ret = TRUE;
+
+    map_type = gst_map_flags_to_d3d11 (flags);
+
+    gst_d3d11_device_lock (dmem->device);
+    hr = ID3D11DeviceContext_Map (device_context,
+        (ID3D11Resource *) dmem->texture, 0, map_type, 0, &dmem->map);
+    if (!gst_d3d11_result (hr, dmem->device)) {
+      GST_ERROR_OBJECT (GST_MEMORY_CAST (dmem)->allocator,
+          "Failed to map staging texture (0x%x)", (guint) hr);
+      ret = FALSE;
+    }
+    gst_d3d11_device_unlock (dmem->device);
+
+    if (!ret) {
+      g_mutex_unlock (&dmem->lock);
+      return NULL;
+    }
+  }
+
+  dmem->cpu_map_count++;
+  g_mutex_unlock (&dmem->lock);
+
+  return dmem->map.pData;
+}
+
+static gpointer
 gst_d3d11_memory_map (GstMemory * mem, gsize maxsize, GstMapFlags flags)
 {
   GstD3D11Memory *dmem = (GstD3D11Memory *) mem;
 
+  if (dmem->type == GST_D3D11_MEMORY_TYPE_STAGING) {
+    if ((flags & GST_MAP_D3D11) == GST_MAP_D3D11)
+      return dmem->texture;
+
+    return gst_d3d11_memory_map_staging (mem, flags);
+  }
+
   g_mutex_lock (&dmem->lock);
   if ((flags & GST_MAP_D3D11) == GST_MAP_D3D11) {
     if (dmem->staging &&
@@ -336,6 +380,9 @@ unmap_cpu_access_data (GstD3D11Memory * dmem)
   ID3D11DeviceContext *device_context =
       gst_d3d11_device_get_device_context_handle (dmem->device);
 
+  if (dmem->type == GST_D3D11_MEMORY_TYPE_STAGING)
+    staging = (ID3D11Resource *) dmem->texture;
+
   gst_d3d11_device_lock (dmem->device);
   ID3D11DeviceContext_Unmap (device_context, staging, 0);
   gst_d3d11_device_unlock (dmem->device);
@@ -348,14 +395,16 @@ gst_d3d11_memory_unmap_full (GstMemory * mem, GstMapInfo * info)
 
   g_mutex_lock (&dmem->lock);
   if ((info->flags & GST_MAP_D3D11) == GST_MAP_D3D11) {
-    if ((info->flags & GST_MAP_WRITE) == GST_MAP_WRITE)
+    if (dmem->type != GST_D3D11_MEMORY_TYPE_STAGING &&
+        (info->flags & GST_MAP_WRITE) == GST_MAP_WRITE)
       GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD);
 
     g_mutex_unlock (&dmem->lock);
     return;
   }
 
-  if ((info->flags & GST_MAP_WRITE))
+  if (dmem->type != GST_D3D11_MEMORY_TYPE_STAGING &&
+      (info->flags & GST_MAP_WRITE))
     GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_UPLOAD);
 
   dmem->cpu_map_count--;
@@ -498,7 +547,7 @@ gst_d3d11_allocator_new (GstD3D11Device * device)
 
 static gboolean
 calculate_mem_size (GstD3D11Device * device, ID3D11Texture2D * texture,
-    D3D11_TEXTURE2D_DESC * desc, D3D11_MAP map_type,
+    const D3D11_TEXTURE2D_DESC * desc, D3D11_MAP map_type,
     gint stride[GST_VIDEO_MAX_PLANES], gsize * size)
 {
   HRESULT hr;
@@ -684,31 +733,24 @@ error:
 
 GstMemory *
 gst_d3d11_allocator_alloc (GstD3D11Allocator * allocator,
-    GstD3D11AllocationParams * params)
+    const D3D11_TEXTURE2D_DESC * desc, GstD3D11AllocationFlags flags,
+    gsize size)
 {
   GstD3D11Memory *mem;
   GstD3D11Device *device;
   ID3D11Texture2D *texture = NULL;
-  ID3D11Texture2D *staging = NULL;
-  D3D11_TEXTURE2D_DESC *desc;
-  gsize *size;
-  gboolean is_first = FALSE;
   guint index_to_use = 0;
   GstD3D11AllocatorPrivate *priv;
   GstD3D11MemoryType type = GST_D3D11_MEMORY_TYPE_TEXTURE;
 
   g_return_val_if_fail (GST_IS_D3D11_ALLOCATOR (allocator), NULL);
-  g_return_val_if_fail (params != NULL, NULL);
+  g_return_val_if_fail (desc != NULL, NULL);
+  g_return_val_if_fail (size > 0, NULL);
 
   priv = allocator->priv;
   device = allocator->device;
-  desc = &params->desc[params->plane];
-  size = &params->size[params->plane];
-
-  if (*size == 0)
-    is_first = TRUE;
 
-  if ((params->flags & GST_D3D11_ALLOCATION_FLAG_TEXTURE_ARRAY)) {
+  if ((flags & GST_D3D11_ALLOCATION_FLAG_TEXTURE_ARRAY)) {
     gint i;
 
   do_again:
@@ -758,50 +800,69 @@ gst_d3d11_allocator_alloc (GstD3D11Allocator * allocator,
     }
   }
 
-  /* per plane, allocated staging texture to calculate actual size,
-   * stride, and offset */
-  if (is_first) {
-    gint num_plane;
-    gint stride[GST_VIDEO_MAX_PLANES];
-    gsize mem_size;
-    gint i;
+  mem = g_new0 (GstD3D11Memory, 1);
 
-    staging = create_staging_texture (device, desc);
-    if (!staging) {
-      GST_ERROR_OBJECT (allocator, "Couldn't create staging texture");
-      goto error;
-    }
+  gst_memory_init (GST_MEMORY_CAST (mem),
+      0, GST_ALLOCATOR_CAST (allocator), NULL, size, 0, 0, size);
 
-    if (!calculate_mem_size (device,
-            staging, desc, D3D11_MAP_READ, stride, &mem_size))
-      goto error;
+  g_mutex_init (&mem->lock);
+  mem->desc = *desc;
+  mem->texture = texture;
+  mem->device = gst_object_ref (device);
+  mem->type = type;
+  mem->subresource_index = index_to_use;
 
-    num_plane = gst_d3d11_dxgi_format_n_planes (desc->Format);
+  return GST_MEMORY_CAST (mem);
 
-    for (i = 0; i < num_plane; i++) {
-      params->stride[params->plane + i] = stride[i];
-    }
+error:
+  if (texture)
+    gst_d3d11_device_release_texture (device, texture);
+
+  return NULL;
+}
 
-    *size = mem_size;
+GstMemory *
+gst_d3d11_allocator_alloc_staging (GstD3D11Allocator * allocator,
+    const D3D11_TEXTURE2D_DESC * desc, GstD3D11AllocationFlags flags,
+    gint * stride)
+{
+  GstD3D11Memory *mem;
+  GstD3D11Device *device;
+  ID3D11Texture2D *texture = NULL;
+  gsize mem_size = 0;
+  gint mem_stride[GST_VIDEO_MAX_PLANES];
+
+  g_return_val_if_fail (GST_IS_D3D11_ALLOCATOR (allocator), NULL);
+  g_return_val_if_fail (desc != NULL, NULL);
+
+  device = allocator->device;
+
+  texture = create_staging_texture (device, desc);
+  if (!texture) {
+    GST_ERROR_OBJECT (allocator, "Couldn't create staging texture");
+    goto error;
+  }
+
+  if (!calculate_mem_size (device,
+          texture, desc, D3D11_MAP_READ, mem_stride, &mem_size)) {
+    GST_ERROR_OBJECT (allocator, "Couldn't calculate staging texture size");
+    goto error;
   }
 
   mem = g_new0 (GstD3D11Memory, 1);
 
   gst_memory_init (GST_MEMORY_CAST (mem),
-      0, GST_ALLOCATOR_CAST (allocator), NULL, *size, 0, 0, *size);
+      0, GST_ALLOCATOR_CAST (allocator), NULL, mem_size, 0, 0, mem_size);
 
   g_mutex_init (&mem->lock);
-  mem->info = params->info;
-  mem->plane = params->plane;
   mem->desc = *desc;
   mem->texture = texture;
-  mem->staging = staging;
   mem->device = gst_object_ref (device);
-  mem->type = type;
-  mem->subresource_index = index_to_use;
+  mem->type = GST_D3D11_MEMORY_TYPE_STAGING;
 
-  if (staging)
-    GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD);
+  /* every plan will have identical size */
+  if (stride)
+    *stride = mem_stride[0];
 
   return GST_MEMORY_CAST (mem);
 
@@ -809,9 +870,6 @@ error:
   if (texture)
     gst_d3d11_device_release_texture (device, texture);
 
-  if (staging)
-    gst_d3d11_device_release_texture (device, texture);
-
   return NULL;
 }
 

diff --git a/sys/d3d11/gstd3d11memory.h b/sys/d3d11/gstd3d11memory.h
index 178e0fc..42da320 100644 (file)
--- a/sys/d3d11/gstd3d11memory.h
+++ b/sys/d3d11/gstd3d11memory.h
@@ -84,13 +84,6 @@ struct _GstD3D11AllocationParams
   GstVideoInfo aligned_info;
   const GstD3D11Format *d3d11_format;
 
-  /* size and stride of staging texture, set by allocator */
-  gint stride[GST_VIDEO_MAX_PLANES];
-  gsize size[GST_VIDEO_MAX_PLANES];
-
-  /* Current target plane for allocation */
-  guint plane;
-
   GstD3D11AllocationFlags flags;
 
   /*< private >*/
@@ -101,6 +94,7 @@ typedef enum
 {
   GST_D3D11_MEMORY_TYPE_TEXTURE = 0,
   GST_D3D11_MEMORY_TYPE_ARRAY = 1,
+  GST_D3D11_MEMORY_TYPE_STAGING = 2,
 } GstD3D11MemoryType;
 
 struct _GstD3D11Memory
@@ -119,9 +113,6 @@ struct _GstD3D11Memory
   ID3D11RenderTargetView *render_target_view[GST_VIDEO_MAX_PLANES];
   guint num_render_target_views;
 
-  GstVideoInfo info;
-
-  guint plane;
   GstD3D11MemoryType type;
 
   /* > 0 if this is Array typed memory */
@@ -173,7 +164,15 @@ GType               gst_d3d11_allocator_get_type  (void);
 GstD3D11Allocator * gst_d3d11_allocator_new       (GstD3D11Device *device);
 
 GstMemory *         gst_d3d11_allocator_alloc     (GstD3D11Allocator * allocator,
-                                                   GstD3D11AllocationParams * params);
+                                                   const D3D11_TEXTURE2D_DESC * desc,
+                                                   GstD3D11AllocationFlags flags,
+                                                   gsize size);
+
+GstMemory *         gst_d3d11_allocator_alloc_staging (GstD3D11Allocator * allocator,
+                                                       const D3D11_TEXTURE2D_DESC * desc,
+                                                       GstD3D11AllocationFlags flags,
+                                                       gint * stride);
+
 
 void                gst_d3d11_allocator_set_flushing (GstD3D11Allocator * allocator,
                                                       gboolean flushing);

diff --git a/sys/d3d11/gstd3d11utils.c b/sys/d3d11/gstd3d11utils.c
index 1d4d30d..e97bc7e 100644 (file)
--- a/sys/d3d11/gstd3d11utils.c
+++ b/sys/d3d11/gstd3d11utils.c
@@ -23,6 +23,7 @@
 
 #include "gstd3d11utils.h"
 #include "gstd3d11device.h"
+#include "gstd3d11memory.h"
 
 #include <windows.h>
 #include <versionhelpers.h>
@@ -397,6 +398,79 @@ gst_d3d11_get_device_vendor (GstD3D11Device * device)
   return vendor;
 }
 
+GstBuffer *
+gst_d3d11_allocate_staging_buffer (GstD3D11Allocator * allocator,
+    const GstVideoInfo * info, const GstD3D11Format * format,
+    const D3D11_TEXTURE2D_DESC desc[GST_VIDEO_MAX_PLANES],
+    gboolean add_videometa)
+{
+  GstBuffer *buffer;
+  gint i;
+  gint stride[GST_VIDEO_MAX_PLANES] = { 0, };
+  gsize offset[GST_VIDEO_MAX_PLANES] = { 0, };
+  GstMemory *mem;
+
+  g_return_val_if_fail (GST_IS_D3D11_ALLOCATOR (allocator), NULL);
+  g_return_val_if_fail (info != NULL, NULL);
+  g_return_val_if_fail (format != NULL, NULL);
+  g_return_val_if_fail (desc != NULL, NULL);
+
+  buffer = gst_buffer_new ();
+
+  if (format->dxgi_format == DXGI_FORMAT_UNKNOWN) {
+    gsize size[GST_VIDEO_MAX_PLANES] = { 0, };
+
+    for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) {
+      mem = gst_d3d11_allocator_alloc_staging (allocator, &desc[i], 0,
+          &stride[i]);
+
+      if (!mem) {
+        GST_ERROR_OBJECT (allocator, "Couldn't allocate memory for plane %d",
+            i);
+        goto error;
+      }
+
+      size[i] = gst_memory_get_sizes (mem, NULL, NULL);
+      if (i > 0)
+        offset[i] = offset[i - 1] + size[i - 1];
+      gst_buffer_append_memory (buffer, mem);
+    }
+  } else {
+    /* must be YUV semi-planar or single plane */
+    g_assert (GST_VIDEO_INFO_N_PLANES (info) <= 2);
+
+    mem = gst_d3d11_allocator_alloc_staging (allocator, &desc[0], 0,
+        &stride[0]);
+
+    if (!mem) {
+      GST_ERROR_OBJECT (allocator, "Couldn't allocate memory");
+      goto error;
+    }
+
+    gst_memory_get_sizes (mem, NULL, NULL);
+    gst_buffer_append_memory (buffer, mem);
+
+    if (GST_VIDEO_INFO_N_PLANES (info) == 2) {
+      stride[1] = stride[0];
+      offset[1] = stride[0] * desc[0].Height;
+    }
+  }
+
+  if (add_videometa) {
+    gst_buffer_add_video_meta_full (buffer, GST_VIDEO_FRAME_FLAG_NONE,
+        GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
+        GST_VIDEO_INFO_HEIGHT (info), GST_VIDEO_INFO_N_PLANES (info),
+        offset, stride);
+  }
+
+  return buffer;
+
+error:
+  gst_buffer_unref (buffer);
+
+  return NULL;
+}
+
 gboolean
 _gst_d3d11_result (HRESULT hr, GstD3D11Device * device, GstDebugCategory * cat,
     const gchar * file, const gchar * function, gint line)

diff --git a/sys/d3d11/gstd3d11utils.h b/sys/d3d11/gstd3d11utils.h
index 5aa37fb..109b208 100644 (file)
--- a/sys/d3d11/gstd3d11utils.h
+++ b/sys/d3d11/gstd3d11utils.h
@@ -54,6 +54,12 @@ gboolean        gst_d3d11_is_windows_8_or_greater   (void);
 
 GstD3D11DeviceVendor gst_d3d11_get_device_vendor    (GstD3D11Device * device);
 
+GstBuffer *     gst_d3d11_allocate_staging_buffer   (GstD3D11Allocator * allocator,
+                                                     const GstVideoInfo * info,
+                                                     const GstD3D11Format * format,
+                                                     const D3D11_TEXTURE2D_DESC desc[GST_VIDEO_MAX_PLANES],
+                                                     gboolean add_videometa);
+
 gboolean       _gst_d3d11_result                    (HRESULT hr,
                                                      GstD3D11Device * device,
                                                      GstDebugCategory * cat,