d3d11videosink: Use ID3D11VideoProcessor interface

...for color space conversion if available

ID3D11VideoProcessor is equivalent to DXVA-HD video processor
which might use specialized blocks for video processing
instead of general GPU resource. In addition to that feature,
we need to use this API for color space conversion of DXVA2 decoder
output memory, because any d3d11 texture arrays that were
created with D3D11_BIND_DECODER cannot be used for shader resource.

This is prework for d3d11decoder zero-copy rendering and also
for conditional HDR tone-map support.
Note that some Intel platform is known to support tone-mapping
at the driver level using this API on Windows 10.

diff --git a/sys/d3d11/gstd3d11videosink.c b/sys/d3d11/gstd3d11videosink.c
index 59ef022..49a87c9 100644 (file)
--- a/sys/d3d11/gstd3d11videosink.c
+++ b/sys/d3d11/gstd3d11videosink.c
@@ -27,6 +27,7 @@
 #include "gstd3d11device.h"
 #include "gstd3d11bufferpool.h"
 #include "gstd3d11format.h"
+#include "gstd3d11videoprocessor.h"
 
 #if GST_D3D11_WINAPI_ONLY_APP
 #include "gstd3d11window_corewindow.h"
@@ -386,9 +387,10 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
   self->pending_render_rect = FALSE;
   GST_OBJECT_UNLOCK (self);
 
+  self->have_video_processor = FALSE;
   if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self),
           GST_VIDEO_SINK_HEIGHT (self), video_par_n, video_par_d,
-          caps, &error)) {
+          caps, &self->have_video_processor, &error)) {
     GstMessage *error_msg;
 
     GST_ERROR_OBJECT (self, "cannot create swapchain");
@@ -412,15 +414,30 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
 
   {
     GstD3D11AllocationParams *d3d11_params;
+    gint bind_flags = D3D11_BIND_SHADER_RESOURCE;
+
+    if (self->have_video_processor) {
+      /* To create video processor input view, one of following bind flags
+       * is required
+       * NOTE: Any texture arrays which were created with D3D11_BIND_DECODER flag
+       * cannot be used for shader input.
+       *
+       * D3D11_BIND_DECODER
+       * D3D11_BIND_VIDEO_ENCODER
+       * D3D11_BIND_RENDER_TARGET
+       * D3D11_BIND_UNORDERED_ACCESS_VIEW
+       */
+      bind_flags |= D3D11_BIND_RENDER_TARGET;
+    }
 
     d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
     if (!d3d11_params) {
       d3d11_params = gst_d3d11_allocation_params_new (self->device,
-          &self->info, 0, D3D11_BIND_SHADER_RESOURCE);
+          &self->info, 0, bind_flags);
     } else {
       /* Set bind flag */
       for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->info); i++) {
-        d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE;
+        d3d11_params->desc[i].BindFlags |= bind_flags;
       }
     }
 
@@ -836,13 +853,6 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
       break;
     }
 
-    if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) {
-      GST_LOG_OBJECT (sink,
-          "shader resource view is unavailable, need fallback");
-      render_buf = NULL;
-      break;
-    }
-
     if (dmem->desc.Usage == D3D11_USAGE_DEFAULT) {
       if (!gst_memory_map (mem, &map, (GST_MAP_READ | GST_MAP_D3D11))) {
         GST_ERROR_OBJECT (self, "cannot map d3d11 memory");
@@ -851,6 +861,19 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
 
       gst_memory_unmap (mem, &map);
     }
+
+    if (gst_buffer_n_memory (buf) == 1 && self->have_video_processor &&
+        gst_d3d11_video_processor_check_bind_flags_for_input_view
+        (dmem->desc.BindFlags)) {
+      break;
+    }
+
+    if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) {
+      GST_LOG_OBJECT (sink,
+          "shader resource view is unavailable, need fallback");
+      render_buf = NULL;
+      break;
+    }
   }
 
   if (!render_buf) {

diff --git a/sys/d3d11/gstd3d11videosink.h b/sys/d3d11/gstd3d11videosink.h
index c443906..c599f93 100644 (file)
--- a/sys/d3d11/gstd3d11videosink.h
+++ b/sys/d3d11/gstd3d11videosink.h
@@ -66,6 +66,7 @@ struct _GstD3D11VideoSink
 
   GstBufferPool *fallback_pool;
   gboolean can_convert;
+  gboolean have_video_processor;
 };
 
 struct _GstD3D11VideoSinkClass

diff --git a/sys/d3d11/gstd3d11window.cpp b/sys/d3d11/gstd3d11window.cpp
index 922c787..96f26ef 100644 (file)
--- a/sys/d3d11/gstd3d11window.cpp
+++ b/sys/d3d11/gstd3d11window.cpp
@@ -306,6 +306,11 @@ gst_d3d11_window_on_resize (GstD3D11Window * window, guint width, guint height)
     window->rtv = NULL;
   }
 
+  if (window->pov) {
+    window->pov->Release ();
+    window->pov = NULL;
+  }
+
   swap_chain->GetDesc (&swap_desc);
   hr = swap_chain->ResizeBuffers (0, width, height, DXGI_FORMAT_UNKNOWN,
       swap_desc.Flags);
@@ -367,6 +372,17 @@ gst_d3d11_window_on_resize (GstD3D11Window * window, guint width, guint height)
     goto done;
   }
 
+  if (window->processor) {
+    D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC pov_desc;
+
+    pov_desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D;
+    pov_desc.Texture2D.MipSlice = 0;
+
+    if (!gst_d3d11_video_processor_create_output_view (window->processor,
+        &pov_desc, (ID3D11Resource *) backbuffer, &window->pov))
+      goto done;
+  }
+
   window->first_present = TRUE;
 
   /* redraw the last scene if cached buffer exits */
@@ -406,11 +422,14 @@ gst_d3d11_window_on_mouse_event (GstD3D11Window * window, const gchar * event,
 
 gboolean
 gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
-    guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps, GError ** error)
+    guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps,
+    gboolean * video_processor_available, GError ** error)
 {
   GstD3D11WindowClass *klass;
   GstCaps *render_caps;
   guint swapchain_flags = 0;
+  gboolean need_processor_output_configure = FALSE;
+  gboolean need_processor_input_configure = FALSE;
 
   g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE);
   g_return_val_if_fail (aspect_ratio_n > 0, FALSE);
@@ -453,6 +472,10 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
 
   gst_video_info_from_caps (&window->info, caps);
 
+  if (window->processor)
+    gst_d3d11_video_processor_free (window->processor);
+  window->processor = NULL;
+
   if (window->converter)
     gst_d3d11_color_converter_free (window->converter);
   window->converter = NULL;
@@ -469,6 +492,50 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
       window->info.colorimetry.primaries;
   window->render_info.colorimetry.transfer = window->info.colorimetry.transfer;
 
+  window->processor =
+      gst_d3d11_video_processor_new (window->device, width, height, width,
+      height);
+  if (window->processor) {
+    const GstD3D11Format *in_format;
+    const GstD3D11Format *out_format;
+    gboolean input_support = FALSE;
+    gboolean out_support = FALSE;
+
+    in_format = gst_d3d11_device_format_from_gst (window->device,
+        GST_VIDEO_INFO_FORMAT (&window->info));
+    out_format = gst_d3d11_device_format_from_gst (window->device,
+        GST_VIDEO_INFO_FORMAT (&window->render_info));
+
+    if (gst_d3d11_video_processor_supports_input_format (window->processor,
+            in_format->dxgi_format)) {
+      input_support = TRUE;
+    } else {
+      GST_DEBUG_OBJECT (window,
+          "IVideoProcessor cannot support input dxgi format %d",
+          in_format->dxgi_format);
+    }
+
+    if (gst_d3d11_video_processor_supports_output_format (window->processor,
+            out_format->dxgi_format)) {
+      out_support = TRUE;
+    } else {
+      GST_DEBUG_OBJECT (window,
+          "IVideoProcessor cannot support output dxgi format %d",
+          out_format->dxgi_format);
+    }
+
+    if (!input_support || !out_support) {
+      gst_d3d11_video_processor_free (window->processor);
+      window->processor = NULL;
+    } else {
+      GST_DEBUG_OBJECT (window, "IVideoProcessor interface available");
+      *video_processor_available = TRUE;
+      need_processor_input_configure = TRUE;
+      need_processor_output_configure = TRUE;
+    }
+  }
+
+  /* configure shader even if video processor is available for fallback */
   window->converter =
       gst_d3d11_color_converter_new (window->device, &window->info,
       &window->render_info);
@@ -512,6 +579,8 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
   window->render_rect.right = width;
   window->render_rect.bottom = height;
 
+  window->input_rect = window->render_rect;
+
   window->width = width;
   window->height = height;
 
@@ -544,14 +613,46 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
             ctype, (guint) hr);
         } else {
           GST_DEBUG_OBJECT (window, "Set colorspace %d", ctype);
+
+          if (window->processor)
+            need_processor_output_configure =
+                !gst_d3d11_video_processor_set_output_dxgi_color_space
+                (window->processor, ctype);
         }
       }
 
       swapchain3->Release ();
     }
   }
+
+  if (window->processor) {
+    if (need_processor_output_configure) {
+      /* Set most common color space */
+      need_processor_output_configure =
+          !gst_d3d11_video_processor_set_output_dxgi_color_space
+          (window->processor, DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709);
+    }
+
+    if (need_processor_input_configure) {
+      DXGI_COLOR_SPACE_TYPE ctype;
+      gst_d3d11_video_info_to_dxgi_color_space (&window->info, &ctype);
+      need_processor_input_configure =
+          !gst_d3d11_video_processor_set_input_dxgi_color_space
+          (window->processor, ctype);
+    }
+  }
 #endif
 
+  if (window->processor && need_processor_output_configure) {
+    gst_d3d11_video_processor_set_output_color_space (window->processor,
+        &window->render_info.colorimetry);
+  }
+
+  if (window->processor && need_processor_input_configure) {
+    gst_d3d11_video_processor_set_input_color_space (window->processor,
+        &window->info.colorimetry);
+  }
+
 #if (DXGI_HEADER_VERSION >= 5)
   {
     GstVideoMasteringDisplayInfo minfo;
@@ -576,6 +677,11 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
         if (!gst_d3d11_result (hr, window->device)) {
           GST_WARNING_OBJECT (window, "Couldn't set HDR metadata, hr 0x%x",
               (guint) hr);
+        } else if (window->processor) {
+          gst_d3d11_video_processor_set_input_hdr10_metadata (window->processor,
+              &metadata);
+          gst_d3d11_video_processor_set_output_hdr10_metadata (window->processor,
+              &metadata);
         }
 
         swapchain4->Release ();
@@ -615,6 +721,58 @@ gst_d3d11_window_set_render_rectangle (GstD3D11Window * window, gint x, gint y,
   /* TODO: resize window and view */
 }
 
+static gboolean
+gst_d3d11_window_buffer_ensure_processor_input (GstD3D11Window * self,
+    GstBuffer * buffer, ID3D11VideoProcessorInputView ** in_view)
+{
+  GstD3D11Memory *mem;
+  ID3D11VideoProcessorInputView *view;
+  GQuark quark;
+
+  if (!self->processor)
+    return FALSE;
+
+  if (gst_buffer_n_memory (buffer) != 1)
+    return FALSE;
+
+  mem = (GstD3D11Memory *) gst_buffer_peek_memory (buffer, 0);
+
+  if (!gst_d3d11_video_processor_check_bind_flags_for_input_view
+      (mem->desc.BindFlags)) {
+    return FALSE;
+  }
+
+  quark = gst_d3d11_video_processor_input_view_quark ();
+  view = (ID3D11VideoProcessorInputView *)
+      gst_mini_object_get_qdata (GST_MINI_OBJECT (mem), quark);
+
+  if (!view) {
+    D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC in_desc;
+
+    in_desc.FourCC = 0;
+    in_desc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D;
+    in_desc.Texture2D.MipSlice = 0;
+    in_desc.Texture2D.ArraySlice = mem->subresource_index;
+
+    GST_TRACE_OBJECT (self, "Create new processor input view");
+
+    if (!gst_d3d11_video_processor_create_input_view (self->processor,
+         &in_desc, mem->texture, &view)) {
+      GST_LOG_OBJECT (self, "Failed to create processor input view");
+      return FALSE;
+    }
+
+    gst_mini_object_set_qdata (GST_MINI_OBJECT (mem), quark, view,
+        (GDestroyNotify) gst_d3d11_video_processor_input_view_release);
+  } else {
+    GST_TRACE_OBJECT (self, "Reuse existing processor input view %p", view);
+  }
+
+  *in_view = view;
+
+  return TRUE;
+}
+
 static GstFlowReturn
 gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer)
 {
@@ -628,14 +786,18 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer)
 
   if (self->cached_buffer) {
     ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES];
+    ID3D11VideoProcessorInputView *piv = NULL;
     guint i, j, k;
 
-    for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) {
-      GstD3D11Memory *mem =
-          (GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, i);
-      for (k = 0; k < mem->num_shader_resource_views; k++) {
-        srv[j] = mem->shader_resource_view[k];
-        j++;
+    if (!gst_d3d11_window_buffer_ensure_processor_input (self,
+        self->cached_buffer, &piv)) {
+      for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) {
+        GstD3D11Memory *mem =
+            (GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, i);
+        for (k = 0; k < mem->num_shader_resource_views; k++) {
+          srv[j] = mem->shader_resource_view[k];
+          j++;
+        }
       }
     }
 
@@ -646,8 +808,23 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer)
           &self->render_rect);
     }
 
-    gst_d3d11_color_converter_convert_unlocked (self->converter,
-        srv, &self->rtv);
+    if (self->processor && piv && self->pov) {
+      if (!gst_d3d11_video_processor_render_unlocked (self->processor,
+          &self->input_rect, piv, &self->render_rect, self->pov)) {
+        GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using processor");
+        return GST_FLOW_ERROR;
+      } else {
+        GST_TRACE_OBJECT (self, "Rendered using processor");
+      }
+    } else {
+      if (!gst_d3d11_color_converter_convert_unlocked (self->converter,
+          srv, &self->rtv)) {
+        GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using converter");
+        return GST_FLOW_ERROR;
+      } else {
+        GST_TRACE_OBJECT (self, "Rendered using converter");
+      }
+    }
 
     gst_d3d11_overlay_compositor_upload (self->compositor, self->cached_buffer);
     gst_d3d11_overlay_compositor_draw_unlocked (self->compositor, &self->rtv);

diff --git a/sys/d3d11/gstd3d11window.h b/sys/d3d11/gstd3d11window.h
index 2d743d3..60ba7b8 100644 (file)
--- a/sys/d3d11/gstd3d11window.h
+++ b/sys/d3d11/gstd3d11window.h
@@ -27,6 +27,7 @@
 #include "gstd3d11_fwd.h"
 #include "gstd3d11colorconverter.h"
 #include "gstd3d11overlaycompositor.h"
+#include "gstd3d11videoprocessor.h"
 
 G_BEGIN_DECLS
 
@@ -79,12 +80,16 @@ struct _GstD3D11Window
   GstVideoInfo info;
   GstVideoInfo render_info;
   const GstD3D11Format *render_format;
+  GstD3D11VideoProcessor *processor;
   GstD3D11ColorConverter *converter;
   GstD3D11OverlayCompositor *compositor;
 
   /* calculated rect with aspect ratio and window area */
   RECT render_rect;
 
+  /* input resolution */
+  RECT input_rect;
+
   /* requested rect via gst_d3d11_window_render */
   GstVideoRectangle rect;
 
@@ -99,6 +104,7 @@ struct _GstD3D11Window
 
   IDXGISwapChain *swap_chain;
   ID3D11RenderTargetView *rtv;
+  ID3D11VideoProcessorOutputView *pov;
 
   GstBuffer *cached_buffer;
   gboolean first_present;
@@ -144,6 +150,7 @@ gboolean      gst_d3d11_window_prepare              (GstD3D11Window * window,
                                                      guint aspect_ratio_n,
                                                      guint aspect_ratio_d,
                                                      GstCaps * caps,
+                                                     gboolean * video_processor_available,
                                                      GError ** error);
 
 GstFlowReturn gst_d3d11_window_render               (GstD3D11Window * window,