From f58f292f0517e7fb2d4994a51a992be7aba67bda Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha@centricular.com>
Date: Tue, 5 Jul 2022 05:14:01 +0900
Subject: [PATCH] video: convertframe: Add D3D11 specific conversion path

Add d3d11 conversion path to make gst_video_convert_sample() work
for GstD3D11Memory.

Note that just adding "d3d11download" to the exisitng code is
suboptimal from GstD3D11 point of view because:
* d3d11convert element can support crop/colorspace-conversion/scale
  all at once while existing software pipeline needs intermediate steps
  for the conversion
* "Process everything on GPU then download it to CPU memory" would be likely
  faster than "download GPU memory to CPU then processing it on CPU"

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4689>
---
 .../gst-libs/gst/video/convertframe.c              | 85 +++++++++++++++++++++-
 1 file changed, 83 insertions(+), 2 deletions(-)

diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/convertframe.c b/subprojects/gst-plugins-base/gst-libs/gst/video/convertframe.c
index d14c755..144e23b 100644
--- a/subprojects/gst-plugins-base/gst-libs/gst/video/convertframe.c
+++ b/subprojects/gst-plugins-base/gst-libs/gst/video/convertframe.c
@@ -114,6 +114,82 @@ fail:
 }
 
 static GstElement *
+build_convert_frame_pipeline_d3d11 (GstElement ** src_element,
+    GstElement ** sink_element, GstCaps * from_caps, GstCaps * to_caps,
+    GError ** err)
+{
+  GstElement *pipeline = NULL;
+  GstElement *appsrc = NULL;
+  GstElement *d3d11_convert = NULL;
+  GstElement *d3d11_download = NULL;
+  GstElement *convert = NULL;
+  GstElement *enc = NULL;
+  GstElement *appsink = NULL;
+  GError *error = NULL;
+
+  if (!create_element ("appsrc", &appsrc, &error) ||
+      !create_element ("d3d11convert", &d3d11_convert, &error) ||
+      !create_element ("d3d11download", &d3d11_download, &error) ||
+      !create_element ("videoconvert", &convert, &error) ||
+      !create_element ("appsink", &appsink, &error)) {
+    GST_ERROR ("Could not create element");
+    goto failed;
+  }
+
+  if (caps_are_raw (to_caps)) {
+    if (!create_element ("identity", &enc, &error)) {
+      GST_ERROR ("Could not create identity element");
+      goto failed;
+    }
+  } else {
+    enc = get_encoder (to_caps, &error);
+    if (!enc) {
+      GST_ERROR ("Could not create encoder");
+      goto failed;
+    }
+  }
+
+  g_object_set (appsrc, "caps", from_caps, "emit-signals", TRUE,
+      "format", GST_FORMAT_TIME, NULL);
+  g_object_set (appsink, "caps", to_caps, "emit-signals", TRUE, NULL);
+
+  pipeline = gst_pipeline_new ("d3d11-convert-frame-pipeline");
+  gst_bin_add_many (GST_BIN (pipeline), appsrc, d3d11_convert, d3d11_download,
+      convert, enc, appsink, NULL);
+
+  if (!gst_element_link_many (appsrc,
+          d3d11_convert, d3d11_download, convert, enc, appsink, NULL)) {
+    /* Now pipeline takes ownership of all elements, so only top-level
+     * pipeline should be cleared */
+    appsrc = d3d11_convert = convert = enc = appsink = NULL;
+
+    error = g_error_new (GST_CORE_ERROR, GST_CORE_ERROR_NEGOTIATION,
+        "Could not configure pipeline for conversion");
+  }
+
+  *src_element = appsrc;
+  *sink_element = appsink;
+
+  return pipeline;
+
+failed:
+  if (err)
+    *err = error;
+  else
+    g_clear_error (&error);
+
+  gst_clear_object (&pipeline);
+  gst_clear_object (&appsrc);
+  gst_clear_object (&d3d11_convert);
+  gst_clear_object (&d3d11_download);
+  gst_clear_object (&convert);
+  gst_clear_object (&enc);
+  gst_clear_object (&appsink);
+
+  return NULL;
+}
+
+static GstElement *
 build_convert_frame_pipeline (GstElement ** src_element,
     GstElement ** sink_element, GstCaps * from_caps,
     GstVideoCropMeta * cmeta, GstCaps * to_caps, GError ** err)
@@ -123,11 +199,16 @@ build_convert_frame_pipeline (GstElement ** src_element,
   GstElement *dl = NULL;
   GstVideoInfo info;
   GError *error = NULL;
-#ifdef HAVE_GL
   GstCapsFeatures *features;
 
   features = gst_caps_get_features (from_caps, 0);
-  if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
+  if (features && gst_caps_features_contains (features, "memory:D3D11Memory")) {
+    return build_convert_frame_pipeline_d3d11 (src_element, sink_element,
+        from_caps, to_caps, err);
+  }
+#ifdef HAVE_GL
+  if (features &&
+      gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
     if (!create_element ("gldownload", &dl, &error))
       goto no_elements;
 #endif
-- 
2.7.4