d3d11videoprocessor: Fix wrong input/output supportability check

The flag argument of ID3D11VideoProcessorEnumerator::CheckVideoProcessorFormat
method is output value, not input.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1726>

diff --git a/sys/d3d11/gstd3d11videoprocessor.c b/sys/d3d11/gstd3d11videoprocessor.c
index 47bf4c0..7ea0221 100644 (file)
--- a/sys/d3d11/gstd3d11videoprocessor.c
+++ b/sys/d3d11/gstd3d11videoprocessor.c
@@ -177,20 +177,25 @@ gst_d3d11_video_processor_supports_format (GstD3D11VideoProcessor *
     self, DXGI_FORMAT format, gboolean is_input)
 {
   HRESULT hr;
-  UINT flag;
+  UINT flag = 0;
+
+  hr = ID3D11VideoProcessorEnumerator_CheckVideoProcessorFormat
+      (self->enumerator, format, &flag);
+
+  if (!gst_d3d11_result (hr, self->device))
+    return FALSE;
 
   if (is_input) {
     /* D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT, missing in mingw header */
-    flag = 1;
+    if ((flag & 0x1) != 0)
+      return TRUE;
   } else {
     /* D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT, missing in mingw header */
-    flag = 2;
+    if ((flag & 0x2) != 0)
+      return TRUE;
   }
 
-  hr = ID3D11VideoProcessorEnumerator_CheckVideoProcessorFormat
-      (self->enumerator, format, &flag);
-
-  return gst_d3d11_result (hr, self->device);
+  return FALSE;
 }
 
 gboolean