gstvaapiencoder: validate chroma according to the VA's RT format

Before, only YUV420 color space where supported. With this patch, the
encoder is queried to know the supported formats and admits YUV422
color space if its available.

Signed-off-by: Víctor Manuel Jáquez Leal <victorx.jaquez@intel.com>
https://bugzilla.gnome.org/show_bug.cgi?id=744042

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder.c b/gst-libs/gst/vaapi/gstvaapiencoder.c
index ab0d674..ebd5d3a 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapiencoder.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder.c
@@ -559,6 +559,37 @@ get_packed_headers (GstVaapiEncoder * encoder)
   return encoder->packed_headers;
 }
 
+static inline gboolean
+is_chroma_type_supported (GstVaapiEncoder * encoder)
+{
+  GstVaapiContextInfo *const cip = &encoder->context_info;
+  const GstVideoFormat fmt =
+    GST_VIDEO_INFO_FORMAT (GST_VAAPI_ENCODER_VIDEO_INFO (encoder));
+  guint format = 0;
+
+  if (fmt == GST_VIDEO_FORMAT_ENCODED)
+    return TRUE;
+
+  if (cip->chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420 &&
+      cip->chroma_type != GST_VAAPI_CHROMA_TYPE_YUV422)
+    goto unsupported;
+
+  if (!get_config_attribute (encoder, VAConfigAttribRTFormat, &format))
+    return FALSE;
+
+  if (!(format & from_GstVaapiChromaType (cip->chroma_type)))
+    goto unsupported;
+
+  return TRUE;
+
+unsupported:
+  {
+    GST_ERROR ("We only support YUV 4:2:0 and YUV 4:2:2 for encoding. "
+        "Please try to use vaapipostproc to convert the input format.");
+    return FALSE;
+  }
+}
+
 /* Updates video context */
 static gboolean
 set_context_info (GstVaapiEncoder * encoder)
@@ -582,16 +613,9 @@ set_context_info (GstVaapiEncoder * encoder)
   cip->height = GST_VAAPI_ENCODER_HEIGHT (encoder);
   cip->ref_frames = encoder->num_ref_frames;
 
-  if (!cip->chroma_type  && (format != GST_VIDEO_FORMAT_ENCODED))
+  if (!is_chroma_type_supported (encoder))
     goto error_unsupported_format;
 
-  if (cip->chroma_type != GST_VAAPI_CHROMA_TYPE_YUV420 &&
-      format != GST_VIDEO_FORMAT_ENCODED) {
-    GST_ERROR ("We are only supporting YUV:4:2:0 for encoding,"
-        "please try to use vaapipostproc to convert the input format!");
-    goto error_unsupported_format;
-  }
-  
   memset (config, 0, sizeof (*config));
   config->rc_mode = GST_VAAPI_ENCODER_RATE_CONTROL (encoder);
   config->packed_headers = get_packed_headers (encoder);