From 1c965c3340f75388d181fde4ae873e96deb93966 Mon Sep 17 00:00:00 2001
From: gb <gb@5584edef-b1fe-4b99-b61b-dd2bab72e969>
Date: Mon, 15 Mar 2010 17:44:35 +0000
Subject: [PATCH] Implement I420 (resp. YV12) with YV12 (resp. I420) if the
 driver does not.

---
 sys/vaapiconvert/gstvaapiconvert.c | 57 ++++++++++++++++++++++++++++++++++++--
 sys/vaapiconvert/gstvaapiconvert.h |  5 ++++
 2 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/sys/vaapiconvert/gstvaapiconvert.c b/sys/vaapiconvert/gstvaapiconvert.c
index 778ff6b..1c0cb09 100644
--- a/sys/vaapiconvert/gstvaapiconvert.c
+++ b/sys/vaapiconvert/gstvaapiconvert.c
@@ -292,14 +292,41 @@ gst_vaapiconvert_transform_caps(
             return NULL;
         out_caps = gst_caps_from_string(gst_vaapiconvert_yuv_caps_str);
         if (convert->display) {
-            GstCaps *allowed_caps, *inter_caps;
+            GstVaapiImageFormat fixup_format;
+            GstCaps *allowed_caps, *new_caps;
+
             allowed_caps = gst_vaapi_display_get_image_caps(convert->display);
             if (!allowed_caps)
                 return NULL;
-            inter_caps = gst_caps_intersect(out_caps, allowed_caps);
+
+            new_caps = gst_caps_intersect(out_caps, allowed_caps);
             gst_caps_unref(allowed_caps);
             gst_caps_unref(out_caps);
-            out_caps = inter_caps;
+            out_caps = new_caps;
+
+            convert->has_YV12 = gst_vaapi_display_has_image_format(
+                convert->display,
+                GST_VAAPI_IMAGE_YV12
+            );
+            convert->has_I420 = gst_vaapi_display_has_image_format(
+                convert->display,
+                GST_VAAPI_IMAGE_I420
+            );
+            if (convert->has_YV12 && !convert->has_I420)
+                fixup_format = GST_VAAPI_IMAGE_I420;
+            else if (convert->has_I420 && !convert->has_YV12)
+                fixup_format = GST_VAAPI_IMAGE_YV12;
+            else
+                fixup_format = 0;
+            if (fixup_format) {
+                allowed_caps = gst_vaapi_image_format_get_caps(fixup_format);
+                if (allowed_caps) {
+                    new_caps = gst_caps_union(out_caps, allowed_caps);
+                    gst_caps_unref(allowed_caps);
+                    gst_caps_unref(out_caps);
+                    out_caps = new_caps;
+                }
+            }
         }
     }
 
@@ -321,13 +348,34 @@ gst_vaapiconvert_set_caps(
 )
 {
     GstVaapiConvert * const convert = GST_VAAPICONVERT(trans);
+    GstCaps *fixed_incaps = NULL;
     GstStructure *structure;
     gint width, height;
+    guint32 format;
 
     structure = gst_caps_get_structure(incaps, 0);
     gst_structure_get_int(structure, "width",  &width);
     gst_structure_get_int(structure, "height", &height);
 
+#define GST_FORMAT_YV12 GST_MAKE_FOURCC ('Y', 'V', '1', '2')
+#define GST_FORMAT_I420 GST_MAKE_FOURCC ('I', '4', '2', '0')
+
+    /* Fix I420 and YV12 formats */
+    if (gst_structure_get_fourcc(structure, "format", &format)) {
+        if (format == GST_FORMAT_I420 && !convert->has_I420)
+            format = GST_FORMAT_YV12;
+        else if (format == GST_FORMAT_YV12 && !convert->has_YV12)
+            format = GST_FORMAT_I420;
+        else
+            format = 0;
+        if (format) {
+            fixed_incaps = gst_caps_copy(incaps);
+            structure = gst_caps_get_structure(fixed_incaps, 0);
+            gst_structure_set(structure, "format", GST_TYPE_FOURCC, format, NULL);
+            incaps = fixed_incaps;
+        }
+    }
+
     if (width != convert->image_width || height != convert->image_height) {
         if (convert->images)
             g_object_unref(convert->images);
@@ -347,6 +395,9 @@ gst_vaapiconvert_set_caps(
         if (!convert->surfaces)
             return FALSE;
     }
+
+    if (fixed_incaps)
+        gst_caps_unref(fixed_incaps);
     return TRUE;
 }
 
diff --git a/sys/vaapiconvert/gstvaapiconvert.h b/sys/vaapiconvert/gstvaapiconvert.h
index 98487f6..e81009d 100644
--- a/sys/vaapiconvert/gstvaapiconvert.h
+++ b/sys/vaapiconvert/gstvaapiconvert.h
@@ -65,12 +65,17 @@ struct _GstVaapiConvert {
     GstBaseTransform    parent_instance;
 
     GstVaapiDisplay    *display;
+    GstVaapiImageFormat image_format;
     GstVaapiVideoPool  *images;
     guint               image_width;
     guint               image_height;
     GstVaapiVideoPool  *surfaces;
     guint               surface_width;
     guint               surface_height;
+
+    /* XXX: implement YV12 or I420 formats ourselves */
+    guint               has_YV12 : 1;
+    guint               has_I420 : 1;
 };
 
 struct _GstVaapiConvertClass {
-- 
2.7.4