libgstgl: run a custom shader to convert YUV to RGB on mac and ios

When GL_APPLE_ycbcr_422 is available, run a custom shader to convert
GL_TEXTURE_RECTANGLE textures from YUV to RGB.

See https://www.opengl.org/registry/specs/APPLE/ycbcr_422.txt

diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c
index daf2a49..51477dd 100644 (file)
--- a/gst-libs/gst/gl/gstglcolorconvert.c
+++ b/gst-libs/gst/gl/gstglcolorconvert.c
@@ -136,6 +136,23 @@ static const gchar frag_REORDER[] =
       " gl_FragColor = vec4(t.%c, t.%c, t.%c, t.%c);\n"
       "}";
 
+static const gchar frag_APPLE_YUV_TO_RGB[] =
+      "#ifdef GL_ES\n"
+      "precision mediump float;\n"
+      "#endif\n"
+      "varying vec2 v_texcoord;\n"
+      "uniform float width;\n"
+      "uniform float height;\n"
+      "uniform sampler2DRect tex;\n"
+      "uniform vec2 tex_scale0;\n"
+      "uniform vec2 tex_scale1;\n"
+      "uniform vec2 tex_scale2;\n"
+      "void main(void)\n"
+      "{\n"
+      " vec4 t = texture2DRect(tex, v_texcoord * vec2(width, height) * tex_scale0);\n"
+      " gl_FragColor = vec4(t.%c, t.%c, t.%c, t.%c);\n"
+      "}";
+
 /* GRAY16 to RGB conversion
  *  data transfered as GL_LUMINANCE_ALPHA then convert back to GRAY16 
  *  high byte weight as : 255*256/65535 
@@ -816,77 +833,97 @@ _YUV_to_RGB (GstGLColorConvert * convert)
       gst_gl_context_check_feature (convert->context, "GL_EXT_texture_rg")
       || gst_gl_context_check_feature (convert->context, "GL_ARB_texture_rg");
 #endif
+  gboolean apple_ycbcr = gst_gl_context_check_feature (convert->context,
+      "GL_APPLE_ycbcr_422");
+  gboolean in_tex_rectangular = FALSE;
+
+  GstMemory *memory = gst_buffer_peek_memory (convert->inbuf, 0);
+  if (gst_is_gl_memory (memory)) {
+    in_tex_rectangular =
+        ((GstGLMemory *) memory)->tex_target == GL_TEXTURE_RECTANGLE;
+  }
 
   info->out_n_textures = 1;
 
-  switch (GST_VIDEO_INFO_FORMAT (&convert->in_info)) {
-    case GST_VIDEO_FORMAT_AYUV:
-      info->frag_prog = g_strdup_printf (frag_AYUV_to_RGB, pixel_order[0],
-          pixel_order[1], pixel_order[2], pixel_order[3]);
-      info->in_n_textures = 1;
-      info->shader_tex_names[0] = "tex";
-      break;
-    case GST_VIDEO_FORMAT_I420:
-    case GST_VIDEO_FORMAT_Y444:
-    case GST_VIDEO_FORMAT_Y42B:
-    case GST_VIDEO_FORMAT_Y41B:
-      info->frag_prog = g_strdup_printf (frag_PLANAR_YUV_to_RGB, pixel_order[0],
-          pixel_order[1], pixel_order[2], pixel_order[3]);
-      info->in_n_textures = 3;
-      info->shader_tex_names[0] = "Ytex";
-      info->shader_tex_names[1] = "Utex";
-      info->shader_tex_names[2] = "Vtex";
-      break;
-    case GST_VIDEO_FORMAT_YV12:
-      info->frag_prog = g_strdup_printf (frag_PLANAR_YUV_to_RGB, pixel_order[0],
-          pixel_order[1], pixel_order[2], pixel_order[3]);
-      info->in_n_textures = 3;
-      info->shader_tex_names[0] = "Ytex";
-      info->shader_tex_names[1] = "Vtex";
-      info->shader_tex_names[2] = "Utex";
-      break;
-    case GST_VIDEO_FORMAT_YUY2:
-    {
-      char uv_val = texture_rg ? 'g' : 'a';
-      info->frag_prog = g_strdup_printf (frag_YUY2_UYVY_to_RGB, 'r', uv_val,
-          uv_val, 'g', 'a', pixel_order[0], pixel_order[1], pixel_order[2],
-          pixel_order[3]);
-      info->in_n_textures = 1;
-      info->shader_tex_names[0] = "Ytex";
-      break;
-    }
-    case GST_VIDEO_FORMAT_NV12:
-    {
-      char val2 = texture_rg ? 'g' : 'a';
-      info->frag_prog = g_strdup_printf (frag_NV12_NV21_to_RGB, 'r', val2,
-          pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
-      info->in_n_textures = 2;
-      info->shader_tex_names[0] = "Ytex";
-      info->shader_tex_names[1] = "UVtex";
-      break;
-    }
-    case GST_VIDEO_FORMAT_NV21:
-    {
-      char val2 = texture_rg ? 'g' : 'a';
-      info->frag_prog = g_strdup_printf (frag_NV12_NV21_to_RGB, val2, 'r',
-          pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
-      info->in_n_textures = 2;
-      info->shader_tex_names[0] = "Ytex";
-      info->shader_tex_names[1] = "UVtex";
-      break;
-    }
-    case GST_VIDEO_FORMAT_UYVY:
-    {
-      char y_val = texture_rg ? 'g' : 'a';
-      info->frag_prog = g_strdup_printf (frag_YUY2_UYVY_to_RGB, y_val, 'g',
-          'g', 'r', 'b', pixel_order[0], pixel_order[1], pixel_order[2],
-          pixel_order[3]);
-      info->in_n_textures = 1;
-      info->shader_tex_names[0] = "Ytex";
-      break;
+  if (in_tex_rectangular && apple_ycbcr
+      && gst_buffer_n_memory (convert->inbuf) == 1) {
+    info->frag_prog =
+        g_strdup_printf (frag_APPLE_YUV_TO_RGB, pixel_order[0], pixel_order[1],
+        pixel_order[2], pixel_order[3]);
+    info->in_n_textures = 1;
+    info->shader_tex_names[0] = "tex";
+  } else {
+    switch (GST_VIDEO_INFO_FORMAT (&convert->in_info)) {
+      case GST_VIDEO_FORMAT_AYUV:
+        info->frag_prog = g_strdup_printf (frag_AYUV_to_RGB, pixel_order[0],
+            pixel_order[1], pixel_order[2], pixel_order[3]);
+        info->in_n_textures = 1;
+        info->shader_tex_names[0] = "tex";
+        break;
+      case GST_VIDEO_FORMAT_I420:
+      case GST_VIDEO_FORMAT_Y444:
+      case GST_VIDEO_FORMAT_Y42B:
+      case GST_VIDEO_FORMAT_Y41B:
+        info->frag_prog =
+            g_strdup_printf (frag_PLANAR_YUV_to_RGB, pixel_order[0],
+            pixel_order[1], pixel_order[2], pixel_order[3]);
+        info->in_n_textures = 3;
+        info->shader_tex_names[0] = "Ytex";
+        info->shader_tex_names[1] = "Utex";
+        info->shader_tex_names[2] = "Vtex";
+        break;
+      case GST_VIDEO_FORMAT_YV12:
+        info->frag_prog =
+            g_strdup_printf (frag_PLANAR_YUV_to_RGB, pixel_order[0],
+            pixel_order[1], pixel_order[2], pixel_order[3]);
+        info->in_n_textures = 3;
+        info->shader_tex_names[0] = "Ytex";
+        info->shader_tex_names[1] = "Vtex";
+        info->shader_tex_names[2] = "Utex";
+        break;
+      case GST_VIDEO_FORMAT_YUY2:
+      {
+        char uv_val = texture_rg ? 'g' : 'a';
+        info->frag_prog = g_strdup_printf (frag_YUY2_UYVY_to_RGB, 'r', uv_val,
+            uv_val, 'g', 'a', pixel_order[0], pixel_order[1], pixel_order[2],
+            pixel_order[3]);
+        info->in_n_textures = 1;
+        info->shader_tex_names[0] = "Ytex";
+        break;
+      }
+      case GST_VIDEO_FORMAT_NV12:
+      {
+        char val2 = texture_rg ? 'g' : 'a';
+        info->frag_prog = g_strdup_printf (frag_NV12_NV21_to_RGB, 'r', val2,
+            pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
+        info->in_n_textures = 2;
+        info->shader_tex_names[0] = "Ytex";
+        info->shader_tex_names[1] = "UVtex";
+        break;
+      }
+      case GST_VIDEO_FORMAT_NV21:
+      {
+        char val2 = texture_rg ? 'g' : 'a';
+        info->frag_prog = g_strdup_printf (frag_NV12_NV21_to_RGB, val2, 'r',
+            pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
+        info->in_n_textures = 2;
+        info->shader_tex_names[0] = "Ytex";
+        info->shader_tex_names[1] = "UVtex";
+        break;
+      }
+      case GST_VIDEO_FORMAT_UYVY:
+      {
+        char y_val = texture_rg ? 'g' : 'a';
+        info->frag_prog = g_strdup_printf (frag_YUY2_UYVY_to_RGB, y_val, 'g',
+            'g', 'r', 'b', pixel_order[0], pixel_order[1], pixel_order[2],
+            pixel_order[3]);
+        info->in_n_textures = 1;
+        info->shader_tex_names[0] = "Ytex";
+        break;
+      }
+      default:
+        break;
     }
-    default:
-      break;
   }
 
   if (gst_video_colorimetry_matches (&convert->in_info.colorimetry,