opengl: Add NV12_16L32S conversion support

This adds a first detiling shader with initial support for
NV12_16L32S as produced by Mediatek decoders.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2190>

diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/egl/gsteglimage.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/egl/gsteglimage.c
index 1714051..25fd068 100644 (file)
--- a/subprojects/gst-plugins-base/gst-libs/gst/gl/egl/gsteglimage.c
+++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/egl/gsteglimage.c
@@ -512,6 +512,7 @@ _drm_rgba_fourcc_from_info (const GstVideoInfo * info, int plane,
     case GST_VIDEO_FORMAT_NV21:
     case GST_VIDEO_FORMAT_NV16:
     case GST_VIDEO_FORMAT_NV61:
+    case GST_VIDEO_FORMAT_NV12_16L32S:
       *out_format = plane == 0 ? GST_GL_RED : GST_GL_RG;
       return plane == 0 ? DRM_FORMAT_R8 : rg_fourcc;
 

diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c
index 167049a..a59e6aa 100644 (file)
--- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c
+++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.c
@@ -50,6 +50,8 @@
 
 #define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
 #define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
+#define USING_OPENGL30(context) \
+  (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 3, 0) || USING_OPENGL3(context))
 #define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
@@ -357,6 +359,49 @@ static const struct shader_templ templ_AV12_to_RGB =
     GST_GL_TEXTURE_TARGET_2D
   };
 
+#define glsl_func_frag_to_tile \
+    "ivec2 frag_to_tile(ivec2 tile_coord, ivec2 delta_coord, ivec2 dim, int width, int tiles_per_row, int need_offset) {\n" \
+    "  int tile_size = (dim.x * dim.y);\n" \
+    "  int tile_index = tile_coord.y * tiles_per_row + tile_coord.x;\n" \
+    "  int linear_index = tile_index * tile_size + delta_coord.y * dim.x + delta_coord.x;\n" \
+    "  linear_index += need_offset * tile_size / 2;\n" \
+    "  return ivec2(linear_index % width, linear_index / width);\n" \
+    "}\n"
+
+/* TILED semi-planar to RGB conversion */
+static const gchar templ_TILED_SEMI_PLANAR_to_RGB_BODY[] =
+    "  vec4 rgba;\n"
+    "  vec3 yuv;\n"
+    "  ivec2 texel;\n"
+    "\n"
+    "  const ivec2 luma_dim = ivec2(%i, %i);\n"
+    "  const ivec2 chroma_dim = ivec2(%i, %i);\n"
+    "  const int fy = chroma_dim.y * 2 / luma_dim.y;\n"
+    "\n"
+    "  int iwidth = int(width);\n"
+    "  int tiles_per_row = iwidth / luma_dim.x;\n"
+    "\n"
+    "  ivec2 coord = ivec2(gl_FragCoord.xy);\n"
+    "  ivec2 tile_coord = coord / luma_dim;\n"
+    "  ivec2 delta_coord = coord %% luma_dim;\n" \
+    "  texel = frag_to_tile(tile_coord, delta_coord, luma_dim, iwidth, tiles_per_row, 0);\n"
+    "  yuv.x = texelFetch(Ytex, texel, 0).r;\n"
+    "\n"
+    "  ivec2 chroma_tcoord = ivec2(tile_coord.x, tile_coord.y / fy);\n"
+    "  texel = frag_to_tile(chroma_tcoord, delta_coord / 2, chroma_dim, iwidth / 2, tiles_per_row, tile_coord.y %% fy);\n"
+    "  yuv.yz = texelFetch(UVtex, texel, 0).%c%c;\n"
+    "\n"
+    "  rgba.rgb = yuv_to_rgb (yuv, offset, coeff1, coeff2, coeff3);\n"
+    "  rgba.a = 1.0;\n"
+    "  gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
+
+static const struct shader_templ templ_TILED_SEMI_PLANAR_to_RGB =
+  { NULL,
+    DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, UVtex;\n",
+    { glsl_func_yuv_to_rgb, glsl_func_frag_to_tile, NULL, },
+    GST_GL_TEXTURE_TARGET_2D
+  };
+
   /* RGB to NV12/NV21/NV16/NV61 conversion */
   /* NV12/NV16: u, v
      NV21/NV61: v, u */
@@ -1082,6 +1127,10 @@ _init_supported_formats (GstGLContext * context, gboolean output,
     _append_value_string_list (supported_formats, "Y412_BE", NULL);
 #endif
   }
+
+  if (!context || USING_GLES3 (context) || USING_OPENGL30 (context)) {
+    _append_value_string_list (supported_formats, "NV12_16L32S", NULL);
+  }
 }
 
 /* copies the given caps */
@@ -1713,6 +1762,7 @@ _get_n_textures (GstVideoFormat v_format)
     case GST_VIDEO_FORMAT_P012_BE:
     case GST_VIDEO_FORMAT_P016_LE:
     case GST_VIDEO_FORMAT_P016_BE:
+    case GST_VIDEO_FORMAT_NV12_16L32S:
       return 2;
     case GST_VIDEO_FORMAT_I420:
     case GST_VIDEO_FORMAT_Y444:
@@ -2066,6 +2116,17 @@ _YUV_to_RGB (GstGLColorConvert * convert)
         info->shader_tex_names[1] = "UVtex";
         break;
       }
+      case GST_VIDEO_FORMAT_NV12_16L32S:
+      {
+        char val2 = convert->priv->in_tex_formats[1] == GST_GL_RG ? 'g' : 'a';
+        info->templ = &templ_TILED_SEMI_PLANAR_to_RGB;
+        info->frag_body = g_strdup_printf (templ_TILED_SEMI_PLANAR_to_RGB_BODY,
+            16, 32, 8, 16, 'r', val2,
+            pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
+        info->shader_tex_names[0] = "Ytex";
+        info->shader_tex_names[1] = "UVtex";
+        break;
+      }
       default:
         break;
     }
@@ -2313,13 +2374,14 @@ _bind_buffer (GstGLColorConvert * convert)
   /* Load the vertex position */
   gl->VertexAttribPointer (convert->priv->attr_position, 3, GL_FLOAT, GL_FALSE,
       5 * sizeof (GLfloat), (void *) 0);
-
-  /* Load the texture coordinate */
-  gl->VertexAttribPointer (convert->priv->attr_texture, 2, GL_FLOAT, GL_FALSE,
-      5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
-
   gl->EnableVertexAttribArray (convert->priv->attr_position);
-  gl->EnableVertexAttribArray (convert->priv->attr_texture);
+
+  if (convert->priv->attr_texture != -1) {
+    /* Load the texture coordinate */
+    gl->VertexAttribPointer (convert->priv->attr_texture, 2, GL_FLOAT, GL_FALSE,
+        5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
+    gl->EnableVertexAttribArray (convert->priv->attr_texture);
+  }
 }
 
 static void
@@ -2328,10 +2390,12 @@ _unbind_buffer (GstGLColorConvert * convert)
   const GstGLFuncs *gl = convert->context->gl_vtable;
 
   gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
-  gl->BindBuffer (GL_ARRAY_BUFFER, 0);
-
   gl->DisableVertexAttribArray (convert->priv->attr_position);
-  gl->DisableVertexAttribArray (convert->priv->attr_texture);
+
+  if (convert->priv->attr_texture != -1) {
+    gl->BindBuffer (GL_ARRAY_BUFFER, 0);
+    gl->DisableVertexAttribArray (convert->priv->attr_texture);
+  }
 }
 
 static GstGLShader *
@@ -2573,13 +2637,25 @@ _init_convert (GstGLColorConvert * convert)
     goto incompatible_api;
   }
 
+  /* Requires texelFetch() function... */
+  if (!(USING_GLES3 (convert->context) || USING_OPENGL30 (convert->context)) &&
+      GST_VIDEO_FORMAT_INFO_IS_TILED (convert->in_info.finfo)) {
+    GST_ERROR ("Conversion requires texelFetch() function available since "
+        "GLSL 1.30");
+    goto incompatible_api;
+  }
+
   if (!(convert->shader = _create_shader (convert)))
     goto error;
 
   convert->priv->attr_position =
       gst_gl_shader_get_attribute_location (convert->shader, "a_position");
-  convert->priv->attr_texture =
-      gst_gl_shader_get_attribute_location (convert->shader, "a_texcoord");
+
+  if (!GST_VIDEO_FORMAT_INFO_IS_TILED (convert->in_info.finfo))
+    convert->priv->attr_texture =
+        gst_gl_shader_get_attribute_location (convert->shader, "a_texcoord");
+  else
+    convert->priv->attr_texture = -1;
 
   gst_gl_shader_use (convert->shader);
 
@@ -2601,10 +2677,25 @@ _init_convert (GstGLColorConvert * convert)
           i);
   }
 
-  gst_gl_shader_set_uniform_1f (convert->shader, "width",
-      GST_VIDEO_INFO_WIDTH (&convert->in_info));
-  gst_gl_shader_set_uniform_1f (convert->shader, "height",
-      GST_VIDEO_INFO_HEIGHT (&convert->in_info));
+  if (GST_VIDEO_FORMAT_INFO_IS_TILED (convert->in_info.finfo)) {
+    guint ws, hs;
+    gsize stride;
+    gfloat width, height;
+
+    stride = GST_VIDEO_INFO_PLANE_STRIDE (&convert->in_info, 0);
+    gst_video_format_info_get_tile_sizes (convert->in_info.finfo, 0, &ws, &hs);
+
+    width = GST_VIDEO_TILE_X_TILES (stride) << ws;
+    height = GST_VIDEO_TILE_Y_TILES (stride) << hs;
+
+    gst_gl_shader_set_uniform_1f (convert->shader, "width", width);
+    gst_gl_shader_set_uniform_1f (convert->shader, "height", height);
+  } else {
+    gst_gl_shader_set_uniform_1f (convert->shader, "width",
+        GST_VIDEO_INFO_WIDTH (&convert->in_info));
+    gst_gl_shader_set_uniform_1f (convert->shader, "height",
+        GST_VIDEO_INFO_HEIGHT (&convert->in_info));
+  }
 
   if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE) {
     gst_gl_shader_set_uniform_1f (convert->shader, "poffset_x", 1.);

diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.h b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.h
index 4512fe1..3bf1f42 100644 (file)
--- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.h
+++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglcolorconvert.h
@@ -103,7 +103,7 @@ struct _GstGLColorConvertClass
                                "xBGR, ARGB, ABGR, GBRA, GBR, RGBP, BGRP, Y444, I420, YV12, Y42B, " \
                                "Y41B, NV12, NV21, NV16, NV61, YUY2, UYVY, Y210, AYUV, " \
                                "VUYA, Y410, GRAY8, GRAY16_LE, GRAY16_BE, " \
-                               "RGB16, BGR16, ARGB64, A420, AV12" \
+                               "RGB16, BGR16, ARGB64, A420, AV12, NV12_16L32S" \
                                GST_GL_COLOR_CONVERT_EXT_FORMATS "}"
 
 /**

diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c
index d1ce3f0..b624ac3 100644 (file)
--- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c
+++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglformat.c
@@ -192,6 +192,7 @@ gst_gl_format_from_video_info (GstGLContext * context,
     case GST_VIDEO_FORMAT_NV21:
     case GST_VIDEO_FORMAT_NV16:
     case GST_VIDEO_FORMAT_NV61:
+    case GST_VIDEO_FORMAT_NV12_16L32S:
       n_plane_components = plane == 0 ? 1 : 2;
       break;
     case GST_VIDEO_FORMAT_AV12:

diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglmemory.h b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglmemory.h
index 8dff412..0916c4f 100644 (file)
--- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglmemory.h
+++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglmemory.h
@@ -64,7 +64,7 @@ GType gst_gl_memory_allocator_get_type(void);
 #define GST_GL_MEMORY_VIDEO_FORMATS_STR \
     "{ RGBA, BGRA, RGBx, BGRx, ARGB, ABGR, xRGB, xBGR, GBRA, GBR, RGBP, BGRP, RGB, BGR, RGB16, BGR16, " \
     "AYUV, VUYA, Y410, I420, YV12, NV12, NV21, NV16, NV61, YUY2, UYVY, Y210, Y41B, " \
-    "Y42B, Y444, GRAY8, GRAY16_LE, GRAY16_BE, ARGB64, A420, AV12" \
+    "Y42B, Y444, GRAY8, GRAY16_LE, GRAY16_BE, ARGB64, A420, AV12, NV12_16L32S" \
     GST_GL_MEMORY_VIDEO_EXT_FORMATS "}"
 
 /**