gltransformation: use the affine transformation meta if available downstream

We can avoid a render pass if downstream supports the affine transformation meta
and increase the performance of some pipelines involving gltransformation.

Implemented by checking for the affine transformation in the allocation query
from downstream and combining our matrix with that of upstream's (or creating
our own).

diff --git a/ext/gl/gstgltransformation.c b/ext/gl/gstgltransformation.c
index 779447c..4a32138 100644 (file)
--- a/ext/gl/gstgltransformation.c
+++ b/ext/gl/gstgltransformation.c
@@ -89,6 +89,10 @@ static gboolean gst_gl_transformation_set_caps (GstGLFilter * filter,
     GstCaps * incaps, GstCaps * outcaps);
 static gboolean gst_gl_transformation_src_event (GstBaseTransform * trans,
     GstEvent * event);
+static gboolean gst_gl_transformation_filter_meta (GstBaseTransform * trans,
+    GstQuery * query, GType api, const GstStructure * params);
+static gboolean gst_gl_transformation_decide_allocation (GstBaseTransform *
+    trans, GstQuery * query);
 
 static void gst_gl_transformation_reset_gl (GstGLFilter * filter);
 static gboolean gst_gl_transformation_stop (GstBaseTransform * trans);
@@ -97,33 +101,14 @@ static void gst_gl_transformation_callback (gpointer stuff);
 static void gst_gl_transformation_build_mvp (GstGLTransformation *
     transformation);
 
+static GstFlowReturn
+gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
+    GstBuffer * inbuf, GstBuffer ** outbuf);
+static gboolean gst_gl_transformation_filter (GstGLFilter * filter,
+    GstBuffer * inbuf, GstBuffer * outbuf);
 static gboolean gst_gl_transformation_filter_texture (GstGLFilter * filter,
     guint in_tex, guint out_tex);
 
-/* vertex source */
-static const gchar *cube_v_src =
-    "attribute vec4 position;                     \n"
-    "attribute vec2 uv;                           \n"
-    "uniform mat4 mvp;                            \n"
-    "varying vec2 out_uv;                         \n"
-    "void main()                                  \n"
-    "{                                            \n"
-    "   gl_Position = mvp * position;             \n"
-    "   out_uv = uv;                              \n"
-    "}                                            \n";
-
-/* fragment source */
-static const gchar *cube_f_src =
-    "#ifdef GL_ES                                 \n"
-    "  precision mediump float;                   \n"
-    "#endif                                       \n"
-    "varying vec2 out_uv;                         \n"
-    "uniform sampler2D texture;                   \n"
-    "void main()                                  \n"
-    "{                                            \n"
-    "  gl_FragColor = texture2D (texture, out_uv);\n"
-    "}                                            \n";
-
 static void
 gst_gl_transformation_class_init (GstGLTransformationClass * klass)
 {
@@ -139,14 +124,20 @@ gst_gl_transformation_class_init (GstGLTransformationClass * klass)
   gobject_class->get_property = gst_gl_transformation_get_property;
 
   base_transform_class->src_event = gst_gl_transformation_src_event;
+  base_transform_class->decide_allocation =
+      gst_gl_transformation_decide_allocation;
+  base_transform_class->filter_meta = gst_gl_transformation_filter_meta;
 
   GST_GL_FILTER_CLASS (klass)->init_fbo = gst_gl_transformation_init_shader;
   GST_GL_FILTER_CLASS (klass)->display_reset_cb =
       gst_gl_transformation_reset_gl;
   GST_GL_FILTER_CLASS (klass)->set_caps = gst_gl_transformation_set_caps;
+  GST_GL_FILTER_CLASS (klass)->filter = gst_gl_transformation_filter;
   GST_GL_FILTER_CLASS (klass)->filter_texture =
       gst_gl_transformation_filter_texture;
   GST_BASE_TRANSFORM_CLASS (klass)->stop = gst_gl_transformation_stop;
+  GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer =
+      gst_gl_transformation_prepare_output_buffer;
 
   g_object_class_install_property (gobject_class, PROP_FOV,
       g_param_spec_float ("fov", "Fov", "Field of view angle in degrees",
@@ -627,6 +618,39 @@ gst_gl_transformation_src_event (GstBaseTransform * trans, GstEvent * event)
   return ret;
 }
 
+static gboolean
+gst_gl_transformation_filter_meta (GstBaseTransform * trans, GstQuery * query,
+    GType api, const GstStructure * params)
+{
+  if (api == GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE)
+    return TRUE;
+
+  if (api == GST_GL_SYNC_META_API_TYPE)
+    return TRUE;
+
+  return FALSE;
+}
+
+static gboolean
+gst_gl_transformation_decide_allocation (GstBaseTransform * trans,
+    GstQuery * query)
+{
+  GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
+
+  if (!GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans,
+          query))
+    return FALSE;
+
+  if (gst_query_find_allocation_meta (query,
+          GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, NULL)) {
+    transformation->downstream_supports_affine_meta = TRUE;
+  } else {
+    transformation->downstream_supports_affine_meta = FALSE;
+  }
+
+  return TRUE;
+}
+
 static void
 gst_gl_transformation_reset_gl (GstGLFilter * filter)
 {
@@ -682,11 +706,81 @@ gst_gl_transformation_init_shader (GstGLFilter * filter)
   if (gst_gl_context_get_gl_api (GST_GL_BASE_FILTER (filter)->context)) {
     /* blocking call, wait until the opengl thread has compiled the shader */
     return gst_gl_context_gen_shader (GST_GL_BASE_FILTER (filter)->context,
-        cube_v_src, cube_f_src, &transformation->shader);
+        gst_gl_shader_string_vertex_mat4_vertex_transform,
+        gst_gl_shader_string_fragment_default, &transformation->shader);
   }
   return TRUE;
 }
 
+static const gfloat from_ndc_matrix[] = {
+  0.5f, 0.0f, 0.0, 0.5f,
+  0.0f, 0.5f, 0.0, 0.5f,
+  0.0f, 0.0f, 0.5, 0.5f,
+  0.0f, 0.0f, 0.0, 1.0f,
+};
+
+static const gfloat to_ndc_matrix[] = {
+  2.0f, 0.0f, 0.0, -1.0f,
+  0.0f, 2.0f, 0.0, -1.0f,
+  0.0f, 0.0f, 2.0, -1.0f,
+  0.0f, 0.0f, 0.0, 1.0f,
+};
+
+static GstFlowReturn
+gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
+    GstBuffer * inbuf, GstBuffer ** outbuf)
+{
+  GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
+  GstGLFilter *filter = GST_GL_FILTER (trans);
+
+  if (transformation->downstream_supports_affine_meta &&
+      gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
+    GstVideoAffineTransformationMeta *af_meta;
+    graphene_matrix_t upstream_matrix, from_ndc, to_ndc, tmp, tmp2, inv_aspect;
+
+    *outbuf = gst_buffer_make_writable (inbuf);
+
+    af_meta = gst_buffer_get_video_affine_transformation_meta (inbuf);
+    if (!af_meta)
+      af_meta = gst_buffer_add_video_affine_transformation_meta (*outbuf);
+
+    GST_LOG_OBJECT (trans, "applying transformation to existing affine "
+        "transformation meta");
+
+    /* apply the transformation to the existing affine meta */
+    graphene_matrix_init_from_float (&from_ndc, from_ndc_matrix);
+    graphene_matrix_init_from_float (&to_ndc, to_ndc_matrix);
+    graphene_matrix_init_from_float (&upstream_matrix, af_meta->matrix);
+
+    graphene_matrix_init_scale (&inv_aspect, transformation->aspect, 1., 1.);
+
+    graphene_matrix_multiply (&from_ndc, &upstream_matrix, &tmp);
+    graphene_matrix_multiply (&tmp, &transformation->mvp_matrix, &tmp2);
+    graphene_matrix_multiply (&tmp2, &inv_aspect, &tmp);
+    graphene_matrix_multiply (&tmp, &to_ndc, &tmp2);
+
+    graphene_matrix_to_float (&tmp2, af_meta->matrix);
+    return GST_FLOW_OK;
+  }
+
+  return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (trans,
+      inbuf, outbuf);
+}
+
+static gboolean
+gst_gl_transformation_filter (GstGLFilter * filter,
+    GstBuffer * inbuf, GstBuffer * outbuf)
+{
+  GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
+
+  if (transformation->downstream_supports_affine_meta &&
+      gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
+    return TRUE;
+  } else {
+    return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
+  }
+}
+
 static gboolean
 gst_gl_transformation_filter_texture (GstGLFilter * filter, guint in_tex,
     guint out_tex)
@@ -784,16 +878,17 @@ gst_gl_transformation_callback (gpointer stuff)
   gst_gl_shader_set_uniform_1i (transformation->shader, "texture", 0);
 
   graphene_matrix_to_float (&transformation->mvp_matrix, temp_matrix);
-  gst_gl_shader_set_uniform_matrix_4fv (transformation->shader, "mvp",
-      1, GL_FALSE, temp_matrix);
+  gst_gl_shader_set_uniform_matrix_4fv (transformation->shader,
+      "u_transformation", 1, GL_FALSE, temp_matrix);
 
   if (!transformation->vertex_buffer) {
     transformation->attr_position =
         gst_gl_shader_get_attribute_location (transformation->shader,
-        "position");
+        "a_position");
 
     transformation->attr_texture =
-        gst_gl_shader_get_attribute_location (transformation->shader, "uv");
+        gst_gl_shader_get_attribute_location (transformation->shader,
+        "a_texcoord");
 
     if (gl->GenVertexArrays) {
       gl->GenVertexArrays (1, &transformation->vao);

diff --git a/ext/gl/gstgltransformation.h b/ext/gl/gstgltransformation.h
index 7d41c61..ed75ff6 100644 (file)
--- a/ext/gl/gstgltransformation.h
+++ b/ext/gl/gstgltransformation.h
@@ -76,6 +76,7 @@ struct _GstGLTransformation
     graphene_matrix_t projection_matrix;
     graphene_matrix_t mvp_matrix;
 
+    gboolean downstream_supports_affine_meta;
     gboolean caps_change;
 };