3 * Copyright (C) 2014 Lubosz Sarnecki <lubosz@gmail.com>
4 * Copyright (C) 2016 Matthew Waters <matthew@centricular.com>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Library General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Library General Public License for more details.
16 * You should have received a copy of the GNU Library General Public
17 * License along with this library; if not, write to the
18 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
19 * Boston, MA 02110-1301, USA.
23 * SECTION:element-gltransformation
24 * @title: gltransformation
26 * Transforms video on the GPU.
30 * gst-launch-1.0 gltestsrc ! gltransformation rotation-z=45 ! glimagesink
31 * ]| A pipeline to rotate by 45 degrees
33 * gst-launch-1.0 gltestsrc ! gltransformation translation-x=0.5 ! glimagesink
34 * ]| Translate the video by 0.5
36 * gst-launch-1.0 gltestsrc ! gltransformation scale-y=0.5 scale-x=0.5 ! glimagesink
37 * ]| Resize the video by 0.5
39 * gst-launch-1.0 gltestsrc ! gltransformation rotation-x=-45 ortho=True ! glimagesink
40 * ]| Rotate the video around the X-Axis by -45° with an orthographic projection
48 #include "gstglelements.h"
49 #include "gstgltransformation.h"
51 #include <gst/gl/gstglapi.h>
52 #include <graphene-gobject.h>
53 #include "gstglutils.h"
55 #define GST_CAT_DEFAULT gst_gl_transformation_debug
56 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
58 #define gst_gl_transformation_parent_class parent_class
60 #define VEC4_FORMAT "%f,%f,%f,%f"
61 #define VEC4_ARGS(v) graphene_vec4_get_x (v), graphene_vec4_get_y (v), graphene_vec4_get_z (v), graphene_vec4_get_w (v)
62 #define VEC3_FORMAT "%f,%f,%f"
63 #define VEC3_ARGS(v) graphene_vec3_get_x (v), graphene_vec3_get_y (v), graphene_vec3_get_z (v)
64 #define VEC2_FORMAT "%f,%f"
65 #define VEC2_ARGS(v) graphene_vec2_get_x (v), graphene_vec2_get_y (v)
66 #define POINT3D_FORMAT "%f,%f,%f"
67 #define POINT3D_ARGS(p) (p)->x, (p)->y, (p)->z
89 GST_DEBUG_CATEGORY_INIT (gst_gl_transformation_debug, "gltransformation", 0, "gltransformation element");
91 G_DEFINE_TYPE_WITH_CODE (GstGLTransformation, gst_gl_transformation,
92 GST_TYPE_GL_FILTER, DEBUG_INIT);
93 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (gltransformation, "gltransformation",
94 GST_RANK_NONE, GST_TYPE_GL_TRANSFORMATION, gl_element_init (plugin));
96 static void gst_gl_transformation_set_property (GObject * object, guint prop_id,
97 const GValue * value, GParamSpec * pspec);
98 static void gst_gl_transformation_get_property (GObject * object, guint prop_id,
99 GValue * value, GParamSpec * pspec);
101 static gboolean gst_gl_transformation_set_caps (GstGLFilter * filter,
102 GstCaps * incaps, GstCaps * outcaps);
103 static gboolean gst_gl_transformation_src_event (GstBaseTransform * trans,
105 static gboolean gst_gl_transformation_filter_meta (GstBaseTransform * trans,
106 GstQuery * query, GType api, const GstStructure * params);
107 static gboolean gst_gl_transformation_decide_allocation (GstBaseTransform *
108 trans, GstQuery * query);
110 static void gst_gl_transformation_gl_stop (GstGLBaseFilter * filter);
111 static gboolean gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter);
112 static gboolean gst_gl_transformation_callback (gpointer stuff);
113 static void gst_gl_transformation_build_mvp (GstGLTransformation *
117 gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
118 GstBuffer * inbuf, GstBuffer ** outbuf);
119 static gboolean gst_gl_transformation_filter (GstGLFilter * filter,
120 GstBuffer * inbuf, GstBuffer * outbuf);
121 static gboolean gst_gl_transformation_filter_texture (GstGLFilter * filter,
122 GstGLMemory * in_tex, GstGLMemory * out_tex);
125 gst_gl_transformation_class_init (GstGLTransformationClass * klass)
127 GObjectClass *gobject_class;
128 GstElementClass *element_class;
129 GstBaseTransformClass *base_transform_class;
131 gobject_class = (GObjectClass *) klass;
132 element_class = GST_ELEMENT_CLASS (klass);
133 base_transform_class = GST_BASE_TRANSFORM_CLASS (klass);
135 gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));
137 gobject_class->set_property = gst_gl_transformation_set_property;
138 gobject_class->get_property = gst_gl_transformation_get_property;
140 base_transform_class->src_event = gst_gl_transformation_src_event;
141 base_transform_class->decide_allocation =
142 gst_gl_transformation_decide_allocation;
143 base_transform_class->filter_meta = gst_gl_transformation_filter_meta;
145 GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_transformation_gl_start;
146 GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_transformation_gl_stop;
148 GST_GL_FILTER_CLASS (klass)->set_caps = gst_gl_transformation_set_caps;
149 GST_GL_FILTER_CLASS (klass)->filter = gst_gl_transformation_filter;
150 GST_GL_FILTER_CLASS (klass)->filter_texture =
151 gst_gl_transformation_filter_texture;
152 GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer =
153 gst_gl_transformation_prepare_output_buffer;
155 g_object_class_install_property (gobject_class, PROP_FOV,
156 g_param_spec_float ("fov", "Fov", "Field of view angle in degrees",
157 0.0, G_MAXFLOAT, 90.0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
159 g_object_class_install_property (gobject_class, PROP_ORTHO,
160 g_param_spec_boolean ("ortho", "Orthographic",
161 "Use orthographic projection", FALSE,
162 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
165 g_object_class_install_property (gobject_class, PROP_ROTATION_X,
166 g_param_spec_float ("rotation-x", "X Rotation",
167 "Rotates the video around the X-Axis in degrees.",
168 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
169 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
171 g_object_class_install_property (gobject_class, PROP_ROTATION_Y,
172 g_param_spec_float ("rotation-y", "Y Rotation",
173 "Rotates the video around the Y-Axis in degrees.",
174 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
175 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
177 g_object_class_install_property (gobject_class, PROP_ROTATION_Z,
178 g_param_spec_float ("rotation-z", "Z Rotation",
179 "Rotates the video around the Z-Axis in degrees.",
180 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
181 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
184 g_object_class_install_property (gobject_class, PROP_TRANSLATION_X,
185 g_param_spec_float ("translation-x", "X Translation",
186 "Translates the video at the X-Axis, in universal [0-1] coordinate.",
187 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
188 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
190 g_object_class_install_property (gobject_class, PROP_TRANSLATION_Y,
191 g_param_spec_float ("translation-y", "Y Translation",
192 "Translates the video at the Y-Axis, in universal [0-1] coordinate.",
193 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
194 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
196 g_object_class_install_property (gobject_class, PROP_TRANSLATION_Z,
197 g_param_spec_float ("translation-z", "Z Translation",
198 "Translates the video at the Z-Axis, in universal [0-1] coordinate.",
199 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
200 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
203 g_object_class_install_property (gobject_class, PROP_SCALE_X,
204 g_param_spec_float ("scale-x", "X Scale",
205 "Scale multiplier for the X-Axis.",
206 -G_MAXFLOAT, G_MAXFLOAT, 1.0,
207 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
209 g_object_class_install_property (gobject_class, PROP_SCALE_Y,
210 g_param_spec_float ("scale-y", "Y Scale",
211 "Scale multiplier for the Y-Axis.",
212 -G_MAXFLOAT, G_MAXFLOAT, 1.0,
213 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
216 g_object_class_install_property (gobject_class, PROP_PIVOT_X,
217 g_param_spec_float ("pivot-x", "X Pivot",
218 "Rotation pivot point X coordinate, where 0 is the center,"
219 " -1 the left border, +1 the right border and <-1, >1 outside.",
220 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
221 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
223 g_object_class_install_property (gobject_class, PROP_PIVOT_Y,
224 g_param_spec_float ("pivot-y", "Y Pivot",
225 "Rotation pivot point X coordinate, where 0 is the center,"
226 " -1 the left border, +1 the right border and <-1, >1 outside.",
227 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
228 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
230 g_object_class_install_property (gobject_class, PROP_PIVOT_Z,
231 g_param_spec_float ("pivot-z", "Z Pivot",
232 "Relevant for rotation in 3D space. You look into the negative Z axis direction",
233 -G_MAXFLOAT, G_MAXFLOAT, 0.0,
234 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
237 g_object_class_install_property (gobject_class, PROP_MVP,
238 g_param_spec_boxed ("mvp-matrix",
239 "Modelview Projection Matrix",
240 "The final Graphene 4x4 Matrix for transformation",
241 GRAPHENE_TYPE_MATRIX, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
243 gst_element_class_set_metadata (element_class, "OpenGL transformation filter",
244 "Filter/Effect/Video", "Transform video on the GPU",
245 "Lubosz Sarnecki <lubosz@gmail.com>\n"
246 "Matthew Waters <matthew@centricular.com>");
248 GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
249 GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2;
253 gst_gl_transformation_init (GstGLTransformation * filter)
255 filter->shader = NULL;
257 filter->aspect = 1.0;
258 filter->znear = 0.1f;
259 filter->zfar = 100.0;
261 filter->xscale = 1.0;
262 filter->yscale = 1.0;
266 gst_gl_transformation_build_mvp (filter);
270 gst_gl_transformation_build_mvp (GstGLTransformation * transformation)
272 GstGLFilter *filter = GST_GL_FILTER (transformation);
273 graphene_matrix_t modelview_matrix;
275 if (!filter->out_info.finfo) {
276 graphene_matrix_init_identity (&transformation->model_matrix);
277 graphene_matrix_init_identity (&transformation->view_matrix);
278 graphene_matrix_init_identity (&transformation->projection_matrix);
280 graphene_point3d_t translation_vector =
281 GRAPHENE_POINT3D_INIT (transformation->xtranslation * 2.0 *
282 transformation->aspect,
283 transformation->ytranslation * 2.0,
284 transformation->ztranslation * 2.0);
286 graphene_point3d_t pivot_vector =
287 GRAPHENE_POINT3D_INIT (-transformation->xpivot * transformation->aspect,
288 transformation->ypivot,
289 -transformation->zpivot);
291 graphene_point3d_t negative_pivot_vector;
293 graphene_vec3_t center;
296 gboolean current_passthrough;
297 gboolean passthrough;
299 graphene_vec3_init (&transformation->camera_position, 0.f, 0.f, 1.f);
300 graphene_vec3_init (¢er, 0.f, 0.f, 0.f);
301 graphene_vec3_init (&up, 0.f, 1.f, 0.f);
303 /* Translate into pivot origin */
304 graphene_matrix_init_translate (&transformation->model_matrix,
308 graphene_matrix_scale (&transformation->model_matrix,
309 transformation->xscale, transformation->yscale, 1.0f);
312 graphene_matrix_rotate (&transformation->model_matrix,
313 transformation->xrotation, graphene_vec3_x_axis ());
314 graphene_matrix_rotate (&transformation->model_matrix,
315 transformation->yrotation, graphene_vec3_y_axis ());
316 graphene_matrix_rotate (&transformation->model_matrix,
317 transformation->zrotation, graphene_vec3_z_axis ());
319 /* Translate back from pivot origin */
320 graphene_point3d_scale (&pivot_vector, -1.0, &negative_pivot_vector);
321 graphene_matrix_translate (&transformation->model_matrix,
322 &negative_pivot_vector);
325 graphene_matrix_translate (&transformation->model_matrix,
326 &translation_vector);
328 if (transformation->ortho) {
329 graphene_matrix_init_ortho (&transformation->projection_matrix,
330 -transformation->aspect, transformation->aspect,
331 -1, 1, transformation->znear, transformation->zfar);
333 graphene_matrix_init_perspective (&transformation->projection_matrix,
335 transformation->aspect, transformation->znear, transformation->zfar);
338 graphene_matrix_init_look_at (&transformation->view_matrix,
339 &transformation->camera_position, ¢er, &up);
341 current_passthrough =
342 gst_base_transform_is_passthrough (GST_BASE_TRANSFORM (transformation));
343 passthrough = transformation->xtranslation == 0.
344 && transformation->ytranslation == 0.
345 && transformation->ztranslation == 0. && transformation->xrotation == 0.
346 && transformation->yrotation == 0. && transformation->zrotation == 0.
347 && transformation->xscale == 1. && transformation->yscale == 1.
348 && gst_video_info_is_equal (&filter->in_info, &filter->out_info);
349 gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (transformation),
351 if (current_passthrough != passthrough) {
352 gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (transformation));
356 graphene_matrix_multiply (&transformation->model_matrix,
357 &transformation->view_matrix, &modelview_matrix);
358 graphene_matrix_multiply (&modelview_matrix,
359 &transformation->projection_matrix, &transformation->mvp_matrix);
361 graphene_matrix_inverse (&transformation->model_matrix,
362 &transformation->inv_model_matrix);
363 graphene_matrix_inverse (&transformation->view_matrix,
364 &transformation->inv_view_matrix);
365 graphene_matrix_inverse (&transformation->projection_matrix,
366 &transformation->inv_projection_matrix);
370 gst_gl_transformation_set_property (GObject * object, guint prop_id,
371 const GValue * value, GParamSpec * pspec)
373 GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
377 filter->fov = g_value_get_float (value);
380 filter->ortho = g_value_get_boolean (value);
382 case PROP_TRANSLATION_X:
383 filter->xtranslation = g_value_get_float (value);
385 case PROP_TRANSLATION_Y:
386 filter->ytranslation = g_value_get_float (value);
388 case PROP_TRANSLATION_Z:
389 filter->ztranslation = g_value_get_float (value);
391 case PROP_ROTATION_X:
392 filter->xrotation = g_value_get_float (value);
394 case PROP_ROTATION_Y:
395 filter->yrotation = g_value_get_float (value);
397 case PROP_ROTATION_Z:
398 filter->zrotation = g_value_get_float (value);
401 filter->xscale = g_value_get_float (value);
404 filter->yscale = g_value_get_float (value);
407 filter->xpivot = g_value_get_float (value);
410 filter->ypivot = g_value_get_float (value);
413 filter->zpivot = g_value_get_float (value);
416 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
419 gst_gl_transformation_build_mvp (filter);
423 gst_gl_transformation_get_property (GObject * object, guint prop_id,
424 GValue * value, GParamSpec * pspec)
426 GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
430 g_value_set_float (value, filter->fov);
433 g_value_set_boolean (value, filter->ortho);
435 case PROP_TRANSLATION_X:
436 g_value_set_float (value, filter->xtranslation);
438 case PROP_TRANSLATION_Y:
439 g_value_set_float (value, filter->ytranslation);
441 case PROP_TRANSLATION_Z:
442 g_value_set_float (value, filter->ztranslation);
444 case PROP_ROTATION_X:
445 g_value_set_float (value, filter->xrotation);
447 case PROP_ROTATION_Y:
448 g_value_set_float (value, filter->yrotation);
450 case PROP_ROTATION_Z:
451 g_value_set_float (value, filter->zrotation);
454 g_value_set_float (value, filter->xscale);
457 g_value_set_float (value, filter->yscale);
460 g_value_set_float (value, filter->xpivot);
463 g_value_set_float (value, filter->ypivot);
466 g_value_set_float (value, filter->zpivot);
469 /* FIXME: need to decompose this to support navigation events */
470 g_value_set_boxed (value, (gconstpointer) & filter->mvp_matrix);
473 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
479 gst_gl_transformation_set_caps (GstGLFilter * filter, GstCaps * incaps,
482 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
484 transformation->aspect =
485 (gdouble) GST_VIDEO_INFO_WIDTH (&filter->out_info) /
486 (gdouble) GST_VIDEO_INFO_HEIGHT (&filter->out_info);
488 transformation->caps_change = TRUE;
490 gst_gl_transformation_build_mvp (transformation);
496 _intersect_plane_and_ray (graphene_plane_t * video_plane, graphene_ray_t * ray,
497 graphene_point3d_t * result)
499 float t = graphene_ray_get_distance_to_plane (ray, video_plane);
500 GST_TRACE ("Calculated a distance of %f to the plane", t);
501 graphene_ray_get_position_at (ray, t, result);
505 _screen_coord_to_world_ray (GstGLTransformation * transformation, float x,
506 float y, graphene_ray_t * ray)
508 GstGLFilter *filter = GST_GL_FILTER (transformation);
509 gfloat w = (gfloat) GST_VIDEO_INFO_WIDTH (&filter->in_info);
510 gfloat h = (gfloat) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
511 graphene_vec3_t ray_eye_vec3, ray_world_dir, *ray_origin, *ray_direction;
512 graphene_vec3_t ray_ortho_dir;
513 graphene_point3d_t ray_clip, ray_eye;
514 graphene_vec2_t screen_coord;
516 /* GL is y-flipped. i.e. 0, 0 is the bottom left corner in screen space */
517 graphene_vec2_init (&screen_coord, (2. * x / w - 1.) / transformation->aspect,
520 graphene_point3d_init (&ray_clip, graphene_vec2_get_x (&screen_coord),
521 graphene_vec2_get_y (&screen_coord), -1.);
522 graphene_matrix_transform_point3d (&transformation->inv_projection_matrix,
523 &ray_clip, &ray_eye);
525 graphene_vec3_init (&ray_eye_vec3, ray_eye.x, ray_eye.y, -1.);
527 if (transformation->ortho) {
528 graphene_vec3_init (&ray_ortho_dir, 0., 0., 1.);
530 ray_origin = &ray_eye_vec3;
531 ray_direction = &ray_ortho_dir;
533 graphene_matrix_transform_vec3 (&transformation->inv_view_matrix,
534 &ray_eye_vec3, &ray_world_dir);
535 graphene_vec3_normalize (&ray_world_dir, &ray_world_dir);
537 ray_origin = &transformation->camera_position;
538 ray_direction = &ray_world_dir;
541 graphene_ray_init_from_vec3 (ray, ray_origin, ray_direction);
543 GST_TRACE_OBJECT (transformation, "Calculated ray origin: " VEC3_FORMAT
544 " direction: " VEC3_FORMAT " from screen coordinates: " VEC2_FORMAT
545 " with %s projection",
546 VEC3_ARGS (ray_origin), VEC3_ARGS (ray_direction),
547 VEC2_ARGS (&screen_coord),
548 transformation->ortho ? "ortho" : "perspection");
552 _init_world_video_plane (GstGLTransformation * transformation,
553 graphene_plane_t * video_plane)
555 graphene_point3d_t bottom_left, bottom_right, top_left, top_right;
556 graphene_point3d_t world_bottom_left, world_bottom_right;
557 graphene_point3d_t world_top_left, world_top_right;
559 graphene_point3d_init (&top_left, -transformation->aspect, 1., 0.);
560 graphene_point3d_init (&top_right, transformation->aspect, 1., 0.);
561 graphene_point3d_init (&bottom_left, -transformation->aspect, -1., 0.);
562 graphene_point3d_init (&bottom_right, transformation->aspect, -1., 0.);
564 graphene_matrix_transform_point3d (&transformation->model_matrix,
565 &bottom_left, &world_bottom_left);
566 graphene_matrix_transform_point3d (&transformation->model_matrix,
567 &bottom_right, &world_bottom_right);
568 graphene_matrix_transform_point3d (&transformation->model_matrix,
569 &top_left, &world_top_left);
570 graphene_matrix_transform_point3d (&transformation->model_matrix,
571 &top_right, &world_top_right);
573 graphene_plane_init_from_points (video_plane, &world_bottom_left,
574 &world_top_right, &world_top_left);
578 _screen_coord_to_model_coord (GstGLTransformation * transformation,
579 double x, double y, double *res_x, double *res_y)
581 GstGLFilter *filter = GST_GL_FILTER (transformation);
582 double w = (double) GST_VIDEO_INFO_WIDTH (&filter->in_info);
583 double h = (double) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
584 graphene_point3d_t world_point, model_coord;
585 graphene_plane_t video_plane;
589 _init_world_video_plane (transformation, &video_plane);
590 _screen_coord_to_world_ray (transformation, x, y, &ray);
591 _intersect_plane_and_ray (&video_plane, &ray, &world_point);
592 graphene_matrix_transform_point3d (&transformation->inv_model_matrix,
593 &world_point, &model_coord);
595 /* ndc to pixels. We render the frame Y-flipped so need to unflip the
597 new_x = (model_coord.x + 1.) * w / 2;
598 new_y = (1. - model_coord.y) * h / 2;
600 if (new_x < 0. || new_x > w || new_y < 0. || new_y > h)
601 /* coords off video surface */
604 GST_DEBUG_OBJECT (transformation, "converted %f,%f to %f,%f", x, y, new_x,
616 /* debugging facilities for transforming vertices from model space to screen
619 _ndc_to_viewport (GstGLTransformation * transformation, graphene_vec3_t * ndc,
620 int x, int y, int w, int h, float near, float far, graphene_vec3_t * result)
622 GstGLFilter *filter = GST_GL_FILTER (transformation);
623 /* center of the viewport */
627 graphene_vec3_init (result, graphene_vec3_get_x (ndc) * w / 2 + o_x,
628 graphene_vec3_get_y (ndc) * h / 2 + o_y,
629 (far - near) * graphene_vec3_get_z (ndc) / 2 + (far + near) / 2);
633 _perspective_division (graphene_vec4_t * clip, graphene_vec3_t * result)
635 float w = graphene_vec4_get_w (clip);
637 graphene_vec3_init (result, graphene_vec4_get_x (clip) / w,
638 graphene_vec4_get_y (clip) / w, graphene_vec4_get_z (clip) / w);
642 _vertex_to_screen_coord (GstGLTransformation * transformation,
643 graphene_vec4_t * vertex, graphene_vec3_t * view)
645 GstGLFilter *filter = GST_GL_FILTER (transformation);
646 gint w = GST_VIDEO_INFO_WIDTH (&filter->in_info);
647 gint h = GST_VIDEO_INFO_HEIGHT (&filter->in_info);
648 graphene_vec4_t clip;
651 graphene_matrix_transform_vec4 (&transformation->mvp_matrix, vertex, &clip);
652 _perspective_division (&clip, &ndc);
653 _ndc_to_viewport (transformation, &ndc, 0, 0, w, h, 0., 1., view);
658 gst_gl_transformation_src_event (GstBaseTransform * trans, GstEvent * event)
660 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
661 GstStructure *structure;
664 GST_DEBUG_OBJECT (trans, "handling %s event", GST_EVENT_TYPE_NAME (event));
666 switch (GST_EVENT_TYPE (event)) {
667 case GST_EVENT_NAVIGATION:{
670 GST_EVENT (gst_mini_object_make_writable (GST_MINI_OBJECT (event)));
672 structure = (GstStructure *) gst_event_get_structure (event);
673 if (gst_structure_get_double (structure, "pointer_x", &x) &&
674 gst_structure_get_double (structure, "pointer_y", &y)) {
675 gdouble new_x, new_y;
677 if (!_screen_coord_to_model_coord (transformation, x, y, &new_x,
679 gst_event_unref (event);
683 gst_structure_set (structure, "pointer_x", G_TYPE_DOUBLE, new_x,
684 "pointer_y", G_TYPE_DOUBLE, new_y, NULL);
692 ret = GST_BASE_TRANSFORM_CLASS (parent_class)->src_event (trans, event);
698 gst_gl_transformation_filter_meta (GstBaseTransform * trans, GstQuery * query,
699 GType api, const GstStructure * params)
701 if (api == GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE)
704 if (api == GST_GL_SYNC_META_API_TYPE)
711 gst_gl_transformation_decide_allocation (GstBaseTransform * trans,
714 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
716 if (!GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans,
720 if (gst_query_find_allocation_meta (query,
721 GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, NULL)) {
722 transformation->downstream_supports_affine_meta = TRUE;
724 transformation->downstream_supports_affine_meta = FALSE;
731 gst_gl_transformation_gl_stop (GstGLBaseFilter * base_filter)
733 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
734 const GstGLFuncs *gl = base_filter->context->gl_vtable;
736 if (transformation->vao) {
737 gl->DeleteVertexArrays (1, &transformation->vao);
738 transformation->vao = 0;
741 if (transformation->vertex_buffer) {
742 gl->DeleteBuffers (1, &transformation->vertex_buffer);
743 transformation->vertex_buffer = 0;
746 if (transformation->vbo_indices) {
747 gl->DeleteBuffers (1, &transformation->vbo_indices);
748 transformation->vbo_indices = 0;
751 if (transformation->shader) {
752 gst_object_unref (transformation->shader);
753 transformation->shader = NULL;
756 GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base_filter);
760 gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter)
762 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
764 if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base_filter))
767 if (gst_gl_context_get_gl_api (base_filter->context)) {
772 gst_gl_shader_string_fragment_get_default (base_filter->context,
773 GST_GLSL_VERSION_NONE,
774 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY);
776 /* blocking call, wait until the opengl thread has compiled the shader */
777 ret = gst_gl_context_gen_shader (base_filter->context,
778 gst_gl_shader_string_vertex_mat4_vertex_transform,
779 frag_str, &transformation->shader);
789 gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
790 GstBuffer * inbuf, GstBuffer ** outbuf)
792 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
793 GstGLFilter *filter = GST_GL_FILTER (trans);
795 if (transformation->downstream_supports_affine_meta &&
796 gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
797 GstVideoAffineTransformationMeta *af_meta;
798 graphene_matrix_t upstream_matrix, tmp, tmp2, inv_aspect, yflip;
799 float upstream[16], downstream[16];
801 *outbuf = gst_buffer_make_writable (inbuf);
803 af_meta = gst_buffer_get_video_affine_transformation_meta (inbuf);
805 af_meta = gst_buffer_add_video_affine_transformation_meta (*outbuf);
807 GST_LOG_OBJECT (trans, "applying transformation to existing affine "
808 "transformation meta");
810 gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, upstream);
812 /* apply the transformation to the existing affine meta */
813 graphene_matrix_init_from_float (&upstream_matrix, upstream);
814 graphene_matrix_init_scale (&inv_aspect, transformation->aspect, -1., 1.);
815 graphene_matrix_init_scale (&yflip, 1., -1., 1.);
817 /* invert the aspect effects */
818 graphene_matrix_multiply (&upstream_matrix, &inv_aspect, &tmp2);
819 /* apply the transformation */
820 graphene_matrix_multiply (&tmp2, &transformation->mvp_matrix, &tmp);
822 graphene_matrix_multiply (&tmp, &yflip, &tmp2);
824 graphene_matrix_to_float (&tmp2, downstream);
825 gst_gl_set_affine_transformation_meta_from_ndc_ext (af_meta, downstream);
830 return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (trans,
835 gst_gl_transformation_filter (GstGLFilter * filter,
836 GstBuffer * inbuf, GstBuffer * outbuf)
838 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
840 if (transformation->downstream_supports_affine_meta &&
841 gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
844 return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
849 gst_gl_transformation_filter_texture (GstGLFilter * filter,
850 GstGLMemory * in_tex, GstGLMemory * out_tex)
852 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
854 transformation->in_tex = in_tex;
856 gst_gl_framebuffer_draw_to_texture (filter->fbo, out_tex,
857 (GstGLFramebufferFunc) gst_gl_transformation_callback, transformation);
862 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
865 _upload_vertices (GstGLTransformation * transformation)
867 const GstGLFuncs *gl =
868 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
871 GLfloat vertices[] = {
872 -transformation->aspect, -1.0, 0.0, 1.0, 0.0, 0.0,
873 transformation->aspect, -1.0, 0.0, 1.0, 1.0, 0.0,
874 transformation->aspect, 1.0, 0.0, 1.0, 1.0, 1.0,
875 -transformation->aspect, 1.0, 0.0, 1.0, 0.0, 1.0,
879 gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
881 gl->BufferData (GL_ARRAY_BUFFER, 4 * 6 * sizeof (GLfloat), vertices,
886 _bind_buffer (GstGLTransformation * transformation)
888 const GstGLFuncs *gl =
889 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
891 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
892 gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
894 /* Load the vertex position */
895 gl->VertexAttribPointer (transformation->attr_position, 4, GL_FLOAT,
896 GL_FALSE, 6 * sizeof (GLfloat), (void *) 0);
898 /* Load the texture coordinate */
899 gl->VertexAttribPointer (transformation->attr_texture, 2, GL_FLOAT, GL_FALSE,
900 6 * sizeof (GLfloat), (void *) (4 * sizeof (GLfloat)));
902 gl->EnableVertexAttribArray (transformation->attr_position);
903 gl->EnableVertexAttribArray (transformation->attr_texture);
907 _unbind_buffer (GstGLTransformation * transformation)
909 const GstGLFuncs *gl =
910 GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
912 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
913 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
915 gl->DisableVertexAttribArray (transformation->attr_position);
916 gl->DisableVertexAttribArray (transformation->attr_texture);
920 gst_gl_transformation_callback (gpointer stuff)
922 GstGLFilter *filter = GST_GL_FILTER (stuff);
923 GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
924 GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
926 GLfloat temp_matrix[16];
928 gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
929 gl->BindTexture (GL_TEXTURE_2D, 0);
931 gl->ClearColor (0.f, 0.f, 0.f, 0.f);
932 gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
934 gst_gl_shader_use (transformation->shader);
936 gl->ActiveTexture (GL_TEXTURE0);
937 gl->BindTexture (GL_TEXTURE_2D, transformation->in_tex->tex_id);
938 gst_gl_shader_set_uniform_1i (transformation->shader, "texture", 0);
940 graphene_matrix_to_float (&transformation->mvp_matrix, temp_matrix);
941 gst_gl_shader_set_uniform_matrix_4fv (transformation->shader,
942 "u_transformation", 1, GL_FALSE, temp_matrix);
944 if (!transformation->vertex_buffer) {
945 transformation->attr_position =
946 gst_gl_shader_get_attribute_location (transformation->shader,
949 transformation->attr_texture =
950 gst_gl_shader_get_attribute_location (transformation->shader,
953 if (gl->GenVertexArrays) {
954 gl->GenVertexArrays (1, &transformation->vao);
955 gl->BindVertexArray (transformation->vao);
958 gl->GenBuffers (1, &transformation->vertex_buffer);
960 gl->GenBuffers (1, &transformation->vbo_indices);
961 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
962 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
965 transformation->caps_change = TRUE;
968 if (gl->GenVertexArrays)
969 gl->BindVertexArray (transformation->vao);
971 if (transformation->caps_change)
972 _upload_vertices (transformation);
973 _bind_buffer (transformation);
975 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
977 if (gl->GenVertexArrays)
978 gl->BindVertexArray (0);
980 _unbind_buffer (transformation);
982 gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
983 transformation->caps_change = FALSE;