3 * Copyright (C) 2009 Julien Isorce <julien.isorce@mail.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
22 * SECTION:element-deinterlace
25 * Deinterlacing using based on fragment shaders.
29 * gst-launch-1.0 videotestsrc ! glupload ! gldeinterlace ! glimagesink
31 * FBO (Frame Buffer Object) and GLSL (OpenGL Shading Language) are required.
39 #include <gst/gl/gstglfuncs.h>
41 #include "gstgldeinterlace.h"
43 #define GST_CAT_DEFAULT gst_gl_deinterlace_debug
44 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
53 GST_DEBUG_CATEGORY_INIT (gst_gl_deinterlace_debug, "gldeinterlace", 0, "gldeinterlace element");
54 #define gst_gl_deinterlace_parent_class parent_class
55 G_DEFINE_TYPE_WITH_CODE (GstGLDeinterlace, gst_gl_deinterlace,
56 GST_TYPE_GL_FILTER, DEBUG_INIT);
58 static void gst_gl_deinterlace_set_property (GObject * object,
59 guint prop_id, const GValue * value, GParamSpec * pspec);
60 static void gst_gl_deinterlace_get_property (GObject * object,
61 guint prop_id, GValue * value, GParamSpec * pspec);
63 static gboolean gst_gl_deinterlace_start (GstBaseTransform * trans);
64 static gboolean gst_gl_deinterlace_reset (GstBaseTransform * trans);
65 static GstCaps *gst_gl_deinterlace_transform_internal_caps (GstGLFilter *
66 filter, GstPadDirection direction, GstCaps * caps, GstCaps * caps_filter);
67 static gboolean gst_gl_deinterlace_init_fbo (GstGLFilter * filter);
68 static gboolean gst_gl_deinterlace_filter (GstGLFilter * filter,
69 GstBuffer * inbuf, GstBuffer * outbuf);
70 static gboolean gst_gl_deinterlace_filter_texture (GstGLFilter * filter,
71 GstGLMemory * in_tex, GstGLMemory * out_tex);
72 static gboolean gst_gl_deinterlace_vfir_callback (GstGLFilter * filter,
73 GstGLMemory * in_tex, gpointer stuff);
74 static gboolean gst_gl_deinterlace_greedyh_callback (GstGLFilter * filter,
75 GstGLMemory * in_tex, gpointer stuff);
78 static const gchar *greedyh_fragment_source =
79 "uniform sampler2D tex;\n"
80 "uniform sampler2D tex_prev;\n"
81 "uniform float max_comb;\n"
82 "uniform float motion_threshold;\n"
83 "uniform float motion_sense;\n"
84 "uniform float width;\n"
85 "uniform float height;\n"
86 "varying vec2 v_texcoord;\n"
89 " if (int(mod(v_texcoord.y * height, 2.0)) == 0) {\n"
90 " gl_FragColor = vec4(texture2D(tex_prev, v_texcoord).rgb, 1.0);\n"
92 " vec2 texcoord_L1_a1, texcoord_L3_a1, texcoord_L1, texcoord_L3, texcoord_L1_1, texcoord_L3_1;\n"
93 " vec3 L1_a1, L3_a1, L1, L3, L1_1, L3_1;\n"
95 " texcoord_L1 = vec2(v_texcoord.x, v_texcoord.y - 1.0 / height);\n"
96 " texcoord_L3 = vec2(v_texcoord.x, v_texcoord.y + 1.0 / height);\n"
97 " L1 = texture2D(tex_prev, texcoord_L1).rgb;\n"
98 " L3 = texture2D(tex_prev, texcoord_L3).rgb;\n"
99 " if (v_texcoord.x == 1.0 && v_texcoord.y == 1.0) {\n"
103 " texcoord_L1_1 = vec2(v_texcoord.x + 1.0 / width, v_texcoord.y - 1.0 / height);\n"
104 " texcoord_L3_1 = vec2(v_texcoord.x + 1.0 / width, v_texcoord.y + 1.0 / height);\n"
105 " L1_1 = texture2D(tex_prev, texcoord_L1_1).rgb;\n"
106 " L3_1 = texture2D(tex_prev, texcoord_L3_1).rgb;\n"
109 " if (int(ceil(v_texcoord.x + v_texcoord.y)) == 0) {\n"
113 " texcoord_L1_a1 = vec2(v_texcoord.x - 1.0 / width, v_texcoord.y - 1.0 / height);\n"
114 " texcoord_L3_a1 = vec2(v_texcoord.x - 1.0 / width, v_texcoord.y + 1.0 / height);\n"
115 " L1_a1 = texture2D(tex_prev, texcoord_L1_a1).rgb;\n"
116 " L3_a1 = texture2D(tex_prev, texcoord_L3_a1).rgb;\n"
119 " vec3 avg_a1 = (L1_a1 + L3_a1) / 2.0;\n"
120 " vec3 avg = (L1 + L3) / 2.0;\n"
121 " vec3 avg_1 = (L1_1 + L3_1) / 2.0;\n"
122 " vec3 avg_s = (avg_a1 + avg_1) / 2.0;\n"
123 " vec3 avg_sc = (avg_s + avg) / 2.0;\n"
124 " vec3 L2 = texture2D(tex, v_texcoord).rgb;\n"
125 " vec3 LP2 = texture2D(tex_prev, v_texcoord).rgb;\n"
127 " if (abs(L2.r - avg_sc.r) < abs(LP2.r - avg_sc.r)) {\n"
128 " best.r = L2.r;\n" " } else {\n"
132 " if (abs(L2.g - avg_sc.g) < abs(LP2.g - avg_sc.g)) {\n"
138 " if (abs(L2.b - avg_sc.b) < abs(LP2.b - avg_sc.b)) {\n"
145 " last.r = clamp(best.r, max(min(L1.r, L3.r) - max_comb, 0.0), min(max(L1.r, L3.r) + max_comb, 1.0));\n"
146 " last.g = clamp(best.g, max(min(L1.g, L3.g) - max_comb, 0.0), min(max(L1.g, L3.g) + max_comb, 1.0));\n"
147 " last.b = clamp(best.b, max(min(L1.b, L3.b) - max_comb, 0.0), min(max(L1.b, L3.b) + max_comb, 1.0));\n"
149 " const vec3 luma = vec3 (0.299011, 0.586987, 0.114001);"
150 " float mov = min(max(abs(dot(L2 - LP2, luma)) - motion_threshold, 0.0) * motion_sense, 1.0);\n"
151 " last = last * (1.0 - mov) + avg_sc * mov;\n"
152 " gl_FragColor = vec4(last, 1.0);\n"
156 const gchar *vfir_fragment_source =
157 "uniform sampler2D tex;\n"
158 "uniform float width;\n"
159 "uniform float height;\n"
160 "varying vec2 v_texcoord;\n"
163 " vec2 topcoord, botcoord;\n"
164 " vec4 cur_color, top_color, bot_color;\n"
165 " topcoord.x = v_texcoord.x;\n"
166 " botcoord.x = v_texcoord.x;\n"
167 " if (v_texcoord.y == 0.0 || v_texcoord.y == 1.0) {\n"
168 " topcoord.y = v_texcoord.y ;\n"
169 " botcoord.y = v_texcoord.y ;\n"
172 " topcoord.y = v_texcoord.y - 1.0/height;\n"
173 " botcoord.y = v_texcoord.y + 1.0/height;\n"
175 " cur_color = texture2D(tex, v_texcoord);\n"
176 " top_color = texture2D(tex, topcoord);\n"
177 " bot_color = texture2D(tex, botcoord);\n"
178 " gl_FragColor = 0.5*cur_color + 0.25*top_color + 0.25*bot_color;\n"
182 /* dont' forget to edit the following when a new method is added */
185 GST_GL_DEINTERLACE_VFIR,
186 GST_GL_DEINTERLACE_GREEDYH
187 } GstGLDeinterlaceMethod;
189 static const GEnumValue *
190 gst_gl_deinterlace_get_methods (void)
192 static const GEnumValue method_types[] = {
193 {GST_GL_DEINTERLACE_VFIR, "Blur Vertical", "vfir"},
194 {GST_GL_DEINTERLACE_GREEDYH, "Motion Adaptive: Advanced Detection",
201 #define GST_TYPE_GL_DEINTERLACE_METHODS (gst_gl_deinterlace_method_get_type ())
203 gst_gl_deinterlace_method_get_type (void)
205 static GType gl_deinterlace_method_type = 0;
206 if (!gl_deinterlace_method_type) {
207 gl_deinterlace_method_type =
208 g_enum_register_static ("GstGLDeinterlaceMethod",
209 gst_gl_deinterlace_get_methods ());
211 return gl_deinterlace_method_type;
215 gst_gl_deinterlace_set_method (GstGLDeinterlace * deinterlace,
218 switch (method_types) {
219 case GST_GL_DEINTERLACE_VFIR:
220 deinterlace->deinterlacefunc = gst_gl_deinterlace_vfir_callback;
221 deinterlace->current_method = method_types;
223 case GST_GL_DEINTERLACE_GREEDYH:
224 deinterlace->deinterlacefunc = gst_gl_deinterlace_greedyh_callback;
225 deinterlace->current_method = method_types;
228 g_assert_not_reached ();
234 gst_gl_deinterlace_class_init (GstGLDeinterlaceClass * klass)
236 GObjectClass *gobject_class;
237 GstElementClass *element_class;
239 gobject_class = (GObjectClass *) klass;
240 element_class = GST_ELEMENT_CLASS (klass);
242 gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));
244 gobject_class->set_property = gst_gl_deinterlace_set_property;
245 gobject_class->get_property = gst_gl_deinterlace_get_property;
247 gst_element_class_set_metadata (element_class,
248 "OpenGL deinterlacing filter", "Deinterlace",
249 "Deinterlacing based on fragment shaders",
250 "Julien Isorce <julien.isorce@mail.com>");
252 g_object_class_install_property (gobject_class,
254 g_param_spec_enum ("method",
255 "Deinterlace Method",
256 "Select which deinterlace method apply to GL video texture",
257 GST_TYPE_GL_DEINTERLACE_METHODS,
258 GST_GL_DEINTERLACE_VFIR, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
260 GST_BASE_TRANSFORM_CLASS (klass)->start = gst_gl_deinterlace_start;
261 GST_BASE_TRANSFORM_CLASS (klass)->stop = gst_gl_deinterlace_reset;
263 GST_GL_FILTER_CLASS (klass)->transform_internal_caps =
264 gst_gl_deinterlace_transform_internal_caps;
265 GST_GL_FILTER_CLASS (klass)->filter = gst_gl_deinterlace_filter;
266 GST_GL_FILTER_CLASS (klass)->filter_texture =
267 gst_gl_deinterlace_filter_texture;
268 GST_GL_FILTER_CLASS (klass)->init_fbo = gst_gl_deinterlace_init_fbo;
270 GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
271 GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3;
273 gst_type_mark_as_plugin_api (GST_TYPE_GL_DEINTERLACE_METHODS, 0);
277 gst_gl_deinterlace_init (GstGLDeinterlace * filter)
279 filter->shaderstable = NULL;
280 filter->deinterlacefunc = gst_gl_deinterlace_vfir_callback;
281 filter->current_method = GST_GL_DEINTERLACE_VFIR;
282 filter->prev_buffer = NULL;
283 filter->prev_tex = NULL;
287 gst_gl_deinterlace_start (GstBaseTransform * trans)
289 GstGLDeinterlace *deinterlace_filter = GST_GL_DEINTERLACE (trans);
291 deinterlace_filter->shaderstable = g_hash_table_new (g_str_hash, g_str_equal);
293 return GST_BASE_TRANSFORM_CLASS (parent_class)->start (trans);
297 gst_gl_deinterlace_ghash_func_clean (gpointer key, gpointer value,
300 GstGLShader *shader = (GstGLShader *) value;
302 gst_object_unref (shader);
308 gst_gl_deinterlace_reset (GstBaseTransform * trans)
310 GstGLDeinterlace *deinterlace_filter = GST_GL_DEINTERLACE (trans);
312 gst_buffer_replace (&deinterlace_filter->prev_buffer, NULL);
314 //blocking call, wait the opengl thread has destroyed the shader
315 if (deinterlace_filter->shaderstable) {
316 /* release shaders in the gl thread */
317 g_hash_table_foreach (deinterlace_filter->shaderstable,
318 gst_gl_deinterlace_ghash_func_clean, deinterlace_filter);
320 /* clean the htable without calling values destructors
321 * because shaders have been released in the glthread
322 * through the foreach func */
323 g_hash_table_unref (deinterlace_filter->shaderstable);
324 deinterlace_filter->shaderstable = NULL;
327 return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (trans);
331 gst_gl_deinterlace_transform_internal_caps (GstGLFilter * filter,
332 GstPadDirection direction, GstCaps * caps, GstCaps * caps_filter)
338 res = gst_caps_copy (caps);
340 for (len = gst_caps_get_size (res); len > 0; len--) {
341 s = gst_caps_get_structure (res, len - 1);
342 if (direction == GST_PAD_SINK) {
343 gst_structure_remove_field (s, "interlace-mode");
351 gst_gl_deinterlace_set_property (GObject * object, guint prop_id,
352 const GValue * value, GParamSpec * pspec)
354 GstGLDeinterlace *filter = GST_GL_DEINTERLACE (object);
358 gst_gl_deinterlace_set_method (filter, g_value_get_enum (value));
361 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
367 gst_gl_deinterlace_get_property (GObject * object, guint prop_id,
368 GValue * value, GParamSpec * pspec)
370 GstGLDeinterlace *filter = GST_GL_DEINTERLACE (object);
374 g_value_set_enum (value, filter->current_method);
377 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
383 gst_gl_deinterlace_init_fbo (GstGLFilter * filter)
389 gst_gl_deinterlace_filter_texture (GstGLFilter * filter, GstGLMemory * in_tex,
390 GstGLMemory * out_tex)
392 GstGLDeinterlace *deinterlace_filter = GST_GL_DEINTERLACE (filter);
394 //blocking call, use a FBO
395 gst_gl_filter_render_to_target (filter, in_tex, out_tex,
396 deinterlace_filter->deinterlacefunc, deinterlace_filter);
402 gst_gl_deinterlace_filter (GstGLFilter * filter, GstBuffer * inbuf,
405 GstGLDeinterlace *deinterlace_filter = GST_GL_DEINTERLACE (filter);
407 gst_gl_filter_filter_texture (filter, inbuf, outbuf);
409 gst_buffer_replace (&deinterlace_filter->prev_buffer, inbuf);
415 gst_gl_deinterlace_get_fragment_shader (GstGLFilter * filter,
416 const gchar * shader_name, const gchar * shader_source)
418 GstGLShader *shader = NULL;
419 GstGLDeinterlace *deinterlace_filter = GST_GL_DEINTERLACE (filter);
420 GstGLContext *context = GST_GL_BASE_FILTER (filter)->context;
421 const gchar *frags[2];
423 shader = g_hash_table_lookup (deinterlace_filter->shaderstable, shader_name);
426 gst_gl_shader_string_get_highest_precision (context,
427 GST_GLSL_VERSION_NONE,
428 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY);
429 frags[1] = shader_source;
432 GError *error = NULL;
434 if (!(shader = gst_gl_shader_new_link_with_stages (context, &error,
435 gst_glsl_stage_new_default_vertex (context),
436 gst_glsl_stage_new_with_strings (context, GL_FRAGMENT_SHADER,
437 GST_GLSL_VERSION_NONE,
438 GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY, 2,
440 GST_ELEMENT_ERROR (deinterlace_filter, RESOURCE, NOT_FOUND,
441 ("Failed to initialize %s shader", shader_name), (NULL));
444 filter->draw_attr_position_loc =
445 gst_gl_shader_get_attribute_location (shader, "a_position");
446 filter->draw_attr_texture_loc =
447 gst_gl_shader_get_attribute_location (shader, "a_texcoord");
450 g_hash_table_insert (deinterlace_filter->shaderstable, (gchar *) shader_name,
457 gst_gl_deinterlace_vfir_callback (GstGLFilter * filter, GstGLMemory * in_tex,
460 GstGLContext *context = GST_GL_BASE_FILTER (filter)->context;
461 const GstGLFuncs *gl = context->gl_vtable;
464 shader = gst_gl_deinterlace_get_fragment_shader (filter, "vfir",
465 vfir_fragment_source);
470 #if GST_GL_HAVE_OPENGL
471 if (USING_OPENGL (context)) {
472 gl->MatrixMode (GL_PROJECTION);
477 gst_gl_shader_use (shader);
479 gl->ActiveTexture (GL_TEXTURE0);
480 gl->BindTexture (GL_TEXTURE_2D, gst_gl_memory_get_texture_id (in_tex));
482 gst_gl_shader_set_uniform_1i (shader, "tex", 0);
483 gst_gl_shader_set_uniform_1f (shader, "width",
484 GST_VIDEO_INFO_WIDTH (&filter->out_info));
485 gst_gl_shader_set_uniform_1f (shader, "height",
486 GST_VIDEO_INFO_HEIGHT (&filter->out_info));
488 gst_gl_filter_draw_fullscreen_quad (filter);
494 gst_gl_deinterlace_greedyh_callback (GstGLFilter * filter, GstGLMemory * in_tex,
498 GstGLDeinterlace *deinterlace_filter = GST_GL_DEINTERLACE (filter);
499 GstGLContext *context = GST_GL_BASE_FILTER (filter)->context;
500 GstGLFuncs *gl = context->gl_vtable;
503 gst_gl_deinterlace_get_fragment_shader (filter, "greedhy",
504 greedyh_fragment_source);
509 #if GST_GL_HAVE_OPENGL
510 if (USING_OPENGL (context)) {
511 gl->MatrixMode (GL_PROJECTION);
516 gst_gl_shader_use (shader);
518 if (G_LIKELY (deinterlace_filter->prev_tex != NULL)) {
519 gl->ActiveTexture (GL_TEXTURE1);
520 gst_gl_shader_set_uniform_1i (shader, "tex_prev", 1);
521 gl->BindTexture (GL_TEXTURE_2D,
522 gst_gl_memory_get_texture_id (deinterlace_filter->prev_tex));
525 gl->ActiveTexture (GL_TEXTURE0);
526 gl->BindTexture (GL_TEXTURE_2D, gst_gl_memory_get_texture_id (in_tex));
528 gst_gl_shader_set_uniform_1i (shader, "tex", 0);
529 gst_gl_shader_set_uniform_1f (shader, "max_comb", 5.0f / 255.0f);
530 gst_gl_shader_set_uniform_1f (shader, "motion_threshold", 25.0f / 255.0f);
531 gst_gl_shader_set_uniform_1f (shader, "motion_sense", 30.0f / 255.0f);
533 gst_gl_shader_set_uniform_1f (shader, "width",
534 GST_VIDEO_INFO_WIDTH (&filter->out_info));
535 gst_gl_shader_set_uniform_1f (shader, "height",
536 GST_VIDEO_INFO_HEIGHT (&filter->out_info));
538 gst_gl_filter_draw_fullscreen_quad (filter);
540 /* we keep the previous buffer around so this is safe */
541 deinterlace_filter->prev_tex = in_tex;