3 * Copyright (C) 2012-2014 Matthew Waters <ystree00@gmail.com>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
18 * Boston, MA 02110-1301, USA.
28 #include "gstglcolorconvert.h"
31 #include "gstglfuncs.h"
32 #include "gstglsl_private.h"
35 * SECTION:gstglcolorconvert
36 * @title: GstGLColorConvert
37 * @short_description: convert between video color spaces and formats
38 * @see_also: #GstGLUpload, #GstGLMemory, #GstGLBaseMemory
40 * #GstGLColorConvert is an object that converts between color spaces and/or
41 * formats using OpenGL Shaders.
43 * A #GstGLColorConvert can be created with gst_gl_color_convert_new(), the
44 * configuration negotiated with gst_gl_color_convert_transform_caps() and the
45 * conversion performed with gst_gl_color_convert_perform().
47 * The glcolorconvertelement provides a GStreamer element that uses
48 * #GstGLColorConvert to convert between video formats and color spaces.
51 #define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
52 #define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
53 #define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
54 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
55 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
57 static void _do_convert (GstGLContext * context, GstGLColorConvert * convert);
58 static gboolean _init_convert (GstGLColorConvert * convert);
59 static gboolean _init_convert_fbo (GstGLColorConvert * convert);
60 static GstBuffer *_gst_gl_color_convert_perform_unlocked (GstGLColorConvert *
61 convert, GstBuffer * inbuf);
63 static gboolean _do_convert_draw (GstGLContext * context,
64 GstGLColorConvert * convert);
68 #define YUV_TO_RGB_COEFFICIENTS \
69 "uniform vec3 offset;\n" \
70 "uniform vec3 coeff1;\n" \
71 "uniform vec3 coeff2;\n" \
72 "uniform vec3 coeff3;\n"
74 /* FIXME: use the colormatrix support from videoconvert */
76 /* BT. 601 standard with the following ranges:
77 * Y = [16..235] (of 255)
78 * Cb/Cr = [16..240] (of 255)
80 static const gfloat from_yuv_bt601_offset[] = {-0.0625f, -0.5f, -0.5f};
81 static const gfloat from_yuv_bt601_rcoeff[] = {1.164f, 0.000f, 1.596f};
82 static const gfloat from_yuv_bt601_gcoeff[] = {1.164f,-0.391f,-0.813f};
83 static const gfloat from_yuv_bt601_bcoeff[] = {1.164f, 2.018f, 0.000f};
85 /* BT. 709 standard with the following ranges:
86 * Y = [16..235] (of 255)
87 * Cb/Cr = [16..240] (of 255)
89 static const gfloat from_yuv_bt709_offset[] = {-0.0625f, -0.5f, -0.5f};
90 static const gfloat from_yuv_bt709_rcoeff[] = {1.164f, 0.000f, 1.787f};
91 static const gfloat from_yuv_bt709_gcoeff[] = {1.164f,-0.213f,-0.531f};
92 static const gfloat from_yuv_bt709_bcoeff[] = {1.164f,2.112f, 0.000f};
94 #define RGB_TO_YUV_COEFFICIENTS \
95 "uniform vec3 offset;\n" \
96 "uniform vec3 coeff1;\n" \
97 "uniform vec3 coeff2;\n" \
98 "uniform vec3 coeff3;\n"
100 /* Matrix inverses of the color matrices found above */
101 /* BT. 601 standard with the following ranges:
102 * Y = [16..235] (of 255)
103 * Cb/Cr = [16..240] (of 255)
105 static const gfloat from_rgb_bt601_offset[] = {0.0625f, 0.5f, 0.5f};
106 static const gfloat from_rgb_bt601_ycoeff[] = {0.256816f, 0.504154f, 0.0979137f};
107 static const gfloat from_rgb_bt601_ucoeff[] = {-0.148246f, -0.29102f, 0.439266f};
108 static const gfloat from_rgb_bt601_vcoeff[] = {0.439271f, -0.367833f, -0.071438f};
110 /* BT. 709 standard with the following ranges:
111 * Y = [16..235] (of 255)
112 * Cb/Cr = [16..240] (of 255)
114 static const gfloat from_rgb_bt709_offset[] = {0.0625f, 0.5f, 0.5f};
115 static const gfloat from_rgb_bt709_ycoeff[] = {0.182604f, 0.614526f, 0.061976f};
116 static const gfloat from_rgb_bt709_ucoeff[] = {-0.100640f, -0.338688f, 0.439327f};
117 static const gfloat from_rgb_bt709_vcoeff[] = {0.440654f, -0.400285f, -0.040370f};
119 /* GRAY16 to RGB conversion
120 * data transferred as GL_LUMINANCE_ALPHA then convert back to GRAY16
121 * high byte weight as : 255*256/65535
122 * ([0~1] denormalize to [0~255],shift to high byte,normalize to [0~1])
123 * low byte weight as : 255/65535 (similar)
125 #define COMPOSE_WEIGHT \
126 "const vec2 compose_weight = vec2(0.996109, 0.003891);\n"
128 #define DEFAULT_UNIFORMS \
129 "uniform vec2 tex_scale0;\n" \
130 "uniform vec2 tex_scale1;\n" \
131 "uniform vec2 tex_scale2;\n" \
132 "uniform float width;\n" \
133 "uniform float height;\n" \
134 "uniform float poffset_x;\n" \
135 "uniform float poffset_y;\n"
137 #define MAX_FUNCTIONS 4
139 #define glsl_OES_extension_string "#extension GL_OES_EGL_image_external : require \n"
143 const gchar *extensions;
144 const gchar *uniforms;
145 const gchar *functions[MAX_FUNCTIONS];
147 GstGLTextureTarget target;
150 #define glsl_func_yuv_to_rgb \
151 "vec3 yuv_to_rgb (vec3 val, vec3 offset, vec3 ycoeff, vec3 ucoeff, vec3 vcoeff) {\n" \
153 " val += offset;\n" \
154 " rgb.r = dot(val, ycoeff);\n" \
155 " rgb.g = dot(val, ucoeff);\n" \
156 " rgb.b = dot(val, vcoeff);\n" \
160 #define glsl_func_rgb_to_yuv \
161 "vec3 rgb_to_yuv (vec3 val, vec3 offset, vec3 rcoeff, vec3 gcoeff, vec3 bcoeff) {\n" \
163 " yuv.r = dot(val.rgb, rcoeff);\n" \
164 " yuv.g = dot(val.rgb, gcoeff);\n" \
165 " yuv.b = dot(val.rgb, bcoeff);\n" \
166 " yuv += offset;\n" \
170 /* Channel reordering for XYZ <-> ZYX conversion */
171 static const gchar templ_REORDER_BODY[] =
172 "vec4 t = texture2D(tex, texcoord * tex_scale0);\n"
173 "%s\n" /* clobber alpha channel? */
174 "gl_FragColor = vec4(t.%c, t.%c, t.%c, t.%c);\n";
176 static const struct shader_templ templ_REORDER =
178 DEFAULT_UNIFORMS "uniform sampler2D tex;\n",
180 GST_GL_TEXTURE_TARGET_2D
183 /* GRAY16 to RGB conversion
184 * data transferred as GL_LUMINANCE_ALPHA then convert back to GRAY16
185 * high byte weight as : 255*256/65535
186 * ([0~1] denormalize to [0~255],shift to high byte,normalize to [0~1])
187 * low byte weight as : 255/65535 (similar)
189 static const gchar templ_COMPOSE_BODY[] =
191 "vec4 t = texture2D(tex, texcoord * tex_scale0);\n"
192 "rgba.rgb = vec3 (dot(t.%c%c, compose_weight));"
194 "gl_FragColor = vec4(rgba.%c, rgba.%c, rgba.%c, rgba.%c);\n";
196 static const struct shader_templ templ_COMPOSE =
198 DEFAULT_UNIFORMS COMPOSE_WEIGHT "uniform sampler2D tex;\n",
200 GST_GL_TEXTURE_TARGET_2D
203 /* Shaders for AYUV and varieties */
204 static const gchar templ_AYUV_to_RGB_BODY[] =
205 "vec4 texel, rgba;\n"
206 "texel = texture2D(tex, texcoord * tex_scale0);\n"
207 "rgba.rgb = yuv_to_rgb (texel.%s, offset, coeff1, coeff2, coeff3);\n"
208 "rgba.a = texel.%c;\n"
209 "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
211 static const struct shader_templ templ_AYUV_to_RGB =
213 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D tex;\n",
214 { glsl_func_yuv_to_rgb, NULL, },
215 GST_GL_TEXTURE_TARGET_2D
218 static const gchar templ_RGB_to_AYUV_BODY[] =
219 "vec4 texel, ayuv;\n"
220 "texel = texture2D(tex, texcoord).%c%c%c%c;\n"
221 "ayuv.%s = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3);\n"
223 "gl_FragColor = ayuv;\n";
225 static const struct shader_templ templ_RGB_to_AYUV =
227 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n",
228 { glsl_func_rgb_to_yuv, NULL, },
229 GST_GL_TEXTURE_TARGET_2D
232 /* YUV to RGB conversion */
233 static const gchar templ_PLANAR_YUV_to_RGB_BODY[] =
234 "vec4 texel, rgba;\n"
235 /* FIXME: should get the sampling right... */
236 "texel.x = texture2D(Ytex, texcoord * tex_scale0).r;\n"
237 "texel.y = texture2D(Utex, texcoord * tex_scale1).r;\n"
238 "texel.z = texture2D(Vtex, texcoord * tex_scale2).r;\n"
239 "rgba.rgb = yuv_to_rgb (texel.xyz, offset, coeff1, coeff2, coeff3);\n"
241 "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
243 static const struct shader_templ templ_PLANAR_YUV_to_RGB =
245 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, Utex, Vtex;\n",
246 { glsl_func_yuv_to_rgb, NULL, },
247 GST_GL_TEXTURE_TARGET_2D
250 static const gchar templ_RGB_to_PLANAR_YUV_BODY[] =
253 "texel = texture2D(tex, texcoord).%c%c%c%c;\n"
254 /* FIXME: this is not quite correct yet */
255 "vec4 uv_texel = vec4(0.0);\n"
256 /* One u and v sample can be generated by a nxm sized block given by */
257 /* @chroma_sampling. The result is the average of all the values in the */
258 /* block computed with a rolling average. */
259 "vec2 unnormalization;\n"
260 "if (texcoord.x == v_texcoord.x) {\n"
261 " unnormalization = vec2(width, height);\n"
263 " unnormalization = vec2 (1.0);\n"
265 /* scale for chroma size */
266 "vec2 chroma_pos = texcoord * chroma_sampling * unnormalization;\n"
267 /* offset chroma to the center of the first texel in the block */
268 "chroma_pos -= clamp(chroma_sampling * 0.5 - 0.5, vec2(0.0), chroma_sampling);\n"
269 "if (chroma_pos.x < width && chroma_pos.y < height) {\n"
270 " for (int i = 0; i < int(chroma_sampling.x); i++) {\n"
271 " vec2 delta = vec2 (float(i), 0.0);\n"
272 " for (int j = 0; j < int(chroma_sampling.y); j++) {\n"
273 " int n = (i+1)*(j+1);\n"
274 " delta.y = float(j);\n"
275 " vec4 s = texture2D(tex, (chroma_pos + delta) / unnormalization).%c%c%c%c;\n"
276 /* rolling average */
277 " uv_texel = (float(n-1) * uv_texel + s) / float(n);\n"
281 "yuv.x = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3).x;\n"
282 "yuv.yz = rgb_to_yuv (uv_texel.rgb, offset, coeff1, coeff2, coeff3).yz;\n"
283 "gl_FragData[0] = vec4(yuv.x, 0.0, 0.0, 1.0);\n"
284 "gl_FragData[1] = vec4(yuv.y, 0.0, 0.0, 1.0);\n"
285 "gl_FragData[2] = vec4(yuv.z, 0.0, 0.0, 1.0);\n";
287 static const struct shader_templ templ_RGB_to_PLANAR_YUV =
289 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n"
290 "uniform vec2 chroma_sampling;\n",
291 { glsl_func_rgb_to_yuv, NULL, },
292 GST_GL_TEXTURE_TARGET_2D
295 /* semi-planar to RGB conversion */
296 static const gchar templ_SEMI_PLANAR_to_RGB_BODY[] =
299 /* FIXME: should get the sampling right... */
300 "yuv.x=texture2D(Ytex, texcoord * tex_scale0).r;\n"
301 "yuv.yz=texture2D(UVtex, texcoord * tex_scale1).%c%c;\n"
302 "rgba.rgb = yuv_to_rgb (yuv, offset, coeff1, coeff2, coeff3);\n"
304 "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
306 static const struct shader_templ templ_SEMI_PLANAR_to_RGB =
308 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, UVtex;\n",
309 { glsl_func_yuv_to_rgb, NULL, },
310 GST_GL_TEXTURE_TARGET_2D
313 /* RGB to NV12/NV21 conversion */
316 static const gchar templ_RGB_to_NV12_NV21_BODY[] =
317 "vec4 texel, uv_texel;\n"
319 "texel = texture2D(tex, texcoord).%c%c%c%c;\n"
320 "uv_texel = texture2D(tex, texcoord * tex_scale0 * 2.0).%c%c%c%c;\n"
321 "yuv.x = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3).x;\n"
322 "yuv.yz = rgb_to_yuv (uv_texel.rgb, offset, coeff1, coeff2, coeff3).yz;\n"
323 "gl_FragData[0] = vec4(yuv.x, 0.0, 0.0, 1.0);\n"
324 "gl_FragData[1] = vec4(yuv.%c, yuv.%c, 0.0, 1.0);\n";
326 static const struct shader_templ templ_RGB_to_NV12_NV21 =
328 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n",
329 { glsl_func_rgb_to_yuv, NULL, },
330 GST_GL_TEXTURE_TARGET_2D
335 static const gchar templ_YUY2_UYVY_to_RGB_BODY[] =
336 "vec4 rgba, uv_texel;\n"
338 /* FIXME: should get the sampling right... */
339 "float dx1 = -poffset_x;\n"
341 "yuv.x = texture2D(Ytex, texcoord * tex_scale0).%c;\n"
342 /* v_texcoord are normalized, texcoord may not be e.g. rectangle textures */
343 "float inorder = mod (v_texcoord.x * width, 2.0);\n"
344 "if (inorder < 1.0) {\n"
348 "uv_texel.rg = texture2D(Ytex, texcoord * tex_scale0 + vec2(dx1, 0.0)).r%c;\n"
349 "uv_texel.ba = texture2D(Ytex, texcoord * tex_scale0 + vec2(dx2, 0.0)).r%c;\n"
350 "yuv.yz = uv_texel.%c%c;\n"
351 "rgba.rgb = yuv_to_rgb (yuv, offset, coeff1, coeff2, coeff3);\n"
353 "gl_FragColor = vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n";
355 static const struct shader_templ templ_YUY2_UYVY_to_RGB =
357 DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex;\n",
358 { glsl_func_yuv_to_rgb, NULL, },
359 GST_GL_TEXTURE_TARGET_2D
362 static const gchar templ_RGB_to_YUY2_UYVY_BODY[] =
363 "vec4 texel1, texel2;\n"
364 "vec3 yuv, yuv1, yuv2;\n"
365 "float fx, dx, fy;\n"
366 /* v_texcoord are normalized, texcoord may not be e.g. rectangle textures */
367 "float inorder = mod (v_texcoord.x * width, 2.0);\n"
370 "if (inorder > 1.0) {\n"
374 "texel1 = texture2D(tex, vec2(fx, fy)).%c%c%c%c;\n"
375 "texel2 = texture2D(tex, vec2(fx + dx, fy)).%c%c%c%c;\n"
376 "yuv1 = rgb_to_yuv (texel1.rgb, offset, coeff1, coeff2, coeff3);\n"
377 "yuv2 = rgb_to_yuv (texel2.rgb, offset, coeff1, coeff2, coeff3);\n"
379 "yuv.yz = (yuv1.yz + yuv2.yz) * 0.5;\n"
380 "if (inorder < 1.0) {\n"
381 " gl_FragColor = vec4(yuv.%c, yuv.%c, 0.0, 0.0);\n"
383 " gl_FragColor = vec4(yuv.%c, yuv.%c, 0.0, 0.0);\n"
386 static const struct shader_templ templ_RGB_to_YUY2_UYVY =
388 DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n",
389 { glsl_func_rgb_to_yuv, NULL, },
390 GST_GL_TEXTURE_TARGET_2D
393 static const gchar text_vertex_shader[] =
394 "attribute vec4 a_position; \n"
395 "attribute vec2 a_texcoord; \n"
396 "varying vec2 v_texcoord; \n"
399 " gl_Position = a_position; \n"
400 " v_texcoord = a_texcoord; \n"
403 static const GLfloat vertices[] = {
404 1.0f, -1.0f, 0.0f, 1.0f, 0.0f,
405 -1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
406 -1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
407 1.0f, 1.0f, 0.0f, 1.0f, 1.0f
410 static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
418 const struct shader_templ *templ;
421 const gchar *shader_tex_names[GST_VIDEO_MAX_PLANES];
423 gfloat *cms_coeff1; /* r,y */
424 gfloat *cms_coeff2; /* g,u */
425 gfloat *cms_coeff3; /* b,v */
426 gfloat chroma_sampling[2];
429 struct _GstGLColorConvertPrivate
433 struct ConvertInfo convert_info;
435 GstGLTextureTarget from_texture_target;
436 GstGLTextureTarget to_texture_target;
438 GstGLMemory *in_tex[GST_VIDEO_MAX_PLANES];
439 GstGLMemory *out_tex[GST_VIDEO_MAX_PLANES];
441 GstGLFormat in_tex_formats[GST_VIDEO_MAX_PLANES];
444 GLuint vertex_buffer;
446 GLuint attr_position;
453 gboolean pool_started;
456 GST_DEBUG_CATEGORY_STATIC (gst_gl_color_convert_debug);
457 #define GST_CAT_DEFAULT gst_gl_color_convert_debug
460 GST_DEBUG_CATEGORY_INIT (gst_gl_color_convert_debug, "glconvert", 0, "convert");
462 G_DEFINE_TYPE_WITH_CODE (GstGLColorConvert, gst_gl_color_convert,
463 GST_TYPE_OBJECT, G_ADD_PRIVATE (GstGLColorConvert) DEBUG_INIT);
465 static void gst_gl_color_convert_finalize (GObject * object);
466 static void gst_gl_color_convert_reset (GstGLColorConvert * convert);
468 #define GST_GL_COLOR_CONVERT_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), \
469 GST_TYPE_GL_COLOR_CONVERT, GstGLColorConvertPrivate))
472 gst_gl_color_convert_class_init (GstGLColorConvertClass * klass)
474 G_OBJECT_CLASS (klass)->finalize = gst_gl_color_convert_finalize;
478 gst_gl_color_convert_init (GstGLColorConvert * convert)
480 convert->priv = gst_gl_color_convert_get_instance_private (convert);
482 gst_gl_color_convert_reset (convert);
486 * gst_gl_color_convert_new:
487 * @context: a #GstGLContext
489 * Returns: (transfer full): a new #GstGLColorConvert object
494 gst_gl_color_convert_new (GstGLContext * context)
496 GstGLColorConvert *convert;
498 convert = g_object_new (GST_TYPE_GL_COLOR_CONVERT, NULL);
499 gst_object_ref_sink (convert);
501 convert->context = gst_object_ref (context);
503 gst_video_info_set_format (&convert->in_info, GST_VIDEO_FORMAT_ENCODED, 0, 0);
504 gst_video_info_set_format (&convert->out_info, GST_VIDEO_FORMAT_ENCODED, 0,
507 GST_DEBUG_OBJECT (convert,
508 "Created new colorconvert for context %" GST_PTR_FORMAT, context);
514 gst_gl_color_convert_finalize (GObject * object)
516 GstGLColorConvert *convert;
518 convert = GST_GL_COLOR_CONVERT (object);
520 gst_gl_color_convert_reset (convert);
522 if (convert->context) {
523 gst_object_unref (convert->context);
524 convert->context = NULL;
527 G_OBJECT_CLASS (gst_gl_color_convert_parent_class)->finalize (object);
531 _reset_gl (GstGLContext * context, GstGLColorConvert * convert)
533 const GstGLFuncs *gl = context->gl_vtable;
535 if (convert->priv->vao) {
536 gl->DeleteVertexArrays (1, &convert->priv->vao);
537 convert->priv->vao = 0;
540 if (convert->priv->vertex_buffer) {
541 gl->DeleteBuffers (1, &convert->priv->vertex_buffer);
542 convert->priv->vertex_buffer = 0;
545 if (convert->priv->vbo_indices) {
546 gl->DeleteBuffers (1, &convert->priv->vbo_indices);
547 convert->priv->vbo_indices = 0;
552 gst_gl_color_convert_reset_shader (GstGLColorConvert * convert)
554 convert->priv->convert_info.chroma_sampling[0] = 1.0f;
555 convert->priv->convert_info.chroma_sampling[1] = 1.0f;
557 if (convert->priv->convert_info.frag_prog) {
558 g_free (convert->priv->convert_info.frag_prog);
559 convert->priv->convert_info.frag_prog = NULL;
561 if (convert->priv->convert_info.frag_body) {
562 g_free (convert->priv->convert_info.frag_body);
563 convert->priv->convert_info.frag_body = NULL;
565 if (convert->shader) {
566 gst_object_unref (convert->shader);
567 convert->shader = NULL;
570 convert->initted = FALSE;
574 gst_gl_color_convert_reset (GstGLColorConvert * convert)
579 gst_object_unref (convert->fbo);
583 for (i = 0; i < convert->priv->convert_info.out_n_textures; i++) {
584 if (convert->priv->out_tex[i])
585 gst_memory_unref ((GstMemory *) convert->priv->out_tex[i]);
586 convert->priv->out_tex[i] = NULL;
589 if (convert->priv->pool) {
590 convert->priv->pool_started = FALSE;
592 gst_object_unref (convert->priv->pool);
593 convert->priv->pool = NULL;
596 gst_caps_replace (&convert->priv->in_caps, NULL);
597 gst_caps_replace (&convert->priv->out_caps, NULL);
599 if (convert->context) {
600 gst_gl_context_thread_add (convert->context,
601 (GstGLContextThreadFunc) _reset_gl, convert);
604 gst_gl_color_convert_reset_shader (convert);
608 _gst_gl_color_convert_can_passthrough_info (GstVideoInfo * in,
613 if (GST_VIDEO_INFO_FORMAT (in) != GST_VIDEO_INFO_FORMAT (out))
615 if (GST_VIDEO_INFO_WIDTH (in) != GST_VIDEO_INFO_WIDTH (out))
617 if (GST_VIDEO_INFO_HEIGHT (in) != GST_VIDEO_INFO_HEIGHT (out))
619 if (GST_VIDEO_INFO_SIZE (in) != GST_VIDEO_INFO_SIZE (out))
622 for (i = 0; i < in->finfo->n_planes; i++) {
623 if (in->stride[i] != out->stride[i])
625 if (in->offset[i] != out->offset[i])
629 if (!gst_video_colorimetry_is_equal (&in->colorimetry, &out->colorimetry))
631 if (in->chroma_site != out->chroma_site)
638 _gst_gl_color_convert_set_caps_unlocked (GstGLColorConvert * convert,
639 GstCaps * in_caps, GstCaps * out_caps)
641 GstVideoInfo in_info, out_info;
642 GstCapsFeatures *in_features, *out_features;
643 GstGLTextureTarget from_target, to_target;
644 gboolean passthrough;
646 g_return_val_if_fail (convert != NULL, FALSE);
647 g_return_val_if_fail (in_caps, FALSE);
648 g_return_val_if_fail (out_caps, FALSE);
650 GST_LOG_OBJECT (convert, "Setting caps in %" GST_PTR_FORMAT
651 " out %" GST_PTR_FORMAT, in_caps, out_caps);
653 if (!gst_video_info_from_caps (&in_info, in_caps))
654 g_assert_not_reached ();
656 if (!gst_video_info_from_caps (&out_info, out_caps))
657 g_assert_not_reached ();
659 g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&in_info) !=
660 GST_VIDEO_FORMAT_UNKNOWN, FALSE);
661 g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&in_info) !=
662 GST_VIDEO_FORMAT_ENCODED, FALSE);
663 g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&out_info) !=
664 GST_VIDEO_FORMAT_UNKNOWN, FALSE);
665 g_return_val_if_fail (GST_VIDEO_INFO_FORMAT (&out_info) !=
666 GST_VIDEO_FORMAT_ENCODED, FALSE);
668 in_features = gst_caps_get_features (in_caps, 0);
669 out_features = gst_caps_get_features (out_caps, 0);
671 if (!gst_caps_features_contains (in_features,
672 GST_CAPS_FEATURE_MEMORY_GL_MEMORY)
673 || !gst_caps_features_contains (out_features,
674 GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
679 GstStructure *in_s = gst_caps_get_structure (in_caps, 0);
680 GstStructure *out_s = gst_caps_get_structure (out_caps, 0);
682 if (gst_structure_has_field_typed (in_s, "texture-target", G_TYPE_STRING))
684 gst_gl_texture_target_from_string (gst_structure_get_string (in_s,
687 from_target = GST_GL_TEXTURE_TARGET_2D;
689 if (gst_structure_has_field_typed (out_s, "texture-target", G_TYPE_STRING))
691 gst_gl_texture_target_from_string (gst_structure_get_string (out_s,
694 to_target = GST_GL_TEXTURE_TARGET_2D;
696 if (to_target == GST_GL_TEXTURE_TARGET_NONE
697 || from_target == GST_GL_TEXTURE_TARGET_NONE)
702 if (gst_video_info_is_equal (&convert->in_info, &in_info) &&
703 gst_video_info_is_equal (&convert->out_info, &out_info) &&
704 convert->priv->from_texture_target == from_target &&
705 convert->priv->to_texture_target == to_target)
708 /* If input and output are identical, pass through directly */
710 _gst_gl_color_convert_can_passthrough_info (&in_info, &out_info) &&
711 from_target == to_target;
713 if (!passthrough && to_target != GST_GL_TEXTURE_TARGET_2D
714 && to_target != GST_GL_TEXTURE_TARGET_RECTANGLE)
718 guint yuv_gray_flags, in_flags, out_flags;
720 in_flags = GST_VIDEO_FORMAT_INFO_FLAGS (in_info.finfo);
721 out_flags = GST_VIDEO_FORMAT_INFO_FLAGS (out_info.finfo);
722 yuv_gray_flags = GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY;
724 /* GRAY/YUV -> GRAY/YUV is not supported for non-passthrough */
725 if (!passthrough && (in_flags & yuv_gray_flags) != 0
726 && (out_flags & yuv_gray_flags) != 0)
730 gst_gl_color_convert_reset (convert);
731 convert->in_info = in_info;
732 convert->out_info = out_info;
733 gst_caps_replace (&convert->priv->in_caps, in_caps);
734 gst_caps_replace (&convert->priv->out_caps, out_caps);
735 convert->priv->from_texture_target = from_target;
736 convert->priv->to_texture_target = to_target;
737 convert->initted = FALSE;
739 convert->passthrough = passthrough;
740 #ifndef GST_DISABLE_GST_DEBUG
741 if (G_UNLIKELY (convert->passthrough))
742 GST_DEBUG_OBJECT (convert,
743 "Configuring passthrough mode for same in/out caps");
745 GST_DEBUG_OBJECT (convert, "Color converting %" GST_PTR_FORMAT
746 " to %" GST_PTR_FORMAT, in_caps, out_caps);
754 * gst_gl_color_convert_set_caps:
755 * @convert: a #GstGLColorConvert
756 * @in_caps: input #GstCaps
757 * @out_caps: output #GstCaps
759 * Initializes @convert with the information required for conversion.
764 gst_gl_color_convert_set_caps (GstGLColorConvert * convert,
765 GstCaps * in_caps, GstCaps * out_caps)
769 GST_OBJECT_LOCK (convert);
770 ret = _gst_gl_color_convert_set_caps_unlocked (convert, in_caps, out_caps);
771 GST_OBJECT_UNLOCK (convert);
777 * gst_gl_color_convert_decide_allocation:
778 * @convert: a #GstGLColorConvert
779 * @query: a completed ALLOCATION #GstQuery
781 * Provides an implementation of #GstBaseTransformClass.decide_allocation()
783 * Returns: whether the allocation parameters were successfully chosen
788 gst_gl_color_convert_decide_allocation (GstGLColorConvert * convert,
791 GstBufferPool *pool = NULL;
792 GstStructure *config;
794 guint min, max, size, n, i;
795 gboolean update_pool;
796 GstGLVideoAllocationParams *params;
799 gst_query_parse_allocation (query, &caps, NULL);
803 gst_video_info_from_caps (&vinfo, caps);
805 n = gst_query_get_n_allocation_pools (query);
808 for (i = 0; i < n; i++) {
809 gst_query_parse_nth_allocation_pool (query, i, &pool, &size, &min, &max);
811 if (!pool || !GST_IS_GL_BUFFER_POOL (pool)) {
813 gst_object_unref (pool);
822 gst_video_info_init (&vinfo);
829 pool = gst_gl_buffer_pool_new (convert->context);
831 config = gst_buffer_pool_get_config (pool);
833 gst_buffer_pool_config_set_params (config, caps, size, min, max);
834 gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
835 if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL))
836 gst_buffer_pool_config_add_option (config,
837 GST_BUFFER_POOL_OPTION_GL_SYNC_META);
839 params = gst_gl_video_allocation_params_new (convert->context, NULL, &vinfo,
840 0, NULL, convert->priv->to_texture_target, 0);
841 gst_buffer_pool_config_set_gl_allocation_params (config,
842 (GstGLAllocationParams *) params);
843 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
845 if (!gst_buffer_pool_set_config (pool, config))
846 GST_WARNING_OBJECT (convert, "Failed to set buffer pool config");
849 gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
851 gst_query_add_allocation_pool (query, pool, size, min, max);
853 if (convert->priv->pool) {
854 gst_object_unref (convert->priv->pool);
855 convert->priv->pool_started = FALSE;
857 convert->priv->pool = pool;
863 _init_value_string_list (GValue * list, ...)
865 GValue item = G_VALUE_INIT;
869 g_value_init (list, GST_TYPE_LIST);
871 va_start (args, list);
872 while ((str = va_arg (args, gchar *))) {
873 g_value_init (&item, G_TYPE_STRING);
874 g_value_set_string (&item, str);
876 gst_value_list_append_value (list, &item);
877 g_value_unset (&item);
883 _append_value_string_list (GValue * list, ...)
885 GValue item = G_VALUE_INIT;
889 va_start (args, list);
890 while ((str = va_arg (args, gchar *))) {
891 g_value_init (&item, G_TYPE_STRING);
892 g_value_set_string (&item, str);
894 gst_value_list_append_value (list, &item);
895 g_value_unset (&item);
901 _init_supported_formats (GstGLContext * context, gboolean output,
902 GValue * supported_formats)
904 /* Assume if context == NULL that we don't have a GL context and can
905 * do the conversion */
907 /* Always supported input and output formats */
908 _init_value_string_list (supported_formats, "RGBA", "RGB", "RGBx", "BGR",
909 "BGRx", "BGRA", "xRGB", "xBGR", "ARGB", "ABGR", "GRAY8", "GRAY16_LE",
910 "GRAY16_BE", "AYUV", "VUYA", "YUY2", "UYVY", NULL);
912 /* Always supported input formats or output with multiple draw buffers */
913 if (!output || (!context || context->gl_vtable->DrawBuffers))
914 _append_value_string_list (supported_formats, "Y444", "I420", "YV12",
915 "Y42B", "Y41B", "NV12", "NV21", NULL);
917 /* Requires reading from a RG/LA framebuffer... */
918 if (!context || (USING_GLES3 (context) || USING_OPENGL (context)))
919 _append_value_string_list (supported_formats, "YUY2", "UYVY", NULL);
921 if (!context || gst_gl_format_is_supported (context, GST_GL_RGBA16))
922 _append_value_string_list (supported_formats, "ARGB64", NULL);
924 if (!context || gst_gl_format_is_supported (context, GST_GL_RGB565))
925 _append_value_string_list (supported_formats, "RGB16", "BGR16", NULL);
927 if (!context || gst_gl_format_is_supported (context, GST_GL_RGB10_A2)) {
928 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
929 _append_value_string_list (supported_formats, "BGR10A2_LE", "RGB10A2_LE",
932 _append_value_string_list (supported_formats, "Y410", NULL);
936 if (!context || (gst_gl_format_is_supported (context, GST_GL_R16) &&
937 gst_gl_format_is_supported (context, GST_GL_RG16)))
938 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
939 _append_value_string_list (supported_formats, "P010_10LE", "P016_LE", NULL);
941 _append_value_string_list (supported_formats, "P010_10BE", "P016_BE", NULL);
945 /* copies the given caps */
947 gst_gl_color_convert_caps_transform_format_info (GstGLContext * context,
948 gboolean output, GstCaps * caps)
954 GValue supported_formats = G_VALUE_INIT;
955 GValue rgb_formats = G_VALUE_INIT;
956 GValue supported_rgb_formats = G_VALUE_INIT;
958 /* There are effectively two modes here with the RGB/YUV transition:
959 * 1. There is a RGB-like format as input and we can transform to YUV or,
960 * 2. No RGB-like format as input so we can only transform to RGB-like formats
962 * We also filter down the list of formats depending on what the OpenGL
963 * context supports (when provided).
966 _init_value_string_list (&rgb_formats, "RGBA", "ARGB", "BGRA", "ABGR", "RGBx",
967 "xRGB", "BGRx", "xBGR", "RGB", "BGR", "ARGB64", "BGR10A2_LE",
969 _init_supported_formats (context, output, &supported_formats);
970 gst_value_intersect (&supported_rgb_formats, &rgb_formats,
973 res = gst_caps_new_empty ();
975 n = gst_caps_get_size (caps);
976 for (i = 0; i < n; i++) {
977 const GValue *format;
979 st = gst_caps_get_structure (caps, i);
980 f = gst_caps_get_features (caps, i);
982 format = gst_structure_get_value (st, "format");
983 st = gst_structure_copy (st);
984 if (GST_VALUE_HOLDS_LIST (format)) {
985 gboolean have_rgb_formats = FALSE;
986 GValue passthrough_formats = G_VALUE_INIT;
989 g_value_init (&passthrough_formats, GST_TYPE_LIST);
990 len = gst_value_list_get_size (format);
991 for (j = 0; j < len; j++) {
994 val = gst_value_list_get_value (format, j);
995 if (G_VALUE_HOLDS_STRING (val)) {
996 const gchar *format_str = g_value_get_string (val);
997 GstVideoFormat v_format = gst_video_format_from_string (format_str);
998 const GstVideoFormatInfo *t_info =
999 gst_video_format_get_info (v_format);
1000 if (GST_VIDEO_FORMAT_INFO_FLAGS (t_info) & (GST_VIDEO_FORMAT_FLAG_YUV
1001 | GST_VIDEO_FORMAT_FLAG_GRAY)) {
1002 gst_value_list_append_value (&passthrough_formats, val);
1003 } else if (GST_VIDEO_FORMAT_INFO_FLAGS (t_info) &
1004 GST_VIDEO_FORMAT_FLAG_RGB) {
1005 have_rgb_formats = TRUE;
1010 if (have_rgb_formats) {
1011 gst_structure_set_value (st, "format", &supported_formats);
1013 /* add passthrough structure, then the rgb conversion structure */
1014 gst_structure_set_value (st, "format", &passthrough_formats);
1015 gst_caps_append_structure_full (res, gst_structure_copy (st),
1016 gst_caps_features_copy (f));
1017 gst_structure_set_value (st, "format", &supported_rgb_formats);
1019 g_value_unset (&passthrough_formats);
1020 } else if (G_VALUE_HOLDS_STRING (format)) {
1021 const gchar *format_str = g_value_get_string (format);
1022 GstVideoFormat v_format = gst_video_format_from_string (format_str);
1023 const GstVideoFormatInfo *t_info = gst_video_format_get_info (v_format);
1024 if (GST_VIDEO_FORMAT_INFO_FLAGS (t_info) & (GST_VIDEO_FORMAT_FLAG_YUV |
1025 GST_VIDEO_FORMAT_FLAG_GRAY)) {
1026 /* add passthrough structure, then the rgb conversion structure */
1027 gst_structure_set_value (st, "format", format);
1028 gst_caps_append_structure_full (res, gst_structure_copy (st),
1029 gst_caps_features_copy (f));
1030 gst_structure_set_value (st, "format", &supported_rgb_formats);
1032 gst_structure_set_value (st, "format", &supported_formats);
1035 gst_structure_remove_fields (st, "colorimetry", "chroma-site",
1036 "texture-target", NULL);
1038 gst_caps_append_structure_full (res, st, gst_caps_features_copy (f));
1041 g_value_unset (&supported_formats);
1042 g_value_unset (&rgb_formats);
1043 g_value_unset (&supported_rgb_formats);
1049 * gst_gl_color_convert_transform_caps:
1050 * @context: a #GstGLContext to use for transforming @caps
1051 * @direction: a #GstPadDirection
1052 * @caps: (transfer none): the #GstCaps to transform
1053 * @filter: (transfer none): a set of filter #GstCaps
1055 * Provides an implementation of #GstBaseTransformClass.transform_caps()
1057 * Returns: (transfer full): the converted #GstCaps
1062 gst_gl_color_convert_transform_caps (GstGLContext * context,
1063 GstPadDirection direction, GstCaps * caps, GstCaps * filter)
1065 caps = gst_gl_color_convert_caps_transform_format_info (context,
1066 direction == GST_PAD_SRC, caps);
1071 tmp = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
1072 gst_caps_unref (caps);
1079 /* fixation from videoconvert */
1080 #define SCORE_FORMAT_CHANGE 1
1081 #define SCORE_DEPTH_CHANGE 1
1082 #define SCORE_ALPHA_CHANGE 1
1083 #define SCORE_CHROMA_W_CHANGE 1
1084 #define SCORE_CHROMA_H_CHANGE 1
1085 #define SCORE_PALETTE_CHANGE 1
1087 #define SCORE_COLORSPACE_LOSS 2 /* RGB <-> YUV */
1088 #define SCORE_DEPTH_LOSS 4 /* change bit depth */
1089 #define SCORE_ALPHA_LOSS 8 /* lose the alpha channel */
1090 #define SCORE_CHROMA_W_LOSS 16 /* vertical subsample */
1091 #define SCORE_CHROMA_H_LOSS 32 /* horizontal subsample */
1092 #define SCORE_PALETTE_LOSS 64 /* convert to palette format */
1093 #define SCORE_COLOR_LOSS 128 /* convert to GRAY */
1095 #define COLORSPACE_MASK (GST_VIDEO_FORMAT_FLAG_YUV | \
1096 GST_VIDEO_FORMAT_FLAG_RGB | GST_VIDEO_FORMAT_FLAG_GRAY)
1097 #define ALPHA_MASK (GST_VIDEO_FORMAT_FLAG_ALPHA)
1098 #define PALETTE_MASK (GST_VIDEO_FORMAT_FLAG_PALETTE)
1100 static GstGLTextureTarget
1101 _texture_target_demask (guint target_mask)
1103 if (target_mask & (1 << GST_GL_TEXTURE_TARGET_2D)) {
1104 return GST_GL_TEXTURE_TARGET_2D;
1106 if (target_mask & (1 << GST_GL_TEXTURE_TARGET_RECTANGLE)) {
1107 return GST_GL_TEXTURE_TARGET_RECTANGLE;
1109 if (target_mask & (1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES)) {
1110 return GST_GL_TEXTURE_TARGET_EXTERNAL_OES;
1116 /* calculate how much loss a conversion would be */
1118 score_format_target (const GstVideoFormatInfo * in_info, guint targets_mask,
1119 GstVideoFormat v_format, guint other_targets_mask, gint * min_loss,
1120 const GstVideoFormatInfo ** out_info, GstGLTextureTarget * result)
1122 const GstVideoFormatInfo *t_info;
1123 GstVideoFormatFlags in_flags, t_flags;
1126 t_info = gst_video_format_get_info (v_format);
1130 /* accept input format immediately without loss */
1131 if (in_info == t_info && (targets_mask & other_targets_mask) != 0) {
1134 *result = _texture_target_demask (targets_mask & other_targets_mask);
1138 /* can only passthrough external-oes textures */
1139 other_targets_mask &= ~(1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES);
1140 if (other_targets_mask == 0)
1142 /* try to keep the same target */
1143 if (targets_mask & other_targets_mask)
1144 other_targets_mask = targets_mask & other_targets_mask;
1146 loss = SCORE_FORMAT_CHANGE;
1148 in_flags = GST_VIDEO_FORMAT_INFO_FLAGS (in_info);
1149 in_flags &= ~GST_VIDEO_FORMAT_FLAG_LE;
1150 in_flags &= ~GST_VIDEO_FORMAT_FLAG_COMPLEX;
1151 in_flags &= ~GST_VIDEO_FORMAT_FLAG_UNPACK;
1153 t_flags = GST_VIDEO_FORMAT_INFO_FLAGS (t_info);
1154 t_flags &= ~GST_VIDEO_FORMAT_FLAG_LE;
1155 t_flags &= ~GST_VIDEO_FORMAT_FLAG_COMPLEX;
1156 t_flags &= ~GST_VIDEO_FORMAT_FLAG_UNPACK;
1158 /* GRAY/YUV -> GRAY/YUV is not supported */
1159 if ((in_flags & (GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY)) != 0
1160 && (t_flags & (GST_VIDEO_FORMAT_FLAG_YUV | GST_VIDEO_FORMAT_FLAG_GRAY)) !=
1164 if ((t_flags & PALETTE_MASK) != (in_flags & PALETTE_MASK)) {
1165 loss += SCORE_PALETTE_CHANGE;
1166 if (t_flags & PALETTE_MASK)
1167 loss += SCORE_PALETTE_LOSS;
1170 if ((t_flags & COLORSPACE_MASK) != (in_flags & COLORSPACE_MASK)) {
1171 loss += SCORE_COLORSPACE_LOSS;
1172 if (t_flags & GST_VIDEO_FORMAT_FLAG_GRAY)
1173 loss += SCORE_COLOR_LOSS;
1176 if ((t_flags & ALPHA_MASK) != (in_flags & ALPHA_MASK)) {
1177 loss += SCORE_ALPHA_CHANGE;
1178 if (in_flags & ALPHA_MASK)
1179 loss += SCORE_ALPHA_LOSS;
1182 if ((in_info->h_sub[1]) != (t_info->h_sub[1])) {
1183 loss += SCORE_CHROMA_H_CHANGE;
1184 if ((in_info->h_sub[1]) < (t_info->h_sub[1]))
1185 loss += SCORE_CHROMA_H_LOSS;
1187 if ((in_info->w_sub[1]) != (t_info->w_sub[1])) {
1188 loss += SCORE_CHROMA_W_CHANGE;
1189 if ((in_info->w_sub[1]) < (t_info->w_sub[1]))
1190 loss += SCORE_CHROMA_W_LOSS;
1193 if ((in_info->bits) != (t_info->bits)) {
1194 loss += SCORE_DEPTH_CHANGE;
1195 if ((in_info->bits) > (t_info->bits))
1196 loss += SCORE_DEPTH_LOSS;
1199 if (loss < *min_loss) {
1200 GstGLTextureTarget target = _texture_target_demask (other_targets_mask);
1211 gst_gl_color_convert_fixate_format_target (GstCaps * caps, GstCaps * result)
1213 GstStructure *ins, *outs;
1214 const gchar *in_format;
1215 const GstVideoFormatInfo *in_info, *out_info = NULL;
1216 const GValue *targets;
1217 guint targets_mask = 0;
1218 GstGLTextureTarget target;
1219 gint min_loss = G_MAXINT;
1222 ins = gst_caps_get_structure (caps, 0);
1223 in_format = gst_structure_get_string (ins, "format");
1226 targets = gst_structure_get_value (ins, "texture-target");
1227 targets_mask = gst_gl_value_get_texture_target_mask (targets);
1232 gst_video_format_get_info (gst_video_format_from_string (in_format));
1236 outs = gst_caps_get_structure (result, 0);
1238 capslen = gst_caps_get_size (result);
1239 for (i = 0; i < capslen; i++) {
1240 GstStructure *tests;
1241 const GValue *format;
1242 const GValue *other_targets;
1243 guint other_targets_mask = 0;
1245 tests = gst_caps_get_structure (result, i);
1247 format = gst_structure_get_value (tests, "format");
1248 other_targets = gst_structure_get_value (tests, "texture-target");
1249 /* should not happen */
1250 if (format == NULL || other_targets == NULL)
1253 other_targets_mask = gst_gl_value_get_texture_target_mask (other_targets);
1255 if (GST_VALUE_HOLDS_LIST (format)) {
1258 len = gst_value_list_get_size (format);
1259 for (j = 0; j < len; j++) {
1262 val = gst_value_list_get_value (format, j);
1263 if (G_VALUE_HOLDS_STRING (val)) {
1264 const gchar *format_str = g_value_get_string (val);
1265 GstVideoFormat v_format = gst_video_format_from_string (format_str);
1266 score_format_target (in_info, targets_mask, v_format,
1267 other_targets_mask, &min_loss, &out_info, &target);
1272 } else if (G_VALUE_HOLDS_STRING (format)) {
1273 const gchar *format_str = g_value_get_string (format);
1274 GstVideoFormat v_format = gst_video_format_from_string (format_str);
1275 score_format_target (in_info, targets_mask, v_format, other_targets_mask,
1276 &min_loss, &out_info, &target);
1280 gst_structure_set (outs, "format", G_TYPE_STRING,
1281 GST_VIDEO_FORMAT_INFO_NAME (out_info), NULL);
1283 gst_structure_set (outs, "texture-target", G_TYPE_STRING,
1284 gst_gl_texture_target_to_string (target), NULL);
1288 * gst_gl_color_convert_fixate_caps:
1289 * @context: a #GstGLContext to use for transforming @caps
1290 * @direction: a #GstPadDirection
1291 * @caps: (transfer none): the #GstCaps of @direction
1292 * @other: (transfer full): the #GstCaps to fixate
1294 * Provides an implementation of #GstBaseTransformClass.fixate_caps()
1296 * Returns: (transfer full): the fixated #GstCaps
1301 gst_gl_color_convert_fixate_caps (GstGLContext * context,
1302 GstPadDirection direction, GstCaps * caps, GstCaps * other)
1306 result = gst_caps_intersect (other, caps);
1307 if (gst_caps_is_empty (result)) {
1308 gst_caps_unref (result);
1311 gst_caps_unref (other);
1314 result = gst_caps_make_writable (result);
1315 gst_gl_color_convert_fixate_format_target (caps, result);
1317 result = gst_caps_fixate (result);
1319 if (direction == GST_PAD_SINK) {
1320 if (gst_caps_is_subset (caps, result)) {
1321 gst_caps_replace (&result, caps);
1329 * gst_gl_color_convert_perform:
1330 * @convert: a #GstGLColorConvert
1331 * @inbuf: (transfer none): the #GstGLMemory filled #GstBuffer to convert
1333 * Converts the data contained by @inbuf using the formats specified by the
1334 * #GstCaps passed to gst_gl_color_convert_set_caps()
1336 * Returns: (transfer full): a converted #GstBuffer or %NULL
1341 gst_gl_color_convert_perform (GstGLColorConvert * convert, GstBuffer * inbuf)
1345 g_return_val_if_fail (convert != NULL, FALSE);
1347 GST_OBJECT_LOCK (convert);
1348 ret = _gst_gl_color_convert_perform_unlocked (convert, inbuf);
1349 GST_OBJECT_UNLOCK (convert);
1355 _gst_gl_color_convert_perform_unlocked (GstGLColorConvert * convert,
1358 g_return_val_if_fail (convert != NULL, FALSE);
1359 g_return_val_if_fail (inbuf, FALSE);
1361 if (G_UNLIKELY (convert->passthrough))
1362 return gst_buffer_ref (inbuf);
1364 convert->inbuf = inbuf;
1366 gst_gl_context_thread_add (convert->context,
1367 (GstGLContextThreadFunc) _do_convert, convert);
1369 if (!convert->priv->result) {
1370 if (convert->outbuf)
1371 gst_buffer_unref (convert->outbuf);
1372 convert->outbuf = NULL;
1376 return convert->outbuf;
1379 static inline gboolean
1380 _is_RGBx (GstVideoFormat v_format)
1383 case GST_VIDEO_FORMAT_RGBx:
1384 case GST_VIDEO_FORMAT_xRGB:
1385 case GST_VIDEO_FORMAT_BGRx:
1386 case GST_VIDEO_FORMAT_xBGR:
1394 _index_to_shader_swizzle (int idx)
1410 /* attempts to transform expected to want using swizzling */
1412 _RGB_pixel_order (const gchar * expected, const gchar * wanted)
1414 GString *ret = g_string_sized_new (4);
1415 gchar *expect, *want;
1418 gboolean discard_output = TRUE;
1420 if (g_ascii_strcasecmp (expected, wanted) == 0) {
1421 g_string_free (ret, TRUE);
1422 return g_ascii_strdown (expected, -1);
1425 expect = g_ascii_strdown (expected, -1);
1426 orig_want = want = g_ascii_strdown (wanted, -1);
1428 if (strcmp (expect, "rgb16") == 0 || strcmp (expect, "bgr16") == 0) {
1429 gchar *temp = expect;
1430 expect = g_strndup (temp, 3);
1432 } else if (strcmp (expect, "bgr10a2_le") == 0) {
1433 gchar *temp = expect;
1434 expect = g_strndup ("bgra", 4);
1436 } else if (strcmp (expect, "rgb10a2_le") == 0) {
1437 gchar *temp = expect;
1438 expect = g_strndup ("rgba", 4);
1442 if (strcmp (want, "rgb16") == 0 || strcmp (want, "bgr16") == 0) {
1444 orig_want = want = g_strndup (temp, 3);
1446 } else if (strcmp (want, "bgr10a2_le") == 0) {
1448 orig_want = want = g_strndup ("bgra", 4);
1450 } else if (strcmp (want, "rgb10a2_le") == 0) {
1452 orig_want = want = g_strndup ("rgba", 4);
1456 /* pad want with 'a's */
1457 if ((len = strlen (want)) < 4) {
1458 gchar *new_want = g_strndup (want, 4);
1460 new_want[len] = 'a';
1464 orig_want = want = new_want;
1467 /* pad expect with 'a's */
1468 if ((len = strlen (expect)) < 4) {
1469 gchar *new_expect = g_strndup (expect, 4);
1471 new_expect[len] = 'a';
1475 expect = new_expect;
1478 /* build the swizzle format */
1479 while (want && want[0] != '\0') {
1482 gchar needle = want[0];
1487 if (!(val = strchr (expect, needle))
1488 && needle == 'a' && !(val = strchr (expect, 'x')))
1491 idx = (gint) (val - expect);
1493 ret = g_string_append_c (ret, _index_to_shader_swizzle (idx));
1497 discard_output = FALSE;
1502 return g_string_free (ret, discard_output);
1506 _get_n_textures (GstVideoFormat v_format)
1509 case GST_VIDEO_FORMAT_RGBA:
1510 case GST_VIDEO_FORMAT_RGBx:
1511 case GST_VIDEO_FORMAT_ARGB:
1512 case GST_VIDEO_FORMAT_xRGB:
1513 case GST_VIDEO_FORMAT_BGRA:
1514 case GST_VIDEO_FORMAT_BGRx:
1515 case GST_VIDEO_FORMAT_ABGR:
1516 case GST_VIDEO_FORMAT_xBGR:
1517 case GST_VIDEO_FORMAT_RGB:
1518 case GST_VIDEO_FORMAT_BGR:
1519 case GST_VIDEO_FORMAT_AYUV:
1520 case GST_VIDEO_FORMAT_VUYA:
1521 case GST_VIDEO_FORMAT_GRAY8:
1522 case GST_VIDEO_FORMAT_GRAY16_LE:
1523 case GST_VIDEO_FORMAT_GRAY16_BE:
1524 case GST_VIDEO_FORMAT_YUY2:
1525 case GST_VIDEO_FORMAT_UYVY:
1526 case GST_VIDEO_FORMAT_RGB16:
1527 case GST_VIDEO_FORMAT_BGR16:
1528 case GST_VIDEO_FORMAT_ARGB64:
1529 case GST_VIDEO_FORMAT_BGR10A2_LE:
1530 case GST_VIDEO_FORMAT_RGB10A2_LE:
1531 case GST_VIDEO_FORMAT_Y410:
1533 case GST_VIDEO_FORMAT_NV12:
1534 case GST_VIDEO_FORMAT_NV21:
1535 case GST_VIDEO_FORMAT_P010_10LE:
1536 case GST_VIDEO_FORMAT_P010_10BE:
1537 case GST_VIDEO_FORMAT_P016_LE:
1538 case GST_VIDEO_FORMAT_P016_BE:
1540 case GST_VIDEO_FORMAT_I420:
1541 case GST_VIDEO_FORMAT_Y444:
1542 case GST_VIDEO_FORMAT_Y42B:
1543 case GST_VIDEO_FORMAT_Y41B:
1544 case GST_VIDEO_FORMAT_YV12:
1547 g_assert_not_reached ();
1553 _RGB_to_RGB (GstGLColorConvert * convert)
1555 struct ConvertInfo *info = &convert->priv->convert_info;
1556 GstVideoFormat in_format = GST_VIDEO_INFO_FORMAT (&convert->in_info);
1557 const gchar *in_format_str = gst_video_format_to_string (in_format);
1558 GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info);
1559 const gchar *out_format_str = gst_video_format_to_string (out_format);
1560 gchar *pixel_order = _RGB_pixel_order (in_format_str, out_format_str);
1561 gchar *alpha = NULL;
1563 if (_is_RGBx (in_format)) {
1565 char input_alpha_channel = 'a';
1566 for (i = 0; i < GST_VIDEO_MAX_PLANES; i++) {
1567 if (in_format_str[i] == 'X' || in_format_str[i] == 'x') {
1568 input_alpha_channel = _index_to_shader_swizzle (i);
1572 alpha = g_strdup_printf ("t.%c = 1.0;", input_alpha_channel);
1574 info->templ = &templ_REORDER;
1575 info->frag_body = g_strdup_printf (templ_REORDER_BODY, alpha ? alpha : "",
1576 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1577 info->shader_tex_names[0] = "tex";
1580 g_free (pixel_order);
1584 _YUV_to_RGB (GstGLColorConvert * convert)
1586 struct ConvertInfo *info = &convert->priv->convert_info;
1587 GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info);
1588 const gchar *out_format_str = gst_video_format_to_string (out_format);
1589 gchar *pixel_order = _RGB_pixel_order ("rgba", out_format_str);
1590 gboolean apple_ycbcr = gst_gl_context_check_feature (convert->context,
1591 "GL_APPLE_ycbcr_422");
1592 gboolean in_tex_rectangular = FALSE;
1594 #if GST_GL_HAVE_OPENGL
1595 GstMemory *memory = gst_buffer_peek_memory (convert->inbuf, 0);
1596 if (gst_is_gl_memory (memory) && (USING_OPENGL (convert->context)
1597 || USING_OPENGL3 (convert->context))) {
1598 in_tex_rectangular =
1599 convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE;
1603 info->out_n_textures = 1;
1605 if (in_tex_rectangular && apple_ycbcr
1606 && gst_buffer_n_memory (convert->inbuf) == 1) {
1607 /* FIXME: We should probably also check if tex_target actually is using
1608 * the Apple YCbCr422 extension. It could also be a normal UYVY texture
1609 * with RB or Lum/Alpha
1611 /* The mangling will change this to the correct texture2DRect, sampler2DRect
1613 info->templ = &templ_REORDER;
1615 g_strdup_printf (templ_REORDER_BODY, "", pixel_order[0], pixel_order[1],
1616 pixel_order[2], pixel_order[3]);
1617 info->shader_tex_names[0] = "tex";
1619 switch (GST_VIDEO_INFO_FORMAT (&convert->in_info)) {
1620 case GST_VIDEO_FORMAT_AYUV:
1621 info->templ = &templ_AYUV_to_RGB;
1622 info->frag_body = g_strdup_printf (templ_AYUV_to_RGB_BODY, "yzw", 'x',
1623 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1624 info->shader_tex_names[0] = "tex";
1626 case GST_VIDEO_FORMAT_VUYA:
1627 info->templ = &templ_AYUV_to_RGB;
1628 info->frag_body = g_strdup_printf (templ_AYUV_to_RGB_BODY, "zyx", 'w',
1629 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1630 info->shader_tex_names[0] = "tex";
1632 case GST_VIDEO_FORMAT_Y410:
1633 info->templ = &templ_AYUV_to_RGB;
1634 info->frag_body = g_strdup_printf (templ_AYUV_to_RGB_BODY, "yxz", 'w',
1635 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1636 info->shader_tex_names[0] = "tex";
1638 case GST_VIDEO_FORMAT_I420:
1639 case GST_VIDEO_FORMAT_Y444:
1640 case GST_VIDEO_FORMAT_Y42B:
1641 case GST_VIDEO_FORMAT_Y41B:
1642 info->templ = &templ_PLANAR_YUV_to_RGB;
1644 g_strdup_printf (templ_PLANAR_YUV_to_RGB_BODY, pixel_order[0],
1645 pixel_order[1], pixel_order[2], pixel_order[3]);
1646 info->shader_tex_names[0] = "Ytex";
1647 info->shader_tex_names[1] = "Utex";
1648 info->shader_tex_names[2] = "Vtex";
1650 case GST_VIDEO_FORMAT_YV12:
1651 info->templ = &templ_PLANAR_YUV_to_RGB;
1653 g_strdup_printf (templ_PLANAR_YUV_to_RGB_BODY, pixel_order[0],
1654 pixel_order[1], pixel_order[2], pixel_order[3]);
1655 info->shader_tex_names[0] = "Ytex";
1656 info->shader_tex_names[1] = "Vtex";
1657 info->shader_tex_names[2] = "Utex";
1659 case GST_VIDEO_FORMAT_YUY2:
1661 char uv_val = convert->priv->in_tex_formats[0] == GST_GL_RG ? 'g' : 'a';
1662 info->templ = &templ_YUY2_UYVY_to_RGB;
1664 g_strdup_printf (templ_YUY2_UYVY_to_RGB_BODY, 'r', uv_val, uv_val,
1665 'g', 'a', pixel_order[0], pixel_order[1], pixel_order[2],
1667 info->shader_tex_names[0] = "Ytex";
1670 case GST_VIDEO_FORMAT_UYVY:
1672 char y_val = convert->priv->in_tex_formats[0] == GST_GL_RG ? 'g' : 'a';
1673 info->templ = &templ_YUY2_UYVY_to_RGB;
1675 g_strdup_printf (templ_YUY2_UYVY_to_RGB_BODY, y_val, 'g', 'g', 'r',
1676 'b', pixel_order[0], pixel_order[1], pixel_order[2],
1678 info->shader_tex_names[0] = "Ytex";
1681 case GST_VIDEO_FORMAT_NV12:
1683 char val2 = convert->priv->in_tex_formats[1] == GST_GL_RG ? 'g' : 'a';
1684 info->templ = &templ_SEMI_PLANAR_to_RGB;
1686 g_strdup_printf (templ_SEMI_PLANAR_to_RGB_BODY, 'r', val2,
1687 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1688 info->shader_tex_names[0] = "Ytex";
1689 info->shader_tex_names[1] = "UVtex";
1692 case GST_VIDEO_FORMAT_NV21:
1694 char val2 = convert->priv->in_tex_formats[1] == GST_GL_RG ? 'g' : 'a';
1695 info->templ = &templ_SEMI_PLANAR_to_RGB;
1697 g_strdup_printf (templ_SEMI_PLANAR_to_RGB_BODY, val2, 'r',
1698 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1699 info->shader_tex_names[0] = "Ytex";
1700 info->shader_tex_names[1] = "UVtex";
1703 case GST_VIDEO_FORMAT_P010_10LE:
1704 case GST_VIDEO_FORMAT_P010_10BE:
1705 case GST_VIDEO_FORMAT_P016_LE:
1706 case GST_VIDEO_FORMAT_P016_BE:
1708 info->templ = &templ_SEMI_PLANAR_to_RGB;
1710 g_strdup_printf (templ_SEMI_PLANAR_to_RGB_BODY, 'r', 'g',
1711 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1712 info->shader_tex_names[0] = "Ytex";
1713 info->shader_tex_names[1] = "UVtex";
1721 if (gst_video_colorimetry_matches (&convert->in_info.colorimetry,
1722 GST_VIDEO_COLORIMETRY_BT709)) {
1723 info->cms_offset = (gfloat *) from_yuv_bt709_offset;
1724 info->cms_coeff1 = (gfloat *) from_yuv_bt709_rcoeff;
1725 info->cms_coeff2 = (gfloat *) from_yuv_bt709_gcoeff;
1726 info->cms_coeff3 = (gfloat *) from_yuv_bt709_bcoeff;
1728 /* defaults/bt601 */
1729 info->cms_offset = (gfloat *) from_yuv_bt601_offset;
1730 info->cms_coeff1 = (gfloat *) from_yuv_bt601_rcoeff;
1731 info->cms_coeff2 = (gfloat *) from_yuv_bt601_gcoeff;
1732 info->cms_coeff3 = (gfloat *) from_yuv_bt601_bcoeff;
1735 g_free (pixel_order);
1739 _RGB_to_YUV (GstGLColorConvert * convert)
1741 struct ConvertInfo *info = &convert->priv->convert_info;
1742 GstVideoFormat in_format = GST_VIDEO_INFO_FORMAT (&convert->in_info);
1743 const gchar *in_format_str = gst_video_format_to_string (in_format);
1744 GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info);
1745 gchar *pixel_order = _RGB_pixel_order (in_format_str, "rgba");
1748 info->frag_prog = NULL;
1750 info->shader_tex_names[0] = "tex";
1752 switch (out_format) {
1753 case GST_VIDEO_FORMAT_AYUV:
1754 alpha = _is_RGBx (in_format) ? "1.0" : "texel.a";
1755 info->templ = &templ_RGB_to_AYUV;
1756 info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, pixel_order[0],
1757 pixel_order[1], pixel_order[2], pixel_order[3], "yzw", 'x', alpha);
1758 info->out_n_textures = 1;
1760 case GST_VIDEO_FORMAT_VUYA:
1761 alpha = _is_RGBx (in_format) ? "1.0" : "texel.a";
1762 info->templ = &templ_RGB_to_AYUV;
1763 info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, pixel_order[0],
1764 pixel_order[1], pixel_order[2], pixel_order[3], "zyx", 'w', alpha);
1765 info->out_n_textures = 1;
1767 case GST_VIDEO_FORMAT_Y410:
1768 alpha = _is_RGBx (in_format) ? "1.0" : "texel.a";
1769 info->templ = &templ_RGB_to_AYUV;
1770 info->frag_body = g_strdup_printf (templ_RGB_to_AYUV_BODY, pixel_order[0],
1771 pixel_order[1], pixel_order[2], pixel_order[3], "yxz", 'w', alpha);
1772 info->out_n_textures = 1;
1774 case GST_VIDEO_FORMAT_I420:
1775 case GST_VIDEO_FORMAT_YV12:
1776 case GST_VIDEO_FORMAT_Y444:
1777 case GST_VIDEO_FORMAT_Y42B:
1778 case GST_VIDEO_FORMAT_Y41B:
1779 info->templ = &templ_RGB_to_PLANAR_YUV;
1780 info->frag_body = g_strdup_printf (templ_RGB_to_PLANAR_YUV_BODY,
1781 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1782 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1783 info->out_n_textures = 3;
1784 if (out_format == GST_VIDEO_FORMAT_Y444) {
1785 info->chroma_sampling[0] = info->chroma_sampling[1] = 1.0f;
1786 } else if (out_format == GST_VIDEO_FORMAT_Y42B) {
1787 info->chroma_sampling[0] = 2.0f;
1788 info->chroma_sampling[1] = 1.0f;
1789 } else if (out_format == GST_VIDEO_FORMAT_Y41B) {
1790 info->chroma_sampling[0] = 4.0f;
1791 info->chroma_sampling[1] = 1.0f;
1793 info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f;
1796 case GST_VIDEO_FORMAT_YUY2:
1797 info->templ = &templ_RGB_to_YUY2_UYVY;
1798 info->frag_body = g_strdup_printf (templ_RGB_to_YUY2_UYVY_BODY,
1799 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1800 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1801 'x', 'y', 'x', 'z');
1802 info->out_n_textures = 1;
1804 case GST_VIDEO_FORMAT_UYVY:
1805 info->templ = &templ_RGB_to_YUY2_UYVY,
1806 info->frag_body = g_strdup_printf (templ_RGB_to_YUY2_UYVY_BODY,
1807 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1808 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1809 'y', 'x', 'z', 'x');
1810 info->out_n_textures = 1;
1812 case GST_VIDEO_FORMAT_NV12:
1813 info->templ = &templ_RGB_to_NV12_NV21,
1814 info->frag_body = g_strdup_printf (templ_RGB_to_NV12_NV21_BODY,
1815 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1816 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1818 info->out_n_textures = 2;
1820 case GST_VIDEO_FORMAT_NV21:
1821 info->templ = &templ_RGB_to_NV12_NV21,
1822 info->frag_body = g_strdup_printf (templ_RGB_to_NV12_NV21_BODY,
1823 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1824 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3],
1826 info->out_n_textures = 2;
1832 if (gst_video_colorimetry_matches (&convert->in_info.colorimetry,
1833 GST_VIDEO_COLORIMETRY_BT709)) {
1834 info->cms_offset = (gfloat *) from_rgb_bt709_offset;
1835 info->cms_coeff1 = (gfloat *) from_rgb_bt709_ycoeff;
1836 info->cms_coeff2 = (gfloat *) from_rgb_bt709_ucoeff;
1837 info->cms_coeff3 = (gfloat *) from_rgb_bt709_vcoeff;
1839 /* defaults/bt601 */
1840 info->cms_offset = (gfloat *) from_rgb_bt601_offset;
1841 info->cms_coeff1 = (gfloat *) from_rgb_bt601_ycoeff;
1842 info->cms_coeff2 = (gfloat *) from_rgb_bt601_ucoeff;
1843 info->cms_coeff3 = (gfloat *) from_rgb_bt601_vcoeff;
1846 g_free (pixel_order);
1850 _RGB_to_GRAY (GstGLColorConvert * convert)
1852 struct ConvertInfo *info = &convert->priv->convert_info;
1853 GstVideoFormat in_format = GST_VIDEO_INFO_FORMAT (&convert->in_info);
1854 const gchar *in_format_str = gst_video_format_to_string (in_format);
1855 gchar *pixel_order = _RGB_pixel_order (in_format_str, "rgba");
1856 gchar *alpha = NULL;
1858 info->out_n_textures = 1;
1859 info->shader_tex_names[0] = "tex";
1861 if (_is_RGBx (in_format))
1862 alpha = g_strdup_printf ("t.%c = 1.0;", pixel_order[3]);
1864 switch (GST_VIDEO_INFO_FORMAT (&convert->out_info)) {
1865 case GST_VIDEO_FORMAT_GRAY8:
1866 info->templ = &templ_REORDER;
1867 info->frag_body = g_strdup_printf (templ_REORDER_BODY, alpha ? alpha : "",
1868 pixel_order[0], pixel_order[0], pixel_order[0], pixel_order[3]);
1875 g_free (pixel_order);
1879 _GRAY_to_RGB (GstGLColorConvert * convert)
1881 struct ConvertInfo *info = &convert->priv->convert_info;
1882 GstVideoFormat out_format = GST_VIDEO_INFO_FORMAT (&convert->out_info);
1883 const gchar *out_format_str = gst_video_format_to_string (out_format);
1884 gchar *pixel_order = _RGB_pixel_order ("rgba", out_format_str);
1886 info->shader_tex_names[0] = "tex";
1888 switch (GST_VIDEO_INFO_FORMAT (&convert->in_info)) {
1889 case GST_VIDEO_FORMAT_GRAY8:
1890 info->templ = &templ_REORDER;
1891 info->frag_body = g_strdup_printf (templ_REORDER_BODY, "", pixel_order[0],
1892 pixel_order[0], pixel_order[0], pixel_order[3]);
1894 case GST_VIDEO_FORMAT_GRAY16_LE:
1896 char val2 = convert->priv->in_tex_formats[0] == GST_GL_RG ? 'g' : 'a';
1897 info->templ = &templ_COMPOSE;
1898 info->frag_body = g_strdup_printf (templ_COMPOSE_BODY, val2, 'r',
1899 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1902 case GST_VIDEO_FORMAT_GRAY16_BE:
1904 char val2 = convert->priv->in_tex_formats[0] == GST_GL_RG ? 'g' : 'a';
1905 info->templ = &templ_COMPOSE;
1906 info->frag_body = g_strdup_printf (templ_COMPOSE_BODY, 'r', val2,
1907 pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]);
1914 g_free (pixel_order);
1918 _bind_buffer (GstGLColorConvert * convert)
1920 const GstGLFuncs *gl = convert->context->gl_vtable;
1922 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, convert->priv->vbo_indices);
1923 gl->BindBuffer (GL_ARRAY_BUFFER, convert->priv->vertex_buffer);
1925 /* Load the vertex position */
1926 gl->VertexAttribPointer (convert->priv->attr_position, 3, GL_FLOAT, GL_FALSE,
1927 5 * sizeof (GLfloat), (void *) 0);
1929 /* Load the texture coordinate */
1930 gl->VertexAttribPointer (convert->priv->attr_texture, 2, GL_FLOAT, GL_FALSE,
1931 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
1933 gl->EnableVertexAttribArray (convert->priv->attr_position);
1934 gl->EnableVertexAttribArray (convert->priv->attr_texture);
1938 _unbind_buffer (GstGLColorConvert * convert)
1940 const GstGLFuncs *gl = convert->context->gl_vtable;
1942 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
1943 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
1945 gl->DisableVertexAttribArray (convert->priv->attr_position);
1946 gl->DisableVertexAttribArray (convert->priv->attr_texture);
1949 static GstGLShader *
1950 _create_shader (GstGLColorConvert * convert)
1952 struct ConvertInfo *info = &convert->priv->convert_info;
1953 GString *str = g_string_new (NULL);
1954 GstGLShader *ret = NULL;
1955 GstGLSLStage *stage;
1956 GstGLSLVersion version;
1957 GstGLSLProfile profile;
1958 gchar *version_str, *tmp, *tmp1;
1959 const gchar *strings[3];
1960 GError *error = NULL;
1963 ret = gst_gl_shader_new (convert->context);
1966 _gst_glsl_mangle_shader (text_vertex_shader, GL_VERTEX_SHADER,
1967 info->templ->target, convert->priv->from_texture_target, convert->context,
1968 &version, &profile);
1970 tmp1 = gst_glsl_version_profile_to_string (version, profile);
1971 version_str = g_strdup_printf ("#version %s\n", tmp1);
1974 strings[0] = version_str;
1976 if (!(stage = gst_glsl_stage_new_with_strings (convert->context,
1977 GL_VERTEX_SHADER, version, profile, 2, strings))) {
1978 GST_ERROR_OBJECT (convert, "Failed to create vertex stage");
1979 g_free (version_str);
1981 gst_object_unref (ret);
1986 if (!gst_gl_shader_compile_attach_stage (ret, stage, &error)) {
1987 GST_ERROR_OBJECT (convert, "Failed to compile vertex shader %s",
1989 g_clear_error (&error);
1990 g_free (version_str);
1991 gst_object_unref (stage);
1992 gst_object_unref (ret);
1996 if (info->templ->extensions)
1997 g_string_append (str, info->templ->extensions);
1999 if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES
2000 && info->templ->target != GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
2001 g_string_append (str, glsl_OES_extension_string);
2003 g_string_append (str,
2004 gst_gl_shader_string_get_highest_precision (convert->context, version,
2007 if (info->templ->uniforms)
2008 g_string_append (str, info->templ->uniforms);
2010 g_string_append_c (str, '\n');
2012 /* GL 3.3+ and GL ES 3.x */
2013 if ((profile == GST_GLSL_PROFILE_CORE && version >= GST_GLSL_VERSION_330)
2014 || (profile == GST_GLSL_PROFILE_ES && version >= GST_GLSL_VERSION_300)) {
2015 if (info->out_n_textures > 1) {
2018 for (i = 0; i < info->out_n_textures; i++) {
2019 g_string_append_printf (str,
2020 "layout(location = %d) out vec4 fragColor_%d;\n", i, i);
2023 g_string_append (str, "layout (location = 0) out vec4 fragColor;\n");
2025 } else if (profile == GST_GLSL_PROFILE_CORE
2026 && version >= GST_GLSL_VERSION_150) {
2027 /* no layout specifiers, use glBindFragDataLocation instead */
2028 if (info->out_n_textures > 1) {
2031 for (i = 0; i < info->out_n_textures; i++) {
2032 gchar *var_name = g_strdup_printf ("fragColor_%d", i);
2033 g_string_append_printf (str, "out vec4 %s;\n", var_name);
2034 gst_gl_shader_bind_frag_data_location (ret, i, var_name);
2038 g_string_append (str, "out vec4 fragColor;\n");
2039 gst_gl_shader_bind_frag_data_location (ret, 0, "fragColor");
2043 for (i = 0; i < MAX_FUNCTIONS; i++) {
2044 if (info->templ->functions[i] == NULL)
2047 g_string_append_c (str, '\n');
2048 g_string_append (str, info->templ->functions[i]);
2049 g_string_append_c (str, '\n');
2053 const gchar *varying = NULL;
2055 if ((profile == GST_GLSL_PROFILE_ES && version >= GST_GLSL_VERSION_300)
2056 || (profile == GST_GLSL_PROFILE_CORE
2057 && version >= GST_GLSL_VERSION_150)) {
2060 varying = "varying";
2062 g_string_append_printf (str, "\n%s vec2 v_texcoord;\nvoid main (void) {\n",
2065 if (info->frag_body) {
2066 g_string_append (str, "vec2 texcoord;\n");
2067 if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE
2068 && info->templ->target != GST_GL_TEXTURE_TARGET_RECTANGLE) {
2069 g_string_append (str, "texcoord = v_texcoord * vec2 (width, height);\n");
2071 g_string_append (str, "texcoord = v_texcoord;\n");
2074 g_string_append (str, info->frag_body);
2076 g_string_append (str, "\n}");
2077 tmp = g_string_free (str, FALSE);
2078 info->frag_prog = _gst_glsl_mangle_shader (tmp, GL_FRAGMENT_SHADER,
2079 info->templ->target, convert->priv->from_texture_target, convert->context,
2080 &version, &profile);
2083 strings[1] = info->frag_prog;
2084 if (!(stage = gst_glsl_stage_new_with_strings (convert->context,
2085 GL_FRAGMENT_SHADER, version, profile, 2, strings))) {
2086 GST_ERROR_OBJECT (convert, "Failed to create fragment stage");
2087 g_free (info->frag_prog);
2088 info->frag_prog = NULL;
2089 g_free (version_str);
2090 gst_object_unref (ret);
2093 g_free (version_str);
2094 if (!gst_gl_shader_compile_attach_stage (ret, stage, &error)) {
2095 GST_ERROR_OBJECT (convert, "Failed to compile fragment shader %s",
2097 g_clear_error (&error);
2098 g_free (info->frag_prog);
2099 info->frag_prog = NULL;
2100 gst_object_unref (stage);
2101 gst_object_unref (ret);
2105 if (!gst_gl_shader_link (ret, &error)) {
2106 GST_ERROR_OBJECT (convert, "Failed to link shader %s", error->message);
2107 g_clear_error (&error);
2108 g_free (info->frag_prog);
2109 info->frag_prog = NULL;
2110 gst_object_unref (ret);
2117 /* Called in the gl thread */
2119 _init_convert (GstGLColorConvert * convert)
2122 struct ConvertInfo *info = &convert->priv->convert_info;
2125 gl = convert->context->gl_vtable;
2127 if (convert->initted)
2130 GST_INFO ("Initializing color conversion from %s to %s",
2131 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->in_info)),
2132 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->out_info)));
2134 if (!gl->CreateProgramObject && !gl->CreateProgram) {
2135 GST_ERROR_OBJECT (convert, "Cannot perform color conversion without "
2140 if (GST_VIDEO_INFO_IS_RGB (&convert->in_info)) {
2141 if (GST_VIDEO_INFO_IS_RGB (&convert->out_info)) {
2142 _RGB_to_RGB (convert);
2146 if (GST_VIDEO_INFO_IS_YUV (&convert->in_info)) {
2147 if (GST_VIDEO_INFO_IS_RGB (&convert->out_info)) {
2148 _YUV_to_RGB (convert);
2152 if (GST_VIDEO_INFO_IS_RGB (&convert->in_info)) {
2153 if (GST_VIDEO_INFO_IS_YUV (&convert->out_info)) {
2154 _RGB_to_YUV (convert);
2158 if (GST_VIDEO_INFO_IS_RGB (&convert->in_info)) {
2159 if (GST_VIDEO_INFO_IS_GRAY (&convert->out_info)) {
2160 _RGB_to_GRAY (convert);
2164 if (GST_VIDEO_INFO_IS_GRAY (&convert->in_info)) {
2165 if (GST_VIDEO_INFO_IS_RGB (&convert->out_info)) {
2166 _GRAY_to_RGB (convert);
2170 if (!info->frag_body || info->in_n_textures == 0 || info->out_n_textures == 0)
2171 goto unhandled_format;
2173 /* multiple draw targets not supported on GLES2... */
2174 if (info->out_n_textures > 1 && !gl->DrawBuffers) {
2175 GST_ERROR ("Conversion requires output to multiple draw buffers");
2176 goto incompatible_api;
2179 /* Requires reading from a RG/LA framebuffer... */
2180 if (USING_GLES2 (convert->context) && !USING_GLES3 (convert->context) &&
2181 (GST_VIDEO_INFO_FORMAT (&convert->out_info) == GST_VIDEO_FORMAT_YUY2 ||
2182 GST_VIDEO_INFO_FORMAT (&convert->out_info) ==
2183 GST_VIDEO_FORMAT_UYVY)) {
2184 GST_ERROR ("Conversion requires reading with an unsupported format");
2185 goto incompatible_api;
2188 if (!(convert->shader = _create_shader (convert)))
2191 convert->priv->attr_position =
2192 gst_gl_shader_get_attribute_location (convert->shader, "a_position");
2193 convert->priv->attr_texture =
2194 gst_gl_shader_get_attribute_location (convert->shader, "a_texcoord");
2196 gst_gl_shader_use (convert->shader);
2198 if (info->cms_offset && info->cms_coeff1
2199 && info->cms_coeff2 && info->cms_coeff3) {
2200 gst_gl_shader_set_uniform_3fv (convert->shader, "offset", 1,
2202 gst_gl_shader_set_uniform_3fv (convert->shader, "coeff1", 1,
2204 gst_gl_shader_set_uniform_3fv (convert->shader, "coeff2", 1,
2206 gst_gl_shader_set_uniform_3fv (convert->shader, "coeff3", 1,
2210 for (i = info->in_n_textures; i >= 0; i--) {
2211 if (info->shader_tex_names[i])
2212 gst_gl_shader_set_uniform_1i (convert->shader, info->shader_tex_names[i],
2216 gst_gl_shader_set_uniform_1f (convert->shader, "width",
2217 GST_VIDEO_INFO_WIDTH (&convert->in_info));
2218 gst_gl_shader_set_uniform_1f (convert->shader, "height",
2219 GST_VIDEO_INFO_HEIGHT (&convert->in_info));
2221 if (convert->priv->from_texture_target == GST_GL_TEXTURE_TARGET_RECTANGLE) {
2222 gst_gl_shader_set_uniform_1f (convert->shader, "poffset_x", 1.);
2223 gst_gl_shader_set_uniform_1f (convert->shader, "poffset_y", 1.);
2225 gst_gl_shader_set_uniform_1f (convert->shader, "poffset_x",
2226 1. / (gfloat) GST_VIDEO_INFO_WIDTH (&convert->in_info));
2227 gst_gl_shader_set_uniform_1f (convert->shader, "poffset_y",
2228 1. / (gfloat) GST_VIDEO_INFO_HEIGHT (&convert->in_info));
2231 if (info->chroma_sampling[0] > 0.0f && info->chroma_sampling[1] > 0.0f) {
2232 gst_gl_shader_set_uniform_2fv (convert->shader, "chroma_sampling", 1,
2233 info->chroma_sampling);
2236 gst_gl_context_clear_shader (convert->context);
2238 if (convert->fbo == NULL && !_init_convert_fbo (convert)) {
2242 if (!convert->priv->vertex_buffer) {
2243 if (gl->GenVertexArrays) {
2244 gl->GenVertexArrays (1, &convert->priv->vao);
2245 gl->BindVertexArray (convert->priv->vao);
2248 gl->GenBuffers (1, &convert->priv->vertex_buffer);
2249 gl->BindBuffer (GL_ARRAY_BUFFER, convert->priv->vertex_buffer);
2250 gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
2253 gl->GenBuffers (1, &convert->priv->vbo_indices);
2254 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, convert->priv->vbo_indices);
2255 gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
2258 if (gl->GenVertexArrays) {
2259 _bind_buffer (convert);
2260 gl->BindVertexArray (0);
2263 gl->BindBuffer (GL_ARRAY_BUFFER, 0);
2264 gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
2267 gl->BindTexture (GL_TEXTURE_2D, 0);
2269 convert->initted = TRUE;
2274 GST_ERROR_OBJECT (convert, "Don't know how to convert from %s to %s",
2275 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->in_info)),
2276 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->out_info)));
2283 GST_ERROR_OBJECT (convert, "Converting from %s to %s requires "
2284 "functionality that the current OpenGL setup does not support",
2285 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->in_info)),
2286 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
2287 (&convert->out_info)));
2292 /* called by _init_convert (in the gl thread) */
2294 _init_convert_fbo (GstGLColorConvert * convert)
2296 guint out_width, out_height;
2298 out_width = GST_VIDEO_INFO_WIDTH (&convert->out_info);
2299 out_height = GST_VIDEO_INFO_HEIGHT (&convert->out_info);
2302 gst_gl_framebuffer_new_with_default_depth (convert->context, out_width,
2305 return convert->fbo != NULL;
2309 _do_convert_one_view (GstGLContext * context, GstGLColorConvert * convert,
2312 guint in_width, in_height, out_width, out_height;
2313 struct ConvertInfo *c_info = &convert->priv->convert_info;
2314 GstMapInfo out_info[GST_VIDEO_MAX_PLANES], in_info[GST_VIDEO_MAX_PLANES];
2315 gboolean res = TRUE;
2317 const gint in_plane_offset = view_num * c_info->in_n_textures;
2318 const gint out_plane_offset = view_num * c_info->out_n_textures;
2320 out_width = GST_VIDEO_INFO_WIDTH (&convert->out_info);
2321 out_height = GST_VIDEO_INFO_HEIGHT (&convert->out_info);
2322 in_width = GST_VIDEO_INFO_WIDTH (&convert->in_info);
2323 in_height = GST_VIDEO_INFO_HEIGHT (&convert->in_info);
2325 for (i = 0; i < c_info->in_n_textures; i++) {
2326 convert->priv->in_tex[i] =
2327 (GstGLMemory *) gst_buffer_peek_memory (convert->inbuf,
2328 i + in_plane_offset);
2329 if (!gst_is_gl_memory ((GstMemory *) convert->priv->in_tex[i])) {
2330 GST_ERROR_OBJECT (convert, "input must be GstGLMemory");
2334 if (!gst_memory_map ((GstMemory *) convert->priv->in_tex[i], &in_info[i],
2335 GST_MAP_READ | GST_MAP_GL)) {
2336 GST_ERROR_OBJECT (convert, "failed to map input memory %p",
2337 convert->priv->in_tex[i]);
2343 for (j = 0; j < c_info->out_n_textures; j++) {
2344 GstGLMemory *out_tex =
2345 (GstGLMemory *) gst_buffer_peek_memory (convert->outbuf,
2346 j + out_plane_offset);
2347 gint mem_width, mem_height;
2349 if (!gst_is_gl_memory ((GstMemory *) out_tex)) {
2350 GST_ERROR_OBJECT (convert, "output must be GstGLMemory");
2355 mem_width = gst_gl_memory_get_texture_width (out_tex);
2356 mem_height = gst_gl_memory_get_texture_height (out_tex);
2358 if (out_tex->tex_format == GST_GL_LUMINANCE
2359 || out_tex->tex_format == GST_GL_LUMINANCE_ALPHA
2360 || out_width != mem_width || out_height != mem_height) {
2361 /* Luminance formats are not color renderable */
2362 /* rendering to a framebuffer only renders the intersection of all
2363 * the attachments i.e. the smallest attachment size */
2364 if (!convert->priv->out_tex[j]) {
2365 GstGLVideoAllocationParams *params;
2366 GstGLBaseMemoryAllocator *base_mem_allocator;
2367 GstAllocator *allocator;
2368 GstVideoInfo temp_info;
2370 gst_video_info_set_format (&temp_info, GST_VIDEO_FORMAT_RGBA, out_width,
2373 allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR_NAME);
2374 base_mem_allocator = GST_GL_BASE_MEMORY_ALLOCATOR (allocator);
2375 params = gst_gl_video_allocation_params_new (context, NULL, &temp_info,
2376 0, NULL, convert->priv->to_texture_target, GST_GL_RGBA);
2378 convert->priv->out_tex[j] =
2379 (GstGLMemory *) gst_gl_base_memory_alloc (base_mem_allocator,
2380 (GstGLAllocationParams *) params);
2382 gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
2383 gst_object_unref (allocator);
2386 convert->priv->out_tex[j] = out_tex;
2389 if (!gst_memory_map ((GstMemory *) convert->priv->out_tex[j], &out_info[j],
2390 GST_MAP_WRITE | GST_MAP_GL)) {
2391 GST_ERROR_OBJECT (convert, "failed to map output memory %p",
2392 convert->priv->out_tex[i]);
2398 GST_LOG_OBJECT (convert, "converting to textures:%p,%p,%p,%p "
2399 "dimensions:%ux%u, from textures:%p,%p,%p,%p dimensions:%ux%u",
2400 convert->priv->out_tex[0], convert->priv->out_tex[1],
2401 convert->priv->out_tex[2], convert->priv->out_tex[3], out_width,
2402 out_height, convert->priv->in_tex[0], convert->priv->in_tex[1],
2403 convert->priv->in_tex[2], convert->priv->in_tex[3], in_width, in_height);
2405 if (!_do_convert_draw (context, convert))
2409 for (j--; j >= 0; j--) {
2410 GstGLMemory *out_tex =
2411 (GstGLMemory *) gst_buffer_peek_memory (convert->outbuf,
2412 j + out_plane_offset);
2413 gint mem_width, mem_height;
2415 gst_memory_unmap ((GstMemory *) convert->priv->out_tex[j], &out_info[j]);
2417 mem_width = gst_gl_memory_get_texture_width (out_tex);
2418 mem_height = gst_gl_memory_get_texture_height (out_tex);
2420 if (out_tex->tex_format == GST_GL_LUMINANCE
2421 || out_tex->tex_format == GST_GL_LUMINANCE_ALPHA
2422 || out_width != mem_width || out_height != mem_height) {
2423 GstMapInfo to_info, from_info;
2425 if (!gst_memory_map ((GstMemory *) convert->priv->out_tex[j], &from_info,
2426 GST_MAP_READ | GST_MAP_GL)) {
2427 GST_ERROR_OBJECT (convert, "Failed to map intermediate memory");
2431 if (!gst_memory_map ((GstMemory *) out_tex, &to_info,
2432 GST_MAP_WRITE | GST_MAP_GL)) {
2433 GST_ERROR_OBJECT (convert, "Failed to map intermediate memory");
2437 gst_gl_memory_copy_into (convert->priv->out_tex[j],
2438 out_tex->tex_id, convert->priv->to_texture_target,
2439 out_tex->tex_format, mem_width, mem_height);
2440 gst_memory_unmap ((GstMemory *) convert->priv->out_tex[j], &from_info);
2441 gst_memory_unmap ((GstMemory *) out_tex, &to_info);
2443 convert->priv->out_tex[j] = NULL;
2447 /* YV12 the same as I420 except planes 1+2 swapped */
2448 if (GST_VIDEO_INFO_FORMAT (&convert->out_info) == GST_VIDEO_FORMAT_YV12) {
2450 gst_buffer_get_memory (convert->outbuf, 1 + out_plane_offset);
2452 gst_buffer_get_memory (convert->outbuf, 2 + out_plane_offset);
2454 gst_buffer_replace_memory (convert->outbuf, 1 + out_plane_offset, mem2);
2455 gst_buffer_replace_memory (convert->outbuf, 2 + out_plane_offset, mem1);
2458 for (i--; i >= 0; i--) {
2459 gst_memory_unmap ((GstMemory *) convert->priv->in_tex[i], &in_info[i]);
2465 /* Called by the idle function in the gl thread */
2467 _do_convert (GstGLContext * context, GstGLColorConvert * convert)
2469 GstVideoInfo *in_info = &convert->in_info;
2470 struct ConvertInfo *c_info = &convert->priv->convert_info;
2471 gboolean res = TRUE;
2473 GstGLSyncMeta *sync_meta;
2476 convert->outbuf = NULL;
2478 if (GST_VIDEO_INFO_MULTIVIEW_MODE (in_info) ==
2479 GST_VIDEO_MULTIVIEW_MODE_SEPARATED)
2480 views = GST_VIDEO_INFO_VIEWS (in_info);
2484 c_info->in_n_textures =
2485 _get_n_textures (GST_VIDEO_INFO_FORMAT (&convert->in_info));
2486 c_info->out_n_textures =
2487 _get_n_textures (GST_VIDEO_INFO_FORMAT (&convert->out_info));
2490 gboolean tex_format_change = FALSE;
2493 for (v = 0; v < views; v++) {
2494 for (i = 0; i < c_info->in_n_textures; i++) {
2495 guint j = v * c_info->in_n_textures + i;
2496 GstGLMemory *in_tex =
2497 (GstGLMemory *) gst_buffer_peek_memory (convert->inbuf, j);
2498 if (!gst_is_gl_memory ((GstMemory *) in_tex)) {
2499 GST_ERROR_OBJECT (convert, "input must be GstGLMemory");
2500 convert->priv->result = FALSE;
2504 if (j >= GST_VIDEO_MAX_PLANES)
2505 /* our arrays aren't that big */
2506 g_assert_not_reached ();
2508 if (v > 0 && in_tex->tex_format != convert->priv->in_tex_formats[i]) {
2509 GST_ERROR_OBJECT (convert, "Cannot convert textures with "
2511 convert->priv->result = FALSE;
2515 if (convert->priv->in_tex_formats[j] != in_tex->tex_format)
2516 tex_format_change = TRUE;
2518 convert->priv->in_tex_formats[j] = in_tex->tex_format;
2522 if (tex_format_change)
2523 gst_gl_color_convert_reset_shader (convert);
2526 if (!_init_convert (convert)) {
2527 convert->priv->result = FALSE;
2531 sync_meta = gst_buffer_get_gl_sync_meta (convert->inbuf);
2533 gst_gl_sync_meta_wait (sync_meta, convert->context);
2535 if (!convert->priv->pool) {
2538 GstQuery *query = gst_query_new_allocation (convert->priv->out_caps, TRUE);
2539 ret = gst_gl_color_convert_decide_allocation (convert, query);
2540 gst_query_unref (query);
2543 GST_ERROR_OBJECT (convert, "Failed to choose allocation parameters");
2544 convert->priv->result = FALSE;
2548 if (!convert->priv->pool) {
2549 GST_ERROR_OBJECT (convert, "Failed to create a buffer pool");
2550 convert->priv->result = FALSE;
2555 if (!convert->priv->pool_started) {
2556 if (!gst_buffer_pool_set_active (convert->priv->pool, TRUE)) {
2557 GST_ERROR_OBJECT (convert, "Failed to start buffer pool");
2558 convert->priv->result = FALSE;
2561 convert->priv->pool_started = TRUE;
2565 gst_buffer_pool_acquire_buffer (convert->priv->pool, &convert->outbuf,
2567 if (ret != GST_FLOW_OK) {
2568 GST_ERROR_OBJECT (convert, "Failed to acquire buffer from pool: %s",
2569 gst_flow_get_name (ret));
2570 convert->priv->result = FALSE;
2574 gst_gl_insert_debug_marker (context, "%s converting from %s to %s",
2575 GST_OBJECT_NAME (convert),
2576 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (in_info)),
2577 gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&convert->out_info)));
2578 /* Handle all views on input and output one at a time */
2579 for (v = 0; res && v < views; v++)
2580 res = _do_convert_one_view (context, convert, v);
2583 gst_buffer_unref (convert->outbuf);
2584 convert->outbuf = NULL;
2587 if (convert->outbuf) {
2588 GstVideoOverlayCompositionMeta *composition_meta;
2589 GstGLSyncMeta *sync_meta =
2590 gst_buffer_add_gl_sync_meta (convert->context, convert->outbuf);
2593 gst_gl_sync_meta_set_sync_point (sync_meta, convert->context);
2596 gst_buffer_get_video_overlay_composition_meta (convert->inbuf);
2597 if (composition_meta) {
2598 GST_DEBUG ("found video overlay composition meta, applying on output.");
2599 gst_buffer_add_video_overlay_composition_meta
2600 (convert->outbuf, composition_meta->overlay);
2604 convert->priv->result = res;
2609 _do_convert_draw (GstGLContext * context, GstGLColorConvert * convert)
2612 struct ConvertInfo *c_info = &convert->priv->convert_info;
2613 guint out_width, out_height;
2615 gboolean ret = TRUE;
2617 GLenum multipleRT[] = {
2618 GL_COLOR_ATTACHMENT0,
2619 GL_COLOR_ATTACHMENT1,
2620 GL_COLOR_ATTACHMENT2
2623 gl = context->gl_vtable;
2625 gst_gl_framebuffer_bind (convert->fbo);
2627 /* attach the texture to the FBO to renderer to */
2628 for (i = 0; i < c_info->out_n_textures; i++) {
2629 GstGLBaseMemory *tex = (GstGLBaseMemory *) convert->priv->out_tex[i];
2631 gst_gl_framebuffer_attach (convert->fbo, GL_COLOR_ATTACHMENT0 + i, tex);
2634 if (gl->DrawBuffers)
2635 gl->DrawBuffers (c_info->out_n_textures, multipleRT);
2636 else if (gl->DrawBuffer)
2637 gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
2639 gst_gl_framebuffer_get_effective_dimensions (convert->fbo, &out_width,
2641 gl->Viewport (0, 0, out_width, out_height);
2643 gst_gl_shader_use (convert->shader);
2645 if (gl->BindVertexArray)
2646 gl->BindVertexArray (convert->priv->vao);
2647 _bind_buffer (convert);
2649 for (i = c_info->in_n_textures - 1; i >= 0; i--) {
2650 gchar *scale_name = g_strdup_printf ("tex_scale%u", i);
2652 gst_gl_texture_target_to_gl (convert->priv->from_texture_target);
2654 gl->ActiveTexture (GL_TEXTURE0 + i);
2655 gl->BindTexture (gl_target, convert->priv->in_tex[i]->tex_id);
2656 gl->TexParameteri (gl_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
2657 gl->TexParameteri (gl_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
2658 gl->TexParameteri (gl_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
2659 gl->TexParameteri (gl_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
2661 gst_gl_shader_set_uniform_2fv (convert->shader, scale_name, 1,
2662 convert->priv->in_tex[i]->tex_scaling);
2664 g_free (scale_name);
2667 gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
2669 if (gl->BindVertexArray)
2670 gl->BindVertexArray (0);
2672 _unbind_buffer (convert);
2675 gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
2677 /* we are done with the shader */
2678 gst_gl_context_clear_shader (context);
2680 if (!gst_gl_context_check_framebuffer_status (context, GL_FRAMEBUFFER))
2683 gst_gl_context_clear_framebuffer (context);