From: Matthew Waters Date: Wed, 19 Jun 2019 09:09:21 +0000 (+1000) Subject: vulkancolorconvert: support RGB <-> AYUV/YUY2/UYVY X-Git-Tag: 1.19.3~507^2~3202 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=2180fbb498c5780279494a4bb7fd311a01dfc572;p=platform%2Fupstream%2Fgstreamer.git vulkancolorconvert: support RGB <-> AYUV/YUY2/UYVY --- diff --git a/ext/vulkan/shaders/ayuv_to_rgb.frag b/ext/vulkan/shaders/ayuv_to_rgb.frag new file mode 100644 index 0000000..db13187 --- /dev/null +++ b/ext/vulkan/shaders/ayuv_to_rgb.frag @@ -0,0 +1,26 @@ +#version 450 core + +#include "color_convert_generic.glsl" +#include "upsample_ayuv.glsl" +#include "swizzle.glsl" + +layout(location = 0) in vec2 inTexCoord; + +layout(set = 0, binding = 0) uniform sampler2D inTexture0; +layout(set = 0, binding = 1) uniform reorder { + ivec4 in_reorder_idx; + ivec4 out_reorder_idx; + ivec2 texSize; + ColorMatrices matrices; +}; + +layout(location = 0) out vec4 outColor0; + +void main() +{ + vec4 yuva = upsample_AYUV (inTexture0, inTexCoord, in_reorder_idx); + vec4 rgba = vec4(1.0); + rgba.a = yuva.a; + rgba.rgb = color_convert_texel (yuva.xyz, matrices); + outColor0 = /*vec4(yuv.x * 0.0, yuv.y * 0.0, yuv.z * 1.0, 1.0);*/swizzle(rgba, out_reorder_idx); +} diff --git a/ext/vulkan/shaders/color_convert_generic.glsl b/ext/vulkan/shaders/color_convert_generic.glsl new file mode 100644 index 0000000..0104991 --- /dev/null +++ b/ext/vulkan/shaders/color_convert_generic.glsl @@ -0,0 +1,20 @@ +struct ColorMatrices +{ + mat4 to_RGB_matrix; + mat4 primaries_matrix; + mat4 to_YUV_matrix; +}; + +vec3 color_matrix_convert (in vec3 texel, in mat4 color_matrix) +{ + vec4 rgb_ = color_matrix * vec4(texel, 1.0); + return rgb_.rgb; +} + +vec3 color_convert_texel (in vec3 texel, in ColorMatrices m) +{ + /* FIXME: need to add gamma remapping between these stages */ + vec3 tmp = color_matrix_convert (texel, m.to_RGB_matrix); + tmp = color_matrix_convert (tmp, m.primaries_matrix); + return color_matrix_convert (tmp, m.to_YUV_matrix); +} diff --git a/ext/vulkan/shaders/downsample_ayuv.glsl b/ext/vulkan/shaders/downsample_ayuv.glsl new file mode 100644 index 0000000..c803358 --- /dev/null +++ b/ext/vulkan/shaders/downsample_ayuv.glsl @@ -0,0 +1,8 @@ +#include "swizzle.glsl" + +vec4 downsample_AYUV(in sampler2D tex, in vec2 texCoord, in ivec4 inReorderIdx) +{ + vec4 yuva = texture(tex, texCoord); + + return swizzle(yuva, inReorderIdx); +} diff --git a/ext/vulkan/shaders/meson.build b/ext/vulkan/shaders/meson.build index c1fcbeb..ae92a26 100644 --- a/ext/vulkan/shaders/meson.build +++ b/ext/vulkan/shaders/meson.build @@ -4,6 +4,10 @@ gst_vulkan_shader_sources = [ 'identity.vert', 'swizzle.frag', 'swizzle_and_clobber_alpha.frag', + 'yuy2_to_rgb.frag', + 'rgb_to_yuy2.frag', + 'ayuv_to_rgb.frag', + 'rgb_to_ayuv.frag', ] bin2array = find_program('bin2array.py') diff --git a/ext/vulkan/shaders/rgb_to_ayuv.frag b/ext/vulkan/shaders/rgb_to_ayuv.frag new file mode 100644 index 0000000..da26040 --- /dev/null +++ b/ext/vulkan/shaders/rgb_to_ayuv.frag @@ -0,0 +1,25 @@ +#version 450 core + +#include "color_convert_generic.glsl" +#include "swizzle.glsl" + +layout(location = 0) in vec2 inTexCoord; + +layout(set = 0, binding = 0) uniform sampler2D inTexture0; +layout(set = 0, binding = 1) uniform reorder { + ivec4 in_reorder_idx; + ivec4 out_reorder_idx; + ivec2 texSize; + ColorMatrices matrices; +}; + +layout(location = 0) out vec4 outColor0; + +void main() +{ + vec4 rgba = swizzle (texture (inTexture0, inTexCoord), in_reorder_idx); + vec4 yuva = vec4(1.0); + yuva.w = rgba.a; + yuva.xyz = color_convert_texel (rgba.rgb, matrices); + outColor0 = swizzle(yuva, out_reorder_idx); +} diff --git a/ext/vulkan/shaders/rgb_to_yuy2.frag b/ext/vulkan/shaders/rgb_to_yuy2.frag new file mode 100644 index 0000000..06e9a4e --- /dev/null +++ b/ext/vulkan/shaders/rgb_to_yuy2.frag @@ -0,0 +1,37 @@ +#version 450 core + +#include "color_convert_generic.glsl" +#include "swizzle.glsl" + +layout(location = 0) in vec2 inTexCoord; + +layout(set = 0, binding = 0) uniform sampler2D inTexture0; +layout(set = 0, binding = 1) uniform reorder { + ivec4 in_reorder_idx; + ivec4 out_reorder_idx; + ivec2 texSize; + ColorMatrices matrices; +}; + +layout(location = 0) out vec4 outColor0; + +void main() +{ + float inorder = mod (inTexCoord.x * texSize.x, 2.0); + float fx = inTexCoord.x; + float dx = -1.0 / texSize.x; + if (inorder > 1.0) + dx = -dx; + vec4 texel1 = swizzle (texture(inTexture0, inTexCoord), in_reorder_idx); + vec4 texel2 = swizzle (texture(inTexture0, inTexCoord + vec2(dx, 0.0)), in_reorder_idx); + vec3 yuv1 = color_convert_texel (texel1.rgb, matrices); + vec3 yuv2 = color_convert_texel (texel2.rgb, matrices); + vec3 yuv; + yuv.x = yuv1.x; + yuv.yz = (yuv1.yz + yuv2.yz) * 0.5; + + if (inorder < 1.0) + outColor0 = vec4(yuv[out_reorder_idx[0]], yuv[out_reorder_idx[1]], 0.0, 1.0); + else + outColor0 = vec4(yuv[out_reorder_idx[2]], yuv[out_reorder_idx[3]], 0.0, 1.0); +} diff --git a/ext/vulkan/shaders/swizzle.glsl b/ext/vulkan/shaders/swizzle.glsl index 6869e16..6d4817e 100644 --- a/ext/vulkan/shaders/swizzle.glsl +++ b/ext/vulkan/shaders/swizzle.glsl @@ -1,4 +1,13 @@ +#ifndef _SWIZZLE_H_ +#define _SWIZZLE_H_ + vec4 swizzle(in vec4 texel, in ivec4 swizzle_idx) { return vec4(texel[swizzle_idx[0]], texel[swizzle_idx[1]], texel[swizzle_idx[2]], texel[swizzle_idx[3]]); } + +vec3 swizzle(in vec3 texel, in ivec3 swizzle_idx) +{ + return vec3(texel[swizzle_idx[0]], texel[swizzle_idx[1]], texel[swizzle_idx[2]]); +} +#endif diff --git a/ext/vulkan/shaders/upsample_ayuv.glsl b/ext/vulkan/shaders/upsample_ayuv.glsl new file mode 100644 index 0000000..e10cea8 --- /dev/null +++ b/ext/vulkan/shaders/upsample_ayuv.glsl @@ -0,0 +1,8 @@ +#include "swizzle.glsl" + +vec4 upsample_AYUV(in sampler2D tex, in vec2 texCoord, in ivec4 inReorderIdx) +{ + vec4 yuva = texture(tex, texCoord); + + return swizzle(yuva, inReorderIdx); +} diff --git a/ext/vulkan/shaders/upsample_yuy2.glsl b/ext/vulkan/shaders/upsample_yuy2.glsl new file mode 100644 index 0000000..7033d3a --- /dev/null +++ b/ext/vulkan/shaders/upsample_yuy2.glsl @@ -0,0 +1,19 @@ +vec3 upsample_YUY2(in sampler2D tex, in vec2 texCoord, in vec2 texSize, in ivec4 inReorderIdx) +{ + vec2 dx = vec2(-1.0 / texSize.x, 0.0); + if (mod (texCoord.x * texSize.x, 2.0) < 1.0) { + dx[1] = -dx[0]; + dx[0] = 0.0; + } + + vec3 yuv; + yuv.x = texture(tex, texCoord)[inReorderIdx[0]]; + /* FIXME: should get cosited sampling right... */ + vec4 texel; + texel.xy = texture(tex, texCoord + vec2(dx[0], 0.0)).rg; + texel.zw = texture(tex, texCoord + vec2(dx[1], 0.0)).rg; + yuv.y = texel[inReorderIdx[1]]; + yuv.z = texel[inReorderIdx[3]]; + + return yuv; +} diff --git a/ext/vulkan/shaders/uyvy_to_rgb.frag b/ext/vulkan/shaders/uyvy_to_rgb.frag new file mode 100644 index 0000000..0f245dc --- /dev/null +++ b/ext/vulkan/shaders/uyvy_to_rgb.frag @@ -0,0 +1,30 @@ +#version 450 core + +#include "yuy2_uyvy_to_rgb.glsl" + +layout(location = 0) in vec2 inTexCoord; + +layout(set = 0, binding = 0) uniform sampler2D inTexture0; +layout(set = 0, binding = 1) uniform YUVCoefficients_ubo { YUVCoefficients coeff; }; + +layout(set = 0, binding = 2) uniform TexelOrdering +{ + vec2 tex_size; + vec2 poffset; + ivec4 in_reorder_idx; + ivec4 out_reorder_idx; +} ordering; + +layout(location = 0) out vec4 outColor0; + +void main() +{ + float dx; + if (mod (inTexCoord.x * ordering.tex_size.x, 2.0) < 1.0) { + dx = -ordering.poffset.x; + } else { + dx = 0.0; + } + + outColor0 = YUY2_UYVY_to_RGB (inTexture0, inTexCoord, dx, coeff, ordering.in_reorder_idx, ordering.out_reorder_idx); +} diff --git a/ext/vulkan/shaders/yuy2_to_rgb.frag b/ext/vulkan/shaders/yuy2_to_rgb.frag new file mode 100644 index 0000000..67bb2b3 --- /dev/null +++ b/ext/vulkan/shaders/yuy2_to_rgb.frag @@ -0,0 +1,25 @@ +#version 450 core + +#include "color_convert_generic.glsl" +#include "upsample_yuy2.glsl" +#include "swizzle.glsl" + +layout(location = 0) in vec2 inTexCoord; + +layout(set = 0, binding = 0) uniform sampler2D inTexture0; +layout(set = 0, binding = 1) uniform reorder { + ivec4 in_reorder_idx; + ivec4 out_reorder_idx; + ivec2 texSize; + ColorMatrices matrices; +}; + +layout(location = 0) out vec4 outColor0; + +void main() +{ + vec3 yuv = upsample_YUY2 (inTexture0, inTexCoord, texSize, in_reorder_idx); + vec4 rgba = vec4(1.0); + rgba.rgb = color_convert_texel (yuv, matrices); + outColor0 = swizzle(rgba, out_reorder_idx); +} diff --git a/ext/vulkan/vkcolorconvert.c b/ext/vulkan/vkcolorconvert.c index b97c878..19fa5c5 100644 --- a/ext/vulkan/vkcolorconvert.c +++ b/ext/vulkan/vkcolorconvert.c @@ -37,20 +37,32 @@ #include "shaders/identity.vert.h" #include "shaders/swizzle.frag.h" #include "shaders/swizzle_and_clobber_alpha.frag.h" +#include "shaders/yuy2_to_rgb.frag.h" +#include "shaders/ayuv_to_rgb.frag.h" +#include "shaders/rgb_to_ayuv.frag.h" +#include "shaders/rgb_to_yuy2.frag.h" GST_DEBUG_CATEGORY (gst_debug_vulkan_color_convert); #define GST_CAT_DEFAULT gst_debug_vulkan_color_convert -#define N_SHADER_INFO (8*8) -static struct shader_info shader_infos[N_SHADER_INFO]; +#define N_SHADER_INFO (8*14) +static shader_info shader_infos[N_SHADER_INFO]; + +#define PUSH_CONSTANT_RANGE_NULL_INIT (VkPushConstantRange) { \ + .stageFlags = 0, \ + .offset = 0, \ + .size = 0, \ +} static void -get_rgb_format_swizzle_order (GstVideoFormat format, gint * swizzle) +get_rgb_format_swizzle_order (GstVideoFormat format, + gint swizzle[GST_VIDEO_MAX_COMPONENTS]) { const GstVideoFormatInfo *finfo = gst_video_format_get_info (format); int c_i = 0, i; - g_return_if_fail (finfo->flags & GST_VIDEO_FORMAT_FLAG_RGB); + g_return_if_fail (finfo->flags & GST_VIDEO_FORMAT_FLAG_RGB + || format == GST_VIDEO_FORMAT_AYUV); for (i = 0; i < finfo->n_components; i++) { swizzle[c_i++] = finfo->poffset[i]; @@ -71,17 +83,19 @@ get_rgb_format_swizzle_order (GstVideoFormat format, gint * swizzle) } static void -get_vulkan_rgb_format_swizzle_order (VkFormat format, gint * swizzle) +get_vulkan_rgb_format_swizzle_order (VkFormat format, gint * swizzle, + guint swizzle_count, guint offset) { const GstVulkanFormatInfo *finfo = gst_vulkan_format_get_info (format); int i; g_return_if_fail (finfo->flags & GST_VULKAN_FORMAT_FLAG_RGB); + g_return_if_fail (finfo->n_components <= swizzle_count); for (i = 0; i < finfo->n_components; i++) { - swizzle[i] = finfo->poffset[i]; + swizzle[i] = offset + finfo->poffset[i]; } - for (i = finfo->n_components; i < GST_VULKAN_MAX_COMPONENTS; i++) { + for (i = finfo->n_components; i < swizzle_count; i++) { swizzle[i] = -1; } } @@ -99,21 +113,391 @@ swizzle_identity_order (gint * swizzle, gint * idx) { int i; - for (i = 0; i < 4; i++) { + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) { idx[i] = -1; } - for (i = 0; i < 4; i++) { - if (swizzle[i] >= 0 && swizzle[i] < 4) { + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) { + if (swizzle[i] >= 0 && swizzle[i] < 4 && idx[swizzle[i]] == -1) { idx[swizzle[i]] = i; } } } +typedef struct +{ + double dm[4][4]; +} Matrix4; + +static void +matrix_debug (const Matrix4 * s) +{ + GST_DEBUG ("[%f %f %f %f]", s->dm[0][0], s->dm[0][1], s->dm[0][2], + s->dm[0][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[1][0], s->dm[1][1], s->dm[1][2], + s->dm[1][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[2][0], s->dm[2][1], s->dm[2][2], + s->dm[2][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[3][0], s->dm[3][1], s->dm[3][2], + s->dm[3][3]); +} + +static void +matrix_to_float (const Matrix4 * m, float *ret) +{ + int i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + ret[j * 4 + i] = m->dm[i][j]; + } + } +} + +static void +matrix_set_identity (Matrix4 * m) +{ + int i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + m->dm[i][j] = (i == j); + } + } +} + +static void +matrix_copy (Matrix4 * d, const Matrix4 * s) +{ + gint i, j; + + for (i = 0; i < 4; i++) + for (j = 0; j < 4; j++) + d->dm[i][j] = s->dm[i][j]; +} + +/* Perform 4x4 matrix multiplication: + * - @dst@ = @a@ * @b@ + * - @dst@ may be a pointer to @a@ andor @b@ + */ +static void +matrix_multiply (Matrix4 * dst, Matrix4 * a, Matrix4 * b) +{ + Matrix4 tmp; + int i, j, k; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + double x = 0; + for (k = 0; k < 4; k++) { + x += a->dm[i][k] * b->dm[k][j]; + } + tmp.dm[i][j] = x; + } + } + matrix_copy (dst, &tmp); +} + +#if 0 +static void +matrix_invert (Matrix4 * d, Matrix4 * s) +{ + Matrix4 tmp; + int i, j; + double det; + + matrix_set_identity (&tmp); + for (j = 0; j < 3; j++) { + for (i = 0; i < 3; i++) { + tmp.dm[j][i] = + s->dm[(i + 1) % 3][(j + 1) % 3] * s->dm[(i + 2) % 3][(j + 2) % 3] - + s->dm[(i + 1) % 3][(j + 2) % 3] * s->dm[(i + 2) % 3][(j + 1) % 3]; + } + } + det = + tmp.dm[0][0] * s->dm[0][0] + tmp.dm[0][1] * s->dm[1][0] + + tmp.dm[0][2] * s->dm[2][0]; + for (j = 0; j < 3; j++) { + for (i = 0; i < 3; i++) { + tmp.dm[i][j] /= det; + } + } + matrix_copy (d, &tmp); +} +#endif +static void +matrix_offset_components (Matrix4 * m, double a1, double a2, double a3) +{ + Matrix4 a; + + matrix_set_identity (&a); + a.dm[0][3] = a1; + a.dm[1][3] = a2; + a.dm[2][3] = a3; + matrix_debug (&a); + matrix_multiply (m, &a, m); +} + +static void +matrix_scale_components (Matrix4 * m, double a1, double a2, double a3) +{ + Matrix4 a; + + matrix_set_identity (&a); + a.dm[0][0] = a1; + a.dm[1][1] = a2; + a.dm[2][2] = a3; + matrix_multiply (m, &a, m); +} + +static void +matrix_YCbCr_to_RGB (Matrix4 * m, double Kr, double Kb) +{ + double Kg = 1.0 - Kr - Kb; + Matrix4 k = { + { + {1., 0., 2 * (1 - Kr), 0.}, + {1., -2 * Kb * (1 - Kb) / Kg, -2 * Kr * (1 - Kr) / Kg, 0.}, + {1., 2 * (1 - Kb), 0., 0.}, + {0., 0., 0., 1.}, + } + }; + + matrix_multiply (m, &k, m); +} + +typedef struct +{ + GstVideoInfo in_info; + GstVideoInfo out_info; + + Matrix4 to_RGB_matrix; + Matrix4 to_YUV_matrix; + Matrix4 convert_matrix; +} ConvertInfo; + +static void +convert_to_RGB (ConvertInfo * conv, Matrix4 * m) +{ + GstVideoInfo *info = &conv->in_info; + + { + const GstVideoFormatInfo *uinfo; + gint offset[4], scale[4], depth[4]; + int i; + + uinfo = gst_video_format_get_info (GST_VIDEO_INFO_FORMAT (info)); + + /* bring color components to [0..1.0] range */ + gst_video_color_range_offsets (info->colorimetry.range, uinfo, offset, + scale); + + for (i = 0; i < uinfo->n_components; i++) + depth[i] = (1 << uinfo->depth[i]) - 1; + + matrix_offset_components (m, -offset[0] / (float) depth[0], + -offset[1] / (float) depth[1], -offset[2] / (float) depth[2]); + matrix_scale_components (m, depth[0] / ((float) scale[0]), + depth[1] / ((float) scale[1]), depth[2] / ((float) scale[2])); + GST_DEBUG ("to RGB scale/offset matrix"); + matrix_debug (m); + } + + if (GST_VIDEO_INFO_IS_YUV (info)) { + gdouble Kr, Kb; + + if (gst_video_color_matrix_get_Kr_Kb (info->colorimetry.matrix, &Kr, &Kb)) + matrix_YCbCr_to_RGB (m, Kr, Kb); + GST_DEBUG ("to RGB matrix"); + matrix_debug (m); + } +} + +static void +matrix_RGB_to_YCbCr (Matrix4 * m, double Kr, double Kb) +{ + double Kg = 1.0 - Kr - Kb; + Matrix4 k; + double x; + + k.dm[0][0] = Kr; + k.dm[0][1] = Kg; + k.dm[0][2] = Kb; + k.dm[0][3] = 0; + + x = 1 / (2 * (1 - Kb)); + k.dm[1][0] = -x * Kr; + k.dm[1][1] = -x * Kg; + k.dm[1][2] = x * (1 - Kb); + k.dm[1][3] = 0; + + x = 1 / (2 * (1 - Kr)); + k.dm[2][0] = x * (1 - Kr); + k.dm[2][1] = -x * Kg; + k.dm[2][2] = -x * Kb; + k.dm[2][3] = 0; + + k.dm[3][0] = 0; + k.dm[3][1] = 0; + k.dm[3][2] = 0; + k.dm[3][3] = 1; + + matrix_multiply (m, &k, m); +} + +static void +convert_to_YUV (ConvertInfo * conv, Matrix4 * m) +{ + GstVideoInfo *info = &conv->out_info; + + if (GST_VIDEO_INFO_IS_YUV (info)) { + gdouble Kr, Kb; + + if (gst_video_color_matrix_get_Kr_Kb (info->colorimetry.matrix, &Kr, &Kb)) + matrix_RGB_to_YCbCr (m, Kr, Kb); + GST_DEBUG ("to YUV matrix"); + matrix_debug (m); + } + + { + const GstVideoFormatInfo *uinfo; + gint offset[4], scale[4], depth[4]; + int i; + + uinfo = gst_video_format_get_info (GST_VIDEO_INFO_FORMAT (info)); + + /* bring color components to nominal range */ + gst_video_color_range_offsets (info->colorimetry.range, uinfo, offset, + scale); + + for (i = 0; i < uinfo->n_components; i++) + depth[i] = (1 << uinfo->depth[i]) - 1; + + matrix_scale_components (m, scale[0] / (float) depth[0], + scale[1] / (float) depth[1], scale[2] / (float) depth[2]); + matrix_offset_components (m, offset[0] / (float) depth[0], + offset[1] / (float) depth[1], offset[2] / (float) depth[2]); + GST_DEBUG ("to YUV scale/offset matrix"); + matrix_debug (m); + } +} + +#if 0 +static void +matrix_RGB_to_XYZ (Matrix4 * dst, double Rx, double Ry, double Gx, + double Gy, double Bx, double By, double Wx, double Wy) +{ + Matrix4 m, im; + double sx, sy, sz; + double wx, wy, wz; + + matrix_set_identity (&m); + + m.dm[0][0] = Rx; + m.dm[1][0] = Ry; + m.dm[2][0] = (1.0 - Rx - Ry); + m.dm[0][1] = Gx; + m.dm[1][1] = Gy; + m.dm[2][1] = (1.0 - Gx - Gy); + m.dm[0][2] = Bx; + m.dm[1][2] = By; + m.dm[2][2] = (1.0 - Bx - By); + + matrix_invert (&im, &m); + + wx = Wx / Wy; + wy = 1.0; + wz = (1.0 - Wx - Wy) / Wy; + + sx = im.dm[0][0] * wx + im.dm[0][1] * wy + im.dm[0][2] * wz; + sy = im.dm[1][0] * wx + im.dm[1][1] * wy + im.dm[1][2] * wz; + sz = im.dm[2][0] * wx + im.dm[2][1] * wy + im.dm[2][2] * wz; + + m.dm[0][0] *= sx; + m.dm[1][0] *= sx; + m.dm[2][0] *= sx; + m.dm[0][1] *= sy; + m.dm[1][1] *= sy; + m.dm[2][1] *= sy; + m.dm[0][2] *= sz; + m.dm[1][2] *= sz; + m.dm[2][2] *= sz; + + matrix_copy (dst, &m); +} + +static void +convert_primaries (ConvertInfo * conv) +{ + gboolean same_matrix, same_primaries; + Matrix4 p1, p2; + + same_matrix = + conv->in_info.colorimetry.matrix == conv->out_info.colorimetry.matrix; + same_primaries = + conv->in_info.colorimetry.primaries == + conv->out_info.colorimetry.primaries; + + GST_DEBUG ("matrix %d -> %d (%d)", conv->in_info.colorimetry.matrix, + conv->out_info.colorimetry.matrix, same_matrix); + GST_DEBUG ("primaries %d -> %d (%d)", conv->in_info.colorimetry.primaries, + conv->out_info.colorimetry.primaries, same_primaries); + + matrix_set_identity (&conv->convert_matrix); + + if (!same_primaries) { + const GstVideoColorPrimariesInfo *pi; + + pi = gst_video_color_primaries_get_info (conv->in_info.colorimetry. + primaries); + matrix_RGB_to_XYZ (&p1, pi->Rx, pi->Ry, pi->Gx, pi->Gy, pi->Bx, pi->By, + pi->Wx, pi->Wy); + GST_DEBUG ("to XYZ matrix"); + matrix_debug (&p1); + GST_DEBUG ("current matrix"); + matrix_multiply (&conv->convert_matrix, &conv->convert_matrix, &p1); + matrix_debug (&conv->convert_matrix); + + pi = gst_video_color_primaries_get_info (conv->out_info.colorimetry. + primaries); + matrix_RGB_to_XYZ (&p2, pi->Rx, pi->Ry, pi->Gx, pi->Gy, pi->Bx, pi->By, + pi->Wx, pi->Wy); + matrix_invert (&p2, &p2); + GST_DEBUG ("to RGB matrix"); + matrix_debug (&p2); + matrix_multiply (&conv->convert_matrix, &conv->convert_matrix, &p2); + GST_DEBUG ("current matrix"); + matrix_debug (&conv->convert_matrix); + } +} +#endif +static ConvertInfo * +convert_info_new (GstVideoInfo * in_info, GstVideoInfo * out_info) +{ + ConvertInfo *conv = g_new0 (ConvertInfo, 1); + + matrix_set_identity (&conv->to_RGB_matrix); + matrix_set_identity (&conv->convert_matrix); + matrix_set_identity (&conv->to_YUV_matrix); + + memcpy (&conv->in_info, in_info, sizeof (*in_info)); + memcpy (&conv->out_info, out_info, sizeof (*out_info)); + + convert_to_RGB (conv, &conv->to_RGB_matrix); + /* by default videoconvert does not convert primaries + convert_primaries (conv); */ + convert_to_YUV (conv, &conv->to_YUV_matrix); + + return conv; +} + static void update_descriptor_set (GstVulkanColorConvert * conv, VkImageView view) { GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + VkDescriptorBufferInfo buffer_info; + VkWriteDescriptorSet writes[5]; + guint i = 0; /* *INDENT-OFF* */ VkDescriptorImageInfo image_info = { @@ -121,47 +505,200 @@ update_descriptor_set (GstVulkanColorConvert * conv, VkImageView view) .imageView = view, .sampler = conv->sampler }; + /* *INDENT-ON* */ - VkWriteDescriptorSet writes = { - .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, - .pNext = NULL, - .dstSet = conv->descriptor_set, - .dstBinding = 0, - .dstArrayElement = 0, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = 1, - .pImageInfo = &image_info + for (; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { + /* *INDENT-OFF* */ + writes[i] = (VkWriteDescriptorSet) { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = NULL, + .dstSet = conv->descriptor_set, + .dstBinding = i, + .dstArrayElement = 0, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .descriptorCount = 1, + .pImageInfo = &image_info + }; + /* *INDENT-ON* */ + } + if (conv->uniform && conv->current_shader->uniform_size) { + /* *INDENT-OFF* */ + buffer_info = (VkDescriptorBufferInfo) { + .buffer = ((GstVulkanBufferMemory *) conv->uniform)->buffer, + .offset = 0, + .range = conv->current_shader->uniform_size + }; + writes[i] = (VkWriteDescriptorSet) { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = NULL, + .dstSet = conv->descriptor_set, + .dstBinding = i, + .dstArrayElement = 0, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .descriptorCount = 1, + .pBufferInfo = &buffer_info + }; + /* *INDENT-ON* */ + i++; }; - /* *INDENT-ON* */ - vkUpdateDescriptorSets (render->device->device, 1, &writes, 0, NULL); + g_assert (i <= G_N_ELEMENTS (writes)); + + vkUpdateDescriptorSets (render->device->device, i, writes, 0, NULL); } -static gboolean -swizzle_rgb_update_command_state (GstVulkanColorConvert * conv, - VkCommandBuffer cmd, GstVulkanImageMemory ** in_mems, - GstVulkanImageMemory ** out_mems) +static void +video_format_to_reorder (GstVideoFormat v_format, gint * reorder, + gboolean input) { - GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - gint in_reorder[4], out_reorder[4], in_vk_order[4], out_vk_order[4], tmp[4], - reorder[8]; + switch (v_format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_RGBx: + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_BGRx: + case GST_VIDEO_FORMAT_ARGB: + case GST_VIDEO_FORMAT_xRGB: + case GST_VIDEO_FORMAT_ABGR: + case GST_VIDEO_FORMAT_xBGR: + case GST_VIDEO_FORMAT_AYUV: + get_rgb_format_swizzle_order (v_format, reorder); + break; + case GST_VIDEO_FORMAT_UYVY: + reorder[0] = 1; + reorder[1] = 0; + reorder[2] = input ? 3 : 2; + reorder[3] = 0; + break; + case GST_VIDEO_FORMAT_YUY2: + reorder[0] = 0; + reorder[1] = 1; + reorder[2] = 0; + reorder[3] = input ? 3 : 2; + break; + default: + g_assert_not_reached (); + break; + } + + GST_TRACE ("swizzle: %u, %u, %u, %u", reorder[0], reorder[1], reorder[2], + reorder[3]); +} + +static guint +finfo_get_plane_n_components (const GstVideoFormatInfo * finfo, guint plane) +{ + guint n_components = 0, i; + + switch (finfo->format) { + case GST_VIDEO_FORMAT_RGBx: + case GST_VIDEO_FORMAT_xRGB: + case GST_VIDEO_FORMAT_BGRx: + case GST_VIDEO_FORMAT_xBGR: + /* fixup spaced RGB formats as we treat the space as a normal alpha + * component */ + return plane == 0 ? 4 : 0; + default: + break; + } + + for (i = 0; i < finfo->n_components; i++) { + if (finfo->plane[i] == plane) + n_components++; + } + + return n_components; +} + +static void +get_vulkan_format_swizzle_order (GstVideoFormat v_format, + VkFormat vk_format[GST_VIDEO_MAX_PLANES], + gint swizzle[GST_VIDEO_MAX_COMPONENTS]) +{ + const GstVideoFormatInfo *finfo; + int i, prev_in_i = 0; + + finfo = gst_video_format_get_info (v_format); + for (i = 0; i < finfo->n_planes; i++) { + guint plane_components = finfo_get_plane_n_components (finfo, i); + get_vulkan_rgb_format_swizzle_order (vk_format[i], + &swizzle[prev_in_i], plane_components, prev_in_i); + prev_in_i += plane_components; + } + + if (v_format == GST_VIDEO_FORMAT_YUY2 || v_format == GST_VIDEO_FORMAT_UYVY) { + /* Fixup these packed YUV formats as we use a two component format for + * a 4-component pixel and access two samples in the shader */ + g_assert (swizzle[0] == 0); + g_assert (swizzle[1] == 1); + swizzle[2] = 2; + swizzle[3] = 3; + } + + GST_TRACE ("%s: %i, %i, %i, %i", finfo->name, swizzle[0], swizzle[1], + swizzle[2], swizzle[3]); +} + +static void +calculate_reorder_indexes (GstVideoFormat in_format, + GstVulkanImageMemory * in_mems[GST_VIDEO_MAX_COMPONENTS], + GstVideoFormat out_format, + GstVulkanImageMemory * out_mems[GST_VIDEO_MAX_COMPONENTS], + int ret_in[GST_VIDEO_MAX_COMPONENTS], int ret_out[GST_VIDEO_MAX_COMPONENTS]) +{ + const GstVideoFormatInfo *in_finfo, *out_finfo; + VkFormat in_vk_formats[GST_VIDEO_MAX_COMPONENTS]; + VkFormat out_vk_formats[GST_VIDEO_MAX_COMPONENTS]; + int in_vk_order[GST_VIDEO_MAX_COMPONENTS], + in_reorder[GST_VIDEO_MAX_COMPONENTS]; + int out_vk_order[GST_VIDEO_MAX_COMPONENTS], + out_reorder[GST_VIDEO_MAX_COMPONENTS]; + int tmp[GST_VIDEO_MAX_PLANES]; int i; - get_vulkan_rgb_format_swizzle_order (in_mems[0]->create_info.format, - in_vk_order); - get_rgb_format_swizzle_order (GST_VIDEO_INFO_FORMAT (&render->in_info), - in_reorder); + in_finfo = gst_video_format_get_info (in_format); + out_finfo = gst_video_format_get_info (out_format); - get_rgb_format_swizzle_order (GST_VIDEO_INFO_FORMAT (&render->out_info), - out_reorder); - get_vulkan_rgb_format_swizzle_order (out_mems[0]->create_info.format, - out_vk_order); + for (i = 0; i < in_finfo->n_planes; i++) + in_vk_formats[i] = in_mems[i]->create_info.format; + for (i = 0; i < out_finfo->n_planes; i++) + out_vk_formats[i] = out_mems[i]->create_info.format; - for (i = 0; i < 4; i++) + get_vulkan_format_swizzle_order (in_format, in_vk_formats, in_vk_order); + video_format_to_reorder (in_format, in_reorder, TRUE); + + video_format_to_reorder (out_format, out_reorder, FALSE); + get_vulkan_format_swizzle_order (out_format, out_vk_formats, out_vk_order); + + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) tmp[i] = out_vk_order[out_reorder[i]]; - for (i = 0; i < 4; i++) - reorder[i] = in_vk_order[in_reorder[i]]; /* find the identity order for RGBA->$format */ - swizzle_identity_order (tmp, &reorder[4]); + GST_TRACE ("pre-invert: %u, %u, %u, %u", tmp[0], tmp[1], tmp[2], tmp[3]); + if (out_format == GST_VIDEO_FORMAT_YUY2 + || out_format == GST_VIDEO_FORMAT_UYVY) { + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) + ret_out[i] = tmp[i]; + } else { + swizzle_identity_order (tmp, ret_out); + } + + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) + ret_in[i] = in_reorder[in_vk_order[i]]; + GST_TRACE ("in reorder: %u, %u, %u, %u", ret_in[0], ret_in[1], ret_in[2], + ret_in[3]); + GST_TRACE ("out reorder: %u, %u, %u, %u", ret_out[0], ret_out[1], ret_out[2], + ret_out[3]); +} + +static gboolean +swizzle_rgb_update_command_state (GstVulkanColorConvert * conv, + VkCommandBuffer cmd, shader_info * sinfo, + GstVulkanImageMemory ** in_mems, GstVulkanImageMemory ** out_mems) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + gint reorder[8]; + + calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), + in_mems, GST_VIDEO_INFO_FORMAT (&render->out_info), + out_mems, reorder, &reorder[4]); vkCmdPushConstants (cmd, render->pipeline_layout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof (reorder), @@ -173,6 +710,70 @@ swizzle_rgb_update_command_state (GstVulkanColorConvert * conv, return TRUE; } +struct ColorMatrices +{ + float to_RGB[16]; + float primaries[16]; + float to_YUV[16]; +}; + +struct YUVUpdateData +{ + int in_reorder[4]; + int out_reorder[4]; + int tex_size[2]; + /* each member is aligned on 4x previous component size boundaries */ + int _padding[2]; + struct ColorMatrices matrices; +}; + +static gboolean +yuv_to_rgb_update_command_state (GstVulkanColorConvert * conv, + VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageMemory ** in_mems, + GstVulkanImageMemory ** out_mems) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + + if (!GPOINTER_TO_INT (sinfo->user_data)) { + struct YUVUpdateData data; + ConvertInfo *conv_info; + GstMapInfo map_info; + + calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), + in_mems, GST_VIDEO_INFO_FORMAT (&render->out_info), + out_mems, data.in_reorder, data.out_reorder); + + conv_info = convert_info_new (&render->in_info, &render->out_info); + matrix_to_float (&conv_info->to_RGB_matrix, data.matrices.to_RGB); + matrix_to_float (&conv_info->convert_matrix, data.matrices.primaries); + matrix_to_float (&conv_info->to_YUV_matrix, data.matrices.to_YUV); + /* FIXME: keep this around */ + g_free (conv_info); + + data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&render->in_info); + data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&render->in_info); + + if (!gst_memory_map (conv->uniform, &map_info, GST_MAP_WRITE)) { + return FALSE; + } + memcpy (map_info.data, &data, sizeof (data)); + gst_memory_unmap (conv->uniform, &map_info); + + update_descriptor_set (conv, in_mems[0]->view); + sinfo->user_data = GINT_TO_POINTER (1); + } + vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, + render->pipeline_layout, 0, 1, &conv->descriptor_set, 0, NULL); + + return TRUE; +} + +static void +clear_user_data_flag (shader_info * sinfo) +{ + sinfo->user_data = NULL; +} + static gboolean gst_vulkan_color_convert_start (GstBaseTransform * bt); static gboolean gst_vulkan_color_convert_stop (GstBaseTransform * bt); @@ -205,7 +806,7 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, - "{ BGRA, RGBA, ABGR, ARGB, BGRx, RGBx, xBGR, xRGB }"))); + "{ BGRA, RGBA, ABGR, ARGB, BGRx, RGBx, xBGR, xRGB, AYUV, YUY2, UYVY }"))); static GstStaticPadTemplate gst_vulkan_src_template = GST_STATIC_PAD_TEMPLATE ("src", @@ -213,7 +814,7 @@ GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, - "{ BGRA, RGBA, ABGR, ARGB, BGRx, RGBx, xBGR, xRGB }"))); + "{ BGRA, RGBA, ABGR, ARGB, BGRx, RGBx, xBGR, xRGB, AYUV, YUY2, UYVY }"))); enum { @@ -234,75 +835,98 @@ G_DEFINE_TYPE_WITH_CODE (GstVulkanColorConvert, gst_vulkan_color_convert, GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_color_convert, "vulkancolorconvert", 0, "Vulkan Color Convert")); +struct yuv_info +{ + GstVideoFormat format; + gchar *from_frag; + gsize from_frag_size; + gchar *to_frag; + gsize to_frag_size; +}; + static void fill_shader_info (void) { - GstVideoFormat rgba[] = { GST_VIDEO_FORMAT_RGBA, GST_VIDEO_FORMAT_ARGB, - GST_VIDEO_FORMAT_BGRA, GST_VIDEO_FORMAT_ABGR + GstVideoFormat rgbs[] = { GST_VIDEO_FORMAT_RGBA, GST_VIDEO_FORMAT_ARGB, + GST_VIDEO_FORMAT_BGRA, GST_VIDEO_FORMAT_ABGR, GST_VIDEO_FORMAT_RGBx, + GST_VIDEO_FORMAT_xRGB, GST_VIDEO_FORMAT_BGRx, GST_VIDEO_FORMAT_xBGR }; - GstVideoFormat rgbx[] = { GST_VIDEO_FORMAT_RGBx, GST_VIDEO_FORMAT_xRGB, - GST_VIDEO_FORMAT_BGRx, GST_VIDEO_FORMAT_xBGR + struct yuv_info yuvs[] = { + {GST_VIDEO_FORMAT_AYUV, ayuv_to_rgb_frag, ayuv_to_rgb_frag_size, + rgb_to_ayuv_frag, rgb_to_ayuv_frag_size}, + {GST_VIDEO_FORMAT_YUY2, yuy2_to_rgb_frag, yuy2_to_rgb_frag_size, + rgb_to_yuy2_frag, rgb_to_yuy2_frag_size}, + {GST_VIDEO_FORMAT_UYVY, yuy2_to_rgb_frag, yuy2_to_rgb_frag_size, + rgb_to_yuy2_frag, rgb_to_yuy2_frag_size}, }; - int info_i = 0; - int i, j; + guint info_i = 0; + guint i, j; /* standard RGB with alpha conversion all components are copied */ - for (i = 0; i < G_N_ELEMENTS (rgba); i++) { - for (j = 0; j < G_N_ELEMENTS (rgba); j++) { - /* *INDENT-OFF* */ - shader_infos[info_i++] = (struct shader_info) { - .from = rgba[i], - .to = rgba[j], - .cmd_state_update = swizzle_rgb_update_command_state, - .frag_code = swizzle_frag, - .frag_size = swizzle_frag_size, - }; - /* *INDENT-ON* */ - } - /* copying to an RGBx variant means we can store whatever we like in the 'x' - * component we choose to copy the alpha component like a standard RGBA->RGBA - * swizzle */ - for (j = 0; j < G_N_ELEMENTS (rgbx); j++) { - /* *INDENT-OFF* */ - shader_infos[info_i++] = (struct shader_info) { - .from = rgba[i], - .to = rgbx[j], + /* *INDENT-OFF* */ + for (i = 0; i < G_N_ELEMENTS (rgbs); i++) { + const GstVideoFormatInfo *from_finfo = gst_video_format_get_info (rgbs[i]); + + for (j = 0; j < G_N_ELEMENTS (rgbs); j++) { + const GstVideoFormatInfo *to_finfo = gst_video_format_get_info (rgbs[j]); + gboolean clobber_alpha = FALSE; + + GST_TRACE ("Initializing info for %s -> %s", from_finfo->name, to_finfo->name); + + /* copying to an RGBx variant means we can store whatever we like in the 'x' + * component we choose to copy the alpha component like a standard RGBA->RGBA + * swizzle. + * Copying from an rgbx to a rgba format means we need to reset the + * alpha value */ + clobber_alpha = !GST_VIDEO_FORMAT_INFO_HAS_ALPHA (from_finfo) && GST_VIDEO_FORMAT_INFO_HAS_ALPHA (to_finfo); + shader_infos[info_i++] = (shader_info) { + .from = rgbs[i], + .to = rgbs[j], .cmd_state_update = swizzle_rgb_update_command_state, - .frag_code = swizzle_frag, - .frag_size = swizzle_frag_size, + .frag_code = clobber_alpha ? swizzle_and_clobber_alpha_frag : swizzle_frag, + .frag_size = clobber_alpha ? swizzle_and_clobber_alpha_frag_size : swizzle_frag_size, + .push_constant_ranges = { + { + .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT, + .offset = 0, + .size = 8 * sizeof(gint32), + }, PUSH_CONSTANT_RANGE_NULL_INIT, + }, + .notify = (GDestroyNotify) clear_user_data_flag, + .user_data = NULL, }; - /* *INDENT-ON* */ } - } - for (i = 0; i < G_N_ELEMENTS (rgbx); i++) { - /* copying to an RGBx variant means we can store whatever we like in the 'x' - * component we choose to copy the 'x' component like a standard RGBA->RGBA - * swizzle */ - for (j = 0; j < G_N_ELEMENTS (rgbx); j++) { - /* *INDENT-OFF* */ - shader_infos[info_i++] = (struct shader_info) { - .from = rgbx[i], - .to = rgbx[j], - .cmd_state_update = swizzle_rgb_update_command_state, - .frag_code = swizzle_frag, - .frag_size = swizzle_frag_size, + + for (j = 0; j < G_N_ELEMENTS (yuvs); j++) { + const GstVideoFormatInfo *to_finfo = gst_video_format_get_info (yuvs[j].format); + GST_TRACE ("Initializing info for %s -> %s", from_finfo->name, to_finfo->name); + shader_infos[info_i++] = (shader_info) { + .from = rgbs[i], + .to = yuvs[j].format, + .cmd_state_update = yuv_to_rgb_update_command_state, + .frag_code = yuvs[j].to_frag, + .frag_size = yuvs[j].to_frag_size, + .push_constant_ranges = { PUSH_CONSTANT_RANGE_NULL_INIT }, + .uniform_size = sizeof(struct YUVUpdateData), + .notify = (GDestroyNotify) clear_user_data_flag, + .user_data = NULL, }; - /* *INDENT-ON* */ - } - /* copying from RGBx to RGBA requires clobbering the destination alpha - * with 1.0 */ - for (j = 0; j < G_N_ELEMENTS (rgba); j++) { - /* *INDENT-OFF* */ - shader_infos[info_i++] = (struct shader_info) { - .from = rgbx[i], - .to = rgba[j], - .cmd_state_update = swizzle_rgb_update_command_state, - .frag_code = swizzle_and_clobber_alpha_frag, - .frag_size = swizzle_and_clobber_alpha_frag_size, + GST_TRACE ("Initializing info for %s -> %s", to_finfo->name, from_finfo->name); + shader_infos[info_i++] = (shader_info) { + .from = yuvs[j].format, + .to = rgbs[i], + .cmd_state_update = yuv_to_rgb_update_command_state, + .frag_code = yuvs[j].from_frag, + .frag_size = yuvs[j].from_frag_size, + .push_constant_ranges = { PUSH_CONSTANT_RANGE_NULL_INIT }, + .uniform_size = sizeof(struct YUVUpdateData), + .notify = (GDestroyNotify) clear_user_data_flag, + .user_data = NULL, }; - /* *INDENT-ON* */ } } + /* *INDENT-ON* */ + GST_TRACE ("initialized %u formats", info_i); g_assert (info_i == N_SHADER_INFO); } @@ -402,9 +1026,9 @@ _init_supported_formats (GstVulkanDevice * device, gboolean output, /* Always supported input and output formats */ _init_value_string_list (supported_formats, "RGBA", "RGB", "RGBx", "BGR", - "BGRx", "BGRA", "xRGB", "xBGR", "ARGB", NULL); + "BGRx", "BGRA", "xRGB", "xBGR", "ARGB", "ABGR", NULL); - _append_value_string_list (supported_formats, "ABGR", NULL); + _append_value_string_list (supported_formats, "AYUV", "YUY2", "UYVY", NULL); } /* copies the given caps */ @@ -582,18 +1206,27 @@ static VkPushConstantRange * gst_vulkan_color_convert_push_constant_ranges (GstVulkanFullScreenRender * render, guint * n_constants) { + GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (render); VkPushConstantRange *ranges; + int i; - *n_constants = 1; - ranges = g_new0 (VkPushConstantRange, *n_constants); + for (i = 0; i < G_N_ELEMENTS (conv->current_shader->push_constant_ranges); + i++) { + if (conv->current_shader->push_constant_ranges[i].stageFlags == 0) { + break; + } + } - /* *INDENT-OFF* */ - ranges[0] = (VkPushConstantRange) { - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT, - .offset = 0, - .size = 4 * sizeof(gint32), - }; - /* *INDENT-ON* */ + *n_constants = i; + GST_DEBUG_OBJECT (conv, "%s->%s has %u push constants", + gst_video_format_to_string (conv->current_shader->from), + gst_video_format_to_string (conv->current_shader->to), *n_constants); + if (*n_constants <= 0) + ranges = NULL; + else + ranges = + g_memdup (conv->current_shader->push_constant_ranges, + sizeof (VkPushConstantRange) * *n_constants); return ranges; } @@ -602,20 +1235,45 @@ static VkDescriptorSetLayoutBinding * gst_vulkan_color_convert_descriptor_set_layout_bindings (GstVulkanFullScreenRender * render, guint * n_bindings) { + GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (render); VkDescriptorSetLayoutBinding *bindings; + guint i; - *n_bindings = 1; + *n_bindings = 0; + *n_bindings += GST_VIDEO_INFO_N_PLANES (&render->in_info); + if (conv->current_shader->uniform_size) + *n_bindings += 1; bindings = g_new0 (VkDescriptorSetLayoutBinding, *n_bindings); - /* *INDENT-OFF* */ - bindings[0] = (VkDescriptorSetLayoutBinding) { - .binding = 0, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .pImmutableSamplers = NULL, - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT - }; - /* *INDENT-ON* */ + GST_DEBUG_OBJECT (conv, "%s->%s has %u descriptor set layout bindings", + gst_video_format_to_string (conv->current_shader->from), + gst_video_format_to_string (conv->current_shader->to), *n_bindings); + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { + /* *INDENT-OFF* */ + bindings[i] = (VkDescriptorSetLayoutBinding) { + .binding = i, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT + }; + /* *INDENT-ON* */ + } + if (conv->current_shader->uniform_size) { + /* *INDENT-OFF* */ + bindings[i] = (VkDescriptorSetLayoutBinding) { + .binding = i, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT + }; + /* *INDENT-ON* */ + i++; + } + + g_assert (i == *n_bindings); return bindings; } @@ -654,7 +1312,7 @@ static VkAttachmentDescription for (i = 0; i < *n_descriptions; i++) { /* *INDENT-OFF* */ color_attachments[i] = (VkAttachmentDescription) { - .format = gst_vulkan_format_from_video_format (GST_VIDEO_INFO_FORMAT (&render->in_info), i), + .format = gst_vulkan_format_from_video_format (GST_VIDEO_INFO_FORMAT (&render->out_info), i), .samples = VK_SAMPLE_COUNT_1_BIT, .loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR, .storeOp = VK_ATTACHMENT_STORE_OP_STORE, @@ -670,6 +1328,59 @@ static VkAttachmentDescription return color_attachments; } +static VkSampler +_create_sampler (GstVulkanColorConvert * conv) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + + /* *INDENT-OFF* */ + VkSamplerCreateInfo samplerInfo = { + .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, + .magFilter = VK_FILTER_LINEAR, + .minFilter = VK_FILTER_LINEAR, + .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, + .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, + .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, + .anisotropyEnable = VK_FALSE, + .maxAnisotropy = 1, + .borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK, + .unnormalizedCoordinates = VK_FALSE, + .compareEnable = VK_FALSE, + .compareOp = VK_COMPARE_OP_ALWAYS, + .mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR, + .mipLodBias = 0.0f, + .minLod = 0.0f, + .maxLod = 0.0f + }; + /* *INDENT-ON* */ + GError *error = NULL; + VkSampler sampler; + VkResult err; + + err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler); + if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) { + GST_ERROR_OBJECT (conv, "Failed to create sampler: %s", error->message); + g_clear_error (&error); + return NULL; + } + + return sampler; +} + +static gboolean +gst_vulkan_color_convert_start (GstBaseTransform * bt) +{ + GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (bt); + + if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt)) + return FALSE; + + if (!(conv->sampler = _create_sampler (conv))) + return FALSE; + + return TRUE; +} + static VkDescriptorPool _create_descriptor_pool (GstVulkanColorConvert * conv) { @@ -679,7 +1390,7 @@ _create_descriptor_pool (GstVulkanColorConvert * conv) VkDescriptorPoolSize pool_sizes[] = { { .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = 1 + .descriptorCount = GST_VIDEO_INFO_N_PLANES (&render->in_info), }, { .type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, @@ -742,60 +1453,20 @@ _create_descriptor_set (GstVulkanColorConvert * conv) return descriptor; } -static VkSampler -_create_sampler (GstVulkanColorConvert * conv) +static gboolean +_create_uniform_buffer (GstVulkanColorConvert * conv) { GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); - /* *INDENT-OFF* */ - VkSamplerCreateInfo samplerInfo = { - .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, - .magFilter = VK_FILTER_LINEAR, - .minFilter = VK_FILTER_LINEAR, - .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, - .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, - .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, - .anisotropyEnable = VK_FALSE, - .maxAnisotropy = 1, - .borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK, - .unnormalizedCoordinates = VK_FALSE, - .compareEnable = VK_FALSE, - .compareOp = VK_COMPARE_OP_ALWAYS, - .mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR, - .mipLodBias = 0.0f, - .minLod = 0.0f, - .maxLod = 0.0f - }; - /* *INDENT-ON* */ - GError *error = NULL; - VkSampler sampler; - VkResult err; - - err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) { - GST_ERROR_OBJECT (conv, "Failed to create sampler: %s", error->message); - g_clear_error (&error); - return NULL; + if (conv->current_shader->uniform_size) { + conv->uniform = + gst_vulkan_buffer_memory_alloc (render->device, VK_FORMAT_R8_UNORM, + conv->current_shader->uniform_size, + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); } - return sampler; -} - -static gboolean -gst_vulkan_color_convert_start (GstBaseTransform * bt) -{ - GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (bt); - - if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt)) - return FALSE; - - if (!(conv->sampler = _create_sampler (conv))) - return FALSE; - if (!(conv->descriptor_pool = _create_descriptor_pool (conv))) - return FALSE; - if (!(conv->descriptor_set = _create_descriptor_set (conv))) - return FALSE; - return TRUE; } @@ -804,6 +1475,7 @@ gst_vulkan_color_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, GstCaps * out_caps) { GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (bt); + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); GstVideoInfo in_info, out_info; int i; @@ -814,6 +1486,11 @@ gst_vulkan_color_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, if (!gst_video_info_from_caps (&out_info, out_caps)) return FALSE; + if (conv->current_shader) { + conv->current_shader->notify (conv->current_shader); + conv->current_shader = NULL; + } + for (i = 0; i < G_N_ELEMENTS (shader_infos); i++) { if (shader_infos[i].from != GST_VIDEO_INFO_FORMAT (&in_info)) continue; @@ -833,10 +1510,41 @@ gst_vulkan_color_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, return FALSE; } + if (render->last_fence) { + if (conv->descriptor_pool) + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_descriptor_pool (gst_vulkan_fence_ref + (render->last_fence), conv->descriptor_pool)); + conv->descriptor_set = NULL; + conv->descriptor_pool = NULL; + if (conv->uniform) + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref + (render->last_fence), (GstMiniObject *) conv->uniform)); + conv->uniform = NULL; + } else { + if (conv->descriptor_pool) + vkDestroyDescriptorPool (render->device->device, + conv->descriptor_pool, NULL); + conv->descriptor_set = NULL; + conv->descriptor_pool = NULL; + if (conv->uniform) + gst_memory_unref (conv->uniform); + conv->uniform = NULL; + } + if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps, out_caps)) return FALSE; + if (!(conv->descriptor_pool = _create_descriptor_pool (conv))) + return FALSE; + if (!(conv->descriptor_set = _create_descriptor_set (conv))) + return FALSE; + + if (!_create_uniform_buffer (conv)) + return FALSE; + return TRUE; } @@ -850,22 +1558,34 @@ gst_vulkan_color_convert_stop (GstBaseTransform * bt) if (render->device) { if (render->last_fence) { - render->trash_list = g_list_prepend (render->trash_list, - gst_vulkan_trash_new_free_descriptor_pool (gst_vulkan_fence_ref - (render->last_fence), conv->descriptor_pool)); + if (conv->descriptor_pool) + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_descriptor_pool (gst_vulkan_fence_ref + (render->last_fence), conv->descriptor_pool)); conv->descriptor_set = NULL; conv->descriptor_pool = NULL; - render->trash_list = g_list_prepend (render->trash_list, - gst_vulkan_trash_new_free_sampler (gst_vulkan_fence_ref - (render->last_fence), conv->sampler)); + if (conv->sampler) + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_sampler (gst_vulkan_fence_ref + (render->last_fence), conv->sampler)); conv->sampler = NULL; + if (conv->uniform) + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref + (render->last_fence), (GstMiniObject *) conv->uniform)); + conv->uniform = NULL; } else { - vkDestroyDescriptorPool (render->device->device, - conv->descriptor_pool, NULL); + if (conv->descriptor_pool) + vkDestroyDescriptorPool (render->device->device, + conv->descriptor_pool, NULL); conv->descriptor_set = NULL; conv->descriptor_pool = NULL; - vkDestroySampler (render->device->device, conv->sampler, NULL); + if (conv->sampler) + vkDestroySampler (render->device->device, conv->sampler, NULL); conv->sampler = NULL; + if (conv->uniform) + gst_memory_unref (conv->uniform); + conv->uniform = NULL; } } @@ -1046,7 +1766,8 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, out_img_mems[i]->barrier.image_layout = out_image_memory_barrier.newLayout; } - conv->current_shader->cmd_state_update (conv, cmd, in_img_mems, out_img_mems); + conv->current_shader->cmd_state_update (conv, cmd, conv->current_shader, + in_img_mems, out_img_mems); if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd, framebuffer)) return GST_FLOW_ERROR; diff --git a/ext/vulkan/vkcolorconvert.h b/ext/vulkan/vkcolorconvert.h index 9d6d4e9..3e9e178 100644 --- a/ext/vulkan/vkcolorconvert.h +++ b/ext/vulkan/vkcolorconvert.h @@ -37,15 +37,23 @@ G_BEGIN_DECLS typedef struct _GstVulkanColorConvert GstVulkanColorConvert; typedef struct _GstVulkanColorConvertClass GstVulkanColorConvertClass; -typedef gboolean (*CommandStateUpdate) (GstVulkanColorConvert * conv, VkCommandBuffer cmd, GstVulkanImageMemory ** src_mems, GstVulkanImageMemory ** dest_mems); +#define MAX_PUSH_CONSTANTS 4 -struct shader_info +typedef struct _shader_info shader_info; + +typedef gboolean (*CommandStateUpdate) (GstVulkanColorConvert * conv, VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageMemory ** src_mems, GstVulkanImageMemory ** dest_mems); + +struct _shader_info { GstVideoFormat from; GstVideoFormat to; CommandStateUpdate cmd_state_update; gchar *frag_code; gsize frag_size; + VkPushConstantRange push_constant_ranges[MAX_PUSH_CONSTANTS]; + gsize uniform_size; + GDestroyNotify notify; + gpointer user_data; }; struct _GstVulkanColorConvert @@ -64,7 +72,8 @@ struct _GstVulkanColorConvert VkDescriptorSetLayoutBinding sampler_layout_binding; VkDescriptorSetLayoutCreateInfo layout_info; - struct shader_info *current_shader; + shader_info *current_shader; + GstMemory *uniform; }; struct _GstVulkanColorConvertClass