From 5d0e1917107f96b59ab2db3b017fb750e73b98d7 Mon Sep 17 00:00:00 2001 From: Michael Olbrich Date: Tue, 30 Oct 2018 12:25:02 +0100 Subject: [PATCH] gl/egl: Add gst_egl_image_from_dmabuf_direct() function The colorspace conversion happens during the upload so the necessary hints must be provided to ensure that the conversion works correctly. At least the Mesa Intel driver will create a texture without error but produces an incorrect result. Use eglQueryDmaBufModifiersEXT() to check if non-external upload is supported for the given format. Based on a patch from Carlos Rafael Giani . https://bugzilla.gnome.org/show_bug.cgi?id=783521 --- gst-libs/gst/gl/egl/gstegl.h | 4 + gst-libs/gst/gl/egl/gsteglimage.c | 346 +++++++++++++++++++++++++++++++++++++- gst-libs/gst/gl/egl/gsteglimage.h | 6 + 3 files changed, 349 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/gl/egl/gstegl.h b/gst-libs/gst/gl/egl/gstegl.h index 877e668..694912e 100644 --- a/gst-libs/gst/gl/egl/gstegl.h +++ b/gst-libs/gst/gl/egl/gstegl.h @@ -52,6 +52,10 @@ typedef gintptr EGLAttrib; #endif +#if !GST_GL_HAVE_EGLUINT64KHR +typedef guint64 EGLuint64KHR; +#endif + GST_GL_API const gchar * gst_egl_get_error_string (EGLint err); diff --git a/gst-libs/gst/gl/egl/gsteglimage.c b/gst-libs/gst/gl/egl/gsteglimage.c index a445666..a68bb09 100644 --- a/gst-libs/gst/gl/egl/gsteglimage.c +++ b/gst-libs/gst/gl/egl/gsteglimage.c @@ -29,9 +29,9 @@ * * #GstEGLImage represents and holds an #EGLImage handle. * - * A #GstEGLImage can be created from a dmabuf with gst_egl_image_from_dmabuf() - * or #GstGLMemoryEGL provides a #GstAllocator to allocate #EGLImage's bound to - * and OpenGL texture. + * A #GstEGLImage can be created from a dmabuf with gst_egl_image_from_dmabuf(), + * or gst_egl_image_from_dmabuf_direct(), or #GstGLMemoryEGL provides a + * #GstAllocator to allocate #EGLImage's bound to and OpenGL texture. */ #ifdef HAVE_CONFIG_H @@ -353,7 +353,7 @@ gst_egl_image_from_texture (GstGLContext * context, GstGLMemory * gl_mem, * target. */ static int -_drm_fourcc_from_info (GstVideoInfo * info, int plane) +_drm_rgba_fourcc_from_info (GstVideoInfo * info, int plane) { GstVideoFormat format = GST_VIDEO_INFO_FORMAT (info); #if G_BYTE_ORDER == G_LITTLE_ENDIAN @@ -423,6 +423,14 @@ _drm_fourcc_from_info (GstVideoInfo * info, int plane) * @plane: the plane in @in_info to create and #GstEGLImage for * @offset: the byte-offset in the data * + * Creates an EGL image that imports the dmabuf FD. The dmabuf data + * is passed as RGBA data. Shaders later take this "RGBA" data and + * convert it from its true format (described by in_info) to actual + * RGBA output. For example, with I420, three EGL images are created, + * one for each plane, each EGL image with a single-channel R format. + * With NV12, two EGL images are created, one with R format, one + * with RG format etc. + * * Returns: a #GstEGLImage wrapping @dmabuf or %NULL on failure */ GstEGLImage * @@ -436,7 +444,7 @@ gst_egl_image_from_dmabuf (GstGLContext * context, gint fourcc; gint i; - fourcc = _drm_fourcc_from_info (in_info, plane); + fourcc = _drm_rgba_fourcc_from_info (in_info, plane); format = gst_gl_format_from_video_info (context, in_info, plane); GST_DEBUG ("fourcc %.4s (%d) plane %d (%dx%d)", @@ -457,12 +465,11 @@ gst_egl_image_from_dmabuf (GstGLContext * context, attribs[atti++] = EGL_DMA_BUF_PLANE0_PITCH_EXT; attribs[atti++] = GST_VIDEO_INFO_PLANE_STRIDE (in_info, plane); attribs[atti] = EGL_NONE; + g_assert (atti == G_N_ELEMENTS (attribs) - 1); for (i = 0; i < atti; i++) GST_LOG ("attr %i: %" G_GINTPTR_FORMAT, i, attribs[i]); - g_assert (atti == 12); - img = _gst_egl_image_create (context, EGL_LINUX_DMA_BUF_EXT, NULL, attribs); if (!img) { GST_WARNING ("eglCreateImage failed: %s", @@ -474,6 +481,331 @@ gst_egl_image_from_dmabuf (GstGLContext * context, (GstEGLImageDestroyNotify) _destroy_egl_image); } +/* + * Variant of _drm_rgba_fourcc_from_info() that is used in case the GPU can + * handle YUV formats directly (by using internal shaders, or hardwired + * YUV->RGB conversion matrices etc.) + */ +static int +_drm_direct_fourcc_from_info (GstVideoInfo * info) +{ + GstVideoFormat format = GST_VIDEO_INFO_FORMAT (info); + + GST_DEBUG ("Getting DRM fourcc for %s", gst_video_format_to_string (format)); + + switch (format) { + case GST_VIDEO_FORMAT_YUY2: + return DRM_FORMAT_YUYV; + + case GST_VIDEO_FORMAT_YVYU: + return DRM_FORMAT_YVYU; + + case GST_VIDEO_FORMAT_UYVY: + return DRM_FORMAT_UYVY; + + case GST_VIDEO_FORMAT_VYUY: + return DRM_FORMAT_VYUY; + + case GST_VIDEO_FORMAT_AYUV: + return DRM_FORMAT_AYUV; + + case GST_VIDEO_FORMAT_NV12: + return DRM_FORMAT_NV12; + + case GST_VIDEO_FORMAT_NV21: + return DRM_FORMAT_NV21; + + case GST_VIDEO_FORMAT_NV16: + return DRM_FORMAT_NV16; + + case GST_VIDEO_FORMAT_NV61: + return DRM_FORMAT_NV61; + + case GST_VIDEO_FORMAT_NV24: + return DRM_FORMAT_NV24; + + case GST_VIDEO_FORMAT_YUV9: + return DRM_FORMAT_YUV410; + + case GST_VIDEO_FORMAT_YVU9: + return DRM_FORMAT_YVU410; + + case GST_VIDEO_FORMAT_Y41B: + return DRM_FORMAT_YUV411; + + case GST_VIDEO_FORMAT_I420: + return DRM_FORMAT_YUV420; + + case GST_VIDEO_FORMAT_YV12: + return DRM_FORMAT_YVU420; + + case GST_VIDEO_FORMAT_Y42B: + return DRM_FORMAT_YUV422; + + case GST_VIDEO_FORMAT_Y444: + return DRM_FORMAT_YUV444; + + case GST_VIDEO_FORMAT_RGB16: + return DRM_FORMAT_RGB565; + + case GST_VIDEO_FORMAT_BGR16: + return DRM_FORMAT_BGR565; + + case GST_VIDEO_FORMAT_RGBA: + return DRM_FORMAT_ABGR8888; + + case GST_VIDEO_FORMAT_RGBx: + return DRM_FORMAT_XBGR8888; + + case GST_VIDEO_FORMAT_BGRA: + return DRM_FORMAT_ARGB8888; + + case GST_VIDEO_FORMAT_BGRx: + return DRM_FORMAT_XRGB8888; + + case GST_VIDEO_FORMAT_ARGB: + return DRM_FORMAT_BGRA8888; + + case GST_VIDEO_FORMAT_xRGB: + return DRM_FORMAT_BGRX8888; + + case GST_VIDEO_FORMAT_ABGR: + return DRM_FORMAT_RGBA8888; + + case GST_VIDEO_FORMAT_xBGR: + return DRM_FORMAT_RGBX8888; + + default: + GST_INFO ("Unsupported format for direct DMABuf."); + return -1; + } +} + +static gboolean +_gst_egl_image_check_dmabuf_direct (GstGLContext * context, + GstVideoInfo * in_info) +{ + EGLDisplay egl_display = EGL_DEFAULT_DISPLAY; + GstGLDisplayEGL *display_egl; + EGLuint64KHR *modifiers; + EGLBoolean *external_only; + int num_modifiers; + gboolean ret; + int i; + + EGLBoolean (*gst_eglQueryDmaBufModifiersEXT) (EGLDisplay dpy, + int format, int max_modifiers, EGLuint64KHR * modifiers, + EGLBoolean * external_only, int *num_modifiers); + + gst_eglQueryDmaBufModifiersEXT = + gst_gl_context_get_proc_address (context, "eglQueryDmaBufModifiersEXT"); + + if (!gst_eglQueryDmaBufModifiersEXT) + return FALSE; + + display_egl = gst_gl_display_egl_from_gl_display (context->display); + if (!display_egl) { + GST_WARNING_OBJECT (context, + "Failed to retrieve GstGLDisplayEGL from %" GST_PTR_FORMAT, + context->display); + return FALSE; + } + egl_display = + (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (display_egl)); + gst_object_unref (display_egl); + + ret = gst_eglQueryDmaBufModifiersEXT (egl_display, + _drm_direct_fourcc_from_info (in_info), 0, NULL, NULL, &num_modifiers); + if (!ret || num_modifiers == 0) + return FALSE; + + modifiers = g_new (EGLuint64KHR, num_modifiers); + external_only = g_new (EGLBoolean, num_modifiers); + + ret = gst_eglQueryDmaBufModifiersEXT (egl_display, + _drm_direct_fourcc_from_info (in_info), num_modifiers, modifiers, + external_only, &num_modifiers); + if (!ret || num_modifiers == 0) { + g_free (modifiers); + g_free (external_only); + return FALSE; + } + + for (i = 0; i < num_modifiers; ++i) { + if (modifiers[i] == DRM_FORMAT_MOD_LINEAR) { + ret = !external_only[i]; + g_free (modifiers); + g_free (external_only); + return ret; + } + } + g_free (modifiers); + g_free (external_only); + return FALSE; +} + +/** + * gst_egl_image_from_dmabuf_direct: + * @context: a #GstGLContext (must be an EGL context) + * @fd: Array of DMABuf file descriptors + * @offset: Array of offsets, relative to the DMABuf + * @in_info: the #GstVideoInfo + * + * Creates an EGL image that imports the dmabuf FD. The dmabuf data + * is passed directly as the format described in in_info. This is + * useful if the hardware is capable of performing color space conversions + * internally. The appropriate DRM format is picked, and the EGL image + * is created with this DRM format. + * + * Another notable difference to gst_egl_image_from_dmabuf() + * is that this function creates one EGL image for all planes, not one for + * a single plane. + * + * Returns: a #GstEGLImage wrapping @dmabuf or %NULL on failure + */ +GstEGLImage * +gst_egl_image_from_dmabuf_direct (GstGLContext * context, + gint * fd, gsize * offset, GstVideoInfo * in_info) +{ + + EGLImageKHR img; + guint n_planes = GST_VIDEO_INFO_N_PLANES (in_info); + gint i; + gboolean with_modifiers; + + /* Explanation of array length: + * - 6 plane independent values are at the start (width, height, format FourCC) + * - 10 values per plane, and there are up to MAX_NUM_DMA_BUF_PLANES planes + * - 4 values for color space and range + * - 1 extra value for the EGL_NONE sentinel + */ + guintptr attribs[41]; /* 6 + 10 * 3 + 4 + 1 */ + gint atti = 0; + + if (!_gst_egl_image_check_dmabuf_direct (context, in_info)) + return NULL; + + with_modifiers = gst_gl_context_check_feature (context, + "EGL_EXT_image_dma_buf_import_with_modifiers"); + + /* EGL DMABuf importation supports a maximum of 3 planes */ + if (G_UNLIKELY (n_planes > 3)) + return NULL; + + attribs[atti++] = EGL_WIDTH; + attribs[atti++] = GST_VIDEO_INFO_WIDTH (in_info); + attribs[atti++] = EGL_HEIGHT; + attribs[atti++] = GST_VIDEO_INFO_HEIGHT (in_info); + attribs[atti++] = EGL_LINUX_DRM_FOURCC_EXT; + attribs[atti++] = _drm_direct_fourcc_from_info (in_info); + + /* first plane */ + { + attribs[atti++] = EGL_DMA_BUF_PLANE0_FD_EXT; + attribs[atti++] = fd[0]; + attribs[atti++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT; + attribs[atti++] = offset[0]; + attribs[atti++] = EGL_DMA_BUF_PLANE0_PITCH_EXT; + attribs[atti++] = in_info->stride[0]; + if (with_modifiers) { + attribs[atti++] = EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT; + attribs[atti++] = DRM_FORMAT_MOD_LINEAR & 0xffffffff; + attribs[atti++] = EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT; + attribs[atti++] = (DRM_FORMAT_MOD_LINEAR >> 32) & 0xffffffff; + } + } + + /* second plane */ + if (n_planes >= 2) { + attribs[atti++] = EGL_DMA_BUF_PLANE1_FD_EXT; + attribs[atti++] = fd[1]; + attribs[atti++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT; + attribs[atti++] = offset[1]; + attribs[atti++] = EGL_DMA_BUF_PLANE1_PITCH_EXT; + attribs[atti++] = in_info->stride[1]; + if (with_modifiers) { + attribs[atti++] = EGL_DMA_BUF_PLANE1_MODIFIER_LO_EXT; + attribs[atti++] = DRM_FORMAT_MOD_LINEAR & 0xffffffff; + attribs[atti++] = EGL_DMA_BUF_PLANE1_MODIFIER_HI_EXT; + attribs[atti++] = (DRM_FORMAT_MOD_LINEAR >> 32) & 0xffffffff; + } + } + + /* third plane */ + if (n_planes == 3) { + attribs[atti++] = EGL_DMA_BUF_PLANE2_FD_EXT; + attribs[atti++] = fd[2]; + attribs[atti++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT; + attribs[atti++] = offset[2]; + attribs[atti++] = EGL_DMA_BUF_PLANE2_PITCH_EXT; + attribs[atti++] = in_info->stride[2]; + if (with_modifiers) { + attribs[atti++] = EGL_DMA_BUF_PLANE2_MODIFIER_LO_EXT; + attribs[atti++] = DRM_FORMAT_MOD_LINEAR & 0xffffffff; + attribs[atti++] = EGL_DMA_BUF_PLANE2_MODIFIER_HI_EXT; + attribs[atti++] = (DRM_FORMAT_MOD_LINEAR >> 32) & 0xffffffff; + } + } + + { + uint32_t color_space; + switch (in_info->colorimetry.matrix) { + case GST_VIDEO_COLOR_MATRIX_BT601: + color_space = EGL_ITU_REC601_EXT; + break; + case GST_VIDEO_COLOR_MATRIX_BT709: + color_space = EGL_ITU_REC709_EXT; + break; + case GST_VIDEO_COLOR_MATRIX_BT2020: + color_space = EGL_ITU_REC2020_EXT; + break; + default: + color_space = 0; + break; + } + if (color_space != 0) { + attribs[atti++] = EGL_YUV_COLOR_SPACE_HINT_EXT; + attribs[atti++] = color_space; + } + } + + { + uint32_t range; + switch (in_info->colorimetry.range) { + case GST_VIDEO_COLOR_RANGE_0_255: + range = EGL_YUV_FULL_RANGE_EXT; + break; + case GST_VIDEO_COLOR_RANGE_16_235: + range = EGL_YUV_NARROW_RANGE_EXT; + break; + default: + range = 0; + break; + } + if (range != 0) { + attribs[atti++] = EGL_SAMPLE_RANGE_HINT_EXT; + attribs[atti++] = range; + } + } + + /* Add the EGL_NONE sentinel */ + attribs[atti] = EGL_NONE; + g_assert (atti <= G_N_ELEMENTS (attribs) - 1); + + for (i = 0; i < atti; i++) + GST_LOG ("attr %i: %" G_GINTPTR_FORMAT, i, attribs[i]); + + img = _gst_egl_image_create (context, EGL_LINUX_DMA_BUF_EXT, NULL, attribs); + if (!img) { + GST_WARNING ("eglCreateImage failed: %s", + gst_egl_get_error_string (eglGetError ())); + return NULL; + } + + return gst_egl_image_new_wrapped (context, img, GST_GL_RGBA, NULL, + (GstEGLImageDestroyNotify) _destroy_egl_image); +} + gboolean gst_egl_image_export_dmabuf (GstEGLImage * image, int *fd, gint * stride, gsize * offset) diff --git a/gst-libs/gst/gl/egl/gsteglimage.h b/gst-libs/gst/gl/egl/gsteglimage.h index eda9c7c..b86788a 100644 --- a/gst-libs/gst/gl/egl/gsteglimage.h +++ b/gst-libs/gst/gl/egl/gsteglimage.h @@ -89,6 +89,12 @@ GstEGLImage * gst_egl_image_from_dmabuf (GstGLContext * gint plane, gsize offset); GST_GL_API +GstEGLImage * gst_egl_image_from_dmabuf_direct (GstGLContext * context, + gint *fd, + gsize *offset, + GstVideoInfo * in_info); + +GST_GL_API gboolean gst_egl_image_export_dmabuf (GstEGLImage *image, int *fd, gint *stride, gsize *offset); #endif -- 2.7.4