From: Matthew Waters Date: Thu, 27 Sep 2018 03:35:15 +0000 (+1000) Subject: gl: add a new overlay compositor element X-Git-Tag: 1.16.2~411 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=f960beaa98ddd144a8f3d1974ef0093d20a00450;p=platform%2Fupstream%2Fgst-plugins-base.git gl: add a new overlay compositor element Flattens all the overlays from the GstVideoOverlayCompositionMeta into the video stream. https://bugzilla.gnome.org/show_bug.cgi?id=759867 --- diff --git a/ext/gl/Makefile.am b/ext/gl/Makefile.am index a6367b5..7c8c886 100644 --- a/ext/gl/Makefile.am +++ b/ext/gl/Makefile.am @@ -52,6 +52,7 @@ libgstopengl_la_SOURCES = \ gstgldeinterlace.c \ gltestsrc.c \ gstgltestsrc.c \ + gstgloverlaycompositorelement.c \ gstglutils.c noinst_HEADERS = \ @@ -75,6 +76,7 @@ noinst_HEADERS = \ gstglviewconvert.h \ gltestsrc.h \ gstgltestsrc.h \ + gstgloverlaycompositorelement.h \ gstglutils.h # full opengl required diff --git a/ext/gl/gstgloverlaycompositorelement.c b/ext/gl/gstgloverlaycompositorelement.c new file mode 100644 index 0000000..1380d89 --- /dev/null +++ b/ext/gl/gstgloverlaycompositorelement.c @@ -0,0 +1,343 @@ +/* + * gloverlaycompositor element + * Copyrithg (C) 2018 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-glcompositoroverlay + * @title: glcompositoroverlay + * + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "gstgloverlaycompositorelement.h" + +enum +{ + PROP_0, + PROP_LAST, +}; + +#define GST_CAT_DEFAULT gst_gl_overlay_compositor_element_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +#define DEBUG_INIT \ + GST_DEBUG_CATEGORY_INIT (gst_gl_overlay_compositor_element_debug, "gloverlaycompositorelement", 0, "gloverlaycompositor element"); +#define gst_gl_overlay_compositor_element_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstGLOverlayCompositorElement, + gst_gl_overlay_compositor_element, GST_TYPE_GL_FILTER, DEBUG_INIT); + +static GstStaticPadTemplate overlay_sink_pad_template = + GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + "RGBA") ", texture-target=(string) { 2D, rectangle } ; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA") ", texture-target=(string) { 2D, rectangle } ; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY", + "RGBA") ", texture-target=(string) { 2D, rectangle } ")); + +static GstStaticPadTemplate overlay_src_pad_template = + GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY "," + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + "RGBA") ", texture-target=(string) { 2D, rectangle } ; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA") ", texture-target=(string) { 2D, rectangle } ; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY", + "RGBA") ", texture-target=(string) { 2D, rectangle } ")); + +static gboolean +gst_gl_overlay_compositor_element_propose_allocation (GstBaseTransform * trans, + GstQuery * decide_query, GstQuery * query); +static GstFlowReturn _oce_prepare_output_buffer (GstBaseTransform * bt, + GstBuffer * buffer, GstBuffer ** outbuf); + +static gboolean gst_gl_overlay_compositor_element_gl_start (GstGLBaseFilter * + base); +static void gst_gl_overlay_compositor_element_gl_stop (GstGLBaseFilter * base); + +static GstCaps *_oce_transform_internal_caps (GstGLFilter * + filter, GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps); +static gboolean gst_gl_overlay_compositor_element_filter (GstGLFilter * filter, + GstBuffer * inbuf, GstBuffer * outbuf); +static gboolean gst_gl_overlay_compositor_element_filter_texture (GstGLFilter * + filter, GstGLMemory * in_tex, GstGLMemory * out_tex); +static gboolean gst_gl_overlay_compositor_element_callback (GstGLFilter * + filter, GstGLMemory * in_tex, gpointer stuff); + +static void +gst_gl_overlay_compositor_element_class_init (GstGLOverlayCompositorElementClass + * klass) +{ + GstElementClass *element_class; + + element_class = GST_ELEMENT_CLASS (klass); + + gst_element_class_set_metadata (element_class, + "OpenGL overlaying filter", "Filter/Effect", + "Flatten a stream containing GstVideoOverlayCompositionMeta", + ""); + + gst_element_class_add_static_pad_template (element_class, + &overlay_src_pad_template); + gst_element_class_add_static_pad_template (element_class, + &overlay_sink_pad_template); + + GST_BASE_TRANSFORM_CLASS (klass)->passthrough_on_same_caps = TRUE; + GST_BASE_TRANSFORM_CLASS (klass)->propose_allocation = + gst_gl_overlay_compositor_element_propose_allocation; + GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer = + _oce_prepare_output_buffer; + + GST_GL_FILTER_CLASS (klass)->filter = + gst_gl_overlay_compositor_element_filter; + GST_GL_FILTER_CLASS (klass)->filter_texture = + gst_gl_overlay_compositor_element_filter_texture; + GST_GL_FILTER_CLASS (klass)->transform_internal_caps = + _oce_transform_internal_caps; + + GST_GL_BASE_FILTER_CLASS (klass)->gl_start = + gst_gl_overlay_compositor_element_gl_start; + GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = + gst_gl_overlay_compositor_element_gl_stop; + GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api = + GST_GL_API_OPENGL | GST_GL_API_GLES2 | GST_GL_API_OPENGL3; +} + +static void +gst_gl_overlay_compositor_element_init (GstGLOverlayCompositorElement * + overlay_compositor_element) +{ +} + +static GstCaps * +_oce_transform_internal_caps (GstGLFilter * filter, + GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps) +{ + GstCaps *ret; + + /* add/remove the composition overlay meta as necessary */ + if (direction == GST_PAD_SRC) { + ret = gst_gl_overlay_compositor_add_caps (gst_caps_copy (caps)); + } else { + guint i, n; + GstCaps *removed; + + ret = gst_caps_copy (caps); + removed = gst_caps_copy (caps); + n = gst_caps_get_size (removed); + for (i = 0; i < n; i++) { + GstCapsFeatures *feat = gst_caps_get_features (removed, i); + + if (feat) { + feat = gst_caps_features_copy (feat); + + if (gst_caps_features_contains (feat, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) { + /* prefer the passthrough case */ + gst_caps_features_remove (feat, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + gst_caps_set_features (removed, i, feat); + } + } + } + + ret = gst_caps_merge (ret, removed); + } + + GST_DEBUG_OBJECT (filter, "meta modifications returned caps %" GST_PTR_FORMAT, + ret); + return ret; +} + +static gboolean +gst_gl_overlay_compositor_element_propose_allocation (GstBaseTransform * trans, + GstQuery * decide_query, GstQuery * query) +{ + GstStructure *allocation_meta = NULL; + guint width = 0, height = 0; + + if (!GST_BASE_TRANSFORM_CLASS (parent_class)->propose_allocation (trans, + decide_query, query)) + return FALSE; + + if ((width == 0 || height == 0) && decide_query) { + GstCaps *decide_caps; + gst_query_parse_allocation (decide_query, &decide_caps, NULL); + + if (decide_caps) { + GstVideoInfo vinfo; + + if (gst_video_info_from_caps (&vinfo, decide_caps)) { + width = GST_VIDEO_INFO_WIDTH (&vinfo); + height = GST_VIDEO_INFO_HEIGHT (&vinfo); + } + } + } + + if ((width == 0 || height == 0) && query) { + GstCaps *caps; + gst_query_parse_allocation (decide_query, &caps, NULL); + + if (caps) { + GstVideoInfo vinfo; + + if (gst_video_info_from_caps (&vinfo, caps)) { + width = GST_VIDEO_INFO_WIDTH (&vinfo); + height = GST_VIDEO_INFO_HEIGHT (&vinfo); + } + } + } + + if (width != 0 && height != 0) { + allocation_meta = + gst_structure_new ("GstVideoOverlayCompositionMeta", + "width", G_TYPE_UINT, width, "height", G_TYPE_UINT, height, NULL); + } + + GST_DEBUG_OBJECT (trans, "Adding overlay composition meta with size %ux%u", + width, height); + gst_query_add_allocation_meta (query, + GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, allocation_meta); + + return TRUE; +} + +static void +gst_gl_overlay_compositor_element_gl_stop (GstGLBaseFilter * base) +{ + GstGLOverlayCompositorElement *self = + GST_GL_OVERLAY_COMPOSITOR_ELEMENT (base); + + if (self->shader) + gst_object_unref (self->shader); + self->shader = NULL; + + if (self->overlay_compositor) { + gst_gl_overlay_compositor_free_overlays (self->overlay_compositor); + gst_object_unref (self->overlay_compositor); + } + self->overlay_compositor = NULL; + + GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base); +} + +static gboolean +gst_gl_overlay_compositor_element_gl_start (GstGLBaseFilter * base) +{ + GstGLOverlayCompositorElement *self = + GST_GL_OVERLAY_COMPOSITOR_ELEMENT (base); + GError *error = NULL; + + self->overlay_compositor = gst_gl_overlay_compositor_new (base->context); + g_object_set (self->overlay_compositor, "yinvert", TRUE, NULL); + + if (!(self->shader = gst_gl_shader_new_default (base->context, &error))) { + GST_ELEMENT_ERROR (base, RESOURCE, NOT_FOUND, ("%s", + "Failed to compile identity shader"), ("%s", error->message)); + return FALSE; + } + + return GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base); +} + +static GstFlowReturn +_oce_prepare_output_buffer (GstBaseTransform * bt, + GstBuffer * buffer, GstBuffer ** outbuf) +{ + GstGLOverlayCompositorElement *self = GST_GL_OVERLAY_COMPOSITOR_ELEMENT (bt); + GstVideoOverlayCompositionMeta *comp_meta; + + if (gst_base_transform_is_passthrough (bt)) + goto passthrough; + + if (!self->overlay_compositor) + return GST_FLOW_NOT_NEGOTIATED; + + comp_meta = gst_buffer_get_video_overlay_composition_meta (buffer); + if (!comp_meta) + goto passthrough; + + if (gst_video_overlay_composition_n_rectangles (comp_meta->overlay) == 0) + goto passthrough; + + return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (bt, + buffer, outbuf); + +passthrough: + GST_LOG_OBJECT (bt, "passthrough detected, forwarding input buffer"); + *outbuf = buffer; + return GST_FLOW_OK; +} + +static gboolean +gst_gl_overlay_compositor_element_filter (GstGLFilter * filter, + GstBuffer * inbuf, GstBuffer * outbuf) +{ + GstGLOverlayCompositorElement *self = + GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter); + + if (inbuf == outbuf) + return TRUE; + + gst_gl_overlay_compositor_upload_overlays (self->overlay_compositor, inbuf); + + return gst_gl_filter_filter_texture (filter, inbuf, outbuf); +} + +static gboolean +gst_gl_overlay_compositor_element_filter_texture (GstGLFilter * filter, + GstGLMemory * in_tex, GstGLMemory * out_tex) +{ + GstGLOverlayCompositorElement *self = + GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter); + + gst_gl_filter_render_to_target_with_shader (filter, in_tex, out_tex, + self->shader); + + gst_gl_filter_render_to_target (filter, NULL, out_tex, + gst_gl_overlay_compositor_element_callback, NULL); + + return TRUE; +} + +static gboolean +gst_gl_overlay_compositor_element_callback (GstGLFilter * filter, + GstGLMemory * in_tex, gpointer stuff) +{ + GstGLOverlayCompositorElement *self = + GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter); + + GST_LOG_OBJECT (self, "drawing overlays"); + + gst_gl_overlay_compositor_draw_overlays (self->overlay_compositor); + + return TRUE; +} diff --git a/ext/gl/gstgloverlaycompositorelement.h b/ext/gl/gstgloverlaycompositorelement.h new file mode 100644 index 0000000..548fc79 --- /dev/null +++ b/ext/gl/gstgloverlaycompositorelement.h @@ -0,0 +1,51 @@ +/* + * glshader gstreamer plugin + * Copyright (C) 2018 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _GST_GL_OVERLAY_COMPOSITOR_ELEMENT_H_ +#define _GST_GL_OVERLAY_COMPOSITOR_ELEMENT_H_ + +#include + +#define GST_TYPE_GL_OVERLAY_COMPOSITOR_ELEMENT (gst_gl_overlay_compositor_element_get_type()) +#define GST_GL_OVERLAY_COMPOSITOR_ELEMENT(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_OVERLAY_COMPOSITOR_ELEMENT,GstGLOverlayCompositorElement)) +#define GST_IS_GL_OVERLAY_COMPOSITOR_ELEMENT(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_OVERLAY_COMPOSITOR_ELEMENT)) +#define GST_GL_OVERLAY_COMPOSITOR_ELEMENT_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass) ,GST_TYPE_GL_OVERLAY_COMPOSITOR_ELEMENT,GstGLOverlayCompositorElementClass)) +#define GST_IS_GL_OVERLAY_COMPOSITOR_ELEMENT_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass) ,GST_TYPE_GL_OVERLAY_COMPOSITOR_ELEMENT)) +#define GST_GL_OVERLAY_COMPOSITOR_ELEMENT_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj) ,GST_TYPE_GL_OVERLAY_COMPOSITOR_ELEMENT,GstGLOverlayCompositorElementClass)) + +typedef struct _GstGLOverlayCompositorElement GstGLOverlayCompositorElement; +typedef struct _GstGLOverlayCompositorElementClass GstGLOverlayCompositorElementClass; + +struct _GstGLOverlayCompositorElement +{ + GstGLFilter filter; + + GstGLShader *shader; + GstGLOverlayCompositor *overlay_compositor; +}; + +struct _GstGLOverlayCompositorElementClass +{ + GstGLFilterClass filter_class; +}; + +GType gst_gl_overlay_compositor_element_get_type (void); + +#endif /* _GST_GL_OVERLAY_COMPOSITOR_ELEMENT_H_ */ diff --git a/ext/gl/gstopengl.c b/ext/gl/gstopengl.c index ffd8af3..43f6462 100644 --- a/ext/gl/gstopengl.c +++ b/ext/gl/gstopengl.c @@ -71,6 +71,7 @@ #include "gstgltestsrc.h" #include "gstgldeinterlace.h" #include "gstglalpha.h" +#include "gstgloverlaycompositorelement.h" #ifdef HAVE_GRAPHENE #include "gstgltransformation.h" @@ -251,6 +252,11 @@ plugin_init (GstPlugin * plugin) GST_RANK_NONE, GST_TYPE_GL_ALPHA)) { return FALSE; } + + if (!gst_element_register (plugin, "gloverlaycompositor", + GST_RANK_NONE, GST_TYPE_GL_OVERLAY_COMPOSITOR_ELEMENT)) { + return FALSE; + } #if defined(HAVE_JPEG) && defined(HAVE_PNG) if (!gst_element_register (plugin, "gloverlay", GST_RANK_NONE, gst_gl_overlay_get_type ())) { diff --git a/ext/gl/meson.build b/ext/gl/meson.build index f92b82a..766da3e 100644 --- a/ext/gl/meson.build +++ b/ext/gl/meson.build @@ -44,6 +44,7 @@ opengl_sources = [ 'gstgldeinterlace.c', 'gltestsrc.c', 'gstgltestsrc.c', + 'gstgloverlaycompositor.c', 'gstglutils.c' ]