From: Matthew Waters Date: Wed, 26 Feb 2020 07:29:06 +0000 (+1100) Subject: qt: add a qml overlay filter element X-Git-Tag: 1.19.3~509^2~678 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=4778d7166a02caf793df4f845dc35b6933d87c81;p=platform%2Fupstream%2Fgstreamer.git qt: add a qml overlay filter element It takes a qml scene description and renders it using a possible input stream. Currently supported on GLX and WGL. --- diff --git a/ext/qt/gstplugin.cc b/ext/qt/gstplugin.cc index 79fb181..196cf09 100644 --- a/ext/qt/gstplugin.cc +++ b/ext/qt/gstplugin.cc @@ -22,6 +22,7 @@ #include "config.h" #endif +#include "gstqtoverlay.h" #include "gstqtsink.h" #include "gstqtsrc.h" #include @@ -33,11 +34,16 @@ plugin_init (GstPlugin * plugin) GST_RANK_NONE, GST_TYPE_QT_SINK)) { return FALSE; } - + if (!gst_element_register (plugin, "qmlglsrc", GST_RANK_NONE, GST_TYPE_QT_SRC)) { return FALSE; } + + if (!gst_element_register (plugin, "qmlgloverlay", + GST_RANK_NONE, GST_TYPE_QT_OVERLAY)) { + return FALSE; + } /* this means the plugin must be loaded before the qml engine is loaded */ qmlRegisterType ("org.freedesktop.gstreamer.GLVideoItem", 1, 0, "GstGLVideoItem"); diff --git a/ext/qt/gstqtglutility.cc b/ext/qt/gstqtglutility.cc index 20e8ead..ba17282 100644 --- a/ext/qt/gstqtglutility.cc +++ b/ext/qt/gstqtglutility.cc @@ -28,10 +28,13 @@ #if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) #include #include +#include #endif #if GST_GL_HAVE_WINDOW_WAYLAND && GST_GL_HAVE_PLATFORM_EGL && defined (HAVE_QT_WAYLAND) +#include #include +#include #include #endif @@ -48,11 +51,19 @@ #endif #endif +#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) +#include +#include +#endif + #include #define GST_CAT_DEFAULT qt_gl_utils_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); +G_LOCK_DEFINE_STATIC (display_lock); +static GWeakRef qt_display; + GstGLDisplay * gst_qt_get_gl_display () { @@ -67,6 +78,16 @@ gst_qt_get_gl_display () "Qt gl utility functions"); g_once_init_leave (&_debug, 1); } + + G_LOCK (display_lock); + /* XXX: this assumes that only one display will ever be created by Qt */ + display = static_cast(g_weak_ref_get (&qt_display)); + if (display) { + GST_INFO ("returning previously created display"); + G_UNLOCK (display_lock); + return display; + } + GST_INFO ("QGuiApplication::instance()->platformName() %s", app->platformName().toUtf8().data()); #if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) @@ -145,6 +166,9 @@ gst_qt_get_gl_display () if (!display) display = gst_gl_display_new (); + g_weak_ref_set (&qt_display, display); + G_UNLOCK (display_lock); + return display; } @@ -159,6 +183,17 @@ gst_qt_get_gl_wrapcontext (GstGLDisplay * display, g_return_val_if_fail (display != NULL && wrap_glcontext != NULL, FALSE); + /* see if we already have a current GL context in GStreamer for this thread */ + { + GstGLContext *current = gst_gl_context_get_current (); + if (current) { + if (current->display == display) { + *wrap_glcontext = static_cast (gst_object_ref (current)); + return TRUE; + } + } + } + #if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) if (GST_IS_GL_DISPLAY_X11 (display)) { #if GST_GL_HAVE_PLATFORM_GLX @@ -262,3 +297,56 @@ gst_qt_get_gl_wrapcontext (GstGLDisplay * display, return TRUE; } + +QVariant +qt_opengl_native_context_from_gst_gl_context (GstGLContext * context) +{ + guintptr handle; + GstGLPlatform platform; + + handle = gst_gl_context_get_gl_context (context); + platform = gst_gl_context_get_gl_platform (context); + +#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) + if (platform == GST_GL_PLATFORM_GLX) { + GstGLDisplay *display = gst_gl_context_get_display (context); + Display *xdisplay = (Display *) gst_gl_display_get_handle (display); + gst_object_unref (display); + return QVariant::fromValue(QGLXNativeContext((GLXContext) handle, xdisplay)); + } +#endif +#if GST_GL_HAVE_PLATFORM_EGL + if (platform == GST_GL_PLATFORM_EGL) { +#if GST_GL_HAVE_WINDOW_WAYLAND && defined (HAVE_QT_WAYLAND) + GstGLDisplay *display = gst_gl_context_get_display (context); + if (gst_gl_display_get_handle_type (display) == GST_GL_DISPLAY_TYPE_WAYLAND) { + GstGLDisplayEGL *display_egl = gst_gl_display_egl_from_gl_display (display); + if (display_egl) { + EGLDisplay egl_display = (EGLDisplay) gst_gl_display_get_handle ((GstGLDisplay *) display_egl); + gst_object_unref (display_egl); + gst_object_unref (display); + return QVariant::fromValue(QEGLNativeContext((EGLContext) handle, egl_display)); + } + } +#endif + } +#endif +#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) + if (platform == GST_GL_PLATFORM_WGL) { + GstGLWindow *window = gst_gl_context_get_window (context); + guintptr hwnd = gst_gl_window_get_window_handle (window); + gst_object_unref (window); + return QVariant::fromValue(QWGLNativeContext((HGLRC) handle, (HWND) hwnd)); + } +#endif + { + gchar *platform_s = gst_gl_platform_to_string (platform); + g_warning ("Unimplemented configuration! This means either:\n" + "1. The qmlgl plugin was built without support for your platform.\n" + "2. The necessary code to convert from a GstGLContext to Qt's " + "native context type for \'%s\' currently does not exist.", + platform_s); + g_free (platform_s); + } + return QVariant::fromValue(nullptr); +} diff --git a/ext/qt/gstqtglutility.h b/ext/qt/gstqtglutility.h index ca90237..ecf3e00 100644 --- a/ext/qt/gstqtglutility.h +++ b/ext/qt/gstqtglutility.h @@ -24,11 +24,15 @@ #include #include +#include + G_BEGIN_DECLS GstGLDisplay * gst_qt_get_gl_display (); gboolean gst_qt_get_gl_wrapcontext (GstGLDisplay * display, GstGLContext **wrap_glcontext, GstGLContext **context); +QVariant qt_opengl_native_context_from_gst_gl_context (GstGLContext * context); + G_END_DECLS #endif /* __QT_GL_UTILS_H__ */ diff --git a/ext/qt/gstqtoverlay.cc b/ext/qt/gstqtoverlay.cc new file mode 100644 index 0000000..0c52634 --- /dev/null +++ b/ext/qt/gstqtoverlay.cc @@ -0,0 +1,305 @@ +/* + * GStreamer + * Copyright (C) 2020 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstqtoverlay + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqtoverlay.h" +#include "qtglrenderer.h" + +#include + +#define GST_CAT_DEFAULT gst_debug_qt_gl_overlay +GST_DEBUG_CATEGORY (GST_CAT_DEFAULT); + +static void gst_qt_overlay_finalize (GObject * object); +static void gst_qt_overlay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * param_spec); +static void gst_qt_overlay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * param_spec); + +static gboolean gst_qt_overlay_gl_start (GstGLBaseFilter * bfilter); +static void gst_qt_overlay_gl_stop (GstGLBaseFilter * bfilter); +static gboolean gst_qt_overlay_gl_set_caps (GstGLBaseFilter * bfilter, + GstCaps * in_caps, GstCaps * out_caps); + +static GstFlowReturn gst_qt_overlay_prepare_output_buffer (GstBaseTransform * btrans, + GstBuffer * buffer, GstBuffer ** outbuf); +static GstFlowReturn gst_qt_overlay_transform (GstBaseTransform * btrans, + GstBuffer * inbuf, GstBuffer * outbuf); + +enum +{ + PROP_0, + PROP_WIDGET, + PROP_QML_SCENE, +}; + +enum +{ + SIGNAL_0, + SIGNAL_QML_SCENE_INITIALIZED, + LAST_SIGNAL +}; + +static guint gst_qt_overlay_signals[LAST_SIGNAL] = { 0 }; + +#define gst_qt_overlay_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstQtOverlay, gst_qt_overlay, + GST_TYPE_GL_FILTER, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, + "qtoverlay", 0, "Qt Video Overlay")); + +static void +gst_qt_overlay_class_init (GstQtOverlayClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstBaseTransformClass *btrans_class; + GstGLBaseFilterClass *glbasefilter_class; + GstGLFilterClass *glfilter_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + glbasefilter_class = (GstGLBaseFilterClass *) klass; + glfilter_class = (GstGLFilterClass *) klass; + btrans_class = (GstBaseTransformClass *) klass; + + gobject_class->set_property = gst_qt_overlay_set_property; + gobject_class->get_property = gst_qt_overlay_get_property; + gobject_class->finalize = gst_qt_overlay_finalize; + + gst_element_class_set_metadata (gstelement_class, "Qt Video Overlay", + "Filter/QML/Overlay", "A filter that renders a QML scene onto a video stream", + "Matthew Waters "); + + g_object_class_install_property (gobject_class, PROP_QML_SCENE, + g_param_spec_string ("qml-scene", "QML Scene", + "The contents of the QML scene", NULL, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (gobject_class, PROP_WIDGET, + g_param_spec_pointer ("widget", "QQuickItem", + "The QQuickItem to place the input video in the object hierarchy", + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** + * GstQmlGLOverlay::qml-scene-initialized + * @element: the #GstQmlGLOverlay + * @root_item: the `QQuickItem` found to be the root item + * @user_data: user provided data + */ + gst_qt_overlay_signals[SIGNAL_QML_SCENE_INITIALIZED] = + g_signal_new ("qml-scene-initialized", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_POINTER); + + gst_gl_filter_add_rgba_pad_templates (glfilter_class); + + btrans_class->prepare_output_buffer = gst_qt_overlay_prepare_output_buffer; + btrans_class->transform = gst_qt_overlay_transform; + + glbasefilter_class->gl_start = gst_qt_overlay_gl_start; + glbasefilter_class->gl_stop = gst_qt_overlay_gl_stop; + glbasefilter_class->gl_set_caps = gst_qt_overlay_gl_set_caps; +} + +static void +gst_qt_overlay_init (GstQtOverlay * qt_overlay) +{ + qt_overlay->widget = QSharedPointer(); +} + +static void +gst_qt_overlay_finalize (GObject * object) +{ + GstQtOverlay *qt_overlay = GST_QT_OVERLAY (object); + + qt_overlay->widget.clear(); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_qt_overlay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQtOverlay *qt_overlay = GST_QT_OVERLAY (object); + + switch (prop_id) { + case PROP_WIDGET: { + QtGLVideoItem *qt_item = static_cast (g_value_get_pointer (value)); + if (qt_item) + qt_overlay->widget = qt_item->getInterface(); + else + qt_overlay->widget.clear(); + break; + } + case PROP_QML_SCENE: + g_free (qt_overlay->qml_scene); + qt_overlay->qml_scene = g_value_dup_string (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qt_overlay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstQtOverlay *qt_overlay = GST_QT_OVERLAY (object); + + switch (prop_id) { + case PROP_WIDGET: + /* This is not really safe - the app needs to be + * sure the widget is going to be kept alive or + * this can crash */ + if (qt_overlay->widget) + g_value_set_pointer (value, qt_overlay->widget->videoItem()); + else + g_value_set_pointer (value, NULL); + break; + case PROP_QML_SCENE: + g_value_set_string (value, qt_overlay->qml_scene); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_qt_overlay_gl_start (GstGLBaseFilter * bfilter) +{ + GstQtOverlay *qt_overlay = GST_QT_OVERLAY (bfilter); + QQuickItem *root; + GError *error = NULL; + + GST_TRACE_OBJECT (bfilter, "using scene:\n%s", qt_overlay->qml_scene); + + if (!qt_overlay->qml_scene || g_strcmp0 (qt_overlay->qml_scene, "") == 0) { + GST_ELEMENT_ERROR (bfilter, RESOURCE, NOT_FOUND, ("qml-scene property not set"), (NULL)); + return FALSE; + } + + if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (bfilter)) + return FALSE; + + qt_overlay->renderer = new GstQuickRenderer; + if (!qt_overlay->renderer->init (bfilter->context, &error)) { + GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND, + ("%s", error->message), (NULL)); + return FALSE; + } + /* FIXME: Qml may do async loading and we need to propagate qml errors in that case as well */ + if (!qt_overlay->renderer->setQmlScene (qt_overlay->qml_scene, &error)) { + GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND, + ("%s", error->message), (NULL)); + return FALSE; + } + + root = qt_overlay->renderer->rootItem(); + if (!root) { + GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND, + ("Qml scene does not have a root item"), (NULL)); + return FALSE; + } + + g_signal_emit (qt_overlay, gst_qt_overlay_signals[SIGNAL_QML_SCENE_INITIALIZED], 0, root); + + return TRUE; +} + +static void +gst_qt_overlay_gl_stop (GstGLBaseFilter * bfilter) +{ + GstQtOverlay *qt_overlay = GST_QT_OVERLAY (bfilter); + + if (qt_overlay->renderer) { + qt_overlay->renderer->cleanup(); + delete qt_overlay->renderer; + } + qt_overlay->renderer = NULL; + + GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (bfilter); +} + +static gboolean +gst_qt_overlay_gl_set_caps (GstGLBaseFilter * bfilter, GstCaps * in_caps, + GstCaps * out_caps) +{ + GstGLFilter *filter = GST_GL_FILTER (bfilter); + GstQtOverlay *qt_overlay = GST_QT_OVERLAY (bfilter); + + if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_set_caps (bfilter, in_caps, out_caps)) + return FALSE; + + qt_overlay->renderer->setSize (GST_VIDEO_INFO_WIDTH (&filter->out_info), + GST_VIDEO_INFO_HEIGHT (&filter->out_info)); + + return TRUE; +} + +static GstFlowReturn +gst_qt_overlay_prepare_output_buffer (GstBaseTransform * btrans, + GstBuffer * buffer, GstBuffer ** outbuf) +{ + GstBaseTransformClass *bclass = GST_BASE_TRANSFORM_GET_CLASS (btrans); + GstGLBaseFilter *bfilter = GST_GL_BASE_FILTER (btrans); + GstGLFilter *filter = GST_GL_FILTER (btrans); + GstQtOverlay *qt_overlay = GST_QT_OVERLAY (btrans); + GstGLMemory *out_mem; + + if (qt_overlay->widget) { + qt_overlay->widget->setCaps (bfilter->in_caps); + qt_overlay->widget->setBuffer (buffer); + } + + /* XXX: is this the correct ts to drive the animation */ + out_mem = qt_overlay->renderer->generateOutput (GST_BUFFER_PTS (buffer)); + if (!out_mem) { + GST_ERROR_OBJECT (qt_overlay, "Failed to generate output"); + return GST_FLOW_ERROR; + } + + *outbuf = gst_buffer_new (); + gst_buffer_append_memory (*outbuf, (GstMemory *) out_mem); + gst_buffer_add_video_meta (*outbuf, (GstVideoFrameFlags) 0, + GST_VIDEO_INFO_FORMAT (&filter->out_info), + GST_VIDEO_INFO_WIDTH (&filter->in_info), + GST_VIDEO_INFO_HEIGHT (&filter->out_info)); + + bclass->copy_metadata (btrans, buffer, *outbuf); + + return GST_FLOW_OK; +} + +static GstFlowReturn +gst_qt_overlay_transform (GstBaseTransform * btrans, GstBuffer * inbuf, + GstBuffer * outbuf) +{ + return GST_FLOW_OK; +} diff --git a/ext/qt/gstqtoverlay.h b/ext/qt/gstqtoverlay.h new file mode 100644 index 0000000..6e438f3 --- /dev/null +++ b/ext/qt/gstqtoverlay.h @@ -0,0 +1,76 @@ +/* + * GStreamer + * Copyright (C) 2020 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QT_OVERLAY_H__ +#define __GST_QT_OVERLAY_H__ + +#include +#include +#include +#include "qtglrenderer.h" +#include "qtitem.h" + +typedef struct _GstQtOverlay GstQtOverlay; +typedef struct _GstQtOverlayClass GstQtOverlayClass; +typedef struct _GstQtOverlayPrivate GstQtOverlayPrivate; + +G_BEGIN_DECLS + +GType gst_qt_overlay_get_type (void); +#define GST_TYPE_QT_OVERLAY (gst_qt_overlay_get_type()) +#define GST_QT_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_QT_OVERLAY,GstQtOverlay)) +#define GST_QT_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_QT_OVERLAY,GstQtOverlayClass)) +#define GST_IS_QT_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_QT_OVERLAY)) +#define GST_IS_QT_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_QT_OVERLAY)) +#define GST_QT_OVERLAY_CAST(obj) ((GstQtOverlay*)(obj)) + +/** + * GstQtOverlay: + * + * Opaque #GstQtOverlay object + */ +struct _GstQtOverlay +{ + /* */ + GstGLFilter parent; + + gchar *qml_scene; + + GstQuickRenderer *renderer; + + QSharedPointer widget; +}; + +/** + * GstQtOverlayClass: + * + * The #GstQtOverlayClass struct only contains private data + */ +struct _GstQtOverlayClass +{ + /* */ + GstGLFilterClass parent_class; +}; + +GstQtOverlay * gst_qt_overlay_new (void); + +G_END_DECLS + +#endif /* __GST_QT_OVERLAY_H__ */ diff --git a/ext/qt/meson.build b/ext/qt/meson.build index badcae2..99fc1ee 100644 --- a/ext/qt/meson.build +++ b/ext/qt/meson.build @@ -2,8 +2,10 @@ sources = [ 'gstplugin.cc', 'gstqsgtexture.cc', 'gstqtglutility.cc', + 'gstqtoverlay.cc', 'gstqtsink.cc', 'gstqtsrc.cc', + 'qtglrenderer.cc', 'qtitem.cc', 'qtwindow.cc', ] @@ -12,6 +14,7 @@ moc_headers = [ 'qtitem.h', 'qtwindow.h', 'gstqsgtexture.h', + 'qtglrenderer.h', ] # FIXME: -Dqt5=enabled is silently ignored if a c++ compiler is not found diff --git a/ext/qt/qtglrenderer.cc b/ext/qt/qtglrenderer.cc new file mode 100644 index 0000000..0e1b3ae --- /dev/null +++ b/ext/qt/qtglrenderer.cc @@ -0,0 +1,446 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "qtglrenderer.h" +#include "gstqtglutility.h" + +#define GST_CAT_DEFAULT gst_qt_gl_renderer_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +static void +init_debug (void) +{ + static volatile gsize _debug; + + if (g_once_init_enter (&_debug)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtglrenderer", 0, + "Qt OpenGL Renderer"); + g_once_init_leave (&_debug, 1); + } +} + +/* Needs to be based on QWindow otherwise (at least) windows and nvidia + * proprietary on linux does not work + * We also need to override the size handling to get the correct output size + */ +class GstBackingSurface : public QWindow +{ +public: + GstBackingSurface(); + ~GstBackingSurface(); + + void setSize (int width, int height); + QSize size() const override; + +private: + QSize m_size; +}; + +GstBackingSurface::GstBackingSurface() + : m_size(QSize()) +{ + /* we do OpenGL things so need an OpenGL surface */ + setSurfaceType(QSurface::OpenGLSurface); +} + +GstBackingSurface::~GstBackingSurface() +{ +} + +QSize GstBackingSurface::size () const +{ + return m_size; +} + +void GstBackingSurface::setSize (int width, int height) +{ + m_size = QSize (width, height); +} + +class GstAnimationDriver : public QAnimationDriver +{ +public: + GstAnimationDriver(); + + void setNextTime(qint64 ms); + void advance() override; + qint64 elapsed() const override; +private: + qint64 m_elapsed; + qint64 m_next; +}; + +GstAnimationDriver::GstAnimationDriver() + : m_elapsed(0), + m_next(0) +{ +} + +void GstAnimationDriver::advance() +{ + m_elapsed = m_next; + advanceAnimation(); +} + +qint64 GstAnimationDriver::elapsed() const +{ + return m_elapsed; +} + +void GstAnimationDriver::setNextTime(qint64 ms) +{ + m_next = ms; +} + +void +GstQuickRenderer::deactivateContext () +{ +} + +void +GstQuickRenderer::activateContext () +{ +} + +static void +delete_cxx (QOpenGLFramebufferObject * cxx) +{ + GST_TRACE ("freeing Qfbo %p", cxx); + delete cxx; +} + +GstQuickRenderer::GstQuickRenderer() + : gl_context(NULL), + m_context(nullptr), + m_renderThread(nullptr), + m_surface(nullptr), + m_fbo(nullptr), + m_quickWindow(nullptr), + m_renderControl(nullptr), + m_qmlEngine(nullptr), + m_qmlComponent(nullptr), + m_rootItem(nullptr), + m_animationDriver(nullptr), + gl_allocator(NULL), + gl_params(NULL), + gl_mem(NULL) +{ + init_debug (); +} + +bool GstQuickRenderer::init (GstGLContext * context, GError ** error) +{ + g_return_val_if_fail (gst_gl_context_get_current () == context, false); + + QVariant qt_native_context = qt_opengl_native_context_from_gst_gl_context (context); + + if (qt_native_context.isNull()) { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND, + "Could not convert from the provided GstGLContext to a Qt " + "native context"); + return false; + } + + m_context = new QOpenGLContext; + m_context->setNativeHandle(qt_native_context); + + m_surface = new GstBackingSurface; + m_surface->create(); /* FIXME: may need to be called on Qt's main thread */ + + m_renderThread = QThread::currentThread(); + gst_gl_context_activate (context, FALSE); + + /* Qt does some things that it may require the OpenGL context current in + * ->create() so that it has the necessry information to create the + * QOpenGLContext from the native handle. This may fail if the OpenGL + * context is already current in another thread so we need to deactivate + * the context from GStreamer's thread before asking Qt to create the + * QOpenGLContext with ->create(). + */ + m_context->create(); + m_context->doneCurrent(); + + m_context->moveToThread (m_renderThread); + if (!m_context->makeCurrent(m_surface)) { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND, + "Could not make Qt OpenGL context current"); + /* try to keep the same OpenGL context state */ + gst_gl_context_activate (context, TRUE); + return false; + } + + if (!gst_gl_context_activate (context, TRUE)) { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND, + "Could not make OpenGL context current again"); + return false; + } + + m_renderControl = new QQuickRenderControl(); + /* Create a QQuickWindow that is associated with our render control. Note that this + * window never gets created or shown, meaning that it will never get an underlying + * native (platform) window. + */ + m_quickWindow = new QQuickWindow(m_renderControl); + /* after QQuickWindow creation as QQuickRenderControl requires it */ + m_renderControl->prepareThread (m_renderThread); + + /* Create a QML engine. */ + m_qmlEngine = new QQmlEngine; + if (!m_qmlEngine->incubationController()) + m_qmlEngine->setIncubationController(m_quickWindow->incubationController()); + + gl_context = static_cast(gst_object_ref (context)); + gl_allocator = (GstGLBaseMemoryAllocator *) gst_gl_memory_allocator_get_default (gl_context); + gl_params = (GstGLAllocationParams *) (gst_gl_video_allocation_params_new_wrapped_texture (gl_context, + NULL, &this->v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA8, + 0, NULL, (GDestroyNotify) delete_cxx)); + + return true; +} + +GstQuickRenderer::~GstQuickRenderer() +{ + gst_gl_allocation_params_free (gl_params); +} + +void GstQuickRenderer::stop () +{ + g_assert (QOpenGLContext::currentContext() == m_context); + + if (m_renderControl) + m_renderControl->invalidate(); + + if (m_fbo) + delete m_fbo; + m_fbo = nullptr; + + m_context->doneCurrent(); + if (m_animationDriver) + delete m_animationDriver; + m_animationDriver = nullptr; +} + +void GstQuickRenderer::cleanup() +{ + if (gl_context) + gst_gl_context_thread_add (gl_context, + (GstGLContextThreadFunc) GstQuickRenderer::stop_c, this); + + /* Delete the render control first since it will free the scenegraph resources. + * Destroy the QQuickWindow only afterwards. */ + if (m_renderControl) + delete m_renderControl; + m_renderControl = nullptr; + + if (m_qmlComponent) + delete m_qmlComponent; + m_qmlComponent = nullptr; + if (m_quickWindow) + delete m_quickWindow; + m_quickWindow = nullptr; + if (m_qmlEngine) + delete m_qmlEngine; + m_qmlEngine = nullptr; + + gst_clear_object (&gl_context); + + if (m_context) + delete m_context; + m_context = nullptr; +} + +void GstQuickRenderer::ensureFbo() +{ + if (m_fbo && m_fbo->size() != m_surface->size()) { + GST_INFO ("removing old framebuffer created with size %ix%i", + m_fbo->size().width(), m_fbo->size().height()); + delete m_fbo; + m_fbo = nullptr; + } + + if (!m_fbo) { + m_fbo = new QOpenGLFramebufferObject(m_surface->size(), + QOpenGLFramebufferObject::CombinedDepthStencil); + m_quickWindow->setRenderTarget(m_fbo); + GST_DEBUG ("new framebuffer created with size %ix%i", + m_fbo->size().width(), m_fbo->size().height()); + } +} + +void +GstQuickRenderer::renderGstGL () +{ + GST_DEBUG ("current QOpenGLContext %p", QOpenGLContext::currentContext()); + m_quickWindow->resetOpenGLState(); + + m_animationDriver->advance(); + + QEventLoop loop; + if (loop.processEvents()) + GST_LOG ("pending QEvents processed"); + + ensureFbo(); + + /* Synchronization and rendering happens here on the render thread. */ + if (m_renderControl->sync()) + GST_LOG ("sync successful"); + + /* Meanwhile on this thread continue with the actual rendering. */ + m_renderControl->render(); + + GST_DEBUG ("wrapping Qfbo %p with texture %u", m_fbo, m_fbo->texture()); + gl_params->user_data = static_cast (m_fbo); + gl_params->gl_handle = GINT_TO_POINTER (m_fbo->texture()); + gl_mem = (GstGLMemory *) gst_gl_base_memory_alloc (gl_allocator, gl_params); + + m_fbo = nullptr; +} + +GstGLMemory *GstQuickRenderer::generateOutput(GstClockTime input_ns) +{ + m_animationDriver->setNextTime(input_ns / GST_MSECOND); + + /* run an event loop to update any changed values for rendering */ + QEventLoop loop; + if (loop.processEvents()) + GST_LOG ("pending QEvents processed"); + + GST_LOG ("generating output for time %" GST_TIME_FORMAT " ms: %" + G_GUINT64_FORMAT, GST_TIME_ARGS (input_ns), input_ns / GST_MSECOND); + + m_quickWindow->update(); + + /* Polishing happens on the gui thread. */ + m_renderControl->polishItems(); + + /* TODO: an async version could be used where */ + gst_gl_context_thread_add (gl_context, (GstGLContextThreadFunc) GstQuickRenderer::render_gst_gl_c, this); + + GstGLMemory *tmp = gl_mem; + gl_mem = NULL; + + return tmp; +} + +void GstQuickRenderer::initializeGstGL () +{ + GST_TRACE ("current QOpenGLContext %p", QOpenGLContext::currentContext()); + if (!m_context->makeCurrent(m_surface)) { + m_errorString = "Failed to make Qt's wrapped OpenGL context current"; + return; + } + GST_INFO ("current QOpenGLContext %p", QOpenGLContext::currentContext()); + m_renderControl->initialize(m_context); + + /* 1. QAnimationDriver's are thread-specific + * 2. QAnimationDriver controls the 'animation time' that the Qml scene is + * rendered at + */ + /* FIXME: what happens with multiple qmlgloverlay elements? Do we need a + * shared animation driver? */ + m_animationDriver = new GstAnimationDriver; + m_animationDriver->install(); +} + +void GstQuickRenderer::initializeQml() +{ + disconnect(m_qmlComponent, &QQmlComponent::statusChanged, this, &GstQuickRenderer::initializeQml); + + if (m_qmlComponent->isError()) { + const QList errorList = m_qmlComponent->errors(); + for (const QQmlError &error : errorList) + m_errorString += error.toString(); + return; + } + + QObject *rootObject = m_qmlComponent->create(); + if (m_qmlComponent->isError()) { + const QList errorList = m_qmlComponent->errors(); + for (const QQmlError &error : errorList) + m_errorString += error.toString(); + delete rootObject; + return; + } + + m_rootItem = qobject_cast(rootObject); + if (!m_rootItem) { + m_errorString += "root QML item is not a QQuickItem"; + delete rootObject; + return; + } + + /* The root item is ready. Associate it with the window. */ + m_rootItem->setParentItem(m_quickWindow->contentItem()); + + /* Update item and rendering related geometries. */ + updateSizes(); + + /* Initialize the render control and our OpenGL resources. */ + gst_gl_context_thread_add (gl_context, (GstGLContextThreadFunc) GstQuickRenderer::initialize_gst_gl_c, this); +} + +void GstQuickRenderer::updateSizes() +{ + GstBackingSurface *surface = static_cast(m_surface); + /* Behave like SizeRootObjectToView. */ + QSize size = surface->size(); + + m_rootItem->setWidth(size.width()); + m_rootItem->setHeight(size.height()); + + m_quickWindow->setGeometry(0, 0, size.width(), size.height()); + + gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGBA, size.width(), + size.height()); + GstGLVideoAllocationParams *params = (GstGLVideoAllocationParams *) (gl_params); + gst_video_info_set_format (params->v_info, GST_VIDEO_FORMAT_RGBA, size.width(), + size.height()); +} + +void GstQuickRenderer::setSize(int w, int h) +{ + static_cast(m_surface)->setSize(w, h); + updateSizes(); +} + +bool GstQuickRenderer::setQmlScene (const gchar * scene, GError ** error) +{ + /* replacing the scene is not supported */ + g_return_val_if_fail (m_qmlComponent == NULL, false); + + m_errorString = ""; + + m_qmlComponent = new QQmlComponent(m_qmlEngine); + /* XXX: do we need to provide a propper base name? */ + m_qmlComponent->setData(QByteArray (scene), QUrl("")); + if (m_qmlComponent->isLoading()) + /* TODO: handle async properly */ + connect(m_qmlComponent, &QQmlComponent::statusChanged, this, &GstQuickRenderer::initializeQml); + else + initializeQml(); + + if (m_errorString != "") { + g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_SETTINGS, + m_errorString.toUtf8()); + return FALSE; + } + + return TRUE; +} + +QQuickItem * GstQuickRenderer::rootItem() const +{ + return m_rootItem; +} diff --git a/ext/qt/qtglrenderer.h b/ext/qt/qtglrenderer.h new file mode 100644 index 0000000..256b1b0 --- /dev/null +++ b/ext/qt/qtglrenderer.h @@ -0,0 +1,112 @@ +/* + * GStreamer + * Copyright (C) 2020 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __QT_QUICK_RENDER_H__ +#define __QT_QUICK_RENDER_H__ + +#include +#include + +#include + +QT_FORWARD_DECLARE_CLASS(QOpenGLContext) +QT_FORWARD_DECLARE_CLASS(QOpenGLFramebufferObject) +QT_FORWARD_DECLARE_CLASS(QQuickRenderControl) +QT_FORWARD_DECLARE_CLASS(QQuickWindow) +QT_FORWARD_DECLARE_CLASS(QQmlEngine) +QT_FORWARD_DECLARE_CLASS(QQmlComponent) +QT_FORWARD_DECLARE_CLASS(QQuickItem) +QT_FORWARD_DECLARE_CLASS(GstAnimationDriver) +QT_FORWARD_DECLARE_CLASS(GstBackingSurface) + +class GstQuickRenderer : public QObject +{ + Q_OBJECT + +public: + GstQuickRenderer(); + ~GstQuickRenderer(); + + /* initialize the GStreamer/Qt integration. On failure returns false + * and fills @error. + * Must be called with @context not wrapped and current in the current + * thread */ + bool init (GstGLContext * context, GError ** error); + + /* set the qml scene. returns false and fills @error on failure */ + bool setQmlScene (const gchar * scene, GError ** error); + + void setSize(int w, int h); + + GstGLMemory *generateOutput(GstClockTime input_ns); + + /* cleanup any resources. Any use of this object after calling this + * function may result in undefined behaviour */ + void cleanup(); + + /* retrieve the rootItem from the qml scene. Only valid after + * setQmlScene() has been successfully called */ + QQuickItem *rootItem() const; + +private slots: + void initializeQml(); + +private: + void init(); + void ensureFbo(); + + void updateSizes(); + + static void render_gst_gl_c (GstGLContext * context, GstQuickRenderer * self) { self->renderGstGL (); } + void renderGstGL (); + + static void initialize_gst_gl_c (GstGLContext * context, GstQuickRenderer * self) { self->initializeGstGL (); } + void initializeGstGL (); + + static void stop_c (GstGLContext * context, GstQuickRenderer * self) { self->stop (); } + void stop (); + + static void activate_context_c (GstGLContext * context, GstQuickRenderer * self) { self->activateContext (); } + void activateContext (); + + static void deactivate_context_c (GstGLContext * context, GstQuickRenderer * self) { self->deactivateContext (); } + void deactivateContext (); + + GstGLContext *gl_context; + QOpenGLContext *m_context; + QThread *m_renderThread; + GstBackingSurface *m_surface; + QOpenGLFramebufferObject *m_fbo; + QQuickWindow *m_quickWindow; + QQuickRenderControl *m_renderControl; + QQmlEngine *m_qmlEngine; + QQmlComponent *m_qmlComponent; + QQuickItem *m_rootItem; + GstAnimationDriver *m_animationDriver; + + GstGLBaseMemoryAllocator *gl_allocator; + GstGLAllocationParams *gl_params; + GstVideoInfo v_info; + GstGLMemory *gl_mem; + + QString m_errorString; +}; + +#endif /* __QT_QUICK_RENDER_H__ */ diff --git a/tests/examples/qt/meson.build b/tests/examples/qt/meson.build index 650216c..1135f17 100644 --- a/tests/examples/qt/meson.build +++ b/tests/examples/qt/meson.build @@ -1,3 +1,4 @@ +subdir('qmloverlay') subdir('qmlsink') subdir('qmlsink-dynamically-added') subdir('qmlsrc') diff --git a/tests/examples/qt/qmloverlay/main.cpp b/tests/examples/qt/qmloverlay/main.cpp new file mode 100644 index 0000000..ba8f1e8 --- /dev/null +++ b/tests/examples/qt/qmloverlay/main.cpp @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include +#include +#include + +class SetPlaying : public QRunnable +{ +public: + SetPlaying(GstElement *); + ~SetPlaying(); + + void run (); + +private: + GstElement * pipeline_; +}; + +SetPlaying::SetPlaying (GstElement * pipeline) +{ + this->pipeline_ = pipeline ? static_cast (gst_object_ref (pipeline)) : NULL; +} + +SetPlaying::~SetPlaying () +{ + if (this->pipeline_) + gst_object_unref (this->pipeline_); +} + +void +SetPlaying::run () +{ + if (this->pipeline_) + gst_element_set_state (this->pipeline_, GST_STATE_PLAYING); +} + +static void +on_overlay_scene_initialized (GstElement * overlay, gpointer root_item, gpointer unused) +{ + GST_INFO ("scene initialized"); + QQuickItem *rootObject = static_cast (root_item); + QQuickItem *videoItem = rootObject->findChild ("inputVideoItem"); + g_object_set (overlay, "widget", videoItem, NULL); +} + +int main(int argc, char *argv[]) +{ + int ret; + + gst_init (&argc, &argv); + + { + QGuiApplication app(argc, argv); + + GstElement *pipeline = gst_pipeline_new (NULL); + GstElement *src = gst_element_factory_make ("videotestsrc", NULL); + GstElement *glupload = gst_element_factory_make ("glupload", NULL); + /* the plugin must be loaded before loading the qml file to register the + * GstGLVideoItem qml item */ + GstElement *overlay = gst_element_factory_make ("qmlgloverlay", NULL); + GstElement *sink = gst_element_factory_make ("qmlglsink", NULL); + + g_assert (src && glupload && overlay && sink); + + gst_bin_add_many (GST_BIN (pipeline), src, glupload, overlay, sink, NULL); + gst_element_link_many (src, glupload, overlay, sink, NULL); + + /* load qmlglsink output */ + QQmlApplicationEngine engine; + engine.load(QUrl(QStringLiteral("qrc:/main.qml"))); + + QQuickItem *videoItem; + QQuickWindow *rootObject; + + /* find and set the videoItem on the sink */ + rootObject = static_cast (engine.rootObjects().first()); + videoItem = rootObject->findChild ("videoItem"); + g_assert (videoItem); + g_object_set(sink, "widget", videoItem, NULL); + + QDirIterator it(":", QDirIterator::Subdirectories); + while (it.hasNext()) { + qDebug() << it.next(); + } + + QFile f(":/overlay.qml"); + if(!f.open(QIODevice::ReadOnly)) { + qWarning() << "error: " << f.errorString(); + return 1; + } + QByteArray overlay_scene = f.readAll(); + qDebug() << overlay_scene; + + /* load qmlgloverlay contents */ + g_signal_connect (overlay, "qml-scene-initialized", G_CALLBACK (on_overlay_scene_initialized), NULL); + g_object_set (overlay, "qml-scene", overlay_scene.data(), NULL); + + rootObject->scheduleRenderJob (new SetPlaying (pipeline), + QQuickWindow::BeforeSynchronizingStage); + + ret = app.exec(); + + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); + } + + gst_deinit (); + + return ret; +} diff --git a/tests/examples/qt/qmloverlay/main.qml b/tests/examples/qt/qmloverlay/main.qml new file mode 100644 index 0000000..0113a1b --- /dev/null +++ b/tests/examples/qt/qmloverlay/main.qml @@ -0,0 +1,39 @@ +import QtQuick 2.4 +import QtQuick.Controls 1.1 +import QtQuick.Controls.Styles 1.3 +import QtQuick.Dialogs 1.2 +import QtQuick.Window 2.1 + +import org.freedesktop.gstreamer.GLVideoItem 1.0 + +ApplicationWindow { + id: window + visible: true + width: 640 + height: 480 + x: 30 + y: 30 + color: "black" + + Item { + anchors.fill: parent + + GstGLVideoItem { + id: video + objectName: "videoItem" + anchors.centerIn: parent + width: parent.width + height: parent.height + } + + Text { + anchors.bottom: parent.bottom + anchors.horizontalCenter: parent.horizontalCenter + text: "qmlglsink text" + font.pointSize: 20 + color: "yellow" + style: Text.Outline + styleColor: "blue" + } + } +} diff --git a/tests/examples/qt/qmloverlay/meson.build b/tests/examples/qt/qmloverlay/meson.build new file mode 100644 index 0000000..57a5383 --- /dev/null +++ b/tests/examples/qt/qmloverlay/meson.build @@ -0,0 +1,20 @@ +sources = [ + 'main.cpp', +] + +if have_cxx and build_gstgl and gstgl_dep.found() + qt5_mod = import('qt5') + qt5qml_deps = dependency('qt5', modules : ['Core', 'Gui', 'Widgets', 'Qml', 'Quick'], + required: get_option('examples')) + + # FIXME Add a way to get that information out of the qt5 module + moc = find_program('moc-qt5', 'moc', required : get_option('examples')) + if qt5qml_deps.found() and moc.found() + qt_preprocessed = qt5_mod.preprocess(qresources : 'qmloverlay.qrc') + executable('qmlgloverlay', sources, qt_preprocessed, + dependencies : [gst_dep, qt5qml_deps], + c_args : gst_plugins_good_args, + include_directories : [configinc], + install: false) + endif +endif diff --git a/tests/examples/qt/qmloverlay/overlay.qml b/tests/examples/qt/qmloverlay/overlay.qml new file mode 100644 index 0000000..81faeb5 --- /dev/null +++ b/tests/examples/qt/qmloverlay/overlay.qml @@ -0,0 +1,57 @@ +import QtQuick 2.4 + +import org.freedesktop.gstreamer.GLVideoItem 1.0 + +Item { + /* render upside down for GStreamer */ + transform: Scale { origin.x : 0; origin.y : height / 2.; yScale : -1 } + + GstGLVideoItem { + id: video + objectName: "inputVideoItem" + anchors.centerIn: parent + width: parent.width + height: parent.height + } + + Text { + id: rotatingText + anchors.centerIn: parent + text: "Qt Quick\nrendered to\na texture" + font.pointSize: 20 + color: "black" + style: Text.Outline + styleColor: "white" + + RotationAnimator { + target: rotatingText; + from: 0; + to: 360; + duration: 5000 + running: true + loops: Animation.Infinite + } + } + + Text { + property int elapsedTime: 0 + + id: time + anchors.top: rotatingText.bottom + anchors.horizontalCenter: rotatingText.horizontalCenter + font.pointSize: 12 + style: Text.Outline + styleColor: "black" + color: "white" + + Timer { + interval: 1000 + running: true + repeat: true + onTriggered: { + parent.elapsedTime += interval / 1000 + parent.text = "overlay: " + parent.elapsedTime.toString() + " seconds" + } + } + } +} diff --git a/tests/examples/qt/qmloverlay/qmloverlay.qrc b/tests/examples/qt/qmloverlay/qmloverlay.qrc new file mode 100644 index 0000000..099a78d --- /dev/null +++ b/tests/examples/qt/qmloverlay/qmloverlay.qrc @@ -0,0 +1,6 @@ + + + main.qml + overlay.qml + +