3 #include <QQmlComponent>
5 #include <QQuickRenderControl>
6 #include <QQuickWindow>
8 #include <QOpenGLContext>
9 #include <QOpenGLFunctions>
10 #include <QOpenGLFramebufferObject>
11 #include <QAnimationDriver>
13 #include <gst/gl/gl.h>
16 #include "qtglrenderer.h"
17 #include "gstqtglutility.h"
19 #define GST_CAT_DEFAULT gst_qt_gl_renderer_debug
20 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
27 if (g_once_init_enter (&_debug)) {
28 GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtglrenderer", 0,
29 "Qt OpenGL Renderer");
30 g_once_init_leave (&_debug, 1);
34 /* Needs to be based on QWindow otherwise (at least) windows and nvidia
35 * proprietary on linux does not work
36 * We also need to override the size handling to get the correct output size
38 class GstBackingSurface : public QWindow
44 void setSize (int width, int height);
45 QSize size() const override;
51 GstBackingSurface::GstBackingSurface()
54 /* we do OpenGL things so need an OpenGL surface */
55 setSurfaceType(QSurface::OpenGLSurface);
58 GstBackingSurface::~GstBackingSurface()
62 QSize GstBackingSurface::size () const
67 void GstBackingSurface::setSize (int width, int height)
69 m_size = QSize (width, height);
72 class GstAnimationDriver : public QAnimationDriver
77 void setNextTime(qint64 ms);
78 void advance() override;
79 qint64 elapsed() const override;
85 GstAnimationDriver::GstAnimationDriver()
91 void GstAnimationDriver::advance()
97 qint64 GstAnimationDriver::elapsed() const
102 void GstAnimationDriver::setNextTime(qint64 ms)
111 STATE_WAITING_FOR_WINDOW,
112 STATE_WINDOW_CREATED,
114 } SharedRenderDataState;
116 struct SharedRenderData
119 SharedRenderDataState state;
122 GstAnimationDriver *m_animationDriver;
123 QOpenGLContext *m_context;
124 GstBackingSurface *m_surface;
125 QThread *m_renderThread;
128 static struct SharedRenderData *
129 shared_render_data_new (void)
131 struct SharedRenderData *ret = g_new0 (struct SharedRenderData, 1);
133 g_atomic_int_set (&ret->refcount, 1);
134 g_mutex_init (&ret->lock);
140 shared_render_data_free (struct SharedRenderData * data)
142 GST_DEBUG ("%p freeing shared render data", data);
144 g_mutex_clear (&data->lock);
146 if (data->m_animationDriver) {
147 data->m_animationDriver->uninstall();
148 delete data->m_animationDriver;
150 data->m_animationDriver = nullptr;
152 delete data->m_context;
153 data->m_context = nullptr;
155 delete data->m_surface;
156 data->m_surface = nullptr;
159 static struct SharedRenderData *
160 shared_render_data_ref (struct SharedRenderData * data)
162 GST_TRACE ("%p reffing shared render data", data);
163 g_atomic_int_inc (&data->refcount);
168 shared_render_data_unref (struct SharedRenderData * data)
170 GST_TRACE ("%p unreffing shared render data", data);
171 if (g_atomic_int_dec_and_test (&data->refcount))
172 shared_render_data_free (data);
176 GstQuickRenderer::deactivateContext ()
181 GstQuickRenderer::activateContext ()
187 GstGLContext * context;
188 QOpenGLFramebufferObject * fbo;
192 delete_cxx_gl_context (GstGLContext * context, struct FBOUserData * data)
194 GST_TRACE ("freeing Qfbo %p", data->fbo);
199 notify_fbo_delete (struct FBOUserData * data)
201 gst_gl_context_thread_add (data->context,
202 (GstGLContextThreadFunc) delete_cxx_gl_context, data);
203 gst_object_unref (data->context);
207 GstQuickRenderer::GstQuickRenderer()
210 m_quickWindow(nullptr),
211 m_renderControl(nullptr),
212 m_qmlEngine(nullptr),
213 m_qmlComponent(nullptr),
218 m_sharedRenderData(NULL)
224 dup_shared_render_data (gpointer data, gpointer user_data)
226 struct SharedRenderData *render_data = (struct SharedRenderData *) data;
229 return shared_render_data_ref (render_data);
234 class CreateSurfaceEvent : public QEvent
237 CreateSurfaceEvent (CreateSurfaceWorker * worker)
238 : QEvent(CreateSurfaceEvent::type())
243 ~CreateSurfaceEvent()
245 GST_TRACE ("%p destroying create surface event", this);
249 static QEvent::Type type()
251 if (customEventType == QEvent::None) {
252 int generatedType = QEvent::registerEventType();
253 customEventType = static_cast<QEvent::Type>(generatedType);
255 return customEventType;
259 static QEvent::Type customEventType;
260 CreateSurfaceWorker *m_worker;
263 QEvent::Type CreateSurfaceEvent::customEventType = QEvent::None;
266 CreateSurfaceWorker::CreateSurfaceWorker (struct SharedRenderData * rdata)
268 m_sharedRenderData = shared_render_data_ref (rdata);
271 CreateSurfaceWorker::~CreateSurfaceWorker ()
273 shared_render_data_unref (m_sharedRenderData);
276 bool CreateSurfaceWorker::event(QEvent * ev)
278 if (ev->type() == CreateSurfaceEvent::type()) {
279 GST_TRACE ("%p creating surface", m_sharedRenderData);
280 /* create the window surface in the main thread */
281 g_mutex_lock (&m_sharedRenderData->lock);
282 m_sharedRenderData->m_surface = new GstBackingSurface;
283 m_sharedRenderData->m_surface->create();
284 m_sharedRenderData->m_surface->moveToThread (m_sharedRenderData->m_renderThread);
285 GST_TRACE ("%p created surface %p", m_sharedRenderData,
286 m_sharedRenderData->m_surface);
287 g_cond_broadcast (&m_sharedRenderData->cond);
288 g_mutex_unlock (&m_sharedRenderData->lock);
291 return QObject::event(ev);
294 bool GstQuickRenderer::init (GstGLContext * context, GError ** error)
296 g_return_val_if_fail (GST_IS_GL_CONTEXT (context), false);
297 g_return_val_if_fail (gst_gl_context_get_current () == context, false);
299 QVariant qt_native_context = qt_opengl_native_context_from_gst_gl_context (context);
301 if (qt_native_context.isNull()) {
302 g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
303 "Could not convert from the provided GstGLContext to a Qt "
308 struct SharedRenderData *render_data = NULL, *old_render_data;
311 shared_render_data_unref (render_data);
313 old_render_data = render_data = (struct SharedRenderData *)
314 g_object_dup_data (G_OBJECT (context),
315 "qt.gl.render.shared.data", dup_shared_render_data, NULL);
317 render_data = shared_render_data_new ();
318 } while (old_render_data != render_data
319 && !g_object_replace_data (G_OBJECT (context),
320 "qt.gl.render.shared.data", old_render_data, render_data,
322 m_sharedRenderData = render_data;
323 GST_TRACE ("%p retrieved shared render data %p", this, m_sharedRenderData);
325 g_mutex_lock (&m_sharedRenderData->lock);
326 if (m_sharedRenderData->state == STATE_ERROR) {
327 g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
328 "In an error state from a previous attempt");
329 g_mutex_unlock (&m_sharedRenderData->lock);
333 if (m_sharedRenderData->state != STATE_READY) {
334 /* this state handling and locking is so that two qtglrenderer's will
335 * not attempt to create an OpenGL context without freeing the previous
336 * OpenGL context and cause a leak. It also only allows one
337 * CreateSurfaceEvent() to be posted to the main thread
338 * (QCoreApplication::instance()->thread()) while still allowing
339 * multiple waiters to wait for the window to be created */
340 if (m_sharedRenderData->state == STATE_NEW) {
341 QCoreApplication *app = QCoreApplication::instance ();
344 g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
345 "Could not retrieve QCoreApplication instance");
346 m_sharedRenderData->state = STATE_ERROR;
347 g_mutex_unlock (&m_sharedRenderData->lock);
351 m_sharedRenderData->m_renderThread = QThread::currentThread();
352 m_sharedRenderData->m_context = new QOpenGLContext;
353 GST_TRACE ("%p new QOpenGLContext %p", this, m_sharedRenderData->m_context);
354 m_sharedRenderData->m_context->setNativeHandle(qt_native_context);
356 CreateSurfaceWorker *w = new CreateSurfaceWorker (m_sharedRenderData);
357 GST_TRACE ("%p posting create surface event to main thread with "
358 "worker %p", this, w);
359 w->moveToThread (app->thread());
360 app->postEvent (w, new CreateSurfaceEvent (w));
361 m_sharedRenderData->state = STATE_WAITING_FOR_WINDOW;
364 if (m_sharedRenderData->state == STATE_WAITING_FOR_WINDOW) {
365 gint64 end_time = g_get_monotonic_time () + 5 * G_TIME_SPAN_SECOND;
366 while (!m_sharedRenderData->m_surface) {
367 /* XXX: This might deadlock with the main thread if the
368 * QCoreApplication is not running and will not be able to
369 * execute. We only wait for 5 seconds until a better
370 * approach can be found here */
371 if (!g_cond_wait_until (&m_sharedRenderData->cond,
372 &m_sharedRenderData->lock, end_time)) {
373 g_set_error (error, GST_RESOURCE_ERROR,
374 GST_RESOURCE_ERROR_NOT_FOUND,
375 "Could not create Qt window within 5 seconds");
376 m_sharedRenderData->state = STATE_ERROR;
377 g_mutex_unlock (&m_sharedRenderData->lock);
382 GST_TRACE ("%p surface successfully created", this);
383 m_sharedRenderData->state = STATE_WINDOW_CREATED;
386 if (m_sharedRenderData->state == STATE_WINDOW_CREATED) {
387 /* Qt does some things that may require the OpenGL context current
388 * in ->create() so that it has the necessry information to create
389 * the QOpenGLContext from the native handle. This may fail if the
390 * OpenGL context is already current in another thread so we need
391 * to deactivate the context from GStreamer's thread before asking
392 * Qt to create the QOpenGLContext with ->create().
394 gst_gl_context_activate (context, FALSE);
395 m_sharedRenderData->m_context->create();
396 m_sharedRenderData->m_context->doneCurrent();
398 if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) {
399 g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
400 "Could not make Qt OpenGL context current");
401 /* try to keep the same OpenGL context state */
402 gst_gl_context_activate (context, TRUE);
403 m_sharedRenderData->state = STATE_ERROR;
404 g_mutex_unlock (&m_sharedRenderData->lock);
408 if (!gst_gl_context_activate (context, TRUE)) {
409 g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
410 "Could not make OpenGL context current again");
411 m_sharedRenderData->state = STATE_ERROR;
412 g_mutex_unlock (&m_sharedRenderData->lock);
415 m_sharedRenderData->state = STATE_READY;
419 m_renderControl = new QQuickRenderControl();
420 /* Create a QQuickWindow that is associated with our render control. Note that this
421 * window never gets created or shown, meaning that it will never get an underlying
422 * native (platform) window.
424 m_quickWindow = new QQuickWindow(m_renderControl);
425 /* after QQuickWindow creation as QQuickRenderControl requires it */
426 m_renderControl->prepareThread (m_sharedRenderData->m_renderThread);
427 g_mutex_unlock (&m_sharedRenderData->lock);
429 /* Create a QML engine. */
430 m_qmlEngine = new QQmlEngine;
431 if (!m_qmlEngine->incubationController())
432 m_qmlEngine->setIncubationController(m_quickWindow->incubationController());
434 gl_context = static_cast<GstGLContext*>(gst_object_ref (context));
435 gl_allocator = (GstGLBaseMemoryAllocator *) gst_gl_memory_allocator_get_default (gl_context);
436 gl_params = (GstGLAllocationParams *)
437 gst_gl_video_allocation_params_new_wrapped_texture (gl_context,
438 NULL, &this->v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA8,
439 0, NULL, (GDestroyNotify) notify_fbo_delete);
441 /* This is a gross hack relying on the internals of Qt and GStreamer
442 * however it's the only way to remove this warning on shutdown of all
445 * GLib-CRITICAL **: 17:35:24.988: g_main_context_pop_thread_default: assertion 'g_queue_peek_head (stack) == context' failed
447 * The reason is that libgstgl has a GMainContext that it pushes as the
448 * thread default context. Then later, Qt pushes a thread default main
449 * context. The detruction order of the GMainContext's is reversed as
450 * GStreamer will explicitly pop the thread default main context however
451 * Qt pops when the thread is about to be destroyed. GMainContext is
452 * unhappy with the ordering of the pops.
454 GMainContext *gst_main_context = g_main_context_ref_thread_default ();
456 /* make Qt allocate and push a thread-default GMainContext if it is
459 if (loop.processEvents())
460 GST_LOG ("pending QEvents processed");
462 GMainContext *qt_main_context = g_main_context_ref_thread_default ();
464 if (qt_main_context == gst_main_context) {
465 g_main_context_unref (qt_main_context);
466 g_main_context_unref (gst_main_context);
468 /* We flip the order of the GMainContext's so that the destruction
469 * order can be preserved. */
470 g_main_context_pop_thread_default (qt_main_context);
471 g_main_context_pop_thread_default (gst_main_context);
472 g_main_context_push_thread_default (qt_main_context);
473 g_main_context_push_thread_default (gst_main_context);
474 g_main_context_unref (qt_main_context);
475 g_main_context_unref (gst_main_context);
481 GstQuickRenderer::~GstQuickRenderer()
483 gst_gl_allocation_params_free (gl_params);
484 gst_clear_object (&gl_allocator);
487 void GstQuickRenderer::stopGL ()
489 GST_DEBUG ("%p stop QOpenGLContext curent: %p stored: %p", this,
490 QOpenGLContext::currentContext(), m_sharedRenderData->m_context);
491 g_assert (QOpenGLContext::currentContext() == m_sharedRenderData->m_context);
494 m_renderControl->invalidate();
501 if (loop.processEvents())
502 GST_LOG ("%p pending QEvents processed", this);
504 if (m_sharedRenderData)
505 shared_render_data_unref (m_sharedRenderData);
506 m_sharedRenderData = NULL;
508 /* XXX: reset the OpenGL context and drawable as Qt may have clobbered it.
509 * Fixes any attempt to access OpenGL after shutting down qmlgloverlay. */
510 gst_gl_context_activate (gl_context, FALSE);
511 gst_gl_context_activate (gl_context, TRUE);
514 void GstQuickRenderer::cleanup()
517 gst_gl_context_thread_add (gl_context,
518 (GstGLContextThreadFunc) GstQuickRenderer::stop_c, this);
520 /* Delete the render control first since it will free the scenegraph resources.
521 * Destroy the QQuickWindow only afterwards. */
523 delete m_renderControl;
524 m_renderControl = nullptr;
527 delete m_qmlComponent;
528 m_qmlComponent = nullptr;
530 delete m_quickWindow;
531 m_quickWindow = nullptr;
534 m_qmlEngine = nullptr;
537 m_rootItem = nullptr;
539 gst_clear_object (&gl_context);
542 void GstQuickRenderer::ensureFbo()
544 if (m_fbo && m_fbo->size() != m_sharedRenderData->m_surface->size()) {
545 GST_INFO ("%p removing old framebuffer created with size %ix%i",
546 this, m_fbo->size().width(), m_fbo->size().height());
552 m_fbo = new QOpenGLFramebufferObject(m_sharedRenderData->m_surface->size(),
553 QOpenGLFramebufferObject::CombinedDepthStencil);
554 m_quickWindow->setRenderTarget(m_fbo);
555 GST_DEBUG ("%p new framebuffer created with size %ix%i", this,
556 m_fbo->size().width(), m_fbo->size().height());
561 GstQuickRenderer::renderGstGL ()
563 const GstGLFuncs *gl = gl_context->gl_vtable;
565 GST_TRACE ("%p current QOpenGLContext %p", this,
566 QOpenGLContext::currentContext());
567 m_quickWindow->resetOpenGLState();
569 m_sharedRenderData->m_animationDriver->advance();
572 if (loop.processEvents())
573 GST_LOG ("pending QEvents processed");
579 /* Synchronization and rendering happens here on the render thread. */
580 if (m_renderControl->sync())
581 GST_LOG ("sync successful");
583 /* Meanwhile on this thread continue with the actual rendering. */
584 m_renderControl->render();
586 GST_DEBUG ("wrapping Qfbo %p with texture %u", m_fbo, m_fbo->texture());
587 struct FBOUserData *data = g_new0 (struct FBOUserData, 1);
588 data->context = (GstGLContext *) gst_object_ref (gl_context);
590 gl_params->user_data = static_cast<gpointer> (data);
591 gl_params->gl_handle = GINT_TO_POINTER (m_fbo->texture());
592 gl_mem = (GstGLMemory *) gst_gl_base_memory_alloc (gl_allocator, gl_params);
596 m_quickWindow->resetOpenGLState ();
597 /* Qt doesn't seem to reset this, breaking glimagesink */
599 gl->DrawBuffer (GL_BACK);
602 GstGLMemory *GstQuickRenderer::generateOutput(GstClockTime input_ns)
604 m_sharedRenderData->m_animationDriver->setNextTime(input_ns / GST_MSECOND);
606 /* run an event loop to update any changed values for rendering */
608 if (loop.processEvents())
609 GST_LOG ("pending QEvents processed");
611 GST_LOG ("generating output for time %" GST_TIME_FORMAT " ms: %"
612 G_GUINT64_FORMAT, GST_TIME_ARGS (input_ns), input_ns / GST_MSECOND);
614 m_quickWindow->update();
616 /* Polishing happens on the gui thread. */
617 m_renderControl->polishItems();
619 /* TODO: an async version could be used where */
620 gst_gl_context_thread_add (gl_context,
621 (GstGLContextThreadFunc) GstQuickRenderer::render_gst_gl_c, this);
623 GstGLMemory *tmp = gl_mem;
629 void GstQuickRenderer::initializeGstGL ()
631 GST_TRACE ("current QOpenGLContext %p", QOpenGLContext::currentContext());
632 if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) {
633 m_errorString = "Failed to make Qt's wrapped OpenGL context current";
636 GST_INFO ("current QOpenGLContext %p", QOpenGLContext::currentContext());
638 /* XXX: Avoid an assertion inside QSGDefaultRenderContext::initialize()
639 * from an unused (in this scenario) property when using multiple
640 * QQuickRenderControl's with the same QOpenGLContext.
642 * First noticed with Qt 5.15. Idea from:
643 * https://forum.qt.io/topic/55888/is-it-impossible-that-2-qquickrendercontrol-use-same-qopenglcontext/2
645 * ASSERT: "!m_gl->property(QSG_RENDERCONTEXT_PROPERTY).isValid()" in file /path/to/qt5/qtdeclarative/src/quick/scenegraph/qsgdefaultrendercontext.cpp, line 121
647 m_sharedRenderData->m_context->setProperty("_q_sgrendercontext", QVariant());
649 m_renderControl->initialize(m_sharedRenderData->m_context);
651 /* 1. QAnimationDriver's are thread-specific
652 * 2. QAnimationDriver controls the 'animation time' that the Qml scene is
655 /* FIXME: what happens with multiple qmlgloverlay elements? Do we need a
656 * shared animation driver? */
657 g_mutex_lock (&m_sharedRenderData->lock);
658 if (m_sharedRenderData->m_animationDriver == nullptr) {
659 m_sharedRenderData->m_animationDriver = new GstAnimationDriver;
660 m_sharedRenderData->m_animationDriver->install();
662 g_mutex_unlock (&m_sharedRenderData->lock);
663 /* XXX: reset the OpenGL context drawable as Qt may have clobbered it.
664 * Fixes glimagesink output where Qt replaces the Surface to use in its
665 * own MakeCurrent call. Qt does this on it's OpenGL initialisation
666 * the the rendering engine. */
667 gst_gl_context_activate (gl_context, FALSE);
668 gst_gl_context_activate (gl_context, TRUE);
671 void GstQuickRenderer::initializeQml()
673 disconnect(m_qmlComponent, &QQmlComponent::statusChanged, this,
674 &GstQuickRenderer::initializeQml);
676 if (m_qmlComponent->isError()) {
677 const QList<QQmlError> errorList = m_qmlComponent->errors();
678 for (const QQmlError &error : errorList)
679 m_errorString += error.toString();
683 QObject *rootObject = m_qmlComponent->create();
684 if (m_qmlComponent->isError()) {
685 const QList<QQmlError> errorList = m_qmlComponent->errors();
686 for (const QQmlError &error : errorList)
687 m_errorString += error.toString();
692 m_rootItem = qobject_cast<QQuickItem *>(rootObject);
694 m_errorString += "root QML item is not a QQuickItem";
699 /* The root item is ready. Associate it with the window. */
700 m_rootItem->setParentItem(m_quickWindow->contentItem());
702 /* Update item and rendering related geometries. */
705 /* Initialize the render control and our OpenGL resources. */
706 gst_gl_context_thread_add (gl_context,
707 (GstGLContextThreadFunc) GstQuickRenderer::initialize_gst_gl_c, this);
710 void GstQuickRenderer::updateSizes()
712 GstBackingSurface *surface =
713 static_cast<GstBackingSurface *>(m_sharedRenderData->m_surface);
714 /* Behave like SizeRootObjectToView. */
715 QSize size = surface->size();
717 m_rootItem->setWidth(size.width());
718 m_rootItem->setHeight(size.height());
720 m_quickWindow->setGeometry(0, 0, size.width(), size.height());
722 gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGBA, size.width(),
724 GstGLVideoAllocationParams *params = (GstGLVideoAllocationParams *) (gl_params);
725 gst_video_info_set_format (params->v_info, GST_VIDEO_FORMAT_RGBA, size.width(),
729 void GstQuickRenderer::setSize(int w, int h)
731 static_cast<GstBackingSurface *>(m_sharedRenderData->m_surface)->setSize(w, h);
735 bool GstQuickRenderer::setQmlScene (const gchar * scene, GError ** error)
737 /* replacing the scene is not supported */
738 g_return_val_if_fail (m_qmlComponent == NULL, false);
742 m_qmlComponent = new QQmlComponent(m_qmlEngine);
743 /* XXX: do we need to provide a propper base name? */
744 m_qmlComponent->setData(QByteArray (scene), QUrl(""));
745 if (m_qmlComponent->isLoading())
746 /* TODO: handle async properly */
747 connect(m_qmlComponent, &QQmlComponent::statusChanged, this,
748 &GstQuickRenderer::initializeQml);
752 if (m_errorString != "") {
753 QByteArray string = m_errorString.toUtf8();
754 g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_SETTINGS,
755 "%s", string.constData());
762 QQuickItem * GstQuickRenderer::rootItem() const