qml: add some docs on display and contexts

Especially considering some dynamic pipeline scenarios.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/822>

diff --git a/ext/qt/gstqtoverlay.cc b/ext/qt/gstqtoverlay.cc
index 26c2a01..e03dda5 100644 (file)
--- a/ext/qt/gstqtoverlay.cc
+++ b/ext/qt/gstqtoverlay.cc
@@ -21,6 +21,58 @@
 /**
  * SECTION:gstqtoverlay
  *
+ * qmlgloverlay provides a way to render an almost-arbitrary QML scene within
+ * GStreamer pipeline using the same OpenGL context that GStreamer uses
+ * internally.  This avoids attempting to share multiple OpenGL contexts
+ * avoiding increased synchronisation points and attempting to share an OpenGL
+ * context at runtime which some drivers do not like.  The Intel driver on
+ * Windows is a notable example of the last point.
+ *
+ * qmlgloverlay will attempt to retrieve the windowing system display connection
+ * that Qt is using (#GstGLDisplay).  This may be different to any already
+ * existing window system display connection already in use in the pipeline for
+ * a number of reasons.  A couple of examples of this are:
+ *
+ * 1. Adding qmlgloverlay to an already running pipeline
+ * 2. Not having any qmlgloverlay (or qmlglsink) element start up before any
+ *    other OpenGL-based element in the pipeline.
+ *
+ * If one of these scenarios occurs, then there will be multiple OpenGL contexts
+ * in use in the pipeline.  This means that either the pipeline will fail to
+ * start up correctly, a downstream element may reject buffers, or a complete
+ * GPU->System memory->GPU transfer is performed for every buffer.
+ *
+ * The requirement to avoid this is that all elements share the same
+ * #GstGLDisplay object and as Qt cannot currently share an existing window
+ * system display connection, GStreamer must use the window system display
+ * connection provided by Qt.  This window system display connection can be
+ * retrieved by either a qmlglsink element or a qmlgloverlay element. The
+ * recommended usage is to have either element (qmlglsink or qmlgloverlay)
+ * be the first to propagate the #GstGLDisplay for the entire pipeline to use by
+ * setting either element to the READY element state before any other OpenGL
+ * element in the pipeline.
+ *
+ * In a dynamically adding qmlgloverlay (or qmlglsink) to a pipeline case,
+ * there are some considerations for ensuring that the window system display
+ * and OpenGL contexts are compatible with Qt.  When the qmlgloverlay (or
+ * qmlglsink) element is added and brought up to READY, it will propagate it's
+ * own #GstGLDisplay using the #GstContext mechanism regardless of any existing
+ * #GstGLDisplay used by the pipeline previously.  In order for the new
+ * #GstGLDisplay to be used, the application must then set the provided
+ * #GstGLDisplay containing #GstContext on the pipeline.  This may effectively
+ * cause each OpenGL element to replace the window system display and also the
+ * OpenGL context it is using.  As such this process may take a significant
+ * amount of time and resources as objects are recreated in the new OpenGL
+ * context.
+ *
+ * All instances of qmlgloverlay and qmlglsink will return the exact same
+ * #GstGLDisplay object while the pipeline is running regardless of whether
+ * any qmlgloverlay or qmlglsink elements are added or removed from the
+ * pipeline.
+ *
+ * The Qml scene will run at the pace of incoming buffers.  One input buffer
+ * will cause a render of one output buffer.  The timestamps on the input
+ * buffers are used to drive the animation time.
  */
 
 #ifdef HAVE_CONFIG_H

diff --git a/ext/qt/gstqtsink.cc b/ext/qt/gstqtsink.cc
index b447c24..ebf4b24 100644 (file)
--- a/ext/qt/gstqtsink.cc
+++ b/ext/qt/gstqtsink.cc
@@ -21,6 +21,51 @@
 /**
  * SECTION:gstqtsink
  *
+ * qmlglsink provides a way to render a video stream as a Qml object inside
+ * the Qml scene graph.  This is achieved by providing the incoming OpenGL
+ * textures to Qt as a scene graph object.
+ *
+ * qmlglsink will attempt to retrieve the windowing system display connection
+ * that Qt is using (#GstGLDisplay).  This may be different to any already
+ * existing window system display connection already in use in the pipeline for
+ * a number of reasons.  A couple of examples of this are:
+ *
+ * 1. Adding qmlglsink to an already running pipeline
+ * 2. Not having any qmlglsink (or qmlgloverlay) element start up before any
+ *    other OpenGL-based element in the pipeline.
+ *
+ * If one of these scenarios occurs, then there will be multiple OpenGL contexts
+ * in use in the pipeline.  This means that either the pipeline will fail to
+ * start up correctly, a downstream element may reject buffers, or a complete
+ * GPU->System memory->GPU transfer is performed for every buffer.
+ *
+ * The requirement to avoid this is that all elements share the same
+ * #GstGLDisplay object and as Qt cannot currently share an existing window
+ * system display connection, GStreamer must use the window system display
+ * connection provided by Qt.  This window system display connection can be
+ * retrieved by either a qmlglsink element or a qmlgloverlay element. The
+ * recommended usage is to have either element (qmlglsink or qmlgloverlay)
+ * be the first to propagate the #GstGLDisplay for the entire pipeline to use by
+ * setting either element to the READY element state before any other OpenGL
+ * element in the pipeline.
+ *
+ * In a dynamically adding qmlglsink (or qmlgloverlay) to a pipeline case,
+ * there are some considerations for ensuring that the window system display
+ * and OpenGL contexts are compatible with Qt.  When the qmlgloverlay (or
+ * qmlglsink) element is added and brought up to READY, it will propagate it's
+ * own #GstGLDisplay using the #GstContext mechanism regardless of any existing
+ * #GstGLDisplay used by the pipeline previously.  In order for the new
+ * #GstGLDisplay to be used, the application must then set the provided
+ * #GstGLDisplay containing #GstContext on the pipeline.  This may effectively
+ * cause each OpenGL element to replace the window system display and also the
+ * OpenGL context it is using.  As such this process may take a significant
+ * amount of time and resources as objects are recreated in the new OpenGL
+ * context.
+ *
+ * All instances of qmlglsink and qmlgloverlay will return the exact same
+ * #GstGLDisplay object while the pipeline is running regardless of whether
+ * any qmlglsink or qmlgloverlay elements are added or removed from the
+ * pipeline.
  */
 
 #ifdef HAVE_CONFIG_H