From 9ac798ae5e7ebefea626d796eac8d859c3dfdf60 Mon Sep 17 00:00:00 2001 From: Philippe Normand Date: Sat, 8 Feb 2020 12:05:03 +0000 Subject: [PATCH] wpe: Add software rendering support support Starting from WPEBackend-FDO 1.6.x, software rendering support is available. This features allows wpesrc to be used on machines without GPU, and/or for testing purpose. To enable it, set the `LIBGL_ALWAYS_SOFTWARE=true` environment variable and make sure `video/x-raw, format=BGRA` caps are negotiated by the wpesrc element. --- ext/wpe/WPEThreadedView.cpp | 193 +++++++++++++++++++++++++++++++++++++------- ext/wpe/WPEThreadedView.h | 30 ++++++- ext/wpe/gstwpesrc.cpp | 70 ++++++++++++++-- ext/wpe/meson.build | 3 +- 4 files changed, 257 insertions(+), 39 deletions(-) diff --git a/ext/wpe/WPEThreadedView.cpp b/ext/wpe/WPEThreadedView.cpp index 1857b44..e16dc74 100644 --- a/ext/wpe/WPEThreadedView.cpp +++ b/ext/wpe/WPEThreadedView.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -64,7 +65,7 @@ WPEThreadedView::WPEThreadedView() g_mutex_init(&threading.ready_mutex); g_cond_init(&threading.ready_cond); - g_mutex_init(&images.mutex); + g_mutex_init(&images_mutex); { GMutexHolder lock(threading.mutex); @@ -78,15 +79,15 @@ WPEThreadedView::WPEThreadedView() WPEThreadedView::~WPEThreadedView() { { - GMutexHolder lock(images.mutex); + GMutexHolder lock(images_mutex); - if (images.pending) { - gst_egl_image_unref(images.pending); - images.pending = nullptr; + if (egl.pending) { + gst_egl_image_unref(egl.pending); + egl.pending = nullptr; } - if (images.committed) { - gst_egl_image_unref(images.committed); - images.committed = nullptr; + if (egl.committed) { + gst_egl_image_unref(egl.committed); + egl.committed = nullptr; } } @@ -114,7 +115,7 @@ WPEThreadedView::~WPEThreadedView() g_cond_clear(&threading.cond); g_mutex_clear(&threading.ready_mutex); g_cond_clear(&threading.ready_cond); - g_mutex_clear(&images.mutex); + g_mutex_clear(&images_mutex); } gpointer WPEThreadedView::s_viewThread(gpointer data) @@ -186,8 +187,18 @@ bool WPEThreadedView::initialize(GstWpeSrc* src, GstGLContext* context, GstGLDis #endif }); - EGLDisplay eglDisplay = gst_gl_display_egl_get_from_native(GST_GL_DISPLAY_TYPE_WAYLAND, - gst_gl_display_get_handle(display)); + EGLDisplay eglDisplay; + if (context && display) + eglDisplay = gst_gl_display_egl_get_from_native(GST_GL_DISPLAY_TYPE_WAYLAND, + gst_gl_display_get_handle(display)); + else { + eglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); + if (eglDisplay == EGL_NO_DISPLAY) + return false; + + if (!eglInitialize(eglDisplay, nullptr, nullptr) || !eglBindAPI(EGL_OPENGL_ES_API)) + return false; + } GST_DEBUG("eglDisplay %p", eglDisplay); struct InitializeContext { @@ -210,8 +221,10 @@ bool WPEThreadedView::initialize(GstWpeSrc* src, GstGLContext* context, GstGLDis GMutexHolder lock(view.threading.mutex); - view.gst.context = GST_GL_CONTEXT(gst_object_ref(initializeContext.context)); - view.gst.display = GST_GL_DISPLAY(gst_object_ref(initializeContext.display)); + if (initializeContext.context) + view.gst.context = GST_GL_CONTEXT(gst_object_ref(initializeContext.context)); + if (initializeContext.display) + view.gst.display = GST_GL_DISPLAY(gst_object_ref(initializeContext.display)); view.wpe.width = initializeContext.width; view.wpe.height = initializeContext.height; @@ -273,25 +286,58 @@ GstEGLImage* WPEThreadedView::image() bool dispatchFrameComplete = false; { - GMutexHolder lock(images.mutex); + GMutexHolder lock(images_mutex); - GST_TRACE("pending %" GST_PTR_FORMAT " (%d) committed %" GST_PTR_FORMAT " (%d)", images.pending, - GST_IS_EGL_IMAGE(images.pending) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(images.pending)) : 0, - images.committed, - GST_IS_EGL_IMAGE(images.committed) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(images.committed)) : 0); + GST_TRACE("pending %" GST_PTR_FORMAT " (%d) committed %" GST_PTR_FORMAT " (%d)", egl.pending, + GST_IS_EGL_IMAGE(egl.pending) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(egl.pending)) : 0, + egl.committed, + GST_IS_EGL_IMAGE(egl.committed) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(egl.committed)) : 0); - if (images.pending) { - auto* previousImage = images.committed; - images.committed = images.pending; - images.pending = nullptr; + if (egl.pending) { + auto* previousImage = egl.committed; + egl.committed = egl.pending; + egl.pending = nullptr; if (previousImage) gst_egl_image_unref(previousImage); dispatchFrameComplete = true; } - if (images.committed) - ret = images.committed; + if (egl.committed) + ret = egl.committed; + } + + if (dispatchFrameComplete) + frameComplete(); + + return ret; +} + +GstBuffer* WPEThreadedView::buffer() +{ + GstBuffer* ret = nullptr; + bool dispatchFrameComplete = false; + + { + GMutexHolder lock(images_mutex); + + GST_TRACE("pending %" GST_PTR_FORMAT " (%d) committed %" GST_PTR_FORMAT " (%d)", shm.pending, + GST_IS_BUFFER(shm.pending) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(shm.pending)) : 0, + shm.committed, + GST_IS_BUFFER(shm.committed) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(shm.committed)) : 0); + + if (shm.pending) { + auto* previousImage = shm.committed; + shm.committed = shm.pending; + shm.pending = nullptr; + + if (previousImage) + gst_buffer_unref(previousImage); + dispatchFrameComplete = true; + } + + if (shm.committed) + ret = shm.committed; } if (dispatchFrameComplete) @@ -504,13 +550,93 @@ void WPEThreadedView::handleExportedImage(gpointer image) auto* gstImage = gst_egl_image_new_wrapped(gst.context, eglImage, GST_GL_RGBA, imageContext, s_releaseImage); { - GMutexHolder lock(images.mutex); + GMutexHolder lock(images_mutex); GST_TRACE("EGLImage %p wrapped in GstEGLImage %" GST_PTR_FORMAT, eglImage, gstImage); - images.pending = gstImage; + egl.pending = gstImage; } } +#if ENABLE_SHM_BUFFER_SUPPORT +struct SHMBufferContext { + WPEThreadedView* view; + struct wpe_fdo_shm_exported_buffer* buffer; +}; + +void WPEThreadedView::releaseSHMBuffer(gpointer data) +{ + SHMBufferContext* context = static_cast(data); + struct ReleaseBufferContext { + WPEThreadedView& view; + SHMBufferContext* context; + } releaseImageContext{ *this, context }; + + GSource* source = g_idle_source_new(); + g_source_set_callback(source, + [](gpointer data) -> gboolean { + auto& releaseBufferContext = *static_cast(data); + auto& view = releaseBufferContext.view; + GMutexHolder lock(view.threading.mutex); + + struct wpe_fdo_shm_exported_buffer* buffer = static_cast(releaseBufferContext.context->buffer); + GST_TRACE("Dispatch release exported buffer %p", buffer); + wpe_view_backend_exportable_fdo_dispatch_release_shm_exported_buffer(view.wpe.exportable, buffer); + g_cond_signal(&view.threading.cond); + return G_SOURCE_REMOVE; + }, + &releaseImageContext, nullptr); + g_source_set_priority(source, WPE_GLIB_SOURCE_PRIORITY); + + { + GMutexHolder lock(threading.mutex); + g_source_attach(source, glib.context); + g_cond_wait(&threading.cond, &threading.mutex); + } + + g_source_unref(source); +} + +void WPEThreadedView::s_releaseSHMBuffer(gpointer data) +{ + SHMBufferContext* context = static_cast(data); + context->view->releaseSHMBuffer(data); + g_slice_free(SHMBufferContext, context); +} + +void WPEThreadedView::handleExportedBuffer(struct wpe_fdo_shm_exported_buffer* buffer) +{ + struct wl_shm_buffer* shmBuffer = wpe_fdo_shm_exported_buffer_get_shm_buffer(buffer); + auto format = wl_shm_buffer_get_format(shmBuffer); + if (format != WL_SHM_FORMAT_ARGB8888 && format != WL_SHM_FORMAT_XRGB8888) { + GST_ERROR("Unsupported pixel format: %d", format); + return; + } + + int32_t width = wl_shm_buffer_get_width(shmBuffer); + int32_t height = wl_shm_buffer_get_height(shmBuffer); + gint stride = wl_shm_buffer_get_stride(shmBuffer); + gsize size = width * height * 4; + auto* data = static_cast(wl_shm_buffer_get_data(shmBuffer)); + + SHMBufferContext* bufferContext = g_slice_new(SHMBufferContext); + bufferContext->view = this; + bufferContext->buffer = buffer; + + auto* gstBuffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY, data, size, 0, size, bufferContext, s_releaseSHMBuffer); + gsize offsets[1]; + gint strides[1]; + offsets[0] = 0; + strides[0] = stride; + gst_buffer_add_video_meta_full(gstBuffer, GST_VIDEO_FRAME_FLAG_NONE, GST_VIDEO_FORMAT_BGRA, width, height, 1, offsets, strides); + + { + GMutexHolder lock(images_mutex); + GST_TRACE("SHM buffer %p wrapped in buffer %" GST_PTR_FORMAT, buffer, gstBuffer); + shm.pending = gstBuffer; + } +} +#endif + struct wpe_view_backend_exportable_fdo_egl_client WPEThreadedView::s_exportableClient = { #if USE_DEPRECATED_FDO_EGL_IMAGE // export_egl_image @@ -518,7 +644,7 @@ struct wpe_view_backend_exportable_fdo_egl_client WPEThreadedView::s_exportableC auto& view = *static_cast(data); view.handleExportedImage(static_cast(image)); }, - nullptr, + nullptr, nullptr, #else // export_egl_image nullptr, @@ -526,9 +652,18 @@ struct wpe_view_backend_exportable_fdo_egl_client WPEThreadedView::s_exportableC auto& view = *static_cast(data); view.handleExportedImage(static_cast(image)); }, -#endif +#if ENABLE_SHM_BUFFER_SUPPORT + // export_shm_buffer + [](void* data, struct wpe_fdo_shm_exported_buffer* buffer) { + auto& view = *static_cast(data); + view.handleExportedBuffer(buffer); + }, +#else + nullptr, +#endif // ENABLE_SHM_BUFFER_SUPPORT +#endif // USE_DEPRECATED_FDO_EGL_IMAGE // padding - nullptr, nullptr, nullptr + nullptr, nullptr }; void WPEThreadedView::s_releaseImage(GstEGLImage* image, gpointer data) diff --git a/ext/wpe/WPEThreadedView.h b/ext/wpe/WPEThreadedView.h index 95aea9b..c2b77fc 100644 --- a/ext/wpe/WPEThreadedView.h +++ b/ext/wpe/WPEThreadedView.h @@ -31,6 +31,12 @@ typedef struct _GstGLContext GstGLContext; typedef struct _GstGLDisplay GstGLDisplay; typedef struct _GstEGLImage GstEGLImage; +#if defined(WPE_FDO_CHECK_VERSION) && WPE_FDO_CHECK_VERSION(1, 5, 0) +#define ENABLE_SHM_BUFFER_SUPPORT 1 +#else +#define ENABLE_SHM_BUFFER_SUPPORT 0 +#endif + class WPEThreadedView { public: WPEThreadedView(); @@ -44,17 +50,25 @@ public: void setDrawBackground(gboolean); GstEGLImage* image(); + GstBuffer* buffer(); struct wpe_view_backend* backend() const; protected: void handleExportedImage(gpointer); +#if ENABLE_SHM_BUFFER_SUPPORT + void handleExportedBuffer(struct wpe_fdo_shm_exported_buffer*); +#endif private: void frameComplete(); void loadUriUnlocked(const gchar*); void releaseImage(gpointer); +#if ENABLE_SHM_BUFFER_SUPPORT + void releaseSHMBuffer(gpointer); + static void s_releaseSHMBuffer(gpointer); +#endif static void s_loadEvent(WebKitWebView*, WebKitLoadEvent, gpointer); @@ -90,9 +104,17 @@ private: WebKitWebView* view; } webkit = { nullptr, nullptr }; + // This mutex guards access to either egl or shm resources declared below, + // depending on the runtime behavior. + GMutex images_mutex; + struct { - GMutex mutex; - GstEGLImage* pending { nullptr }; - GstEGLImage* committed { nullptr }; - } images; + GstEGLImage* pending; + GstEGLImage* committed; + } egl { nullptr, nullptr }; + + struct { + GstBuffer* pending; + GstBuffer* committed; + } shm { nullptr, nullptr }; }; diff --git a/ext/wpe/gstwpesrc.cpp b/ext/wpe/gstwpesrc.cpp index 8ef816c..a720c1a 100644 --- a/ext/wpe/gstwpesrc.cpp +++ b/ext/wpe/gstwpesrc.cpp @@ -24,7 +24,13 @@ * The wpesrc element is used to produce a video texture representing a web page * rendered off-screen by WPE. * - * ## Example launch line + * Starting from WPEBackend-FDO 1.6.x, software rendering support is available. This + * features allows wpesrc to be used on machines without GPU, and/or for testing + * purpose. To enable it, set the `LIBGL_ALWAYS_SOFTWARE=true` environment + * variable and make sure `video/x-raw, format=BGRA` caps are negotiated by the + * wpesrc element. + * + * ## Example launch lines * * |[ * gst-launch-1.0 -v wpesrc location="https://gstreamer.freedesktop.org" ! queue ! glimagesink @@ -32,6 +38,11 @@ * Shows the GStreamer website homepage * * |[ + * LIBGL_ALWAYS_SOFTWARE=true gst-launch-1.0 -v wpesrc num-buffers=50 location="https://gstreamer.freedesktop.org" ! videoconvert ! pngenc ! multifilesink location=/tmp/snapshot-%05d.png + * ]| + * Saves the first 50 video frames generated for the GStreamer website as PNG files in /tmp. + * + * |[ * gst-play-1.0 --videosink gtkglsink wpe://https://gstreamer.freedesktop.org * ]| * Shows the GStreamer website homepage as played with GstPlayer in a GTK+ window. @@ -102,6 +113,7 @@ struct _GstWpeSrc gboolean draw_background; GBytes *bytes; + gboolean gl_enabled; }; static void gst_wpe_src_uri_handler_init (gpointer iface, gpointer data); @@ -118,9 +130,41 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE ", " "framerate = " GST_VIDEO_FPS_RANGE ", " - "pixel-aspect-ratio = (fraction)1/1," "texture-target = (string)2D") + "pixel-aspect-ratio = (fraction)1/1, texture-target = (string)2D;" +#if ENABLE_SHM_BUFFER_SUPPORT + "video/x-raw, " + "format = (string) BGRA, " + "width = " GST_VIDEO_SIZE_RANGE ", " + "height = " GST_VIDEO_SIZE_RANGE ", " + "framerate = " GST_VIDEO_FPS_RANGE ", " + "pixel-aspect-ratio = (fraction)1/1" +#endif + ) ); +static GstFlowReturn +gst_wpe_src_create (GstBaseSrc * bsrc, guint64 offset, guint length, GstBuffer ** buf) +{ + GstWpeSrc *src = GST_WPE_SRC (bsrc); + GstFlowReturn ret = GST_FLOW_ERROR; + GstBuffer *locked_buffer; + + GST_OBJECT_LOCK (src); + if (src->gl_enabled) { + GST_OBJECT_UNLOCK (src); + return GST_CALL_PARENT_WITH_DEFAULT (GST_BASE_SRC_CLASS, create, (bsrc, offset, length, buf), ret); + } + + locked_buffer = src->view->buffer (); + + if (locked_buffer != NULL) { + *buf = gst_buffer_ref (locked_buffer); + ret = GST_FLOW_OK; + } + GST_OBJECT_UNLOCK (src); + return ret; +} + static gboolean gst_wpe_src_fill_memory (GstGLBaseSrc * bsrc, GstGLMemory * memory) { @@ -160,13 +204,28 @@ gst_wpe_src_gl_start (GstGLBaseSrc * base_src) { GstWpeSrc *src = GST_WPE_SRC (base_src); gboolean result = TRUE; + GstCapsFeatures *caps_features; + GstGLContext *context = NULL; + GstGLDisplay *display = NULL; GST_INFO_OBJECT (src, "Starting up"); GST_OBJECT_LOCK (src); + + caps_features = gst_caps_get_features (base_src->out_caps, 0); + if (caps_features != NULL && gst_caps_features_contains (caps_features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { + src->gl_enabled = TRUE; + context = base_src->context; + display = base_src->display; + } else { + src->gl_enabled = FALSE; + } + + GST_DEBUG_OBJECT (src, "Will fill GLMemories: %d\n", src->gl_enabled); + src->view = new WPEThreadedView; - result = src->view->initialize (src, base_src->context, base_src->display, - GST_VIDEO_INFO_WIDTH (&base_src->out_info), - GST_VIDEO_INFO_HEIGHT (&base_src->out_info)); + result = src->view->initialize (src, context, display, + GST_VIDEO_INFO_WIDTH (&base_src->out_info), + GST_VIDEO_INFO_HEIGHT (&base_src->out_info)); if (src->bytes != NULL) { src->view->loadData (src->bytes); @@ -484,6 +543,7 @@ gst_wpe_src_class_init (GstWpeSrcClass * klass) gst_element_class_add_static_pad_template (gstelement_class, &src_factory); base_src_class->fixate = GST_DEBUG_FUNCPTR (gst_wpe_src_fixate); + base_src_class->create = GST_DEBUG_FUNCPTR (gst_wpe_src_create); gl_base_src_class->supported_gl_api = static_cast < GstGLAPI > diff --git a/ext/wpe/meson.build b/ext/wpe/meson.build index 0dbdf54..9381e9b 100644 --- a/ext/wpe/meson.build +++ b/ext/wpe/meson.build @@ -10,6 +10,7 @@ wpe_dep = dependency('wpe-webkit-1.0', version : '>= 2.24', required : get_optio wpe_fdo_dep = dependency('wpebackend-fdo-1.0', required : get_option('wpe')) egl_dep = dependency('egl', required : get_option('wpe')) xkbcommon_dep = dependency('xkbcommon', version : '>= 0.8', required : get_option('wpe')) +wl_server_dep = dependency('wayland-server', required : get_option('wpe')) if not wpe_dep.found() or not wpe_fdo_dep.found() or not egl_dep.found() or not xkbcommon_dep.found() subdir_done() @@ -17,7 +18,7 @@ endif gstwpe = library('gstwpe', ['WPEThreadedView.cpp', 'gstwpesrc.cpp'], - dependencies : [egl_dep, wpe_dep, wpe_fdo_dep, gstvideo_dep, gstbase_dep, gstgl_dep, xkbcommon_dep], + dependencies : [egl_dep, wpe_dep, wpe_fdo_dep, gstvideo_dep, gstbase_dep, gstgl_dep, xkbcommon_dep, wl_server_dep], cpp_args : gst_plugins_bad_args + ['-DHAVE_CONFIG_H=1'], include_directories : [configinc], install : true, -- 2.7.4