wpe: WebView and WebContext handling fixes
[platform/upstream/gstreamer.git] / ext / wpe / WPEThreadedView.cpp
1 /* Copyright (C) <2018, 2019, 2020> Philippe Normand <philn@igalia.com>
2  * Copyright (C) <2018> Žan Doberšek <zdobersek@igalia.com>
3  *
4  * This library is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Library General Public
6  * License as published by the Free Software Foundation; either
7  * version 2 of the License, or (at your option) any later version.
8  *
9  * This library is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12  * Library General Public License for more details.
13  *
14  * You should have received a copy of the GNU Library General Public
15  * License along with this library; if not, write to the
16  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
17  * Boston, MA 02110-1301, USA.
18  */
19
20 #ifdef HAVE_CONFIG_H
21 #include <config.h>
22 #endif
23
24 #include "WPEThreadedView.h"
25
26 #include <gst/gl/gl.h>
27 #include <gst/gl/egl/gsteglimage.h>
28 #include <gst/gl/egl/gstgldisplay_egl.h>
29 #include <wayland-server.h>
30
31 #include <cstdio>
32 #include <mutex>
33
34 #if ENABLE_SHM_BUFFER_SUPPORT
35 #include <wpe/unstable/fdo-shm.h>
36 #endif
37
38 GST_DEBUG_CATEGORY_EXTERN (wpe_src_debug);
39 #define GST_CAT_DEFAULT wpe_src_debug
40
41 #if defined(WPE_FDO_CHECK_VERSION) && WPE_FDO_CHECK_VERSION(1, 3, 0)
42 #define USE_DEPRECATED_FDO_EGL_IMAGE 0
43 #define WPE_GLIB_SOURCE_PRIORITY G_PRIORITY_DEFAULT
44 #else
45 #define USE_DEPRECATED_FDO_EGL_IMAGE 1
46 #define WPE_GLIB_SOURCE_PRIORITY -70
47 #endif
48
49 class GMutexHolder {
50 public:
51     GMutexHolder(GMutex& mutex)
52         : m(mutex)
53     {
54         g_mutex_lock(&m);
55     }
56     ~GMutexHolder()
57     {
58         g_mutex_unlock(&m);
59     }
60
61 private:
62     GMutex& m;
63 };
64
65 static WPEContextThread *s_view = NULL;
66
67 WPEContextThread& WPEContextThread::singleton()
68 {
69     if (!s_view)
70         s_view = new WPEContextThread;
71
72     return *s_view;
73 }
74
75 WPEContextThread::WPEContextThread()
76 {
77     g_mutex_init(&threading.mutex);
78     g_cond_init(&threading.cond);
79     g_mutex_init(&threading.ready_mutex);
80     g_cond_init(&threading.ready_cond);
81
82     {
83         GMutexHolder lock(threading.mutex);
84         threading.thread = g_thread_new("WPEContextThread", s_viewThread, this);
85         g_cond_wait(&threading.cond, &threading.mutex);
86         GST_DEBUG("thread spawned");
87     }
88 }
89
90 WPEContextThread::~WPEContextThread()
91 {
92     if (threading.thread) {
93         g_thread_unref(threading.thread);
94         threading.thread = nullptr;
95     }
96
97     g_mutex_clear(&threading.mutex);
98     g_cond_clear(&threading.cond);
99     g_mutex_clear(&threading.ready_mutex);
100     g_cond_clear(&threading.ready_cond);
101 }
102
103 template<typename Function>
104 void WPEContextThread::dispatch(Function func)
105 {
106     struct Payload {
107         Function& func;
108     };
109     struct Payload payload { func };
110
111     GSource* source = g_idle_source_new();
112     g_source_set_callback(source, [](gpointer data) -> gboolean {
113         auto& view = WPEContextThread::singleton();
114         GMutexHolder lock(view.threading.mutex);
115
116         auto* payload = static_cast<struct Payload*>(data);
117         payload->func();
118
119         g_cond_signal(&view.threading.cond);
120         return G_SOURCE_REMOVE;
121     }, &payload, nullptr);
122     g_source_set_priority(source, WPE_GLIB_SOURCE_PRIORITY);
123
124     {
125         GMutexHolder lock(threading.mutex);
126         g_source_attach(source, glib.context);
127         g_cond_wait(&threading.cond, &threading.mutex);
128     }
129
130     g_source_unref(source);
131 }
132
133 gpointer WPEContextThread::s_viewThread(gpointer data)
134 {
135     auto& view = *static_cast<WPEContextThread*>(data);
136
137     view.glib.context = g_main_context_new();
138     view.glib.loop = g_main_loop_new(view.glib.context, FALSE);
139
140     g_main_context_push_thread_default(view.glib.context);
141
142     {
143         GSource* source = g_idle_source_new();
144         g_source_set_callback(source,
145             [](gpointer data) -> gboolean {
146                 auto& view = *static_cast<WPEContextThread*>(data);
147                 GMutexHolder lock(view.threading.mutex);
148                 g_cond_signal(&view.threading.cond);
149                 return G_SOURCE_REMOVE;
150             },
151             &view, nullptr);
152         g_source_attach(source, view.glib.context);
153         g_source_unref(source);
154     }
155
156     g_main_loop_run(view.glib.loop);
157
158     g_main_loop_unref(view.glib.loop);
159     view.glib.loop = nullptr;
160
161     g_main_context_pop_thread_default(view.glib.context);
162     g_main_context_unref(view.glib.context);
163     view.glib.context = nullptr;
164     return nullptr;
165 }
166
167 WPEView* WPEContextThread::createWPEView(GstWpeSrc* src, GstGLContext* context, GstGLDisplay* display, int width, int height)
168 {
169     GST_DEBUG("context %p display %p, size (%d,%d)", context, display, width, height);
170     threading.ready = FALSE;
171
172     static std::once_flag s_loaderFlag;
173     std::call_once(s_loaderFlag,
174         [] {
175 #if defined(WPE_BACKEND_CHECK_VERSION) && WPE_BACKEND_CHECK_VERSION(1, 2, 0)
176             wpe_loader_init("libWPEBackend-fdo-1.0.so");
177 #endif
178         });
179
180     WPEView* view = nullptr;
181     dispatch([&]() mutable {
182         if (!glib.web_context) {
183             auto* manager = webkit_website_data_manager_new_ephemeral();
184             glib.web_context = webkit_web_context_new_with_website_data_manager(manager);
185             g_object_unref(manager);
186         }
187
188         view = new WPEView(glib.web_context, src, context, display, width, height);
189     });
190
191     if (view && view->hasUri()) {
192         GST_DEBUG("waiting load to finish");
193         GMutexHolder lock(threading.ready_mutex);
194         while (!threading.ready)
195             g_cond_wait(&threading.ready_cond, &threading.ready_mutex);
196         GST_DEBUG("done");
197     }
198
199     return view;
200 }
201
202 void WPEContextThread::notifyLoadFinished()
203 {
204     GMutexHolder lock(threading.ready_mutex);
205     if (!threading.ready) {
206         threading.ready = TRUE;
207         g_cond_signal(&threading.ready_cond);
208     }
209 }
210
211 static gboolean s_loadFailed(WebKitWebView*, WebKitLoadEvent, gchar* failing_uri, GError* error, gpointer data)
212 {
213     GstWpeSrc* src = GST_WPE_SRC(data);
214     GST_ELEMENT_ERROR (GST_ELEMENT_CAST(src), RESOURCE, FAILED, (NULL), ("Failed to load %s (%s)", failing_uri, error->message));
215     return FALSE;
216 }
217
218 WPEView::WPEView(WebKitWebContext* web_context, GstWpeSrc* src, GstGLContext* context, GstGLDisplay* display, int width, int height)
219 {
220     g_mutex_init(&images_mutex);
221     if (context)
222         gst.context = GST_GL_CONTEXT(gst_object_ref(context));
223     if (display)
224         gst.display = GST_GL_DISPLAY(gst_object_ref(display));
225
226     wpe.width = width;
227     wpe.height = height;
228
229     EGLDisplay eglDisplay = EGL_NO_DISPLAY;
230     if (context && display)
231         eglDisplay = gst_gl_display_egl_get_from_native(GST_GL_DISPLAY_TYPE_WAYLAND, gst_gl_display_get_handle(display));
232     GST_DEBUG("eglDisplay %p", eglDisplay);
233
234     if (eglDisplay) {
235         m_isValid = wpe_fdo_initialize_for_egl_display(eglDisplay);
236         GST_DEBUG("FDO EGL display initialisation result: %d", m_isValid);
237     } else {
238 #if ENABLE_SHM_BUFFER_SUPPORT
239         m_isValid = wpe_fdo_initialize_shm();
240         GST_DEBUG("FDO SHM initialisation result: %d", m_isValid);
241 #else
242         GST_WARNING("FDO SHM support is available only in WPEBackend-FDO 1.7.0");
243 #endif
244     }
245     if (!m_isValid)
246         return;
247
248     if (eglDisplay) {
249         wpe.exportable = wpe_view_backend_exportable_fdo_egl_create(&s_exportableEGLClient, this, wpe.width, wpe.height);
250     } else {
251 #if ENABLE_SHM_BUFFER_SUPPORT
252         wpe.exportable = wpe_view_backend_exportable_fdo_create(&s_exportableClient, this, wpe.width, wpe.height);
253 #endif
254     }
255
256     auto* wpeViewBackend = wpe_view_backend_exportable_fdo_get_view_backend(wpe.exportable);
257     auto* viewBackend = webkit_web_view_backend_new(wpeViewBackend, (GDestroyNotify) wpe_view_backend_exportable_fdo_destroy, wpe.exportable);
258 #if defined(WPE_BACKEND_CHECK_VERSION) && WPE_BACKEND_CHECK_VERSION(1, 1, 0)
259     wpe_view_backend_add_activity_state(wpeViewBackend, wpe_view_activity_state_visible | wpe_view_activity_state_focused | wpe_view_activity_state_in_window);
260 #endif
261
262     webkit.view = WEBKIT_WEB_VIEW(g_object_new(WEBKIT_TYPE_WEB_VIEW, "web-context", web_context, "backend", viewBackend, nullptr));
263
264     g_signal_connect(webkit.view, "load-failed", G_CALLBACK(s_loadFailed), src);
265     g_signal_connect(webkit.view, "load-failed-with-tls-errors", G_CALLBACK(s_loadFailed), src);
266
267     gst_wpe_src_configure_web_view(src, webkit.view);
268
269     const gchar* location;
270     gboolean drawBackground = TRUE;
271     g_object_get(src, "location", &location, "draw-background", &drawBackground, nullptr);
272     setDrawBackground(drawBackground);
273     if (location)
274         loadUriUnlocked(location);
275 }
276
277 WPEView::~WPEView()
278 {
279     {
280         GMutexHolder lock(images_mutex);
281
282         if (egl.pending) {
283             gst_egl_image_unref(egl.pending);
284             egl.pending = nullptr;
285         }
286         if (egl.committed) {
287             gst_egl_image_unref(egl.committed);
288             egl.committed = nullptr;
289         }
290     }
291
292     WPEContextThread::singleton().dispatch([&]() {
293         if (webkit.view) {
294             g_object_unref(webkit.view);
295             webkit.view = nullptr;
296         }
297     });
298
299     if (gst.display) {
300         gst_object_unref(gst.display);
301         gst.display = nullptr;
302     }
303
304     if (gst.context) {
305         gst_object_unref(gst.context);
306         gst.context = nullptr;
307     }
308     if (webkit.uri) {
309         g_free(webkit.uri);
310         webkit.uri = nullptr;
311     }
312
313     g_mutex_clear(&images_mutex);
314 }
315
316 GstEGLImage* WPEView::image()
317 {
318     GstEGLImage* ret = nullptr;
319     bool dispatchFrameComplete = false;
320
321     {
322         GMutexHolder lock(images_mutex);
323
324         GST_TRACE("pending %" GST_PTR_FORMAT " (%d) committed %" GST_PTR_FORMAT " (%d)", egl.pending,
325                   GST_IS_EGL_IMAGE(egl.pending) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(egl.pending)) : 0,
326                   egl.committed,
327                   GST_IS_EGL_IMAGE(egl.committed) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(egl.committed)) : 0);
328
329         if (egl.pending) {
330             auto* previousImage = egl.committed;
331             egl.committed = egl.pending;
332             egl.pending = nullptr;
333
334             if (previousImage)
335                 gst_egl_image_unref(previousImage);
336             dispatchFrameComplete = true;
337         }
338
339         if (egl.committed)
340             ret = egl.committed;
341     }
342
343     if (dispatchFrameComplete)
344         frameComplete();
345
346     return ret;
347 }
348
349 GstBuffer* WPEView::buffer()
350 {
351     GstBuffer* ret = nullptr;
352     bool dispatchFrameComplete = false;
353
354     {
355         GMutexHolder lock(images_mutex);
356
357         GST_TRACE("pending %" GST_PTR_FORMAT " (%d) committed %" GST_PTR_FORMAT " (%d)", shm.pending,
358                   GST_IS_BUFFER(shm.pending) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(shm.pending)) : 0,
359                   shm.committed,
360                   GST_IS_BUFFER(shm.committed) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(shm.committed)) : 0);
361
362         if (shm.pending) {
363             auto* previousImage = shm.committed;
364             shm.committed = shm.pending;
365             shm.pending = nullptr;
366
367             if (previousImage)
368                 gst_buffer_unref(previousImage);
369             dispatchFrameComplete = true;
370         }
371
372         if (shm.committed)
373             ret = shm.committed;
374     }
375
376     if (dispatchFrameComplete)
377         frameComplete();
378
379     return ret;
380 }
381
382 void WPEView::resize(int width, int height)
383 {
384     GST_DEBUG("resize to %dx%d", width, height);
385     wpe.width = width;
386     wpe.height = height;
387
388     s_view->dispatch([&]() {
389         if (wpe.exportable && wpe_view_backend_exportable_fdo_get_view_backend(wpe.exportable))
390           wpe_view_backend_dispatch_set_size(wpe_view_backend_exportable_fdo_get_view_backend(wpe.exportable), wpe.width, wpe.height);
391     });
392 }
393
394 void WPEView::frameComplete()
395 {
396     GST_TRACE("frame complete");
397     s_view->dispatch([&]() {
398         GST_TRACE("dispatching");
399         wpe_view_backend_exportable_fdo_dispatch_frame_complete(wpe.exportable);
400     });
401 }
402
403 void WPEView::loadUriUnlocked(const gchar* uri)
404 {
405     if (webkit.uri)
406         g_free(webkit.uri);
407
408     GST_DEBUG("loading %s", uri);
409     webkit.uri = g_strdup(uri);
410     webkit_web_view_load_uri(webkit.view, webkit.uri);
411 }
412
413 void WPEView::loadUri(const gchar* uri)
414 {
415     s_view->dispatch([&]() {
416         loadUriUnlocked(uri);
417     });
418 }
419
420 void WPEView::loadData(GBytes* bytes)
421 {
422     s_view->dispatch([this, bytes = g_bytes_ref(bytes)]() {
423         webkit_web_view_load_bytes(webkit.view, bytes, nullptr, nullptr, nullptr);
424         g_bytes_unref(bytes);
425     });
426 }
427
428 void WPEView::setDrawBackground(gboolean drawsBackground)
429 {
430 #if WEBKIT_CHECK_VERSION(2, 24, 0)
431     GST_DEBUG("%s background rendering", drawsBackground ? "Enabling" : "Disabling");
432     WebKitColor color;
433     webkit_color_parse(&color, drawsBackground ? "white" : "transparent");
434     webkit_web_view_set_background_color(webkit.view, &color);
435 #else
436     GST_FIXME("webkit_web_view_set_background_color is not implemented in WPE %u.%u. Please upgrade to 2.24", webkit_get_major_version(), webkit_get_minor_version());
437 #endif
438 }
439
440 void WPEView::releaseImage(gpointer imagePointer)
441 {
442     s_view->dispatch([&]() {
443         GST_TRACE("Dispatch release exported image %p", imagePointer);
444 #if USE_DEPRECATED_FDO_EGL_IMAGE
445         wpe_view_backend_exportable_fdo_egl_dispatch_release_image(wpe.exportable,
446                                                                    static_cast<EGLImageKHR>(imagePointer));
447 #else
448         wpe_view_backend_exportable_fdo_egl_dispatch_release_exported_image(wpe.exportable,
449                                                                             static_cast<struct wpe_fdo_egl_exported_image*>(imagePointer));
450 #endif
451     });
452 }
453
454 struct ImageContext {
455     WPEView* view;
456     gpointer image;
457 };
458
459 void WPEView::handleExportedImage(gpointer image)
460 {
461     ImageContext* imageContext = g_slice_new(ImageContext);
462     imageContext->view = this;
463     imageContext->image = static_cast<gpointer>(image);
464     EGLImageKHR eglImage;
465 #if USE_DEPRECATED_FDO_EGL_IMAGE
466     eglImage = static_cast<EGLImageKHR>(image);
467 #else
468     eglImage = wpe_fdo_egl_exported_image_get_egl_image(static_cast<struct wpe_fdo_egl_exported_image*>(image));
469 #endif
470
471     auto* gstImage = gst_egl_image_new_wrapped(gst.context, eglImage, GST_GL_RGBA, imageContext, s_releaseImage);
472     {
473       GMutexHolder lock(images_mutex);
474
475       GST_TRACE("EGLImage %p wrapped in GstEGLImage %" GST_PTR_FORMAT, eglImage, gstImage);
476       egl.pending = gstImage;
477
478       s_view->notifyLoadFinished();
479     }
480 }
481
482 #if ENABLE_SHM_BUFFER_SUPPORT
483 struct SHMBufferContext {
484     WPEView* view;
485     struct wpe_fdo_shm_exported_buffer* buffer;
486 };
487
488 void WPEView::releaseSHMBuffer(gpointer data)
489 {
490     SHMBufferContext* context = static_cast<SHMBufferContext*>(data);
491     s_view->dispatch([&]() {
492         auto* buffer = static_cast<struct wpe_fdo_shm_exported_buffer*>(context->buffer);
493         GST_TRACE("Dispatch release exported buffer %p", buffer);
494         wpe_view_backend_exportable_fdo_dispatch_release_shm_exported_buffer(wpe.exportable, buffer);
495     });
496 }
497
498 void WPEView::s_releaseSHMBuffer(gpointer data)
499 {
500     SHMBufferContext* context = static_cast<SHMBufferContext*>(data);
501     context->view->releaseSHMBuffer(data);
502     g_slice_free(SHMBufferContext, context);
503 }
504
505 void WPEView::handleExportedBuffer(struct wpe_fdo_shm_exported_buffer* buffer)
506 {
507     struct wl_shm_buffer* shmBuffer = wpe_fdo_shm_exported_buffer_get_shm_buffer(buffer);
508     auto format = wl_shm_buffer_get_format(shmBuffer);
509     if (format != WL_SHM_FORMAT_ARGB8888 && format != WL_SHM_FORMAT_XRGB8888) {
510         GST_ERROR("Unsupported pixel format: %d", format);
511         return;
512     }
513
514     int32_t width = wl_shm_buffer_get_width(shmBuffer);
515     int32_t height = wl_shm_buffer_get_height(shmBuffer);
516     gint stride = wl_shm_buffer_get_stride(shmBuffer);
517     gsize size = width * height * 4;
518     auto* data = static_cast<uint8_t*>(wl_shm_buffer_get_data(shmBuffer));
519
520     SHMBufferContext* bufferContext = g_slice_new(SHMBufferContext);
521     bufferContext->view = this;
522     bufferContext->buffer = buffer;
523
524     auto* gstBuffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY, data, size, 0, size, bufferContext, s_releaseSHMBuffer);
525     gsize offsets[1];
526     gint strides[1];
527     offsets[0] = 0;
528     strides[0] = stride;
529     gst_buffer_add_video_meta_full(gstBuffer, GST_VIDEO_FRAME_FLAG_NONE, GST_VIDEO_FORMAT_BGRA, width, height, 1, offsets, strides);
530
531     {
532         GMutexHolder lock(images_mutex);
533         GST_TRACE("SHM buffer %p wrapped in buffer %" GST_PTR_FORMAT, buffer, gstBuffer);
534         shm.pending = gstBuffer;
535         s_view->notifyLoadFinished();
536     }
537 }
538 #endif
539
540 struct wpe_view_backend_exportable_fdo_egl_client WPEView::s_exportableEGLClient = {
541 #if USE_DEPRECATED_FDO_EGL_IMAGE
542     // export_egl_image
543     [](void* data, EGLImageKHR image) {
544         auto& view = *static_cast<WPEView*>(data);
545         view.handleExportedImage(static_cast<gpointer>(image));
546     },
547     nullptr, nullptr,
548 #else
549     // export_egl_image
550     nullptr,
551     [](void* data, struct wpe_fdo_egl_exported_image* image) {
552         auto& view = *static_cast<WPEView*>(data);
553         view.handleExportedImage(static_cast<gpointer>(image));
554     },
555     nullptr,
556 #endif // USE_DEPRECATED_FDO_EGL_IMAGE
557     // padding
558     nullptr, nullptr
559 };
560
561 #if ENABLE_SHM_BUFFER_SUPPORT
562 struct wpe_view_backend_exportable_fdo_client WPEView::s_exportableClient = {
563     nullptr,
564     nullptr,
565     // export_shm_buffer
566     [](void* data, struct wpe_fdo_shm_exported_buffer* buffer) {
567         auto& view = *static_cast<WPEView*>(data);
568         view.handleExportedBuffer(buffer);
569     },
570     nullptr,
571     nullptr,
572 };
573 #endif
574
575 void WPEView::s_releaseImage(GstEGLImage* image, gpointer data)
576 {
577     ImageContext* context = static_cast<ImageContext*>(data);
578     context->view->releaseImage(context->image);
579     g_slice_free(ImageContext, context);
580 }
581
582 struct wpe_view_backend* WPEView::backend() const
583 {
584     return wpe.exportable ? wpe_view_backend_exportable_fdo_get_view_backend(wpe.exportable) : nullptr;
585 }
586
587 void WPEView::dispatchKeyboardEvent(struct wpe_input_keyboard_event& wpe_event)
588 {
589     s_view->dispatch([&]() {
590         wpe_view_backend_dispatch_keyboard_event(backend(), &wpe_event);
591     });
592 }
593
594 void WPEView::dispatchPointerEvent(struct wpe_input_pointer_event& wpe_event)
595 {
596     s_view->dispatch([&]() {
597         wpe_view_backend_dispatch_pointer_event(backend(), &wpe_event);
598     });
599 }
600
601 void WPEView::dispatchAxisEvent(struct wpe_input_axis_event& wpe_event)
602 {
603     s_view->dispatch([&]() {
604         wpe_view_backend_dispatch_axis_event(backend(), &wpe_event);
605     });
606 }