1 /* GStreamer Wayland video sink
3 * Copyright (C) 2011 Intel Corporation
4 * Copyright (C) 2011 Sreerenj Balachandran <sreerenj.balachandran@intel.com>
5 * Copyright (C) 2012 Wim Taymans <wim.taymans@gmail.com>
6 * Copyright (C) 2014 Collabora Ltd.
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Library General Public
10 * License as published by the Free Software Foundation; either
11 * version 2 of the License, or (at your option) any later version.
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Library General Public License for more details.
18 * You should have received a copy of the GNU Library General Public
19 * License along with this library; if not, write to the Free
20 * Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
21 * Boston, MA 02110-1301 USA.
25 * SECTION:element-waylandsink
27 * The waylandsink is creating its own window and render the decoded video frames to that.
28 * Setup the Wayland environment as described in
29 * <ulink url="http://wayland.freedesktop.org/building.html">Wayland</ulink> home page.
30 * The current implementaion is based on weston compositor.
33 * <title>Example pipelines</title>
35 * gst-launch -v videotestsrc ! waylandsink
36 * ]| test the video rendering in wayland
44 #include "gstwaylandsink.h"
45 #ifdef GST_WLSINK_ENHANCEMENT
48 #include "wlvideoformat.h"
50 #include "wlshmallocator.h"
52 #include <gst/wayland/wayland.h>
53 #include <gst/video/videooverlay.h>
74 #ifdef GST_WLSINK_ENHANCEMENT
80 GST_DEBUG_CATEGORY (gstwayland_debug);
81 #define GST_CAT_DEFAULT gstwayland_debug
83 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
86 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE
87 ("{ BGRx, BGRA, RGBx, xBGR, xRGB, RGBA, ABGR, ARGB, RGB, BGR, "
88 "RGB16, BGR16, YUY2, YVYU, UYVY, AYUV, NV12, NV21, NV16, "
89 #ifdef GST_WLSINK_ENHANCEMENT
92 "YUV9, YVU9, Y41B, I420, YV12, Y42B, v308 }"))
95 static void gst_wayland_sink_get_property (GObject * object,
96 guint prop_id, GValue * value, GParamSpec * pspec);
97 static void gst_wayland_sink_set_property (GObject * object,
98 guint prop_id, const GValue * value, GParamSpec * pspec);
99 static void gst_wayland_sink_finalize (GObject * object);
101 static GstStateChangeReturn gst_wayland_sink_change_state (GstElement * element,
102 GstStateChange transition);
103 static void gst_wayland_sink_set_context (GstElement * element,
104 GstContext * context);
106 static GstCaps *gst_wayland_sink_get_caps (GstBaseSink * bsink,
108 static gboolean gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps);
109 static gboolean gst_wayland_sink_preroll (GstBaseSink * bsink,
112 gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query);
113 static gboolean gst_wayland_sink_render (GstBaseSink * bsink,
116 /* VideoOverlay interface */
117 static void gst_wayland_sink_videooverlay_init (GstVideoOverlayInterface *
119 static void gst_wayland_sink_set_window_handle (GstVideoOverlay * overlay,
121 static void gst_wayland_sink_set_render_rectangle (GstVideoOverlay * overlay,
122 gint x, gint y, gint w, gint h);
123 static void gst_wayland_sink_expose (GstVideoOverlay * overlay);
125 /* WaylandVideo interface */
126 static void gst_wayland_sink_waylandvideo_init (GstWaylandVideoInterface *
128 static void gst_wayland_sink_begin_geometry_change (GstWaylandVideo * video);
129 static void gst_wayland_sink_end_geometry_change (GstWaylandVideo * video);
131 #define gst_wayland_sink_parent_class parent_class
132 G_DEFINE_TYPE_WITH_CODE (GstWaylandSink, gst_wayland_sink, GST_TYPE_VIDEO_SINK,
133 G_IMPLEMENT_INTERFACE (GST_TYPE_VIDEO_OVERLAY,
134 gst_wayland_sink_videooverlay_init)
135 G_IMPLEMENT_INTERFACE (GST_TYPE_WAYLAND_VIDEO,
136 gst_wayland_sink_waylandvideo_init));
139 gst_wayland_sink_class_init (GstWaylandSinkClass * klass)
142 GObjectClass *gobject_class;
143 GstElementClass *gstelement_class;
144 GstBaseSinkClass *gstbasesink_class;
146 gobject_class = (GObjectClass *) klass;
147 gstelement_class = (GstElementClass *) klass;
148 gstbasesink_class = (GstBaseSinkClass *) klass;
150 gobject_class->set_property = gst_wayland_sink_set_property;
151 gobject_class->get_property = gst_wayland_sink_get_property;
152 gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_wayland_sink_finalize);
154 gst_element_class_add_pad_template (gstelement_class,
155 gst_static_pad_template_get (&sink_template));
157 gst_element_class_set_static_metadata (gstelement_class,
158 "wayland video sink", "Sink/Video",
159 "Output to wayland surface",
160 "Sreerenj Balachandran <sreerenj.balachandran@intel.com>, "
161 "George Kiagiadakis <george.kiagiadakis@collabora.com>");
163 gstelement_class->change_state =
164 GST_DEBUG_FUNCPTR (gst_wayland_sink_change_state);
165 gstelement_class->set_context =
166 GST_DEBUG_FUNCPTR (gst_wayland_sink_set_context);
168 gstbasesink_class->get_caps = GST_DEBUG_FUNCPTR (gst_wayland_sink_get_caps);
169 gstbasesink_class->set_caps = GST_DEBUG_FUNCPTR (gst_wayland_sink_set_caps);
170 gstbasesink_class->preroll = GST_DEBUG_FUNCPTR (gst_wayland_sink_preroll);
171 gstbasesink_class->propose_allocation =
172 GST_DEBUG_FUNCPTR (gst_wayland_sink_propose_allocation);
173 gstbasesink_class->render = GST_DEBUG_FUNCPTR (gst_wayland_sink_render);
175 g_object_class_install_property (gobject_class, PROP_DISPLAY,
176 g_param_spec_string ("display", "Wayland Display name", "Wayland "
177 "display name to connect to, if not supplied via the GstContext",
178 NULL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
180 g_object_class_install_property (gobject_class, PROP_USE_TBM,
181 g_param_spec_boolean ("use-tbm",
182 "Use Tizen Buffer Memory insted of Shared memory",
183 "When enabled, Memory is alloced by TBM insted of SHM ", TRUE,
184 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
189 __write_rawdata (const char *file, const void *data, unsigned int size)
193 fp = fopen (file, "wb");
197 fwrite ((char *) data, sizeof (char), size, fp);
204 gst_wayland_sink_init (GstWaylandSink * sink)
207 sink->USE_TBM = TRUE;
208 g_mutex_init (&sink->display_lock);
209 g_mutex_init (&sink->render_lock);
213 gst_wayland_sink_get_property (GObject * object,
214 guint prop_id, GValue * value, GParamSpec * pspec)
217 GstWaylandSink *sink = GST_WAYLAND_SINK (object);
221 GST_OBJECT_LOCK (sink);
222 g_value_set_string (value, sink->display_name);
223 GST_OBJECT_UNLOCK (sink);
225 #ifdef GST_WLSINK_ENHANCEMENT
227 g_value_set_boolean (value, sink->USE_TBM);
231 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
237 gst_wayland_sink_set_property (GObject * object,
238 guint prop_id, const GValue * value, GParamSpec * pspec)
241 GstWaylandSink *sink = GST_WAYLAND_SINK (object);
245 GST_OBJECT_LOCK (sink);
246 sink->display_name = g_value_dup_string (value);
247 GST_OBJECT_UNLOCK (sink);
249 #ifdef GST_WLSINK_ENHANCEMENT
251 sink->USE_TBM = g_value_get_boolean (value);
252 GST_LOG ("1:USE TBM 0: USE SHM set(%d)", sink->USE_TBM);
256 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
262 gst_wayland_sink_finalize (GObject * object)
265 GstWaylandSink *sink = GST_WAYLAND_SINK (object);
267 GST_DEBUG_OBJECT (sink, "Finalizing the sink..");
269 if (sink->last_buffer)
270 gst_buffer_unref (sink->last_buffer);
272 g_object_unref (sink->display);
274 g_object_unref (sink->window);
276 gst_object_unref (sink->pool);
278 if (sink->display_name)
279 g_free (sink->display_name);
281 g_mutex_clear (&sink->display_lock);
282 g_mutex_clear (&sink->render_lock);
284 G_OBJECT_CLASS (parent_class)->finalize (object);
287 /* must be called with the display_lock */
289 gst_wayland_sink_set_display_from_context (GstWaylandSink * sink,
290 GstContext * context)
293 struct wl_display *display;
294 GError *error = NULL;
296 display = gst_wayland_display_handle_context_get_handle (context);
297 sink->display = gst_wl_display_new_existing (display, FALSE, &error);
300 GST_ELEMENT_WARNING (sink, RESOURCE, OPEN_READ_WRITE,
301 ("Could not set display handle"),
302 ("Failed to use the external wayland display: '%s'", error->message));
303 g_error_free (error);
305 #ifdef GST_WLSINK_ENHANCEMENT
306 sink->display->USE_TBM = sink->USE_TBM;
311 gst_wayland_sink_find_display (GstWaylandSink * sink)
316 GstContext *context = NULL;
317 GError *error = NULL;
320 g_mutex_lock (&sink->display_lock);
322 if (!sink->display) {
323 /* first query upstream for the needed display handle */
324 query = gst_query_new_context (GST_WAYLAND_DISPLAY_HANDLE_CONTEXT_TYPE);
325 if (gst_pad_peer_query (GST_VIDEO_SINK_PAD (sink), query)) {
326 gst_query_parse_context (query, &context);
327 gst_wayland_sink_set_display_from_context (sink, context);
329 gst_query_unref (query);
331 if (G_LIKELY (!sink->display)) {
332 /* now ask the application to set the display handle */
333 msg = gst_message_new_need_context (GST_OBJECT_CAST (sink),
334 GST_WAYLAND_DISPLAY_HANDLE_CONTEXT_TYPE);
336 g_mutex_unlock (&sink->display_lock);
337 gst_element_post_message (GST_ELEMENT_CAST (sink), msg);
338 /* at this point we expect gst_wayland_sink_set_context
339 * to get called and fill sink->display */
340 g_mutex_lock (&sink->display_lock);
342 if (!sink->display) {
343 /* if the application didn't set a display, let's create it ourselves */
344 GST_OBJECT_LOCK (sink);
345 sink->display = gst_wl_display_new (sink->display_name, &error);
346 GST_OBJECT_UNLOCK (sink);
349 GST_ELEMENT_WARNING (sink, RESOURCE, OPEN_READ_WRITE,
350 ("Could not initialise Wayland output"),
351 ("Failed to create GstWlDisplay: '%s'", error->message));
352 g_error_free (error);
355 #ifdef GST_WLSINK_ENHANCEMENT
356 sink->display->USE_TBM = sink->USE_TBM;
362 g_mutex_unlock (&sink->display_lock);
367 static GstStateChangeReturn
368 gst_wayland_sink_change_state (GstElement * element, GstStateChange transition)
371 GstWaylandSink *sink = GST_WAYLAND_SINK (element);
372 GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
374 switch (transition) {
375 case GST_STATE_CHANGE_NULL_TO_READY:
376 if (!gst_wayland_sink_find_display (sink))
377 return GST_STATE_CHANGE_FAILURE;
383 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
384 if (ret == GST_STATE_CHANGE_FAILURE)
387 switch (transition) {
388 case GST_STATE_CHANGE_PAUSED_TO_READY:
389 gst_buffer_replace (&sink->last_buffer, NULL);
391 if (gst_wl_window_is_toplevel (sink->window)) {
392 g_clear_object (&sink->window);
394 /* remove buffer from surface, show nothing */
395 gst_wl_window_render (sink->window, NULL, NULL);
399 case GST_STATE_CHANGE_READY_TO_NULL:
400 g_mutex_lock (&sink->display_lock);
401 /* If we had a toplevel window, we most likely have our own connection
402 * to the display too, and it is a good idea to disconnect and allow
403 * potentially the application to embed us with GstVideoOverlay
404 * (which requires to re-use the same display connection as the parent
405 * surface). If we didn't have a toplevel window, then the display
406 * connection that we have is definitely shared with the application
407 * and it's better to keep it around (together with the window handle)
408 * to avoid requesting them again from the application if/when we are
409 * restarted (GstVideoOverlay behaves like that in other sinks)
411 if (sink->display && !sink->window) { /* -> the window was toplevel */
412 g_clear_object (&sink->display);
414 g_mutex_unlock (&sink->display_lock);
415 g_clear_object (&sink->pool);
425 gst_wayland_sink_set_context (GstElement * element, GstContext * context)
428 GstWaylandSink *sink = GST_WAYLAND_SINK (element);
430 if (gst_context_has_context_type (context,
431 GST_WAYLAND_DISPLAY_HANDLE_CONTEXT_TYPE)) {
432 g_mutex_lock (&sink->display_lock);
433 if (G_LIKELY (!sink->display))
434 gst_wayland_sink_set_display_from_context (sink, context);
436 GST_WARNING_OBJECT (element, "changing display handle is not supported");
437 #ifdef GST_WLSINK_ENHANCEMENT
438 g_mutex_unlock (&sink->display_lock);
442 g_mutex_unlock (&sink->display_lock);
445 if (GST_ELEMENT_CLASS (parent_class)->set_context)
446 GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
450 gst_wayland_sink_get_caps (GstBaseSink * bsink, GstCaps * filter)
453 GstWaylandSink *sink;
456 sink = GST_WAYLAND_SINK (bsink);
458 caps = gst_pad_get_pad_template_caps (GST_VIDEO_SINK_PAD (sink));
460 g_mutex_lock (&sink->display_lock);
463 GValue list = G_VALUE_INIT;
464 GValue value = G_VALUE_INIT;
467 #ifdef GST_WLSINK_ENHANCEMENT
470 enum wl_shm_format fmt;
472 g_value_init (&list, GST_TYPE_LIST);
473 g_value_init (&value, G_TYPE_STRING);
474 #ifdef GST_WLSINK_ENHANCEMENT
475 if (sink->display->USE_TBM)
476 formats = sink->display->tbm_formats;
479 formats = sink->display->formats;
481 for (i = 0; i < formats->len; i++) {
482 #ifdef GST_WLSINK_ENHANCEMENT
484 tbm_fmt = g_array_index (formats, uint32_t, i);
485 g_value_set_string (&value, gst_wl_tbm_format_to_string (tbm_fmt));
486 gst_value_list_append_value (&list, &value);
487 /* TBM doesn't support SN12. So we add SN12 manually as supported format.
488 * SN12 is exactly same with NV12.
490 if (tbm_fmt == TBM_FORMAT_NV12) {
491 g_value_set_string (&value,
492 gst_video_format_to_string (GST_VIDEO_FORMAT_SN12));
493 gst_value_list_append_value (&list, &value);
495 } else { /* USE SHM */
496 fmt = g_array_index (formats, uint32_t, i);
497 g_value_set_string (&value, gst_wl_shm_format_to_string (fmt));
498 gst_value_list_append_value (&list, &value);
500 #else /* open source */
501 fmt = g_array_index (formats, uint32_t, i);
502 g_value_set_string (&value, gst_wl_shm_format_to_string (fmt));
503 gst_value_list_append_value (&list, &value);
507 caps = gst_caps_make_writable (caps);
508 gst_structure_set_value (gst_caps_get_structure (caps, 0), "format", &list);
510 GST_DEBUG_OBJECT (sink, "display caps: %" GST_PTR_FORMAT, caps);
513 g_mutex_unlock (&sink->display_lock);
516 GstCaps *intersection;
519 gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
520 gst_caps_unref (caps);
528 gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
531 GstWaylandSink *sink;
532 GstBufferPool *newpool;
534 #ifdef GST_WLSINK_ENHANCEMENT
537 enum wl_shm_format format;
541 GstStructure *structure;
543 sink = GST_WAYLAND_SINK (bsink);
545 GST_DEBUG_OBJECT (sink, "set caps %" GST_PTR_FORMAT, caps);
547 /* extract info from caps */
548 if (!gst_video_info_from_caps (&info, caps))
550 #ifdef GST_WLSINK_ENHANCEMENT
551 sink->caps = gst_caps_copy (caps);
554 gst_video_format_to_wl_tbm_format (GST_VIDEO_INFO_FORMAT (&info));
555 if ((gint) tbm_format == -1)
558 format = gst_video_format_to_wl_shm_format (GST_VIDEO_INFO_FORMAT (&info));
559 if ((gint) format == -1)
562 #else /* open source */
563 format = gst_video_format_to_wl_shm_format (GST_VIDEO_INFO_FORMAT (&info));
565 if ((gint) format == -1)
569 /* verify we support the requested format */
570 #ifdef GST_WLSINK_ENHANCEMENT
571 if (sink->display->USE_TBM) {
572 GST_ERROR ("USE TBM FORMAT");
573 formats = sink->display->tbm_formats;
574 for (i = 0; i < formats->len; i++) {
575 if (g_array_index (formats, uint32_t, i) == tbm_format)
578 } else { /* USE SHM */
579 GST_ERROR ("USE SHM FORMAT");
580 formats = sink->display->formats;
581 for (i = 0; i < formats->len; i++) {
582 if (g_array_index (formats, uint32_t, i) == format)
586 #else /* open source */
588 formats = sink->display->formats;
589 for (i = 0; i < formats->len; i++) {
590 if (g_array_index (formats, uint32_t, i) == format)
594 if (i >= formats->len)
595 goto unsupported_format;
597 #ifdef GST_WLSINK_ENHANCEMENT
599 if (GST_VIDEO_INFO_FORMAT (&info) == GST_VIDEO_FORMAT_SN12 ||
600 GST_VIDEO_INFO_FORMAT (&info) == GST_VIDEO_FORMAT_ST12) {
601 sink->display->is_native_format = TRUE;
603 /* store the video info */
604 sink->video_info = info;
605 sink->video_info_changed = TRUE;
607 sink->display->is_native_format = FALSE;
608 GstWlShmAllocator *self =
609 GST_WL_SHM_ALLOCATOR (gst_wl_shm_allocator_get ());
610 self->display = sink->display;
611 /* create a new pool for the new configuration */
612 newpool = gst_video_buffer_pool_new ();
616 structure = gst_buffer_pool_get_config (newpool);
617 gst_buffer_pool_config_set_params (structure, caps, info.size, 6, 0);
618 gst_buffer_pool_config_set_allocator (structure,
619 gst_wl_shm_allocator_get (), NULL);
620 if (!gst_buffer_pool_set_config (newpool, structure))
623 /* store the video info */
624 sink->video_info = info;
625 sink->video_info_changed = TRUE;
627 gst_object_replace ((GstObject **) & sink->pool, (GstObject *) newpool);
628 gst_object_unref (newpool);
630 } else { /* USE SHM */
632 GstWlShmAllocator *self =
633 GST_WL_SHM_ALLOCATOR (gst_wl_shm_allocator_get ());
634 self->display = sink->display;
636 /* create a new pool for the new configuration */
637 newpool = gst_video_buffer_pool_new ();
641 structure = gst_buffer_pool_get_config (newpool);
642 gst_buffer_pool_config_set_params (structure, caps, info.size, 6, 0);
643 gst_buffer_pool_config_set_allocator (structure,
644 gst_wl_shm_allocator_get (), NULL);
645 if (!gst_buffer_pool_set_config (newpool, structure))
648 /* store the video info */
649 sink->video_info = info;
650 sink->video_info_changed = TRUE;
652 gst_object_replace ((GstObject **) & sink->pool, (GstObject *) newpool);
653 gst_object_unref (newpool);
655 #else /*open source */
656 /* create a new pool for the new configuration */
657 newpool = gst_video_buffer_pool_new ();
661 structure = gst_buffer_pool_get_config (newpool);
662 gst_buffer_pool_config_set_params (structure, caps, info.size, 6, 0);
663 gst_buffer_pool_config_set_allocator (structure,
664 gst_wl_shm_allocator_get (), NULL);
665 if (!gst_buffer_pool_set_config (newpool, structure))
668 /* store the video info */
669 sink->video_info = info;
670 sink->video_info_changed = TRUE;
672 gst_object_replace ((GstObject **) & sink->pool, (GstObject *) newpool);
673 gst_object_unref (newpool);
680 GST_DEBUG_OBJECT (sink,
681 "Could not locate image format from caps %" GST_PTR_FORMAT, caps);
686 #ifdef GST_WLSINK_ENHANCEMENT
688 GST_DEBUG_OBJECT (sink, "Format %s is not available on the display",
689 gst_wl_tbm_format_to_string (tbm_format));
691 GST_DEBUG_OBJECT (sink, "Format %s is not available on the display",
692 gst_wl_shm_format_to_string (format));
693 #else /*open source */
694 GST_DEBUG_OBJECT (sink, "Format %s is not available on the display",
695 gst_wl_shm_format_to_string (format));
701 GST_DEBUG_OBJECT (sink, "Failed to create new pool");
706 GST_DEBUG_OBJECT (bsink, "failed setting config");
707 gst_object_unref (newpool);
713 gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
716 GstWaylandSink *sink = GST_WAYLAND_SINK (bsink);
717 GstStructure *config;
718 guint size, min_bufs, max_bufs;
719 #ifdef GST_WLSINK_ENHANCEMENT
723 if (sink->display->is_native_format == TRUE)
726 gst_query_parse_allocation (query, &caps, &need_pool);
729 GST_DEBUG_OBJECT (bsink, "no caps specified");
734 config = gst_buffer_pool_get_config (sink->pool);
735 gst_buffer_pool_config_get_params (config, NULL, &size, &min_bufs, &max_bufs);
737 /* we do have a pool for sure (created in set_caps),
738 * so let's propose it anyway, but also propose the allocator on its own */
739 gst_query_add_allocation_pool (query, sink->pool, size, min_bufs, max_bufs);
740 gst_query_add_allocation_param (query, gst_wl_shm_allocator_get (), NULL);
742 gst_structure_free (config);
748 gst_wayland_sink_preroll (GstBaseSink * bsink, GstBuffer * buffer)
751 GST_DEBUG_OBJECT (bsink, "preroll buffer %p", buffer);
752 return gst_wayland_sink_render (bsink, buffer);
756 frame_redraw_callback (void *data, struct wl_callback *callback, uint32_t time)
759 GstWaylandSink *sink = data;
761 GST_LOG ("frame_redraw_cb");
763 g_atomic_int_set (&sink->redraw_pending, FALSE);
764 wl_callback_destroy (callback);
767 static const struct wl_callback_listener frame_callback_listener = {
768 frame_redraw_callback
771 /* must be called with the render lock */
773 render_last_buffer (GstWaylandSink * sink)
776 GstWlBuffer *wlbuffer;
777 const GstVideoInfo *info = NULL;
778 struct wl_surface *surface;
779 struct wl_callback *callback;
781 wlbuffer = gst_buffer_get_wl_buffer (sink->last_buffer);
782 surface = gst_wl_window_get_wl_surface (sink->window);
784 g_atomic_int_set (&sink->redraw_pending, TRUE);
785 callback = wl_surface_frame (surface);
786 /* frame_callback_listener is called when wayland-client finish rendering the wl_buffer */
787 wl_callback_add_listener (callback, &frame_callback_listener, sink);
789 if (G_UNLIKELY (sink->video_info_changed)) {
790 info = &sink->video_info;
791 sink->video_info_changed = FALSE;
793 gst_wl_window_render (sink->window, wlbuffer, info);
797 gst_wayland_sink_render (GstBaseSink * bsink, GstBuffer * buffer)
800 GstWaylandSink *sink = GST_WAYLAND_SINK (bsink);
801 GstBuffer *to_render;
802 GstWlBuffer *wlbuffer;
803 GstFlowReturn ret = GST_FLOW_OK;
805 g_mutex_lock (&sink->render_lock);
807 GST_LOG_OBJECT (sink, "render buffer %p", buffer);
809 if (G_UNLIKELY (!sink->window)) {
810 /* ask for window handle. Unlock render_lock while doing that because
811 * set_window_handle & friends will lock it in this context */
812 g_mutex_unlock (&sink->render_lock);
813 gst_video_overlay_prepare_window_handle (GST_VIDEO_OVERLAY (sink));
814 g_mutex_lock (&sink->render_lock);
817 /* if we were not provided a window, create one ourselves */
819 gst_wl_window_new_toplevel (sink->display, &sink->video_info);
822 /* drop buffers until we get a frame callback */
823 if (g_atomic_int_get (&sink->redraw_pending) == TRUE)
825 /* make sure that the application has called set_render_rectangle() */
826 if (G_UNLIKELY (sink->window->render_rectangle.w == 0))
829 #ifdef GST_WLSINK_ENHANCEMENT
831 wlbuffer = gst_buffer_get_wl_buffer (buffer);
832 if (G_LIKELY (wlbuffer && wlbuffer->display == sink->display)) {
833 GST_LOG_OBJECT (sink, "buffer %p has a wl_buffer from our display, " "writing directly", buffer); //buffer is from our pool and have wl_buffer
834 GST_INFO ("wl_buffer (%p)", wlbuffer->wlbuffer);
838 GstMapInfo mem_info = GST_MAP_INFO_INIT;
839 int size = GST_VIDEO_INFO_SIZE (&sink->video_info);
840 mem = gst_buffer_peek_memory (to_render, 0);
841 gst_memory_map (mem, &mem_info, GST_MAP_READ);
843 data = mem_info.data;
847 sprintf (file_name, "/home/owner/DUMP/_WLSINK_OUT_DUMP_%2.2d.dump",
849 ret = __write_rawdata (file_name, data, size);
851 GST_ERROR ("_write_rawdata() failed");
853 GST_ERROR ("DUMP IMAGE %d, size (%d)", dump__cnt, size);
854 gst_memory_unmap (mem, &mem_info);
858 struct wl_buffer *wbuf = NULL;
860 GST_LOG_OBJECT (sink, "buffer %p does not have a wl_buffer from our " "display, creating it", buffer); //buffer is from our pool but have not wl_buffer
861 mem = gst_buffer_peek_memory (buffer, 0);
862 if (gst_is_wl_shm_memory (mem)) {
864 wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
867 gst_buffer_add_wl_buffer (buffer, wbuf, sink->display); //careat GstWlBuffer and add gstbuffer, wlbuffer, display and etc
870 } else { //buffer is not from our pool and have not wl_buffer
872 /* we don't know how to create a wl_buffer directly from the provided
873 * memory, so we have to copy the data to a memory that we know how
876 GST_LOG_OBJECT (sink, "buffer %p is not from our pool", buffer);
877 GST_LOG_OBJECT (sink, "buffer %p cannot have a wl_buffer, " "copying",
880 if (sink->USE_TBM && sink->display->is_native_format) {
881 /* in case of SN12 or ST12 */
883 struct wl_buffer *wbuf = NULL;
884 GstMapInfo mem_info = GST_MAP_INFO_INIT;
885 MMVideoBuffer *mm_video_buf = NULL;
887 mem = gst_buffer_peek_memory (buffer, 1);
888 gst_memory_map (mem, &mem_info, GST_MAP_READ);
889 mm_video_buf = (MMVideoBuffer *) mem_info.data;
890 gst_memory_unmap (mem, &mem_info);
892 if (mm_video_buf == NULL) {
893 GST_WARNING_OBJECT (sink, "mm_video_buf is NULL. Skip rendering");
896 /* assign mm_video_buf info */
897 if (mm_video_buf->type == MM_VIDEO_BUFFER_TYPE_TBM_BO) {
898 GST_DEBUG_OBJECT (sink, "TBM bo %p %p %p", mm_video_buf->handle.bo[0],
899 mm_video_buf->handle.bo[1], mm_video_buf->handle.bo[2]);
900 sink->display->native_video_size = 0;
901 for (int i = 0; i < NV_BUF_PLANE_NUM; i++) {
902 if (mm_video_buf->handle.bo[i] != NULL) {
903 sink->display->bo[i] = mm_video_buf->handle.bo[i];
905 sink->display->bo[i] = 0;
907 sink->display->plane_size[i] = mm_video_buf->size[i];
908 sink->display->stride_width[i] = mm_video_buf->stride_width[i];
909 sink->display->stride_height[i] = mm_video_buf->stride_height[i];
910 sink->display->native_video_size += sink->display->plane_size[i];
913 GST_ERROR_OBJECT (sink, "Buffer type is not TBM");
916 wlbuffer = gst_buffer_get_wl_buffer (buffer);
917 if (G_UNLIKELY (!wlbuffer)) {
919 gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
921 if (G_UNLIKELY (!wbuf))
924 gst_buffer_add_wl_buffer (buffer, wbuf, sink->display);
928 else if (sink->USE_TBM && !sink->display->is_native_format) {
930 /* sink->pool always exists (created in set_caps), but it may not
931 * be active if upstream is not using it */
932 if (!gst_buffer_pool_is_active (sink->pool)
933 && !gst_buffer_pool_set_active (sink->pool, TRUE))
934 goto activate_failed;
936 ret = gst_buffer_pool_acquire_buffer (sink->pool, &to_render, NULL);
937 if (ret != GST_FLOW_OK)
941 //mem = gst_buffer_peek_memory (to_render, 0);
942 //if (gst_is_wl_shm_memory (mem)) {
943 GST_INFO ("to_render buffer is our buffer");
945 /* the first time we acquire a buffer,
946 * we need to attach a wl_buffer on it */
947 wlbuffer = gst_buffer_get_wl_buffer (buffer);
948 if (G_UNLIKELY (!wlbuffer)) {
949 mem = gst_buffer_peek_memory (to_render, 0);
950 wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
952 if (G_UNLIKELY (!wbuf))
955 wlbuffer = gst_buffer_add_wl_buffer (to_render, wbuf, sink->display);
958 gst_buffer_map (buffer, &src, GST_MAP_READ);
959 gst_buffer_fill (to_render, 0, src.data, src.size);
960 gst_buffer_unmap (buffer, &src);
961 } else { /* USE SHM */
962 /* sink->pool always exists (created in set_caps), but it may not
963 * be active if upstream is not using it */
964 if (!gst_buffer_pool_is_active (sink->pool) &&
965 !gst_buffer_pool_set_active (sink->pool, TRUE))
966 goto activate_failed;
967 ret = gst_buffer_pool_acquire_buffer (sink->pool, &to_render, NULL);
968 if (ret != GST_FLOW_OK)
970 /* the first time we acquire a buffer,
971 * we need to attach a wl_buffer on it */
972 wlbuffer = gst_buffer_get_wl_buffer (buffer);
973 if (G_UNLIKELY (!wlbuffer)) {
974 mem = gst_buffer_peek_memory (to_render, 0);
975 wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
977 if (G_UNLIKELY (!wbuf))
980 gst_buffer_add_wl_buffer (to_render, wbuf, sink->display);
984 gst_buffer_map (buffer, &src, GST_MAP_READ);
985 gst_buffer_fill (to_render, 0, src.data, src.size);
986 gst_buffer_unmap (buffer, &src);
991 if (sink->USE_TBM && sink->display->is_native_format) {
992 if (G_UNLIKELY (buffer == sink->last_buffer)) {
993 GST_LOG_OBJECT (sink, "Buffer already being rendered");
996 gst_buffer_replace (&sink->last_buffer, buffer);
997 render_last_buffer (sink);
1000 } else { /* USE SHM or normal format */
1001 /* drop double rendering */
1002 if (G_UNLIKELY (buffer == sink->last_buffer)) {
1003 GST_LOG_OBJECT (sink, "Buffer already being rendered");
1006 gst_buffer_replace (&sink->last_buffer, to_render);
1007 render_last_buffer (sink);
1009 if (buffer != to_render)
1010 gst_buffer_unref (to_render);
1015 #else /* open source */
1017 wlbuffer = gst_buffer_get_wl_buffer (buffer);
1019 if (G_LIKELY (wlbuffer && wlbuffer->display == sink->display)) {
1020 GST_LOG_OBJECT (sink,
1021 "buffer %p has a wl_buffer from our display, " "writing directly",
1023 GST_INFO ("wl_buffer (%p)", wlbuffer->wlbuffer);
1028 struct wl_buffer *wbuf = NULL;
1030 GST_LOG_OBJECT (sink,
1031 "buffer %p does not have a wl_buffer from our " "display, creating it",
1033 mem = gst_buffer_peek_memory (buffer, 0);
1034 if (gst_is_wl_shm_memory (mem)) {
1036 wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
1040 gst_buffer_add_wl_buffer (buffer, wbuf, sink->display);
1045 /* we don't know how to create a wl_buffer directly from the provided
1046 * memory, so we have to copy the data to a memory that we know how
1049 GST_LOG_OBJECT (sink, "buffer %p is not from our pool", buffer);
1050 GST_LOG_OBJECT (sink, "buffer %p cannot have a wl_buffer, " "copying",
1052 /* sink->pool always exists (created in set_caps), but it may not
1053 * be active if upstream is not using it */
1054 if (!gst_buffer_pool_is_active (sink->pool) &&
1055 !gst_buffer_pool_set_active (sink->pool, TRUE))
1056 goto activate_failed;
1058 ret = gst_buffer_pool_acquire_buffer (sink->pool, &to_render, NULL);
1059 if (ret != GST_FLOW_OK)
1062 /* the first time we acquire a buffer,
1063 * we need to attach a wl_buffer on it */
1064 wlbuffer = gst_buffer_get_wl_buffer (buffer);
1065 if (G_UNLIKELY (!wlbuffer)) {
1066 mem = gst_buffer_peek_memory (to_render, 0);
1067 wbuf = gst_wl_shm_memory_construct_wl_buffer (mem, sink->display,
1069 if (G_UNLIKELY (!wbuf))
1072 gst_buffer_add_wl_buffer (to_render, wbuf, sink->display);
1075 gst_buffer_map (buffer, &src, GST_MAP_READ);
1076 gst_buffer_fill (to_render, 0, src.data, src.size);
1077 gst_buffer_unmap (buffer, &src);
1080 /* drop double rendering */
1081 if (G_UNLIKELY (buffer == sink->last_buffer)) {
1082 GST_LOG_OBJECT (sink, "Buffer already being rendered");
1086 gst_buffer_replace (&sink->last_buffer, to_render);
1087 render_last_buffer (sink);
1089 if (buffer != to_render)
1090 gst_buffer_unref (to_render);
1094 #endif /* GST_WLSINK_ENHANCEMENT */
1098 GST_ELEMENT_ERROR (sink, RESOURCE, WRITE,
1099 ("Window has no size set"),
1100 ("Make sure you set the size after calling set_window_handle"));
1101 ret = GST_FLOW_ERROR;
1106 GST_WARNING_OBJECT (sink, "could not create buffer");
1111 GST_ERROR_OBJECT (sink, "could not create wl_buffer out of wl_shm memory");
1112 ret = GST_FLOW_ERROR;
1117 GST_ERROR_OBJECT (sink, "failed to activate bufferpool.");
1118 ret = GST_FLOW_ERROR;
1123 g_mutex_unlock (&sink->render_lock);
1129 gst_wayland_sink_videooverlay_init (GstVideoOverlayInterface * iface)
1131 iface->set_window_handle = gst_wayland_sink_set_window_handle;
1132 iface->set_render_rectangle = gst_wayland_sink_set_render_rectangle;
1133 iface->expose = gst_wayland_sink_expose;
1137 gst_wayland_sink_set_window_handle (GstVideoOverlay * overlay, guintptr handle)
1140 GstWaylandSink *sink = GST_WAYLAND_SINK (overlay);
1141 struct wl_surface *surface = (struct wl_surface *) handle;
1143 g_return_if_fail (sink != NULL);
1144 #ifdef GST_WLSINK_ENHANCEMENT
1145 if (sink->window != NULL) {
1146 GST_WARNING_OBJECT (sink, "changing window handle is not supported");
1150 g_mutex_lock (&sink->render_lock);
1152 GST_DEBUG_OBJECT (sink, "Setting window handle %" GST_PTR_FORMAT,
1155 g_clear_object (&sink->window);
1158 if (G_LIKELY (gst_wayland_sink_find_display (sink))) {
1159 /* we cannot use our own display with an external window handle */
1160 if (G_UNLIKELY (sink->display->own_display)) {
1161 GST_ELEMENT_ERROR (sink, RESOURCE, OPEN_READ_WRITE,
1162 ("Application did not provide a wayland display handle"),
1163 ("waylandsink cannot use an externally-supplied surface without "
1164 "an externally-supplied display handle. Consider providing a "
1165 "display handle from your application with GstContext"));
1167 sink->window = gst_wl_window_new_in_surface (sink->display, surface);
1170 GST_ERROR_OBJECT (sink, "Failed to find display handle, "
1171 "ignoring window handle");
1175 g_mutex_unlock (&sink->render_lock);
1179 gst_wayland_sink_set_render_rectangle (GstVideoOverlay * overlay,
1180 gint x, gint y, gint w, gint h)
1183 GstWaylandSink *sink = GST_WAYLAND_SINK (overlay);
1185 g_return_if_fail (sink != NULL);
1187 g_mutex_lock (&sink->render_lock);
1188 if (!sink->window) {
1189 g_mutex_unlock (&sink->render_lock);
1190 GST_WARNING_OBJECT (sink,
1191 "set_render_rectangle called without window, ignoring");
1195 GST_DEBUG_OBJECT (sink, "window geometry changed to (%d, %d) %d x %d",
1197 gst_wl_window_set_render_rectangle (sink->window, x, y, w, h);
1199 g_mutex_unlock (&sink->render_lock);
1203 gst_wayland_sink_expose (GstVideoOverlay * overlay)
1206 GstWaylandSink *sink = GST_WAYLAND_SINK (overlay);
1208 g_return_if_fail (sink != NULL);
1210 GST_DEBUG_OBJECT (sink, "expose");
1212 g_mutex_lock (&sink->render_lock);
1213 if (sink->last_buffer && g_atomic_int_get (&sink->redraw_pending) == FALSE) {
1214 GST_DEBUG_OBJECT (sink, "redrawing last buffer");
1215 render_last_buffer (sink);
1217 g_mutex_unlock (&sink->render_lock);
1221 gst_wayland_sink_waylandvideo_init (GstWaylandVideoInterface * iface)
1223 iface->begin_geometry_change = gst_wayland_sink_begin_geometry_change;
1224 iface->end_geometry_change = gst_wayland_sink_end_geometry_change;
1228 gst_wayland_sink_begin_geometry_change (GstWaylandVideo * video)
1231 GstWaylandSink *sink = GST_WAYLAND_SINK (video);
1232 g_return_if_fail (sink != NULL);
1234 g_mutex_lock (&sink->render_lock);
1235 if (!sink->window || !sink->window->area_subsurface) {
1236 g_mutex_unlock (&sink->render_lock);
1237 GST_INFO_OBJECT (sink,
1238 "begin_geometry_change called without window, ignoring");
1242 wl_subsurface_set_sync (sink->window->area_subsurface);
1243 g_mutex_unlock (&sink->render_lock);
1247 gst_wayland_sink_end_geometry_change (GstWaylandVideo * video)
1250 GstWaylandSink *sink = GST_WAYLAND_SINK (video);
1251 g_return_if_fail (sink != NULL);
1253 g_mutex_lock (&sink->render_lock);
1254 if (!sink->window || !sink->window->area_subsurface) {
1255 g_mutex_unlock (&sink->render_lock);
1256 GST_INFO_OBJECT (sink,
1257 "end_geometry_change called without window, ignoring");
1261 wl_subsurface_set_desync (sink->window->area_subsurface);
1262 g_mutex_unlock (&sink->render_lock);
1266 plugin_init (GstPlugin * plugin)
1268 GST_DEBUG_CATEGORY_INIT (gstwayland_debug, "waylandsink", 0,
1269 " wayland video sink");
1271 gst_wl_shm_allocator_register ();
1273 return gst_element_register (plugin, "waylandsink", GST_RANK_MARGINAL,
1274 GST_TYPE_WAYLAND_SINK);
1277 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
1280 "Wayland Video Sink", plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME,