2 * Copyright (C) 2007 OpenedHand
3 * Copyright (C) 2007 Alp Toker <alp@atoker.com>
4 * Copyright (C) 2009, 2010, 2011, 2012 Igalia S.L
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 * WebKitVideoSink is a GStreamer sink element that triggers
24 * repaints in the WebKit GStreamer media player for the
25 * current video buffer.
30 #include "VideoSinkGStreamer.h"
32 #include "GStreamerVersioning.h"
36 #ifdef GST_API_VERSION_1
37 #include <gst/video/gstvideometa.h>
38 #include <gst/video/gstvideopool.h>
40 #include <wtf/FastAllocBase.h>
42 // CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant.
43 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
44 #ifndef GST_API_VERSION_1
45 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_BGRx ";" GST_VIDEO_CAPS_BGRA
47 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_MAKE("{ BGRx, BGRA }")
50 #ifndef GST_API_VERSION_1
51 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_xRGB ";" GST_VIDEO_CAPS_ARGB
53 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_MAKE("{ xRGB, ARGB }")
56 static GstStaticPadTemplate s_sinkTemplate = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(WEBKIT_VIDEO_SINK_PAD_CAPS));
59 GST_DEBUG_CATEGORY_STATIC(webkitVideoSinkDebug);
60 #define GST_CAT_DEFAULT webkitVideoSinkDebug
64 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
74 static guint webkitVideoSinkSignals[LAST_SIGNAL] = { 0, };
76 struct _WebKitVideoSinkPrivate {
82 #ifdef GST_API_VERSION_1
86 #ifndef GST_API_VERSION_1
87 WebCore::GStreamerGWorld* gstGWorld;
90 // If this is TRUE all processing should finish ASAP
91 // This is necessary because there could be a race between
92 // unlock() and render(), where unlock() wins, signals the
93 // GCond, then render() tries to render a frame although
94 // everything else isn't running anymore. This will lead
95 // to deadlocks because render() holds the stream lock.
97 // Protected by the buffer mutex
99 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
104 #define webkit_video_sink_parent_class parent_class
105 G_DEFINE_TYPE_WITH_CODE(WebKitVideoSink, webkit_video_sink, GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT(webkitVideoSinkDebug, "webkitsink", 0, "webkit video sink"));
108 static void webkit_video_sink_init(WebKitVideoSink* sink)
110 sink->priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);
111 #if GLIB_CHECK_VERSION(2, 31, 0)
112 sink->priv->dataCondition = WTF::fastNew<GCond>();
113 g_cond_init(sink->priv->dataCondition);
114 sink->priv->bufferMutex = WTF::fastNew<GMutex>();
115 g_mutex_init(sink->priv->bufferMutex);
117 sink->priv->dataCondition = g_cond_new();
118 sink->priv->bufferMutex = g_mutex_new();
121 #ifdef GST_API_VERSION_1
122 gst_video_info_init(&sink->priv->info);
126 static gboolean webkitVideoSinkTimeoutCallback(gpointer data)
128 WebKitVideoSink* sink = reinterpret_cast<WebKitVideoSink*>(data);
129 WebKitVideoSinkPrivate* priv = sink->priv;
131 g_mutex_lock(priv->bufferMutex);
132 GstBuffer* buffer = priv->buffer;
136 if (!buffer || priv->unlocked || UNLIKELY(!GST_IS_BUFFER(buffer))) {
137 g_cond_signal(priv->dataCondition);
138 g_mutex_unlock(priv->bufferMutex);
142 g_signal_emit(sink, webkitVideoSinkSignals[REPAINT_REQUESTED], 0, buffer);
143 gst_buffer_unref(buffer);
144 g_cond_signal(priv->dataCondition);
145 g_mutex_unlock(priv->bufferMutex);
150 static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buffer)
152 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
153 WebKitVideoSinkPrivate* priv = sink->priv;
155 g_mutex_lock(priv->bufferMutex);
157 if (priv->unlocked) {
158 g_mutex_unlock(priv->bufferMutex);
162 #ifndef GST_API_VERSION_1
163 // Ignore buffers if the video is already in fullscreen using
165 if (priv->gstGWorld->isFullscreen()) {
166 g_mutex_unlock(priv->bufferMutex);
171 priv->buffer = gst_buffer_ref(buffer);
173 #ifndef GST_API_VERSION_1
174 // For the unlikely case where the buffer has no caps, the caps
175 // are implicitely the caps of the pad. This shouldn't happen.
176 if (UNLIKELY(!GST_BUFFER_CAPS(buffer))) {
177 buffer = priv->buffer = gst_buffer_make_metadata_writable(priv->buffer);
178 gst_buffer_set_caps(priv->buffer, GST_PAD_CAPS(GST_BASE_SINK_PAD(baseSink)));
181 GstCaps* caps = GST_BUFFER_CAPS(buffer);
183 GstCaps* caps = gst_video_info_to_caps(&priv->info);
186 GstVideoFormat format;
187 WebCore::IntSize size;
188 int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride;
189 if (!getVideoSizeAndFormatFromCaps(caps, size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride)) {
190 gst_buffer_unref(buffer);
191 #ifdef GST_API_VERSION_1
192 gst_caps_unref(caps);
194 g_mutex_unlock(priv->bufferMutex);
195 return GST_FLOW_ERROR;
198 #ifdef GST_API_VERSION_1
199 gst_caps_unref(caps);
202 // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't.
203 // Here we convert to Cairo's ARGB.
204 if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) {
205 // Because GstBaseSink::render() only owns the buffer reference in the
206 // method scope we can't use gst_buffer_make_writable() here. Also
207 // The buffer content should not be changed here because the same buffer
208 // could be passed multiple times to this method (in theory).
210 GstBuffer* newBuffer = createGstBuffer(buffer);
212 // Check if allocation failed.
213 if (UNLIKELY(!newBuffer)) {
214 g_mutex_unlock(priv->bufferMutex);
215 return GST_FLOW_ERROR;
218 // We don't use Color::premultipliedARGBFromColor() here because
219 // one function call per video pixel is just too expensive:
220 // For 720p/PAL for example this means 1280*720*25=23040000
221 // function calls per second!
222 #ifndef GST_API_VERSION_1
223 const guint8* source = GST_BUFFER_DATA(buffer);
224 guint8* destination = GST_BUFFER_DATA(newBuffer);
226 GstMapInfo sourceInfo;
227 GstMapInfo destinationInfo;
228 gst_buffer_map(buffer, &sourceInfo, GST_MAP_READ);
229 const guint8* source = const_cast<guint8*>(sourceInfo.data);
230 gst_buffer_map(newBuffer, &destinationInfo, GST_MAP_WRITE);
231 guint8* destination = static_cast<guint8*>(destinationInfo.data);
234 for (int x = 0; x < size.height(); x++) {
235 for (int y = 0; y < size.width(); y++) {
236 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
237 unsigned short alpha = source[3];
238 destination[0] = (source[0] * alpha + 128) / 255;
239 destination[1] = (source[1] * alpha + 128) / 255;
240 destination[2] = (source[2] * alpha + 128) / 255;
241 destination[3] = alpha;
243 unsigned short alpha = source[0];
244 destination[0] = alpha;
245 destination[1] = (source[1] * alpha + 128) / 255;
246 destination[2] = (source[2] * alpha + 128) / 255;
247 destination[3] = (source[3] * alpha + 128) / 255;
254 #ifdef GST_API_VERSION_1
255 gst_buffer_unmap(buffer, &sourceInfo);
256 gst_buffer_unmap(newBuffer, &destinationInfo);
258 gst_buffer_unref(buffer);
259 buffer = priv->buffer = newBuffer;
262 // This should likely use a lower priority, but glib currently starves
263 // lower priority sources.
264 // See: https://bugzilla.gnome.org/show_bug.cgi?id=610830.
265 priv->timeoutId = g_timeout_add_full(G_PRIORITY_DEFAULT, 0, webkitVideoSinkTimeoutCallback,
266 gst_object_ref(sink), reinterpret_cast<GDestroyNotify>(gst_object_unref));
268 g_cond_wait(priv->dataCondition, priv->bufferMutex);
269 g_mutex_unlock(priv->bufferMutex);
273 static void webkitVideoSinkDispose(GObject* object)
275 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
276 WebKitVideoSinkPrivate* priv = sink->priv;
278 if (priv->dataCondition) {
279 #if GLIB_CHECK_VERSION(2, 31, 0)
280 g_cond_clear(priv->dataCondition);
281 WTF::fastDelete(priv->dataCondition);
283 g_cond_free(priv->dataCondition);
285 priv->dataCondition = 0;
288 if (priv->bufferMutex) {
289 #if GLIB_CHECK_VERSION(2, 31, 0)
290 g_mutex_clear(priv->bufferMutex);
291 WTF::fastDelete(priv->bufferMutex);
293 g_mutex_free(priv->bufferMutex);
295 priv->bufferMutex = 0;
298 G_OBJECT_CLASS(parent_class)->dispose(object);
301 static void unlockBufferMutex(WebKitVideoSinkPrivate* priv)
303 g_mutex_lock(priv->bufferMutex);
306 gst_buffer_unref(priv->buffer);
310 priv->unlocked = true;
312 g_cond_signal(priv->dataCondition);
313 g_mutex_unlock(priv->bufferMutex);
316 static gboolean webkitVideoSinkUnlock(GstBaseSink* baseSink)
318 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
320 unlockBufferMutex(sink->priv);
322 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock, (baseSink), TRUE);
325 static gboolean webkitVideoSinkUnlockStop(GstBaseSink* baseSink)
327 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
329 g_mutex_lock(priv->bufferMutex);
330 priv->unlocked = false;
331 g_mutex_unlock(priv->bufferMutex);
333 return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop, (baseSink), TRUE);
336 static gboolean webkitVideoSinkStop(GstBaseSink* baseSink)
338 unlockBufferMutex(WEBKIT_VIDEO_SINK(baseSink)->priv);
342 static gboolean webkitVideoSinkStart(GstBaseSink* baseSink)
344 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
346 g_mutex_lock(priv->bufferMutex);
347 priv->unlocked = false;
348 g_mutex_unlock(priv->bufferMutex);
352 #ifdef GST_API_VERSION_1
353 static gboolean webkitVideoSinkProposeAllocation(GstBaseSink* baseSink, GstQuery* query)
356 gst_query_parse_allocation(query, &caps, 0);
360 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
361 if (!gst_video_info_from_caps(&sink->priv->info, caps))
364 gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, 0);
365 gst_query_add_allocation_meta(query, GST_VIDEO_CROP_META_API_TYPE, 0);
370 #ifndef GST_API_VERSION_1
371 static void webkitVideoSinkMarshalVoidAndMiniObject(GClosure* closure, GValue* returnValue, guint parametersNumber, const GValue* parameterValues, gpointer invocationHint, gpointer marshalData)
373 typedef void (*marshalfunc_VOID__MINIOBJECT) (gpointer obj, gpointer arg1, gpointer data2);
374 marshalfunc_VOID__MINIOBJECT callback;
375 GCClosure* cclosure = reinterpret_cast<GCClosure*>(closure);
376 gpointer data1, data2;
378 g_return_if_fail(parametersNumber == 2);
380 if (G_CCLOSURE_SWAP_DATA(closure)) {
381 data1 = closure->data;
382 data2 = g_value_peek_pointer(parameterValues + 0);
384 data1 = g_value_peek_pointer(parameterValues + 0);
385 data2 = closure->data;
388 callback = (marshalfunc_VOID__MINIOBJECT) (marshalData ? marshalData : cclosure->callback);
389 callback(data1, gst_value_get_mini_object(parameterValues + 1), data2);
393 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
395 webkit_video_sink_event(GstBaseSink* base_sink, GstEvent* event)
399 WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(base_sink)->priv;
400 WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(base_sink);
401 const GstStructure* structure = gst_event_get_structure(event);
402 switch (GST_EVENT_TYPE (event)) {
403 case GST_EVENT_CUSTOM_DOWNSTREAM_OOB:
405 const GValue* dataAddress = gst_structure_get_value(structure, "data-addr");
406 priv->addr = g_value_get_uint(dataAddress);
407 GST_DEBUG_OBJECT(sink, "%d", priv->addr);
408 GstBuffer *buf = gst_buffer_new();
409 gchar *addr = g_strdup_printf("ADDR %d", priv->addr);
410 GST_BUFFER_MALLOCDATA(buf) = (guint8*) g_malloc(sizeof(addr));
411 GST_BUFFER_DATA(buf) = GST_BUFFER_MALLOCDATA(buf);
412 gst_buffer_set_data(buf, (guint8*) addr, sizeof(addr));
413 GST_DEBUG_OBJECT(sink, "%s", addr);
415 g_signal_emit(sink, webkitVideoSinkSignals[IGNORE_REQUESTED], 0, buf);
416 gst_buffer_unref(buf);
420 GST_LOG_OBJECT(sink, "let base class handle event");
421 if (GST_BASE_SINK_CLASS(parent_class)->event) {
422 event = gst_event_ref(event);
423 ret = GST_BASE_SINK_CLASS(parent_class)->event(base_sink, event);
430 static void webkit_video_sink_class_init(WebKitVideoSinkClass* klass)
432 GObjectClass* gobjectClass = G_OBJECT_CLASS(klass);
433 GstBaseSinkClass* baseSinkClass = GST_BASE_SINK_CLASS(klass);
434 GstElementClass* elementClass = GST_ELEMENT_CLASS(klass);
436 gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&s_sinkTemplate));
437 setGstElementClassMetadata(elementClass, "WebKit video sink", "Sink/Video", "Sends video data from a GStreamer pipeline to a Cairo surface", "Alp Toker <alp@atoker.com>");
439 g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate));
441 gobjectClass->dispose = webkitVideoSinkDispose;
443 baseSinkClass->unlock = webkitVideoSinkUnlock;
444 baseSinkClass->unlock_stop = webkitVideoSinkUnlockStop;
445 baseSinkClass->render = webkitVideoSinkRender;
446 baseSinkClass->preroll = webkitVideoSinkRender;
447 baseSinkClass->stop = webkitVideoSinkStop;
448 baseSinkClass->start = webkitVideoSinkStart;
449 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
450 baseSinkClass->event = webkit_video_sink_event;
452 #ifdef GST_API_VERSION_1
453 baseSinkClass->propose_allocation = webkitVideoSinkProposeAllocation;
456 webkitVideoSinkSignals[REPAINT_REQUESTED] = g_signal_new("repaint-requested",
457 G_TYPE_FROM_CLASS(klass),
458 static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
461 0, // Accumulator data
462 #ifndef GST_API_VERSION_1
463 webkitVideoSinkMarshalVoidAndMiniObject,
465 g_cclosure_marshal_generic,
467 G_TYPE_NONE, // Return type
468 1, // Only one parameter
470 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
471 webkitVideoSinkSignals[IGNORE_REQUESTED] = g_signal_new("ignore-requested",
472 G_TYPE_FROM_CLASS(klass),
473 static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
476 0, // Accumulator data
477 webkitVideoSinkMarshalVoidAndMiniObject,
478 G_TYPE_NONE, // Return type
479 1, // Only one parameter
485 #ifndef GST_API_VERSION_1
486 GstElement* webkitVideoSinkNew(WebCore::GStreamerGWorld* gstGWorld)
488 GstElement* element = GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0));
489 WEBKIT_VIDEO_SINK(element)->priv->gstGWorld = gstGWorld;
493 GstElement* webkitVideoSinkNew()
495 return GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0));
499 #endif // USE(GSTREAMER)