Source/WebCore/platform/graphics/gstreamer/VideoSinkGStreamer.cpp

   1 /*
   2  *  Copyright (C) 2007 OpenedHand
   3  *  Copyright (C) 2007 Alp Toker <alp@atoker.com>
   4  *  Copyright (C) 2009, 2010, 2011, 2012 Igalia S.L
   5  *
   6  *  This library is free software; you can redistribute it and/or
   7  *  modify it under the terms of the GNU Lesser General Public
   8  *  License as published by the Free Software Foundation; either
   9  *  version 2 of the License, or (at your option) any later version.
  10  *
  11  *  This library is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  *  Lesser General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU Lesser General Public
  17  *  License along with this library; if not, write to the Free Software
  18  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  19  */
  20
  21 /*
  22  *
  23  * WebKitVideoSink is a GStreamer sink element that triggers
  24  * repaints in the WebKit GStreamer media player for the
  25  * current video buffer.
  26  */
  27
  28 #include "config.h"
  29 #if USE(GSTREAMER)
  30 #include "VideoSinkGStreamer.h"
  31
  32 #include "GStreamerVersioning.h"
  33 #include "IntSize.h"
  34 #include <glib.h>
  35 #include <gst/gst.h>
  36 #ifdef GST_API_VERSION_1
  37 #include <gst/video/gstvideometa.h>
  38 #include <gst/video/gstvideopool.h>
  39 #endif
  40 #include <wtf/FastAllocBase.h>
  41
  42 // CAIRO_FORMAT_RGB24 used to render the video buffers is little/big endian dependant.
  43 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
  44 #ifndef GST_API_VERSION_1
  45 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_BGRx ";" GST_VIDEO_CAPS_BGRA
  46 #else
  47 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_MAKE("{ BGRx, BGRA }")
  48 #endif
  49 #else
  50 #ifndef GST_API_VERSION_1
  51 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_xRGB ";" GST_VIDEO_CAPS_ARGB
  52 #else
  53 #define WEBKIT_VIDEO_SINK_PAD_CAPS GST_VIDEO_CAPS_MAKE("{ xRGB, ARGB }")
  54 #endif
  55 #endif
  56 static GstStaticPadTemplate s_sinkTemplate = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(WEBKIT_VIDEO_SINK_PAD_CAPS));
  57
  58
  59 GST_DEBUG_CATEGORY_STATIC(webkitVideoSinkDebug);
  60 #define GST_CAT_DEFAULT webkitVideoSinkDebug
  61
  62 enum {
  63     REPAINT_REQUESTED,
  64 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
  65     IGNORE_REQUESTED,
  66 #endif
  67     LAST_SIGNAL
  68 };
  69
  70 enum {
  71     PROP_0
  72 };
  73
  74 static guint webkitVideoSinkSignals[LAST_SIGNAL] = { 0, };
  75
  76 struct _WebKitVideoSinkPrivate {
  77     GstBuffer* buffer;
  78     guint timeoutId;
  79     GMutex* bufferMutex;
  80     GCond* dataCondition;
  81
  82 #ifdef GST_API_VERSION_1
  83     GstVideoInfo info;
  84 #endif
  85
  86 #ifndef GST_API_VERSION_1
  87     WebCore::GStreamerGWorld* gstGWorld;
  88 #endif
  89
  90     // If this is TRUE all processing should finish ASAP
  91     // This is necessary because there could be a race between
  92     // unlock() and render(), where unlock() wins, signals the
  93     // GCond, then render() tries to render a frame although
  94     // everything else isn't running anymore. This will lead
  95     // to deadlocks because render() holds the stream lock.
  96     //
  97     // Protected by the buffer mutex
  98     bool unlocked;
  99 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
 100     guint addr;
 101 #endif
 102 };
 103
 104 #define webkit_video_sink_parent_class parent_class
 105 G_DEFINE_TYPE_WITH_CODE(WebKitVideoSink, webkit_video_sink, GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT(webkitVideoSinkDebug, "webkitsink", 0, "webkit video sink"));
 106
 107
 108 static void webkit_video_sink_init(WebKitVideoSink* sink)
 109 {
 110     sink->priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);
 111 #if GLIB_CHECK_VERSION(2, 31, 0)
 112     sink->priv->dataCondition = WTF::fastNew<GCond>();
 113     g_cond_init(sink->priv->dataCondition);
 114     sink->priv->bufferMutex = WTF::fastNew<GMutex>();
 115     g_mutex_init(sink->priv->bufferMutex);
 116 #else
 117     sink->priv->dataCondition = g_cond_new();
 118     sink->priv->bufferMutex = g_mutex_new();
 119 #endif
 120
 121 #ifdef GST_API_VERSION_1
 122     gst_video_info_init(&sink->priv->info);
 123 #endif
 124 }
 125
 126 static gboolean webkitVideoSinkTimeoutCallback(gpointer data)
 127 {
 128     WebKitVideoSink* sink = reinterpret_cast<WebKitVideoSink*>(data);
 129     WebKitVideoSinkPrivate* priv = sink->priv;
 130
 131     g_mutex_lock(priv->bufferMutex);
 132     GstBuffer* buffer = priv->buffer;
 133     priv->buffer = 0;
 134     priv->timeoutId = 0;
 135
 136     if (!buffer || priv->unlocked || UNLIKELY(!GST_IS_BUFFER(buffer))) {
 137         g_cond_signal(priv->dataCondition);
 138         g_mutex_unlock(priv->bufferMutex);
 139         return FALSE;
 140     }
 141
 142     g_signal_emit(sink, webkitVideoSinkSignals[REPAINT_REQUESTED], 0, buffer);
 143     gst_buffer_unref(buffer);
 144     g_cond_signal(priv->dataCondition);
 145     g_mutex_unlock(priv->bufferMutex);
 146
 147     return FALSE;
 148 }
 149
 150 static GstFlowReturn webkitVideoSinkRender(GstBaseSink* baseSink, GstBuffer* buffer)
 151 {
 152     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
 153     WebKitVideoSinkPrivate* priv = sink->priv;
 154
 155     g_mutex_lock(priv->bufferMutex);
 156
 157     if (priv->unlocked) {
 158         g_mutex_unlock(priv->bufferMutex);
 159         return GST_FLOW_OK;
 160     }
 161
 162 #ifndef GST_API_VERSION_1
 163     // Ignore buffers if the video is already in fullscreen using
 164     // another sink.
 165     if (priv->gstGWorld->isFullscreen()) {
 166         g_mutex_unlock(priv->bufferMutex);
 167         return GST_FLOW_OK;
 168     }
 169 #endif
 170
 171     priv->buffer = gst_buffer_ref(buffer);
 172
 173 #ifndef GST_API_VERSION_1
 174     // For the unlikely case where the buffer has no caps, the caps
 175     // are implicitely the caps of the pad. This shouldn't happen.
 176     if (UNLIKELY(!GST_BUFFER_CAPS(buffer))) {
 177         buffer = priv->buffer = gst_buffer_make_metadata_writable(priv->buffer);
 178         gst_buffer_set_caps(priv->buffer, GST_PAD_CAPS(GST_BASE_SINK_PAD(baseSink)));
 179     }
 180
 181     GstCaps* caps = GST_BUFFER_CAPS(buffer);
 182 #else
 183     GstCaps* caps = gst_video_info_to_caps(&priv->info);
 184 #endif
 185
 186     GstVideoFormat format;
 187     WebCore::IntSize size;
 188     int pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride;
 189     if (!getVideoSizeAndFormatFromCaps(caps, size, format, pixelAspectRatioNumerator, pixelAspectRatioDenominator, stride)) {
 190         gst_buffer_unref(buffer);
 191 #ifdef GST_API_VERSION_1
 192         gst_caps_unref(caps);
 193 #endif
 194         g_mutex_unlock(priv->bufferMutex);
 195         return GST_FLOW_ERROR;
 196     }
 197
 198 #ifdef GST_API_VERSION_1
 199     gst_caps_unref(caps);
 200 #endif
 201
 202     // Cairo's ARGB has pre-multiplied alpha while GStreamer's doesn't.
 203     // Here we convert to Cairo's ARGB.
 204     if (format == GST_VIDEO_FORMAT_ARGB || format == GST_VIDEO_FORMAT_BGRA) {
 205         // Because GstBaseSink::render() only owns the buffer reference in the
 206         // method scope we can't use gst_buffer_make_writable() here. Also
 207         // The buffer content should not be changed here because the same buffer
 208         // could be passed multiple times to this method (in theory).
 209
 210         GstBuffer* newBuffer = createGstBuffer(buffer);
 211
 212         // Check if allocation failed.
 213         if (UNLIKELY(!newBuffer)) {
 214             g_mutex_unlock(priv->bufferMutex);
 215             return GST_FLOW_ERROR;
 216         }
 217
 218         // We don't use Color::premultipliedARGBFromColor() here because
 219         // one function call per video pixel is just too expensive:
 220         // For 720p/PAL for example this means 1280*720*25=23040000
 221         // function calls per second!
 222 #ifndef GST_API_VERSION_1
 223         const guint8* source = GST_BUFFER_DATA(buffer);
 224         guint8* destination = GST_BUFFER_DATA(newBuffer);
 225 #else
 226         GstMapInfo sourceInfo;
 227         GstMapInfo destinationInfo;
 228         gst_buffer_map(buffer, &sourceInfo, GST_MAP_READ);
 229         const guint8* source = const_cast<guint8*>(sourceInfo.data);
 230         gst_buffer_map(newBuffer, &destinationInfo, GST_MAP_WRITE);
 231         guint8* destination = static_cast<guint8*>(destinationInfo.data);
 232 #endif
 233
 234         for (int x = 0; x < size.height(); x++) {
 235             for (int y = 0; y < size.width(); y++) {
 236 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
 237                 unsigned short alpha = source[3];
 238                 destination[0] = (source[0] * alpha + 128) / 255;
 239                 destination[1] = (source[1] * alpha + 128) / 255;
 240                 destination[2] = (source[2] * alpha + 128) / 255;
 241                 destination[3] = alpha;
 242 #else
 243                 unsigned short alpha = source[0];
 244                 destination[0] = alpha;
 245                 destination[1] = (source[1] * alpha + 128) / 255;
 246                 destination[2] = (source[2] * alpha + 128) / 255;
 247                 destination[3] = (source[3] * alpha + 128) / 255;
 248 #endif
 249                 source += 4;
 250                 destination += 4;
 251             }
 252         }
 253
 254 #ifdef GST_API_VERSION_1
 255         gst_buffer_unmap(buffer, &sourceInfo);
 256         gst_buffer_unmap(newBuffer, &destinationInfo);
 257 #endif
 258         gst_buffer_unref(buffer);
 259         buffer = priv->buffer = newBuffer;
 260     }
 261
 262     // This should likely use a lower priority, but glib currently starves
 263     // lower priority sources.
 264     // See: https://bugzilla.gnome.org/show_bug.cgi?id=610830.
 265     priv->timeoutId = g_timeout_add_full(G_PRIORITY_DEFAULT, 0, webkitVideoSinkTimeoutCallback,
 266                                           gst_object_ref(sink), reinterpret_cast<GDestroyNotify>(gst_object_unref));
 267
 268     g_cond_wait(priv->dataCondition, priv->bufferMutex);
 269     g_mutex_unlock(priv->bufferMutex);
 270     return GST_FLOW_OK;
 271 }
 272
 273 static void webkitVideoSinkDispose(GObject* object)
 274 {
 275     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
 276     WebKitVideoSinkPrivate* priv = sink->priv;
 277
 278     if (priv->dataCondition) {
 279 #if GLIB_CHECK_VERSION(2, 31, 0)
 280         g_cond_clear(priv->dataCondition);
 281         WTF::fastDelete(priv->dataCondition);
 282 #else
 283         g_cond_free(priv->dataCondition);
 284 #endif
 285         priv->dataCondition = 0;
 286     }
 287
 288     if (priv->bufferMutex) {
 289 #if GLIB_CHECK_VERSION(2, 31, 0)
 290         g_mutex_clear(priv->bufferMutex);
 291         WTF::fastDelete(priv->bufferMutex);
 292 #else
 293         g_mutex_free(priv->bufferMutex);
 294 #endif
 295         priv->bufferMutex = 0;
 296     }
 297
 298     G_OBJECT_CLASS(parent_class)->dispose(object);
 299 }
 300
 301 static void unlockBufferMutex(WebKitVideoSinkPrivate* priv)
 302 {
 303     g_mutex_lock(priv->bufferMutex);
 304
 305     if (priv->buffer) {
 306         gst_buffer_unref(priv->buffer);
 307         priv->buffer = 0;
 308     }
 309
 310     priv->unlocked = true;
 311
 312     g_cond_signal(priv->dataCondition);
 313     g_mutex_unlock(priv->bufferMutex);
 314 }
 315
 316 static gboolean webkitVideoSinkUnlock(GstBaseSink* baseSink)
 317 {
 318     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
 319
 320     unlockBufferMutex(sink->priv);
 321
 322     return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock, (baseSink), TRUE);
 323 }
 324
 325 static gboolean webkitVideoSinkUnlockStop(GstBaseSink* baseSink)
 326 {
 327     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
 328
 329     g_mutex_lock(priv->bufferMutex);
 330     priv->unlocked = false;
 331     g_mutex_unlock(priv->bufferMutex);
 332
 333     return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop, (baseSink), TRUE);
 334 }
 335
 336 static gboolean webkitVideoSinkStop(GstBaseSink* baseSink)
 337 {
 338     unlockBufferMutex(WEBKIT_VIDEO_SINK(baseSink)->priv);
 339     return TRUE;
 340 }
 341
 342 static gboolean webkitVideoSinkStart(GstBaseSink* baseSink)
 343 {
 344     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(baseSink)->priv;
 345
 346     g_mutex_lock(priv->bufferMutex);
 347     priv->unlocked = false;
 348     g_mutex_unlock(priv->bufferMutex);
 349     return TRUE;
 350 }
 351
 352 #ifdef GST_API_VERSION_1
 353 static gboolean webkitVideoSinkProposeAllocation(GstBaseSink* baseSink, GstQuery* query)
 354 {
 355     GstCaps* caps;
 356     gst_query_parse_allocation(query, &caps, 0);
 357     if (!caps)
 358         return FALSE;
 359
 360     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(baseSink);
 361     if (!gst_video_info_from_caps(&sink->priv->info, caps))
 362         return FALSE;
 363
 364     gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, 0);
 365     gst_query_add_allocation_meta(query, GST_VIDEO_CROP_META_API_TYPE, 0);
 366     return TRUE;
 367 }
 368 #endif
 369
 370 #ifndef GST_API_VERSION_1
 371 static void webkitVideoSinkMarshalVoidAndMiniObject(GClosure* closure, GValue* returnValue, guint parametersNumber, const GValue* parameterValues, gpointer invocationHint, gpointer marshalData)
 372 {
 373     typedef void (*marshalfunc_VOID__MINIOBJECT) (gpointer obj, gpointer arg1, gpointer data2);
 374     marshalfunc_VOID__MINIOBJECT callback;
 375     GCClosure* cclosure = reinterpret_cast<GCClosure*>(closure);
 376     gpointer data1, data2;
 377
 378     g_return_if_fail(parametersNumber == 2);
 379
 380     if (G_CCLOSURE_SWAP_DATA(closure)) {
 381         data1 = closure->data;
 382         data2 = g_value_peek_pointer(parameterValues + 0);
 383     } else {
 384         data1 = g_value_peek_pointer(parameterValues + 0);
 385         data2 = closure->data;
 386     }
 387
 388     callback = (marshalfunc_VOID__MINIOBJECT) (marshalData ? marshalData : cclosure->callback);
 389     callback(data1, gst_value_get_mini_object(parameterValues + 1), data2);
 390 }
 391 #endif
 392
 393 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
 394 static gboolean
 395 webkit_video_sink_event(GstBaseSink* base_sink, GstEvent* event)
 396 {
 397     gboolean ret = TRUE;
 398
 399     WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(base_sink)->priv;
 400     WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(base_sink);
 401     const GstStructure* structure = gst_event_get_structure(event);
 402     switch (GST_EVENT_TYPE (event)) {
 403     case GST_EVENT_CUSTOM_DOWNSTREAM_OOB:
 404         if (structure) {
 405             const GValue* dataAddress = gst_structure_get_value(structure, "data-addr");
 406             priv->addr = g_value_get_uint(dataAddress);
 407             GST_DEBUG_OBJECT(sink, "%d", priv->addr);
 408             GstBuffer *buf = gst_buffer_new();
 409             gchar *addr = g_strdup_printf("ADDR %d", priv->addr);
 410             GST_BUFFER_MALLOCDATA(buf) = (guint8*) g_malloc(sizeof(addr));
 411             GST_BUFFER_DATA(buf) = GST_BUFFER_MALLOCDATA(buf);
 412             gst_buffer_set_data(buf, (guint8*) addr, sizeof(addr));
 413             GST_DEBUG_OBJECT(sink, "%s", addr);
 414
 415             g_signal_emit(sink, webkitVideoSinkSignals[IGNORE_REQUESTED], 0, buf);
 416             gst_buffer_unref(buf);
 417         }
 418     break;
 419     default:
 420         GST_LOG_OBJECT(sink, "let base class handle event");
 421         if (GST_BASE_SINK_CLASS(parent_class)->event) {
 422             event = gst_event_ref(event);
 423             ret = GST_BASE_SINK_CLASS(parent_class)->event(base_sink, event);
 424         }
 425     break;
 426     }
 427     return ret;
 428 }
 429 #endif
 430 static void webkit_video_sink_class_init(WebKitVideoSinkClass* klass)
 431 {
 432     GObjectClass* gobjectClass = G_OBJECT_CLASS(klass);
 433     GstBaseSinkClass* baseSinkClass = GST_BASE_SINK_CLASS(klass);
 434     GstElementClass* elementClass = GST_ELEMENT_CLASS(klass);
 435
 436     gst_element_class_add_pad_template(elementClass, gst_static_pad_template_get(&s_sinkTemplate));
 437     setGstElementClassMetadata(elementClass, "WebKit video sink", "Sink/Video", "Sends video data from a GStreamer pipeline to a Cairo surface", "Alp Toker <alp@atoker.com>");
 438
 439     g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate));
 440
 441     gobjectClass->dispose = webkitVideoSinkDispose;
 442
 443     baseSinkClass->unlock = webkitVideoSinkUnlock;
 444     baseSinkClass->unlock_stop = webkitVideoSinkUnlockStop;
 445     baseSinkClass->render = webkitVideoSinkRender;
 446     baseSinkClass->preroll = webkitVideoSinkRender;
 447     baseSinkClass->stop = webkitVideoSinkStop;
 448     baseSinkClass->start = webkitVideoSinkStart;
 449 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
 450     baseSinkClass->event = webkit_video_sink_event;
 451 #endif
 452 #ifdef GST_API_VERSION_1
 453     baseSinkClass->propose_allocation = webkitVideoSinkProposeAllocation;
 454 #endif
 455
 456     webkitVideoSinkSignals[REPAINT_REQUESTED] = g_signal_new("repaint-requested",
 457             G_TYPE_FROM_CLASS(klass),
 458             static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
 459             0, // Class offset
 460             0, // Accumulator
 461             0, // Accumulator data
 462 #ifndef GST_API_VERSION_1
 463             webkitVideoSinkMarshalVoidAndMiniObject,
 464 #else
 465             g_cclosure_marshal_generic,
 466 #endif
 467             G_TYPE_NONE, // Return type
 468             1, // Only one parameter
 469             GST_TYPE_BUFFER);
 470 #if ENABLE(TIZEN_GSTREAMER_VIDEO)
 471     webkitVideoSinkSignals[IGNORE_REQUESTED] = g_signal_new("ignore-requested",
 472             G_TYPE_FROM_CLASS(klass),
 473             static_cast<GSignalFlags>(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
 474             0, // Class offset
 475             0, // Accumulator
 476             0, // Accumulator data
 477             webkitVideoSinkMarshalVoidAndMiniObject,
 478             G_TYPE_NONE, // Return type
 479             1, // Only one parameter
 480             GST_TYPE_BUFFER);
 481 #endif
 482 }
 483
 484
 485 #ifndef GST_API_VERSION_1
 486 GstElement* webkitVideoSinkNew(WebCore::GStreamerGWorld* gstGWorld)
 487 {
 488     GstElement* element = GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0));
 489     WEBKIT_VIDEO_SINK(element)->priv->gstGWorld = gstGWorld;
 490     return element;
 491 }
 492 #else
 493 GstElement* webkitVideoSinkNew()
 494 {
 495     return GST_ELEMENT(g_object_new(WEBKIT_TYPE_VIDEO_SINK, 0));
 496 }
 497 #endif
 498
 499 #endif // USE(GSTREAMER)