emotion: keep last gstreamer buffer around as YUV data are not copied inside evas.
[profile/ivi/emotion.git] / src / modules / gstreamer / emotion_sink.c
1 #include <glib.h>
2 #include <gst/gst.h>
3 #include <gst/video/video.h>
4 #include <gst/video/gstvideosink.h>
5
6 #include <Ecore.h>
7
8 #include "emotion_gstreamer.h"
9
10 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE("sink",
11                                                                    GST_PAD_SINK, GST_PAD_ALWAYS,
12                                                                    GST_STATIC_CAPS(GST_VIDEO_CAPS_BGRx ";" GST_VIDEO_CAPS_BGR ";" GST_VIDEO_CAPS_BGRA ";" GST_VIDEO_CAPS_YUV("{I420,YV12}")));
13
14
15 GST_DEBUG_CATEGORY_STATIC(evas_video_sink_debug);
16 #define GST_CAT_DEFAULT evas_video_sink_debug
17
18 enum {
19   REPAINT_REQUESTED,
20   LAST_SIGNAL
21 };
22
23 enum {
24   PROP_0,
25   PROP_EVAS_OBJECT,
26   PROP_WIDTH,
27   PROP_HEIGHT,
28   PROP_LAST,
29 };
30
31 static guint evas_video_sink_signals[LAST_SIGNAL] = { 0, };
32
33 struct _EvasVideoSinkPrivate {
34    Evas_Object *o;
35    Ecore_Pipe *p;
36
37    int width;
38    int height;
39    GstVideoFormat format;
40
41    GMutex* buffer_mutex;
42    GCond* data_cond;
43
44    GstBuffer *last_buffer; /* We need to keep a copy of the last inserted buffer as evas doesn't copy YUV data around */
45
46    // If this is TRUE all processing should finish ASAP
47    // This is necessary because there could be a race between
48    // unlock() and render(), where unlock() wins, signals the
49    // GCond, then render() tries to render a frame although
50    // everything else isn't running anymore. This will lead
51    // to deadlocks because render() holds the stream lock.
52    //
53    // Protected by the buffer mutex
54    Eina_Bool unlocked : 1;
55    Eina_Bool preroll : 1;
56    Eina_Bool update_size : 1;
57 };
58
59 #define _do_init(bla)                                   \
60   GST_DEBUG_CATEGORY_INIT(evas_video_sink_debug,        \
61                           "emotion-sink",               \
62                           0,                            \
63                           "emotion video sink")
64
65 GST_BOILERPLATE_FULL(EvasVideoSink,
66                      evas_video_sink,
67                      GstVideoSink,
68                      GST_TYPE_VIDEO_SINK,
69                      _do_init);
70
71
72 static void unlock_buffer_mutex(EvasVideoSinkPrivate* priv);
73
74 static void evas_video_sink_render_handler(void *data, void *buf, unsigned int len);
75
76 static void
77 evas_video_sink_base_init(gpointer g_class)
78 {
79    GstElementClass* element_class;
80
81    element_class = GST_ELEMENT_CLASS(g_class);
82    gst_element_class_add_pad_template(element_class, gst_static_pad_template_get(&sinktemplate));
83    gst_element_class_set_details_simple(element_class, "Evas video sink",
84                                         "Sink/Video", "Sends video data from a GStreamer pipeline to an Evas object",
85                                         "Vincent Torri <vtorri@univ-evry.fr>");
86 }
87
88 static void
89 evas_video_sink_init(EvasVideoSink* sink, EvasVideoSinkClass* klass __UNUSED__)
90 {
91    EvasVideoSinkPrivate* priv;
92
93    sink->priv = priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, EVAS_TYPE_VIDEO_SINK, EvasVideoSinkPrivate);
94    priv->o = NULL;
95    priv->p = ecore_pipe_add(evas_video_sink_render_handler, sink);
96    priv->last_buffer = NULL;
97    priv->width = 0;
98    priv->height = 0;
99    priv->format = GST_VIDEO_FORMAT_UNKNOWN;
100    priv->data_cond = g_cond_new();
101    priv->buffer_mutex = g_mutex_new();
102    priv->preroll = EINA_FALSE;
103    priv->unlocked = EINA_FALSE;
104    priv->update_size = EINA_TRUE;
105 }
106
107
108 /**** Object methods ****/
109
110 static void
111 evas_video_sink_set_property(GObject * object, guint prop_id,
112                              const GValue * value, GParamSpec * pspec)
113 {
114    EvasVideoSink* sink;
115    EvasVideoSinkPrivate* priv;
116
117    sink = EVAS_VIDEO_SINK (object);
118    priv = sink->priv;
119
120    switch (prop_id) {
121     case PROP_EVAS_OBJECT:
122        g_mutex_lock(priv->buffer_mutex);
123        priv->o = g_value_get_pointer (value);
124        g_mutex_unlock(priv->buffer_mutex);
125        break;
126     default:
127        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
128        break;
129    }
130 }
131
132 static void
133 evas_video_sink_get_property(GObject * object, guint prop_id,
134                              GValue * value, GParamSpec * pspec)
135 {
136    EvasVideoSink* sink;
137    EvasVideoSinkPrivate* priv;
138
139    sink = EVAS_VIDEO_SINK (object);
140    priv = sink->priv;
141
142    switch (prop_id) {
143     case PROP_EVAS_OBJECT:
144        g_mutex_lock(priv->buffer_mutex);
145        g_value_set_pointer (value, priv->o);
146        g_mutex_unlock(priv->buffer_mutex);
147        break;
148     case PROP_WIDTH:
149        g_mutex_lock(priv->buffer_mutex);
150        g_value_set_int(value, priv->width);
151        g_mutex_unlock(priv->buffer_mutex);
152        break;
153     case PROP_HEIGHT:
154        g_mutex_lock(priv->buffer_mutex);
155        g_value_set_int (value, priv->height);
156        g_mutex_unlock(priv->buffer_mutex);
157        break;
158     default:
159        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
160        break;
161    }
162 }
163
164 static void
165 evas_video_sink_dispose(GObject* object)
166 {
167    EvasVideoSink* sink;
168    EvasVideoSinkPrivate* priv;
169
170    sink = EVAS_VIDEO_SINK(object);
171    priv = sink->priv;
172
173    if (priv->buffer_mutex) {
174       g_mutex_free(priv->buffer_mutex);
175       priv->buffer_mutex = 0;
176    }
177
178    if (priv->data_cond) {
179       g_cond_free(priv->data_cond);
180       priv->data_cond = 0;
181    }
182
183    if (priv->p) {
184       ecore_pipe_del(priv->p);
185       priv->p = NULL;
186    }
187
188    if (priv->last_buffer) {
189       gst_buffer_unref(priv->last_buffer);
190       priv->last_buffer = NULL;
191    }
192
193    G_OBJECT_CLASS(parent_class)->dispose(object);
194 }
195
196
197 /**** BaseSink methods ****/
198
199 gboolean evas_video_sink_set_caps(GstBaseSink *bsink, GstCaps *caps)
200 {
201    EvasVideoSink* sink;
202    EvasVideoSinkPrivate* priv;
203    int width;
204    int height;
205
206    sink = EVAS_VIDEO_SINK(bsink);
207    priv = sink->priv;
208
209    if (G_UNLIKELY(!gst_video_format_parse_caps(caps, &priv->format, &width, &height))) {
210       return FALSE;
211    }
212
213    if ((width != priv->width) || (height != priv->height))
214      {
215         priv->width = width;
216         priv->height = height;
217         priv->update_size = EINA_TRUE;
218      }
219
220    printf("format :");
221    switch (priv->format)
222      {
223       case GST_VIDEO_FORMAT_I420:
224          evas_object_image_size_set(priv->o, priv->width, priv->height);
225          evas_object_image_fill_set(priv->o, 0, 0, priv->width, priv->height);
226          evas_object_image_colorspace_set(priv->o, EVAS_COLORSPACE_YCBCR422P601_PL);
227          evas_object_image_alpha_set(priv->o, 0);
228          printf ("I420\n");
229          break;
230       case GST_VIDEO_FORMAT_YV12:
231          evas_object_image_size_set(priv->o, priv->width, priv->height);
232          evas_object_image_fill_set(priv->o, 0, 0, priv->width, priv->height);
233          evas_object_image_colorspace_set(priv->o, EVAS_COLORSPACE_YCBCR422P601_PL);
234          evas_object_image_alpha_set(priv->o, 0);
235          printf ("YV12\n");
236          break;
237       case GST_VIDEO_FORMAT_BGR:
238          printf ("BGR\n");
239          break;
240       case GST_VIDEO_FORMAT_BGRx:
241          printf ("BGRx\n");
242          break;
243       case GST_VIDEO_FORMAT_BGRA:
244          printf ("BGRA\n");
245          break;
246       default:
247          printf ("unsupported : %d\n", priv->format);
248          return FALSE;
249      }
250
251    return TRUE;
252 }
253
254 static gboolean
255 evas_video_sink_start(GstBaseSink* base_sink)
256 {
257    EvasVideoSinkPrivate* priv;
258    gboolean res = TRUE;
259
260    priv = EVAS_VIDEO_SINK(base_sink)->priv;
261    g_mutex_lock(priv->buffer_mutex);
262    if (!priv->o)
263      res = FALSE;
264    else
265      {
266         if (!priv->p)
267           res = FALSE;
268         else
269           {
270              priv->unlocked = EINA_FALSE;
271           }
272      }
273    g_mutex_unlock(priv->buffer_mutex);
274    return res;
275 }
276
277 static gboolean
278 evas_video_sink_stop(GstBaseSink* base_sink)
279 {
280    EvasVideoSinkPrivate* priv = EVAS_VIDEO_SINK(base_sink)->priv;
281
282    unlock_buffer_mutex(priv);
283    return TRUE;
284 }
285
286 static gboolean
287 evas_video_sink_unlock(GstBaseSink* object)
288 {
289    EvasVideoSink* sink;
290
291    sink = EVAS_VIDEO_SINK(object);
292
293    unlock_buffer_mutex(sink->priv);
294
295    return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock,
296                                        (object), TRUE);
297 }
298
299 static gboolean
300 evas_video_sink_unlock_stop(GstBaseSink* object)
301 {
302    EvasVideoSink* sink;
303    EvasVideoSinkPrivate* priv;
304
305    sink = EVAS_VIDEO_SINK(object);
306    priv = sink->priv;
307
308    g_mutex_lock(priv->buffer_mutex);
309    priv->unlocked = FALSE;
310    g_mutex_unlock(priv->buffer_mutex);
311
312    return GST_CALL_PARENT_WITH_DEFAULT(GST_BASE_SINK_CLASS, unlock_stop,
313                                        (object), TRUE);
314 }
315
316 static GstFlowReturn
317 evas_video_sink_preroll(GstBaseSink* bsink, GstBuffer* buffer)
318 {
319    GstBuffer *send;
320    EvasVideoSink* sink;
321    EvasVideoSinkPrivate* priv;
322
323    sink = EVAS_VIDEO_SINK(bsink);
324    priv = sink->priv;
325
326    send = gst_buffer_ref(buffer);
327
328    priv->preroll = EINA_TRUE;
329
330    ecore_pipe_write(priv->p, &send, sizeof(buffer));
331    return GST_FLOW_OK;
332 }
333
334 static GstFlowReturn
335 evas_video_sink_render(GstBaseSink* bsink, GstBuffer* buffer)
336 {
337    GstBuffer *send;
338    EvasVideoSink* sink;
339    EvasVideoSinkPrivate* priv;
340    Eina_Bool ret;
341
342    sink = EVAS_VIDEO_SINK(bsink);
343    priv = sink->priv;
344
345    g_mutex_lock(priv->buffer_mutex);
346
347    if (priv->unlocked) {
348       g_mutex_unlock(priv->buffer_mutex);
349       return GST_FLOW_OK;
350    }
351
352    priv->preroll = EINA_FALSE;
353
354    send = gst_buffer_ref(buffer);
355    ret = ecore_pipe_write(priv->p, &send, sizeof(buffer));
356    if (!ret)
357      return GST_FLOW_ERROR;
358
359    g_cond_wait(priv->data_cond, priv->buffer_mutex);
360    g_mutex_unlock(priv->buffer_mutex);
361
362    return GST_FLOW_OK;
363 }
364
365 static void evas_video_sink_render_handler(void *data,
366                                            void *buf,
367                                            unsigned int len)
368 {
369    Emotion_Gstreamer_Video *ev;
370    Emotion_Video_Stream *vstream;
371    EvasVideoSink* sink;
372    EvasVideoSinkPrivate* priv;
373    GstBuffer* buffer;
374    unsigned char *evas_data;
375    const guint8 *gst_data;
376    GstQuery *query;
377    GstFormat fmt = GST_FORMAT_TIME;
378    gint64 pos;
379
380    sink = (EvasVideoSink *)data;
381    priv = sink->priv;
382
383    buffer = *((GstBuffer **)buf);
384
385    if (priv->unlocked)
386      goto exit_point;
387
388    gst_data = GST_BUFFER_DATA(buffer);
389    if (!gst_data) goto exit_point;
390    if (priv->update_size)
391      {
392         evas_object_image_size_set(priv->o, priv->width, priv->height);
393         if (!priv->preroll) priv->update_size = FALSE;
394      }
395
396    evas_data = (unsigned char *)evas_object_image_data_get(priv->o, 1);
397
398    // Evas's BGRA has pre-multiplied alpha while GStreamer's doesn't.
399    // Here we convert to Evas's BGRA.
400    if (priv->format == GST_VIDEO_FORMAT_BGR) {
401       unsigned char *evas_tmp;
402       int x;
403       int y;
404
405       evas_tmp = evas_data;
406       /* FIXME: could this be optimized ? */
407       for (x = 0; x < priv->height; x++) {
408          for (y = 0; y < priv->width; y++) {
409             evas_tmp[0] = gst_data[0];
410             evas_tmp[1] = gst_data[1];
411             evas_tmp[2] = gst_data[2];
412             evas_tmp[3] = 255;
413             gst_data += 3;
414             evas_tmp += 4;
415          }
416       }
417    }
418
419    // Evas's BGRA has pre-multiplied alpha while GStreamer's doesn't.
420    // Here we convert to Evas's BGRA.
421    if (priv->format == GST_VIDEO_FORMAT_BGRx) {
422       unsigned char *evas_tmp;
423       int x;
424       int y;
425
426       evas_tmp = evas_data;
427       /* FIXME: could this be optimized ? */
428       for (x = 0; x < priv->height; x++) {
429          for (y = 0; y < priv->width; y++) {
430             evas_tmp[0] = gst_data[0];
431             evas_tmp[1] = gst_data[1];
432             evas_tmp[2] = gst_data[2];
433             evas_tmp[3] = 255;
434             gst_data += 4;
435             evas_tmp += 4;
436          }
437       }
438    }
439
440    // Evas's BGRA has pre-multiplied alpha while GStreamer's doesn't.
441    // Here we convert to Evas's BGRA.
442    if (priv->format == GST_VIDEO_FORMAT_BGRA) {
443       unsigned char *evas_tmp;
444       int x;
445       int y;
446       unsigned char alpha;
447
448       evas_tmp = evas_data;
449       /* FIXME: could this be optimized ? */
450       for (x = 0; x < priv->height; x++) {
451          for (y = 0; y < priv->width; y++) {
452             alpha = gst_data[3];
453             evas_tmp[0] = (gst_data[0] * alpha) / 255;
454             evas_tmp[1] = (gst_data[1] * alpha) / 255;
455             evas_tmp[2] = (gst_data[2] * alpha) / 255;
456             evas_tmp[3] = alpha;
457             gst_data += 4;
458             evas_tmp += 4;
459          }
460       }
461    }
462
463    if (priv->format == GST_VIDEO_FORMAT_I420) {
464       int i;
465       unsigned char **rows;
466
467       evas_object_image_pixels_dirty_set(priv->o, 1);
468       rows = (unsigned char **)evas_data;
469
470       for (i = 0; i < priv->height; i++)
471         rows[i] = &gst_data[i * priv->width];
472
473       rows += priv->height;
474       for (i = 0; i < (priv->height / 2); i++)
475         rows[i] = &gst_data[priv->height * priv->width + i * (priv->width / 2)];
476
477       rows += priv->height / 2;
478       for (i = 0; i < (priv->height / 2); i++)
479         rows[i] = &gst_data[priv->height * priv->width + priv->height * (priv->width /4) + i * (priv->width / 2)];
480    }
481
482    if (priv->format == GST_VIDEO_FORMAT_YV12) {
483       int i;
484       unsigned char **rows;
485
486       evas_object_image_pixels_dirty_set(priv->o, 1);
487
488       rows = (unsigned char **)evas_data;
489
490       for (i = 0; i < priv->height; i++)
491         rows[i] = &gst_data[i * priv->width];
492
493       rows += priv->height;
494       for (i = 0; i < (priv->height / 2); i++)
495         rows[i] = &gst_data[priv->height * priv->width + priv->height * (priv->width /4) + i * (priv->width / 2)];
496
497       rows += priv->height / 2;
498       for (i = 0; i < (priv->height / 2); i++)
499         rows[i] = &gst_data[priv->height * priv->width + i * (priv->width / 2)];
500    }
501
502    evas_object_image_data_update_add(priv->o, 0, 0, priv->width, priv->height);
503    evas_object_image_data_set(priv->o, evas_data);
504    evas_object_image_pixels_dirty_set(priv->o, 0);
505
506    ev = evas_object_data_get(priv->o, "_emotion_gstreamer_video");
507    _emotion_frame_new(ev->obj);
508
509    vstream = eina_list_nth(ev->video_streams, ev->video_stream_nbr - 1);
510
511    gst_element_query_position(ev->pipeline, &fmt, &pos);
512    ev->position = (double)pos / (double)GST_SECOND;
513
514    vstream->width = priv->width;
515    vstream->height = priv->height;
516    ev->ratio = (double) priv->width / (double) priv->height;
517
518    _emotion_video_pos_update(ev->obj, ev->position, vstream->length_time);
519    _emotion_frame_resize(ev->obj, priv->width, priv->height, ev->ratio);
520
521  exit_point:
522    if (priv->last_buffer) gst_buffer_unref(priv->last_buffer);
523    priv->last_buffer = buffer;
524
525    if (priv->preroll) return ;
526
527    g_mutex_lock(priv->buffer_mutex);
528
529    if (priv->unlocked) {
530       g_mutex_unlock(priv->buffer_mutex);
531       return;
532    }
533
534    g_cond_signal(priv->data_cond);
535    g_mutex_unlock(priv->buffer_mutex);
536 }
537
538 static void
539 unlock_buffer_mutex(EvasVideoSinkPrivate* priv)
540 {
541    g_mutex_lock(priv->buffer_mutex);
542
543    priv->unlocked = EINA_TRUE;
544    g_cond_signal(priv->data_cond);
545    g_mutex_unlock(priv->buffer_mutex);
546 }
547
548 static void
549 marshal_VOID__MINIOBJECT(GClosure * closure, GValue * return_value __UNUSED__,
550                          guint n_param_values, const GValue * param_values,
551                          gpointer invocation_hint __UNUSED__, gpointer marshal_data)
552 {
553    typedef void (*marshalfunc_VOID__MINIOBJECT) (gpointer obj, gpointer arg1, gpointer data2);
554    marshalfunc_VOID__MINIOBJECT callback;
555    GCClosure *cc;
556    gpointer data1, data2;
557
558    cc = (GCClosure *) closure;
559
560    g_return_if_fail(n_param_values == 2);
561
562    if (G_CCLOSURE_SWAP_DATA(closure)) {
563       data1 = closure->data;
564       data2 = g_value_peek_pointer(param_values + 0);
565    } else {
566       data1 = g_value_peek_pointer(param_values + 0);
567       data2 = closure->data;
568    }
569    callback = (marshalfunc_VOID__MINIOBJECT) (marshal_data ? marshal_data : cc->callback);
570
571    callback(data1, gst_value_get_mini_object(param_values + 1), data2);
572 }
573
574 static void
575 evas_video_sink_class_init(EvasVideoSinkClass* klass)
576 {
577    GObjectClass* gobject_class;
578    GstBaseSinkClass* gstbase_sink_class;
579
580    gobject_class = G_OBJECT_CLASS(klass);
581    gstbase_sink_class = GST_BASE_SINK_CLASS(klass);
582
583    g_type_class_add_private(klass, sizeof(EvasVideoSinkPrivate));
584
585    gobject_class->set_property = evas_video_sink_set_property;
586    gobject_class->get_property = evas_video_sink_get_property;
587
588    g_object_class_install_property (gobject_class, PROP_EVAS_OBJECT,
589                                     g_param_spec_pointer ("evas-object", "Evas Object",
590                                                           "The Evas object where the display of the video will be done",
591                                                           G_PARAM_READWRITE));
592
593    g_object_class_install_property (gobject_class, PROP_WIDTH,
594                                     g_param_spec_int ("width", "Width",
595                                                       "The width of the video",
596                                                       0, 65536, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
597
598    g_object_class_install_property (gobject_class, PROP_HEIGHT,
599                                     g_param_spec_int ("height", "Height",
600                                                       "The height of the video",
601                                                       0, 65536, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
602
603    gobject_class->dispose = evas_video_sink_dispose;
604
605    gstbase_sink_class->set_caps = evas_video_sink_set_caps;
606    gstbase_sink_class->stop = evas_video_sink_stop;
607    gstbase_sink_class->start = evas_video_sink_start;
608    gstbase_sink_class->unlock = evas_video_sink_unlock;
609    gstbase_sink_class->unlock_stop = evas_video_sink_unlock_stop;
610    gstbase_sink_class->render = evas_video_sink_render;
611    gstbase_sink_class->preroll = evas_video_sink_preroll;
612
613    evas_video_sink_signals[REPAINT_REQUESTED] = g_signal_new("repaint-requested",
614                                                              G_TYPE_FROM_CLASS(klass),
615                                                              (GSignalFlags)(G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
616                                                              0,
617                                                              0,
618                                                              0,
619                                                              marshal_VOID__MINIOBJECT,
620                                                              G_TYPE_NONE, 1, GST_TYPE_BUFFER);
621 }
622
623 gboolean
624 gstreamer_plugin_init (GstPlugin * plugin)
625 {
626    return gst_element_register (plugin,
627                                 "emotion-sink",
628                                 GST_RANK_NONE,
629                                 EVAS_TYPE_VIDEO_SINK);
630 }
631
632 GstElement *
633 gstreamer_video_sink_new(Emotion_Gstreamer_Video *ev,
634                          Evas_Object *o,
635                          const char *uri)
636 {
637    GstElement *playbin;
638    GstElement *sink;
639    Evas_Object *obj;
640    GstStateChangeReturn res;
641
642    obj = _emotion_image_get(o);
643    if (!obj)
644      {
645         ERR("Not Evas_Object specified");
646         return NULL;
647      }
648
649    playbin = gst_element_factory_make("playbin2", "playbin");
650    if (!playbin)
651      {
652         ERR("Unable to create 'playbin' GstElement.");
653         return NULL;
654      }
655
656    sink = gst_element_factory_make("emotion-sink", "sink");
657    if (!sink)
658      {
659         ERR("Unable to create 'emotion-sink' GstElement.");
660         goto unref_pipeline;
661      }
662
663    g_object_set(G_OBJECT(playbin), "video-sink", sink, NULL);
664    g_object_set(G_OBJECT(playbin), "uri", uri, NULL);
665    g_object_set(G_OBJECT(sink), "evas-object", obj, NULL);
666
667    res = gst_element_set_state(playbin, GST_STATE_PAUSED);
668    if (res == GST_STATE_CHANGE_FAILURE)
669      {
670         ERR("Unable to set GST_STATE_PAUSED.");
671         goto unref_pipeline;
672      }
673
674    res = gst_element_get_state(playbin, NULL, NULL, GST_CLOCK_TIME_NONE);
675    if (res != GST_STATE_CHANGE_SUCCESS)
676      {
677         ERR("Unable to get GST_CLOCK_TIME_NONE.");
678         goto unref_pipeline;
679      }
680
681    evas_object_data_set(obj, "_emotion_gstreamer_video", ev);
682
683    return playbin;
684
685  unref_pipeline:
686    gst_object_unref(playbin);
687    return NULL;
688 }