2 * Copyright (c) 2008 Benjamin Schmitz <vortex@wolpzone.de>
3 * Copyright (c) 2009 Sebastian Dröge <sebastian.droege@collabora.co.uk>
5 * This library is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU Library General Public
7 * License as published by the Free Software Foundation; either
8 * version 2 of the License, or (at your option) any later version.
10 * This library is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Library General Public License for more details.
15 * You should have received a copy of the GNU Library General Public
16 * License along with this library; if not, write to the
17 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 * Boston, MA 02111-1307, USA.
22 * SECTION:element-assrender
24 * Renders timestamped SSA/ASS subtitles on top of a video stream.
27 * <title>Example launch line</title>
29 * gst-launch -v filesrc location=/path/to/mkv ! matroskademux name=d ! queue ! mp3parse ! mad ! audioconvert ! autoaudiosink d. ! queue ! ffdec_h264 ! ffmpegcolorspace ! r. d. ! queue ! "application/x-ass" ! assrender name=r ! ffmpegcolorspace ! autovideosink
30 * ]| This pipeline demuxes a Matroska file with h.264 video, MP3 audio and embedded ASS subtitles and renders the subtitles on top of the video.
39 #include "gstassrender.h"
43 GST_DEBUG_CATEGORY_STATIC (gst_ass_render_debug);
44 GST_DEBUG_CATEGORY_STATIC (gst_ass_render_lib_debug);
45 #define GST_CAT_DEFAULT gst_ass_render_debug
47 /* Filter signals and props */
60 #define FORMATS "{ RGB, BGR, xRGB, xBGR, RGBx, BGRx, I420 }"
62 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
65 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS))
68 static GstStaticPadTemplate video_sink_factory =
69 GST_STATIC_PAD_TEMPLATE ("video_sink",
72 GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS))
75 static GstStaticPadTemplate text_sink_factory =
76 GST_STATIC_PAD_TEMPLATE ("text_sink",
79 GST_STATIC_CAPS ("application/x-ass; application/x-ssa")
82 static void gst_ass_render_set_property (GObject * object, guint prop_id,
83 const GValue * value, GParamSpec * pspec);
84 static void gst_ass_render_get_property (GObject * object, guint prop_id,
85 GValue * value, GParamSpec * pspec);
87 static void gst_ass_render_finalize (GObject * object);
89 static GstStateChangeReturn gst_ass_render_change_state (GstElement * element,
90 GstStateChange transition);
92 #define gst_ass_render_parent_class parent_class
93 G_DEFINE_TYPE (GstAssRender, gst_ass_render, GST_TYPE_ELEMENT);
95 static GstCaps *gst_ass_render_getcaps (GstPad * pad, GstCaps * filter);
97 static gboolean gst_ass_render_setcaps_video (GstPad * pad, GstCaps * caps);
98 static gboolean gst_ass_render_setcaps_text (GstPad * pad, GstCaps * caps);
100 static GstFlowReturn gst_ass_render_chain_video (GstPad * pad, GstBuffer * buf);
101 static GstFlowReturn gst_ass_render_chain_text (GstPad * pad, GstBuffer * buf);
103 static gboolean gst_ass_render_event_video (GstPad * pad, GstEvent * event);
104 static gboolean gst_ass_render_event_text (GstPad * pad, GstEvent * event);
105 static gboolean gst_ass_render_event_src (GstPad * pad, GstEvent * event);
107 static gboolean gst_ass_render_query_src (GstPad * pad, GstQuery * query);
109 /* initialize the plugin's class */
111 gst_ass_render_class_init (GstAssRenderClass * klass)
113 GObjectClass *gobject_class = (GObjectClass *) klass;
114 GstElementClass *gstelement_class = (GstElementClass *) klass;
116 gobject_class->set_property = gst_ass_render_set_property;
117 gobject_class->get_property = gst_ass_render_get_property;
118 gobject_class->finalize = gst_ass_render_finalize;
120 g_object_class_install_property (gobject_class, PROP_ENABLE,
121 g_param_spec_boolean ("enable", "Enable",
122 "Enable rendering of subtitles", TRUE,
123 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
124 g_object_class_install_property (gobject_class, PROP_EMBEDDEDFONTS,
125 g_param_spec_boolean ("embeddedfonts", "Embedded Fonts",
126 "Extract and use fonts embedded in the stream", TRUE,
127 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
129 gstelement_class->change_state =
130 GST_DEBUG_FUNCPTR (gst_ass_render_change_state);
132 gst_element_class_add_pad_template (gstelement_class,
133 gst_static_pad_template_get (&src_factory));
134 gst_element_class_add_pad_template (gstelement_class,
135 gst_static_pad_template_get (&video_sink_factory));
136 gst_element_class_add_pad_template (gstelement_class,
137 gst_static_pad_template_get (&text_sink_factory));
139 gst_element_class_set_details_simple (gstelement_class, "ASS/SSA Render",
140 "Mixer/Video/Overlay/Subtitle",
141 "Renders ASS/SSA subtitles with libass",
142 "Benjamin Schmitz <vortex@wolpzone.de>, "
143 "Sebastian Dröge <sebastian.droege@collabora.co.uk>");
146 #if defined(LIBASS_VERSION) && LIBASS_VERSION >= 0x00907000
148 _libass_message_cb (gint level, const gchar * fmt, va_list args,
151 gchar *message = g_strdup_vprintf (fmt, args);
154 GST_CAT_ERROR_OBJECT (gst_ass_render_lib_debug, render, "%s", message);
156 GST_CAT_WARNING_OBJECT (gst_ass_render_lib_debug, render, "%s", message);
158 GST_CAT_INFO_OBJECT (gst_ass_render_lib_debug, render, "%s", message);
160 GST_CAT_DEBUG_OBJECT (gst_ass_render_lib_debug, render, "%s", message);
162 GST_CAT_LOG_OBJECT (gst_ass_render_lib_debug, render, "%s", message);
169 gst_ass_render_init (GstAssRender * render)
171 GST_DEBUG_OBJECT (render, "init");
173 render->srcpad = gst_pad_new_from_static_template (&src_factory, "src");
174 render->video_sinkpad =
175 gst_pad_new_from_static_template (&video_sink_factory, "video_sink");
176 render->text_sinkpad =
177 gst_pad_new_from_static_template (&text_sink_factory, "text_sink");
179 gst_pad_set_getcaps_function (render->srcpad,
180 GST_DEBUG_FUNCPTR (gst_ass_render_getcaps));
181 gst_pad_set_getcaps_function (render->video_sinkpad,
182 GST_DEBUG_FUNCPTR (gst_ass_render_getcaps));
184 gst_pad_set_chain_function (render->video_sinkpad,
185 GST_DEBUG_FUNCPTR (gst_ass_render_chain_video));
186 gst_pad_set_chain_function (render->text_sinkpad,
187 GST_DEBUG_FUNCPTR (gst_ass_render_chain_text));
189 gst_pad_set_event_function (render->video_sinkpad,
190 GST_DEBUG_FUNCPTR (gst_ass_render_event_video));
191 gst_pad_set_event_function (render->text_sinkpad,
192 GST_DEBUG_FUNCPTR (gst_ass_render_event_text));
193 gst_pad_set_event_function (render->srcpad,
194 GST_DEBUG_FUNCPTR (gst_ass_render_event_src));
196 gst_pad_set_query_function (render->srcpad,
197 GST_DEBUG_FUNCPTR (gst_ass_render_query_src));
199 gst_element_add_pad (GST_ELEMENT (render), render->srcpad);
200 gst_element_add_pad (GST_ELEMENT (render), render->video_sinkpad);
201 gst_element_add_pad (GST_ELEMENT (render), render->text_sinkpad);
203 gst_video_info_init (&render->info);
205 render->subtitle_mutex = g_mutex_new ();
206 render->subtitle_cond = g_cond_new ();
208 render->renderer_init_ok = FALSE;
209 render->track_init_ok = FALSE;
210 render->enable = TRUE;
211 render->embeddedfonts = TRUE;
213 gst_segment_init (&render->video_segment, GST_FORMAT_TIME);
214 gst_segment_init (&render->subtitle_segment, GST_FORMAT_TIME);
216 render->ass_mutex = g_mutex_new ();
217 render->ass_library = ass_library_init ();
218 #if defined(LIBASS_VERSION) && LIBASS_VERSION >= 0x00907000
219 ass_set_message_cb (render->ass_library, _libass_message_cb, render);
221 ass_set_extract_fonts (render->ass_library, 1);
223 render->ass_renderer = ass_renderer_init (render->ass_library);
224 if (!render->ass_renderer) {
225 GST_WARNING_OBJECT (render, "cannot create renderer instance");
226 g_assert_not_reached ();
229 render->ass_track = NULL;
231 GST_DEBUG_OBJECT (render, "init complete");
235 gst_ass_render_finalize (GObject * object)
237 GstAssRender *render = GST_ASS_RENDER (object);
239 if (render->subtitle_mutex)
240 g_mutex_free (render->subtitle_mutex);
242 if (render->subtitle_cond)
243 g_cond_free (render->subtitle_cond);
245 if (render->ass_track) {
246 ass_free_track (render->ass_track);
249 if (render->ass_renderer) {
250 ass_renderer_done (render->ass_renderer);
253 if (render->ass_library) {
254 ass_library_done (render->ass_library);
257 if (render->ass_mutex)
258 g_mutex_free (render->ass_mutex);
260 G_OBJECT_CLASS (parent_class)->finalize (object);
264 gst_ass_render_set_property (GObject * object, guint prop_id,
265 const GValue * value, GParamSpec * pspec)
267 GstAssRender *render = GST_ASS_RENDER (object);
271 render->enable = g_value_get_boolean (value);
273 case PROP_EMBEDDEDFONTS:
274 render->embeddedfonts = g_value_get_boolean (value);
275 g_mutex_lock (render->ass_mutex);
276 ass_set_extract_fonts (render->ass_library, render->embeddedfonts);
277 g_mutex_unlock (render->ass_mutex);
280 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
286 gst_ass_render_get_property (GObject * object, guint prop_id,
287 GValue * value, GParamSpec * pspec)
289 GstAssRender *render = GST_ASS_RENDER (object);
293 g_value_set_boolean (value, render->enable);
295 case PROP_EMBEDDEDFONTS:
296 g_value_set_boolean (value, render->embeddedfonts);
299 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
304 static GstStateChangeReturn
305 gst_ass_render_change_state (GstElement * element, GstStateChange transition)
307 GstAssRender *render = GST_ASS_RENDER (element);
308 GstStateChangeReturn ret;
310 switch (transition) {
311 case GST_STATE_CHANGE_READY_TO_PAUSED:
312 render->subtitle_flushing = FALSE;
313 gst_segment_init (&render->video_segment, GST_FORMAT_TIME);
314 gst_segment_init (&render->subtitle_segment, GST_FORMAT_TIME);
316 case GST_STATE_CHANGE_NULL_TO_READY:
317 case GST_STATE_CHANGE_PAUSED_TO_PLAYING:
321 case GST_STATE_CHANGE_PAUSED_TO_READY:
322 g_mutex_lock (render->subtitle_mutex);
323 render->subtitle_flushing = TRUE;
324 if (render->subtitle_pending)
325 gst_buffer_unref (render->subtitle_pending);
326 render->subtitle_pending = NULL;
327 g_cond_signal (render->subtitle_cond);
328 g_mutex_unlock (render->subtitle_mutex);
332 ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
334 switch (transition) {
335 case GST_STATE_CHANGE_PAUSED_TO_READY:
336 g_mutex_lock (render->ass_mutex);
337 if (render->ass_track)
338 ass_free_track (render->ass_track);
339 render->ass_track = NULL;
340 g_mutex_unlock (render->ass_mutex);
341 render->track_init_ok = FALSE;
342 render->renderer_init_ok = FALSE;
344 case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
345 case GST_STATE_CHANGE_READY_TO_NULL:
355 gst_ass_render_query_src (GstPad * pad, GstQuery * query)
357 GstAssRender *render = GST_ASS_RENDER (gst_pad_get_parent (pad));
360 ret = gst_pad_peer_query (render->video_sinkpad, query);
362 gst_object_unref (render);
367 gst_ass_render_event_src (GstPad * pad, GstEvent * event)
369 GstAssRender *render = GST_ASS_RENDER (gst_pad_get_parent (pad));
370 gboolean ret = FALSE;
372 switch (GST_EVENT_TYPE (event)) {
373 case GST_EVENT_SEEK:{
376 GST_DEBUG_OBJECT (render, "seek received, driving from here");
378 gst_event_parse_seek (event, NULL, NULL, &flags, NULL, NULL, NULL, NULL);
380 /* Flush downstream, only for flushing seek */
381 if (flags & GST_SEEK_FLAG_FLUSH)
382 gst_pad_push_event (render->srcpad, gst_event_new_flush_start ());
384 /* Mark subtitle as flushing, unblocks chains */
385 g_mutex_lock (render->subtitle_mutex);
386 if (render->subtitle_pending)
387 gst_buffer_unref (render->subtitle_pending);
388 render->subtitle_pending = NULL;
389 render->subtitle_flushing = TRUE;
390 g_cond_signal (render->subtitle_cond);
391 g_mutex_unlock (render->subtitle_mutex);
393 /* Seek on each sink pad */
394 gst_event_ref (event);
395 ret = gst_pad_push_event (render->video_sinkpad, event);
397 ret = gst_pad_push_event (render->text_sinkpad, event);
399 gst_event_unref (event);
404 gst_event_ref (event);
405 ret = gst_pad_push_event (render->video_sinkpad, event);
406 gst_pad_push_event (render->text_sinkpad, event);
410 gst_object_unref (render);
416 gst_ass_render_getcaps (GstPad * pad, GstCaps * filter)
418 GstAssRender *render = GST_ASS_RENDER (gst_pad_get_parent (pad));
422 if (pad == render->srcpad)
423 otherpad = render->video_sinkpad;
425 otherpad = render->srcpad;
427 /* we can do what the peer can */
428 caps = gst_pad_peer_get_caps (otherpad, filter);
431 const GstCaps *templ;
433 /* filtered against our padtemplate */
434 templ = gst_pad_get_pad_template_caps (otherpad);
435 temp = gst_caps_intersect (caps, templ);
436 gst_caps_unref (caps);
437 /* this is what we can do */
440 /* no peer, our padtemplate is enough then */
441 caps = gst_caps_copy (gst_pad_get_pad_template_caps (pad));
444 gst_object_unref (render);
449 #define CREATE_RGB_BLIT_FUNCTION(name,bpp,R,G,B) \
451 blit_##name (GstAssRender * render, ASS_Image * ass_image, GstVideoFrame * frame) \
454 gint alpha, r, g, b, k; \
456 guint8 *dst, *data; \
464 width = GST_VIDEO_FRAME_WIDTH (frame); \
465 height = GST_VIDEO_FRAME_HEIGHT (frame); \
466 dst_stride = GST_VIDEO_FRAME_PLANE_STRIDE (frame, 0); \
467 data = GST_VIDEO_FRAME_PLANE_DATA (frame, 0); \
469 while (ass_image) { \
470 if (ass_image->dst_y > height || ass_image->dst_x > width) \
473 /* blend subtitles onto the video frame */ \
474 alpha = 255 - ((ass_image->color) & 0xff); \
475 r = ((ass_image->color) >> 24) & 0xff; \
476 g = ((ass_image->color) >> 16) & 0xff; \
477 b = ((ass_image->color) >> 8) & 0xff; \
478 src = ass_image->bitmap; \
479 dst = data + ass_image->dst_y * dst_stride + ass_image->dst_x * bpp; \
481 w = MIN (ass_image->w, width - ass_image->dst_x); \
482 h = MIN (ass_image->h, height - ass_image->dst_y); \
483 src_skip = ass_image->stride - w; \
484 dst_skip = dst_stride - w * bpp; \
486 for (y = 0; y < h; y++) { \
487 for (x = 0; x < w; x++) { \
488 k = src[0] * alpha / 255; \
489 dst[R] = (k * r + (255 - k) * dst[R]) / 255; \
490 dst[G] = (k * g + (255 - k) * dst[G]) / 255; \
491 dst[B] = (k * b + (255 - k) * dst[B]) / 255; \
500 ass_image = ass_image->next; \
502 GST_LOG_OBJECT (render, "amount of rendered ass_image: %u", counter); \
505 CREATE_RGB_BLIT_FUNCTION (rgb, 3, 0, 1, 2);
506 CREATE_RGB_BLIT_FUNCTION (bgr, 3, 2, 1, 0);
507 CREATE_RGB_BLIT_FUNCTION (xrgb, 4, 1, 2, 3);
508 CREATE_RGB_BLIT_FUNCTION (xbgr, 4, 3, 2, 1);
509 CREATE_RGB_BLIT_FUNCTION (rgbx, 4, 0, 1, 2);
510 CREATE_RGB_BLIT_FUNCTION (bgrx, 4, 2, 1, 0);
512 #undef CREATE_RGB_BLIT_FUNCTION
515 rgb_to_y (gint r, gint g, gint b)
519 ret = (gint) (((19595 * r) >> 16) + ((38470 * g) >> 16) + ((7471 * b) >> 16));
520 ret = CLAMP (ret, 0, 255);
525 rgb_to_u (gint r, gint g, gint b)
530 (gint) (-((11059 * r) >> 16) - ((21709 * g) >> 16) + ((32768 * b) >> 16) +
532 ret = CLAMP (ret, 0, 255);
537 rgb_to_v (gint r, gint g, gint b)
542 (gint) (((32768 * r) >> 16) - ((27439 * g) >> 16) - ((5329 * b) >> 16) +
544 ret = CLAMP (ret, 0, 255);
549 blit_i420 (GstAssRender * render, ASS_Image * ass_image, GstVideoFrame * frame)
552 gint alpha, r, g, b, k, k2;
555 guint8 *dst_y, *dst_u, *dst_v;
557 /* FIXME ignoring source image stride might be wrong here */
563 guint8 *y_data, *u_data, *v_data;
564 gint y_stride, u_stride, v_stride;
566 width = GST_VIDEO_FRAME_WIDTH (frame);
567 height = GST_VIDEO_FRAME_HEIGHT (frame);
569 y_data = GST_VIDEO_FRAME_COMP_DATA (frame, 0);
570 u_data = GST_VIDEO_FRAME_COMP_DATA (frame, 1);
571 v_data = GST_VIDEO_FRAME_COMP_DATA (frame, 2);
573 y_stride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0);
574 u_stride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 1);
575 v_stride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 2);
578 if (ass_image->dst_y > height || ass_image->dst_x > width)
581 /* blend subtitles onto the video frame */
582 alpha = 255 - ((ass_image->color) & 0xff);
583 r = ((ass_image->color) >> 24) & 0xff;
584 g = ((ass_image->color) >> 16) & 0xff;
585 b = ((ass_image->color) >> 8) & 0xff;
587 Y = rgb_to_y (r, g, b);
588 U = rgb_to_u (r, g, b);
589 V = rgb_to_v (r, g, b);
591 w = MIN (ass_image->w, width - ass_image->dst_x);
592 h = MIN (ass_image->h, height - ass_image->dst_y);
597 src_stride = ass_image->stride;
600 src = ass_image->bitmap;
602 dst_y = y_data + ass_image->dst_y * y_stride + ass_image->dst_x;
603 dst_u = u_data + (ass_image->dst_y / 2) * u_stride + ass_image->dst_x / 2;
604 dst_v = v_data + (ass_image->dst_y / 2) * v_stride + ass_image->dst_x / 2;
607 for (y = 0; y < h; y++) {
608 dst_y = y_data + (ass_image->dst_y + y) * y_stride + ass_image->dst_x;
609 for (x = 0; x < w; x++) {
610 k = src[y * ass_image->w + x] * alpha / 255;
611 dst_y[x] = (k * Y + (255 - k) * dst_y[x]) / 255;
616 if (ass_image->dst_y & 1) {
617 dst_u = u_data + (ass_image->dst_y / 2) * u_stride + ass_image->dst_x / 2;
618 dst_v = v_data + (ass_image->dst_y / 2) * v_stride + ass_image->dst_x / 2;
620 if (ass_image->dst_x & 1) {
621 k2 = src[y * ass_image->w + x] * alpha / 255;
623 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
624 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
629 for (; x < w - 1; x += 2) {
630 k2 = src[y * ass_image->w + x] * alpha / 255;
631 k2 += src[y * ass_image->w + x + 1] * alpha / 255;
633 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
634 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
639 k2 = src[y * ass_image->w + x] * alpha / 255;
641 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
642 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
646 for (; y < h - 1; y += 2) {
647 dst_u = u_data + ((ass_image->dst_y + y) / 2) * u_stride +
648 ass_image->dst_x / 2;
649 dst_v = v_data + ((ass_image->dst_y + y) / 2) * v_stride +
650 ass_image->dst_x / 2;
652 if (ass_image->dst_x & 1) {
653 k2 = src[y * ass_image->w + x] * alpha / 255;
654 k2 += src[(y + 1) * ass_image->w + x] * alpha / 255;
656 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
657 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
662 for (; x < w - 1; x += 2) {
663 k2 = src[y * ass_image->w + x] * alpha / 255;
664 k2 += src[y * ass_image->w + x + 1] * alpha / 255;
665 k2 += src[(y + 1) * ass_image->w + x] * alpha / 255;
666 k2 += src[(y + 1) * ass_image->w + x + 1] * alpha / 255;
668 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
669 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
674 k2 = src[y * ass_image->w + x] * alpha / 255;
675 k2 += src[(y + 1) * ass_image->w + x] * alpha / 255;
677 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
678 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
683 dst_u = u_data + (ass_image->dst_y / 2) * u_stride + ass_image->dst_x / 2;
684 dst_v = v_data + (ass_image->dst_y / 2) * v_stride + ass_image->dst_x / 2;
686 if (ass_image->dst_x & 1) {
687 k2 = src[y * ass_image->w + x] * alpha / 255;
689 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
690 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
695 for (; x < w - 1; x += 2) {
696 k2 = src[y * ass_image->w + x] * alpha / 255;
697 k2 += src[y * ass_image->w + x + 1] * alpha / 255;
699 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
700 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
705 k2 = src[y * ass_image->w + x] * alpha / 255;
707 dst_u[0] = (k2 * U + (255 - k2) * dst_u[0]) / 255;
708 dst_v[0] = (k2 * V + (255 - k2) * dst_v[0]) / 255;
716 ass_image = ass_image->next;
719 GST_LOG_OBJECT (render, "amount of rendered ass_image: %u", counter);
723 gst_ass_render_setcaps_video (GstPad * pad, GstCaps * caps)
725 GstAssRender *render = GST_ASS_RENDER (gst_pad_get_parent (pad));
726 gboolean ret = FALSE;
727 gint par_n = 1, par_d = 1;
731 if (!gst_video_info_from_caps (&info, caps))
736 ret = gst_pad_set_caps (render->srcpad, caps);
740 switch (GST_VIDEO_INFO_FORMAT (&info)) {
741 case GST_VIDEO_FORMAT_RGB:
742 render->blit = blit_rgb;
744 case GST_VIDEO_FORMAT_BGR:
745 render->blit = blit_bgr;
747 case GST_VIDEO_FORMAT_xRGB:
748 render->blit = blit_xrgb;
750 case GST_VIDEO_FORMAT_xBGR:
751 render->blit = blit_xbgr;
753 case GST_VIDEO_FORMAT_RGBx:
754 render->blit = blit_rgbx;
756 case GST_VIDEO_FORMAT_BGRx:
757 render->blit = blit_bgrx;
759 case GST_VIDEO_FORMAT_I420:
760 render->blit = blit_i420;
767 g_mutex_lock (render->ass_mutex);
768 ass_set_frame_size (render->ass_renderer, info.width, info.height);
770 dar = (((gdouble) par_n) * ((gdouble) info.width))
771 / (((gdouble) par_d) * ((gdouble) info.height));
772 #if !defined(LIBASS_VERSION) || LIBASS_VERSION < 0x00907000
773 ass_set_aspect_ratio (render->ass_renderer, dar);
775 ass_set_aspect_ratio (render->ass_renderer,
776 dar, ((gdouble) info.width) / ((gdouble) info.height));
778 ass_set_font_scale (render->ass_renderer, 1.0);
779 ass_set_hinting (render->ass_renderer, ASS_HINTING_LIGHT);
781 #if !defined(LIBASS_VERSION) || LIBASS_VERSION < 0x00907000
782 ass_set_fonts (render->ass_renderer, "Arial", "sans-serif");
783 ass_set_fonts (render->ass_renderer, NULL, "Sans");
785 ass_set_fonts (render->ass_renderer, "Arial", "sans-serif", 1, NULL, 1);
786 ass_set_fonts (render->ass_renderer, NULL, "Sans", 1, NULL, 1);
788 ass_set_margins (render->ass_renderer, 0, 0, 0, 0);
789 ass_set_use_margins (render->ass_renderer, 0);
790 g_mutex_unlock (render->ass_mutex);
792 render->renderer_init_ok = TRUE;
794 GST_DEBUG_OBJECT (render, "ass renderer setup complete");
797 gst_object_unref (render);
804 GST_ERROR_OBJECT (render, "Can't parse caps: %" GST_PTR_FORMAT, caps);
811 gst_ass_render_setcaps_text (GstPad * pad, GstCaps * caps)
813 GstAssRender *render = GST_ASS_RENDER (gst_pad_get_parent (pad));
814 GstStructure *structure;
817 gchar *codec_private;
818 gsize codec_private_size;
819 gboolean ret = FALSE;
821 structure = gst_caps_get_structure (caps, 0);
823 GST_DEBUG_OBJECT (render, "text pad linked with caps: %" GST_PTR_FORMAT,
826 value = gst_structure_get_value (structure, "codec_data");
828 g_mutex_lock (render->ass_mutex);
830 priv = gst_value_get_buffer (value);
831 g_return_val_if_fail (priv != NULL, FALSE);
834 gst_buffer_map (priv, &codec_private_size, NULL, GST_MAP_READ);
836 if (!render->ass_track)
837 render->ass_track = ass_new_track (render->ass_library);
839 ass_process_codec_private (render->ass_track,
840 codec_private, codec_private_size);
842 gst_buffer_unmap (priv, codec_private, codec_private_size);
844 GST_DEBUG_OBJECT (render, "ass track created");
846 render->track_init_ok = TRUE;
849 } else if (!render->ass_track) {
850 render->ass_track = ass_new_track (render->ass_library);
852 render->track_init_ok = TRUE;
856 g_mutex_unlock (render->ass_mutex);
858 gst_object_unref (render);
865 gst_ass_render_process_text (GstAssRender * render, GstBuffer * buffer,
866 GstClockTime running_time, GstClockTime duration)
870 gdouble pts_start, pts_end;
872 pts_start = running_time;
873 pts_start /= GST_MSECOND;
875 pts_end /= GST_MSECOND;
877 GST_DEBUG_OBJECT (render,
878 "Processing subtitles with running time %" GST_TIME_FORMAT
879 " and duration %" GST_TIME_FORMAT, GST_TIME_ARGS (running_time),
880 GST_TIME_ARGS (duration));
882 data = gst_buffer_map (buffer, &size, NULL, GST_MAP_READ);
884 g_mutex_lock (render->ass_mutex);
885 ass_process_chunk (render->ass_track, data, size, pts_start, pts_end);
886 g_mutex_unlock (render->ass_mutex);
888 gst_buffer_unmap (buffer, data, size);
889 gst_buffer_unref (buffer);
893 gst_ass_render_chain_video (GstPad * pad, GstBuffer * buffer)
895 GstAssRender *render = GST_ASS_RENDER (GST_PAD_PARENT (pad));
896 GstFlowReturn ret = GST_FLOW_OK;
897 gboolean in_seg = FALSE;
898 guint64 start, stop, clip_start = 0, clip_stop = 0;
899 ASS_Image *ass_image;
901 if (!GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) {
902 GST_WARNING_OBJECT (render, "buffer without timestamp, discarding");
903 gst_buffer_unref (buffer);
907 /* ignore buffers that are outside of the current segment */
908 start = GST_BUFFER_TIMESTAMP (buffer);
910 if (!GST_BUFFER_DURATION_IS_VALID (buffer)) {
911 stop = GST_CLOCK_TIME_NONE;
913 stop = start + GST_BUFFER_DURATION (buffer);
916 /* segment_clip() will adjust start unconditionally to segment_start if
917 * no stop time is provided, so handle this ourselves */
918 if (stop == GST_CLOCK_TIME_NONE && start < render->video_segment.start)
922 gst_segment_clip (&render->video_segment, GST_FORMAT_TIME, start, stop,
923 &clip_start, &clip_stop);
928 /* if the buffer is only partially in the segment, fix up stamps */
929 if (clip_start != start || (stop != -1 && clip_stop != stop)) {
930 GST_DEBUG_OBJECT (render, "clipping buffer timestamp/duration to segment");
931 buffer = gst_buffer_make_writable (buffer);
932 GST_BUFFER_TIMESTAMP (buffer) = clip_start;
934 GST_BUFFER_DURATION (buffer) = clip_stop - clip_start;
937 render->video_segment.position = clip_start;
939 g_mutex_lock (render->subtitle_mutex);
940 if (render->subtitle_pending) {
941 GstClockTime sub_running_time, vid_running_time;
942 GstClockTime sub_running_time_end, vid_running_time_end;
945 gst_segment_to_running_time (&render->subtitle_segment, GST_FORMAT_TIME,
946 GST_BUFFER_TIMESTAMP (render->subtitle_pending));
947 sub_running_time_end =
948 gst_segment_to_running_time (&render->subtitle_segment, GST_FORMAT_TIME,
949 GST_BUFFER_TIMESTAMP (render->subtitle_pending) +
950 GST_BUFFER_DURATION (render->subtitle_pending));
952 gst_segment_to_running_time (&render->video_segment, GST_FORMAT_TIME,
953 GST_BUFFER_TIMESTAMP (buffer));
954 vid_running_time_end =
955 gst_segment_to_running_time (&render->video_segment, GST_FORMAT_TIME,
956 GST_BUFFER_TIMESTAMP (buffer) + GST_BUFFER_DURATION (buffer));
958 if (sub_running_time_end < vid_running_time) {
959 gst_buffer_unref (render->subtitle_pending);
960 GST_DEBUG_OBJECT (render,
961 "Too late text buffer, dropping (%" GST_TIME_FORMAT " < %"
962 GST_TIME_FORMAT, GST_TIME_ARGS (sub_running_time_end),
963 GST_TIME_ARGS (vid_running_time));
964 render->subtitle_pending = NULL;
965 g_cond_signal (render->subtitle_cond);
966 } else if (sub_running_time <= vid_running_time_end + GST_SECOND / 2) {
967 gst_ass_render_process_text (render, render->subtitle_pending,
968 sub_running_time, sub_running_time_end - sub_running_time);
969 render->subtitle_pending = NULL;
970 g_cond_signal (render->subtitle_cond);
973 g_mutex_unlock (render->subtitle_mutex);
975 /* now start rendering subtitles, if all conditions are met */
976 if (render->renderer_init_ok && render->track_init_ok && render->enable) {
977 GstClockTime running_time;
979 #ifndef GST_DISABLE_GST_DEBUG
984 gst_segment_to_running_time (&render->video_segment, GST_FORMAT_TIME,
985 GST_BUFFER_TIMESTAMP (buffer));
986 GST_DEBUG_OBJECT (render,
987 "rendering frame for running time %" GST_TIME_FORMAT,
988 GST_TIME_ARGS (running_time));
989 /* libass needs timestamps in ms */
990 timestamp = running_time / GST_MSECOND;
992 g_mutex_lock (render->ass_mutex);
993 #ifndef GST_DISABLE_GST_DEBUG
994 /* only for testing right now. could possibly be used for optimizations? */
995 step = ass_step_sub (render->ass_track, timestamp, 1);
996 GST_DEBUG_OBJECT (render, "Current running time: %" GST_TIME_FORMAT
997 " // Next event: %" GST_TIME_FORMAT,
998 GST_TIME_ARGS (running_time), GST_TIME_ARGS (step * GST_MSECOND));
1001 /* not sure what the last parameter to this call is for (detect_change) */
1002 ass_image = ass_render_frame (render->ass_renderer, render->ass_track,
1004 g_mutex_unlock (render->ass_mutex);
1006 if (ass_image != NULL) {
1007 GstVideoFrame frame;
1009 buffer = gst_buffer_make_writable (buffer);
1011 gst_video_frame_map (&frame, &render->info, buffer, GST_MAP_WRITE);
1012 render->blit (render, ass_image, &frame);
1013 gst_video_frame_unmap (&frame);
1015 GST_LOG_OBJECT (render, "nothing to render right now");
1018 GST_LOG_OBJECT (render, "rendering disabled, doing buffer passthrough");
1021 ret = gst_pad_push (render->srcpad, buffer);
1027 GST_DEBUG_OBJECT (render, "buffer out of segment, discarding");
1028 gst_buffer_unref (buffer);
1033 static GstFlowReturn
1034 gst_ass_render_chain_text (GstPad * pad, GstBuffer * buffer)
1036 GstFlowReturn ret = GST_FLOW_OK;
1037 GstAssRender *render = GST_ASS_RENDER (GST_PAD_PARENT (pad));
1038 GstClockTime timestamp, duration;
1039 GstClockTime sub_running_time, vid_running_time;
1040 GstClockTime sub_running_time_end;
1041 guint64 cstart, cstop;
1044 if (render->subtitle_flushing) {
1045 gst_buffer_unref (buffer);
1046 return GST_FLOW_WRONG_STATE;
1049 timestamp = GST_BUFFER_TIMESTAMP (buffer);
1050 duration = GST_BUFFER_DURATION (buffer);
1052 if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (timestamp)
1053 || !GST_CLOCK_TIME_IS_VALID (duration))) {
1054 GST_WARNING_OBJECT (render,
1055 "Text buffer without valid timestamp" " or duration, dropping");
1056 gst_buffer_unref (buffer);
1061 gst_segment_clip (&render->subtitle_segment, GST_FORMAT_TIME, timestamp,
1062 timestamp + duration, &cstart, &cstop);
1064 GST_DEBUG_OBJECT (render,
1065 "Text buffer before segment start (%" GST_TIME_FORMAT " < %"
1066 GST_TIME_FORMAT ")", GST_TIME_ARGS (timestamp),
1067 GST_TIME_ARGS (render->subtitle_segment.start));
1068 gst_buffer_unref (buffer);
1072 GST_BUFFER_TIMESTAMP (buffer) = timestamp = cstart;
1073 GST_BUFFER_DURATION (buffer) = duration = cstop - cstart;
1075 render->subtitle_segment.position = GST_BUFFER_TIMESTAMP (buffer);
1078 gst_segment_to_running_time (&render->subtitle_segment, GST_FORMAT_TIME,
1080 sub_running_time_end =
1081 gst_segment_to_running_time (&render->subtitle_segment, GST_FORMAT_TIME,
1082 timestamp + duration);
1084 gst_segment_to_running_time (&render->video_segment, GST_FORMAT_TIME,
1085 render->video_segment.position);
1087 if (render->info.fps_n && render->info.fps_d)
1089 gst_util_uint64_scale (GST_SECOND, render->info.fps_d,
1090 render->info.fps_n);
1092 if (sub_running_time > vid_running_time + GST_SECOND / 2) {
1093 g_assert (render->subtitle_pending == NULL);
1094 g_mutex_lock (render->subtitle_mutex);
1095 if (G_UNLIKELY (render->subtitle_flushing)) {
1096 GST_DEBUG_OBJECT (render, "Text pad flushing");
1097 gst_buffer_unref (buffer);
1098 g_mutex_unlock (render->subtitle_mutex);
1099 return GST_FLOW_WRONG_STATE;
1101 GST_DEBUG_OBJECT (render,
1102 "Too early text buffer, waiting (%" GST_TIME_FORMAT " > %"
1103 GST_TIME_FORMAT, GST_TIME_ARGS (sub_running_time),
1104 GST_TIME_ARGS (vid_running_time));
1105 render->subtitle_pending = buffer;
1106 g_cond_wait (render->subtitle_cond, render->subtitle_mutex);
1107 g_mutex_unlock (render->subtitle_mutex);
1108 } else if (sub_running_time_end < vid_running_time) {
1109 GST_DEBUG_OBJECT (render,
1110 "Too late text buffer, dropping (%" GST_TIME_FORMAT " < %"
1111 GST_TIME_FORMAT, GST_TIME_ARGS (sub_running_time_end),
1112 GST_TIME_ARGS (vid_running_time));
1113 gst_buffer_unref (buffer);
1116 gst_ass_render_process_text (render, buffer, sub_running_time,
1117 sub_running_time_end - sub_running_time);
1121 GST_DEBUG_OBJECT (render,
1122 "processed text packet with timestamp %" GST_TIME_FORMAT
1123 " and duration %" GST_TIME_FORMAT,
1124 GST_TIME_ARGS (timestamp), GST_TIME_ARGS (duration));
1130 gst_ass_render_handle_tags (GstAssRender * render, GstTagList * taglist)
1133 static const gchar *mimetypes[] = {
1134 "application/x-font-ttf",
1135 "application/x-font-otf",
1136 "application/x-truetype-font"
1138 static const gchar *extensions[] = {
1148 tag_size = gst_tag_list_get_tag_size (taglist, GST_TAG_ATTACHMENT);
1149 if (tag_size > 0 && render->embeddedfonts) {
1151 const GValue *value;
1154 GstStructure *structure;
1155 gboolean valid_mimetype, valid_extension;
1157 const gchar *filename;
1161 GST_DEBUG_OBJECT (render, "TAG event has attachments");
1163 for (index = 0; index < tag_size; index++) {
1165 value = gst_tag_list_get_value_index (taglist, GST_TAG_ATTACHMENT, index);
1166 buf = gst_value_get_buffer (value);
1167 if (!buf || !GST_BUFFER_CAPS (buf))
1170 caps = GST_BUFFER_CAPS (buf);
1171 structure = gst_caps_get_structure (caps, 0);
1173 valid_mimetype = FALSE;
1174 valid_extension = FALSE;
1176 for (j = 0; j < G_N_ELEMENTS (mimetypes); j++) {
1177 if (gst_structure_has_name (structure, mimetypes[j])) {
1178 valid_mimetype = TRUE;
1182 filename = gst_structure_get_string (structure, "filename");
1186 if (!valid_mimetype) {
1187 guint len = strlen (filename);
1188 const gchar *extension = filename + len - 4;
1189 for (j = 0; j < G_N_ELEMENTS (extensions); j++) {
1190 if (g_ascii_strcasecmp (extension, extensions[j]) == 0) {
1191 valid_extension = TRUE;
1197 if (valid_mimetype || valid_extension) {
1198 g_mutex_lock (render->ass_mutex);
1199 ass_add_font (render->ass_library, (gchar *) filename,
1200 (gchar *) GST_BUFFER_DATA (buf), GST_BUFFER_SIZE (buf));
1201 GST_DEBUG_OBJECT (render, "registered new font %s", filename);
1202 g_mutex_unlock (render->ass_mutex);
1210 gst_ass_render_event_video (GstPad * pad, GstEvent * event)
1212 gboolean ret = FALSE;
1213 GstAssRender *render = GST_ASS_RENDER (gst_pad_get_parent (pad));
1215 GST_DEBUG_OBJECT (pad, "received video event %s",
1216 GST_EVENT_TYPE_NAME (event));
1218 switch (GST_EVENT_TYPE (event)) {
1219 case GST_EVENT_CAPS:
1223 gst_event_parse_caps (event, &caps);
1224 ret = gst_ass_render_setcaps_video (pad, caps);
1225 gst_event_unref (event);
1228 case GST_EVENT_SEGMENT:
1232 GST_DEBUG_OBJECT (render, "received new segment");
1234 gst_event_copy_segment (event, &segment);
1236 if (segment.format == GST_FORMAT_TIME) {
1237 GST_DEBUG_OBJECT (render, "VIDEO SEGMENT now: %" GST_SEGMENT_FORMAT,
1238 &render->video_segment);
1240 render->video_segment = segment;
1242 GST_DEBUG_OBJECT (render, "VIDEO SEGMENT after: %" GST_SEGMENT_FORMAT,
1243 &render->video_segment);
1244 ret = gst_pad_push_event (render->srcpad, event);
1246 GST_ELEMENT_WARNING (render, STREAM, MUX, (NULL),
1247 ("received non-TIME newsegment event on video input"));
1249 gst_event_unref (event);
1255 GstTagList *taglist = NULL;
1257 /* tag events may contain attachments which might be fonts */
1258 GST_DEBUG_OBJECT (render, "got TAG event");
1260 gst_event_parse_tag (event, &taglist);
1261 gst_ass_render_handle_tags (render, taglist);
1262 ret = gst_pad_push_event (render->srcpad, event);
1265 case GST_EVENT_FLUSH_STOP:
1266 gst_segment_init (&render->video_segment, GST_FORMAT_TIME);
1268 ret = gst_pad_push_event (render->srcpad, event);
1272 gst_object_unref (render);
1278 gst_ass_render_event_text (GstPad * pad, GstEvent * event)
1281 gboolean ret = FALSE;
1282 GstAssRender *render = GST_ASS_RENDER (gst_pad_get_parent (pad));
1284 GST_DEBUG_OBJECT (pad, "received text event %s", GST_EVENT_TYPE_NAME (event));
1286 switch (GST_EVENT_TYPE (event)) {
1287 case GST_EVENT_CAPS:
1291 gst_event_parse_caps (event, &caps);
1292 ret = gst_ass_render_setcaps_text (pad, caps);
1293 gst_event_unref (event);
1296 case GST_EVENT_SEGMENT:
1300 GST_DEBUG_OBJECT (render, "received new segment");
1302 gst_event_copy_segment (event, &segment);
1304 if (segment.format == GST_FORMAT_TIME) {
1305 GST_DEBUG_OBJECT (render, "SUBTITLE SEGMENT now: %" GST_SEGMENT_FORMAT,
1306 &render->subtitle_segment);
1308 render->subtitle_segment = segment;
1310 GST_DEBUG_OBJECT (render,
1311 "SUBTITLE SEGMENT after: %" GST_SEGMENT_FORMAT,
1312 &render->subtitle_segment);
1314 gst_event_unref (event);
1316 GST_ELEMENT_WARNING (render, STREAM, MUX, (NULL),
1317 ("received non-TIME newsegment event on subtitle input"));
1319 gst_event_unref (event);
1323 case GST_EVENT_FLUSH_STOP:
1324 gst_segment_init (&render->subtitle_segment, GST_FORMAT_TIME);
1325 render->subtitle_flushing = FALSE;
1326 gst_event_unref (event);
1329 case GST_EVENT_FLUSH_START:
1330 GST_DEBUG_OBJECT (render, "begin flushing");
1331 g_mutex_lock (render->ass_mutex);
1332 if (render->ass_track) {
1333 /* delete any events on the ass_track */
1334 for (i = 0; i < render->ass_track->n_events; i++) {
1335 GST_DEBUG_OBJECT (render, "deleted event with eid %i", i);
1336 ass_free_event (render->ass_track, i);
1338 render->ass_track->n_events = 0;
1339 GST_DEBUG_OBJECT (render, "done flushing");
1341 g_mutex_unlock (render->ass_mutex);
1342 g_mutex_lock (render->subtitle_mutex);
1343 if (render->subtitle_pending)
1344 gst_buffer_unref (render->subtitle_pending);
1345 render->subtitle_pending = NULL;
1346 render->subtitle_flushing = TRUE;
1347 g_cond_signal (render->subtitle_cond);
1348 g_mutex_unlock (render->subtitle_mutex);
1349 gst_event_unref (event);
1353 GST_OBJECT_LOCK (render);
1354 GST_INFO_OBJECT (render, "text EOS");
1355 GST_OBJECT_UNLOCK (render);
1356 gst_event_unref (event);
1361 GstTagList *taglist = NULL;
1363 /* tag events may contain attachments which might be fonts */
1364 GST_DEBUG_OBJECT (render, "got TAG event");
1366 gst_event_parse_tag (event, &taglist);
1367 gst_ass_render_handle_tags (render, taglist);
1368 ret = gst_pad_push_event (render->srcpad, event);
1372 ret = gst_pad_push_event (render->srcpad, event);
1376 gst_object_unref (render);
1382 plugin_init (GstPlugin * plugin)
1384 GST_DEBUG_CATEGORY_INIT (gst_ass_render_debug, "assrender",
1385 0, "ASS/SSA subtitle renderer");
1386 GST_DEBUG_CATEGORY_INIT (gst_ass_render_lib_debug, "assrender_library",
1387 0, "ASS/SSA subtitle renderer library");
1389 return gst_element_register (plugin, "assrender",
1390 GST_RANK_PRIMARY, GST_TYPE_ASS_RENDER);
1393 GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
1396 "ASS/SSA subtitle renderer",
1397 plugin_init, VERSION, "LGPL", GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN)