ttmlparse

From d82ae6949feb100b74ca3781a57cbe19ce38787c Mon Sep 17 00:00:00 2001 From: Chris Bass Date: Wed, 29 Jun 2016 09:58:38 +0100 Subject: [PATCH] ttml: Add plugin that supports TTML subtitles Add a parser (ttmlparse) and renderer (ttmlrender) element that handle subtitles that use the EBU-TT-D profile of TTML1. https://bugzilla.gnome.org/show_bug.cgi?id=758232 --- configure.ac | 12 + docs/plugins/gst-plugins-bad-plugins-docs.sgml | 3 + docs/plugins/gst-plugins-bad-plugins-sections.txt | 19 + docs/plugins/inspect/plugin-ttmlsubs.xml | 61 + ext/Makefile.am | 12 +- ext/ttml/Makefile.am | 36 + ext/ttml/gstttmlparse.c | 570 +++++ ext/ttml/gstttmlparse.h | 76 + ext/ttml/gstttmlplugin.c | 53 + ext/ttml/gstttmlrender.c | 2456 +++++++++++++++++++++ ext/ttml/gstttmlrender.h | 124 ++ ext/ttml/subtitle.c | 312 +++ ext/ttml/subtitle.h | 592 +++++ ext/ttml/subtitlemeta.c | 101 + ext/ttml/subtitlemeta.h | 66 + ext/ttml/ttmlparse.c | 1813 +++++++++++++++ ext/ttml/ttmlparse.h | 34 + 17 files changed, 6338 insertions(+), 2 deletions(-) create mode 100644 docs/plugins/inspect/plugin-ttmlsubs.xml create mode 100644 ext/ttml/Makefile.am create mode 100644 ext/ttml/gstttmlparse.c create mode 100644 ext/ttml/gstttmlparse.h create mode 100644 ext/ttml/gstttmlplugin.c create mode 100644 ext/ttml/gstttmlrender.c create mode 100644 ext/ttml/gstttmlrender.h create mode 100644 ext/ttml/subtitle.c create mode 100644 ext/ttml/subtitle.h create mode 100644 ext/ttml/subtitlemeta.c create mode 100644 ext/ttml/subtitlemeta.h create mode 100644 ext/ttml/ttmlparse.c create mode 100644 ext/ttml/ttmlparse.h diff --git a/configure.ac b/configure.ac index 3aff929..acbe922 100644 --- a/configure.ac +++ b/configure.ac @@ -2456,6 +2456,16 @@ AG_GST_CHECK_FEATURE(DTLS, [DTLS plugin], dtls, [ ]) ]) +dnl *** ttml *** +translit(dnm, m, l) AM_CONDITIONAL(USE_TTML, true) +AG_GST_CHECK_FEATURE(TTML, [TTML plugin], ttml, [ + PKG_CHECK_MODULES(TTML, [ libxml-2.0 pango cairo pangocairo ], [ + HAVE_TTML="yes" + ], [ + HAVE_TTML="no" + ]) +]) + dnl *** linsys *** translit(dnm, m, l) AM_CONDITIONAL(USE_LINSYS, true) AG_GST_CHECK_FEATURE(LINSYS, [Linear Systems SDI plugin], linsys, [ @@ -3529,6 +3539,7 @@ AM_CONDITIONAL(USE_OPENH264, false) AM_CONDITIONAL(USE_X265, false) AM_CONDITIONAL(USE_DTLS, false) AM_CONDITIONAL(USE_VULKAN, false) +AM_CONDITIONAL(USE_TTML, false) fi dnl of EXT plugins @@ -3841,6 +3852,7 @@ ext/xvid/Makefile ext/zbar/Makefile ext/dtls/Makefile ext/webrtcdsp/Makefile +ext/ttml/Makefile po/Makefile.in docs/Makefile docs/plugins/Makefile diff --git a/docs/plugins/gst-plugins-bad-plugins-docs.sgml b/docs/plugins/gst-plugins-bad-plugins-docs.sgml index dea2355..70b1ea0 100644 --- a/docs/plugins/gst-plugins-bad-plugins-docs.sgml +++ b/docs/plugins/gst-plugins-bad-plugins-docs.sgml @@ -152,6 +152,8 @@ + + @@ -203,5 +205,6 @@ + diff --git a/docs/plugins/gst-plugins-bad-plugins-sections.txt b/docs/plugins/gst-plugins-bad-plugins-sections.txt index ba933c1..811c0e5 100644 --- a/docs/plugins/gst-plugins-bad-plugins-sections.txt +++ b/docs/plugins/gst-plugins-bad-plugins-sections.txt @@ -4805,3 +4805,22 @@ GST_TYPE_FFECTS_XRAY gst_ffects_xray_get_type +

+element-ttmlparse +ttmlparse +GstTtmlParse + +GstTtmlParseClass +GST_TYPE_TTML_PARSE +gst_ttml_parse_get_type +

+ +

+element-ttmlrender +ttmlrender +GstTtmlRender + +GstTtmlRenderClass +GST_TYPE_TTML_RENDER +gst_ttml_render_get_type +

diff --git a/docs/plugins/inspect/plugin-ttmlsubs.xml b/docs/plugins/inspect/plugin-ttmlsubs.xml new file mode 100644 index 0000000..4d10360 --- /dev/null +++ b/docs/plugins/inspect/plugin-ttmlsubs.xml @@ -0,0 +1,61 @@ + + ttmlsubs + TTML subtitle handling + ../../ext/ttml/.libs/libgstttmlsubs.so + libgstttmlsubs.so + 1.9.90 + LGPL + gst-plugins-bad + gst-ttml + http://www.bbc.co.uk/rd + + + ttmlparse + TTML subtitle parser + Codec/Parser/Subtitle + Parses TTML subtitle files + GStreamer maintainers <gstreamer-devel@lists.sourceforge.net>, Chris Bass <dash@rd.bbc.co.uk> + + + sink + sink + always +

application/ttml+xml

+ + + src + source + always +

text/x-raw(meta:GstSubtitleMeta)

+ + + + + ttmlrender + TTML subtitle renderer + Overlay/Subtitle + Renders timed-text subtitles on top of video buffers + David Schleef <ds@schleef.org>, Zeeshan Ali <zeeshan.ali@nokia.com>, Chris Bass <dash@rd.bbc.co.uk> + + + text_sink + sink + always +

text/x-raw(meta:GstSubtitleMeta)

+ + + video_sink + sink + always +

video/x-raw, format=(string){ BGRx, RGBx, xRGB, xBGR, RGBA, BGRA, ARGB, ABGR, RGB, BGR, I420, YV12, AYUV, YUY2, UYVY, v308, Y41B, Y42B, Y444, NV12, NV21, A420, YUV9, YVU9, IYU1, GRAY8 }, width=(int)[ 1, 2147483647 ], height=(int)[ 1, 2147483647 ], framerate=(fraction)[ 0/1, 2147483647/1 ]; video/x-raw(ANY), format=(string){ I420, YV12, YUY2, UYVY, AYUV, RGBx, BGRx, xRGB, xBGR, RGBA, BGRA, ARGB, ABGR, RGB, BGR, Y41B, Y42B, YVYU, Y444, v210, v216, NV12, NV21, NV16, NV61, NV24, GRAY8, GRAY16_BE, GRAY16_LE, v308, IYU2, RGB16, BGR16, RGB15, BGR15, UYVP, A420, RGB8P, YUV9, YVU9, IYU1, ARGB64, AYUV64, r210, I420_10LE, I420_10BE, I422_10LE, I422_10BE, Y444_10LE, Y444_10BE, GBR, GBR_10LE, GBR_10BE, NV12_64Z32, A420_10LE, A420_10BE, A422_10LE, A422_10BE, A444_10LE, A444_10BE, P010_10LE, P010_10BE }, width=(int)[ 1, 2147483647 ], height=(int)[ 1, 2147483647 ], framerate=(fraction)[ 0/1, 2147483647/1 ]

+ + + src + source + always +

+ + + + + \ No newline at end of file diff --git a/ext/Makefile.am b/ext/Makefile.am index bae25c2..16821e0 100644 --- a/ext/Makefile.am +++ b/ext/Makefile.am @@ -424,6 +424,12 @@ else WEBRTCDSP_DIR= endif +if USE_TTML +TTML_DIR=ttml +else +TTML_DIR= +endif + SUBDIRS=\ $(VOAACENC_DIR) \ $(ASSRENDER_DIR) \ @@ -495,7 +501,8 @@ SUBDIRS=\ $(X265_DIR) \ $(DTLS_DIR) \ $(VULKAN_DIR) \ - $(WEBRTCDSP_DIR) + $(WEBRTCDSP_DIR) \ + $(TTML_DIR) DIST_SUBDIRS = \ assrender \ @@ -565,6 +572,7 @@ DIST_SUBDIRS = \ x265 \ dtls \ vulkan \ - webrtcdsp + webrtcdsp \ + ttml include $(top_srcdir)/common/parallel-subdirs.mak diff --git a/ext/ttml/Makefile.am b/ext/ttml/Makefile.am new file mode 100644 index 0000000..ce7a197 --- /dev/null +++ b/ext/ttml/Makefile.am @@ -0,0 +1,36 @@ +plugin_LTLIBRARIES = libgstttmlsubs.la + +# sources used to compile this plug-in +libgstttmlsubs_la_SOURCES = \ + subtitle.c \ + subtitlemeta.c \ + gstttmlparse.c \ + gstttmlparse.h \ + ttmlparse.c \ + ttmlparse.h \ + gstttmlrender.c \ + gstttmlplugin.c + +# compiler and linker flags used to compile this plugin, set in configure.ac +libgstttmlsubs_la_CFLAGS = \ + $(GST_PLUGINS_BASE_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_CFLAGS) \ + $(TTML_CFLAGS) + +libgstttmlsubs_la_LIBADD = \ + $(GST_BASE_LIBS) \ + $(GST_LIBS) \ + -lgstvideo-$(GST_API_VERSION) \ + $(TTML_LIBS) + +libgstttmlsubs_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstttmlsubs_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) + +# headers we need but don't want installed +noinst_HEADERS = \ + subtitle.h \ + subtitlemeta.h \ + gstttmlparse.h \ + ttmlparse.h \ + gstttmlrender.h diff --git a/ext/ttml/gstttmlparse.c b/ext/ttml/gstttmlparse.c new file mode 100644 index 0000000..3daeb6e --- /dev/null +++ b/ext/ttml/gstttmlparse.c @@ -0,0 +1,570 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) 2004 Ronald S. Bultje + * Copyright (C) 2006 Tim-Philipp MÃ¼ller + * Copyright (C) <2015> British Broadcasting Corporation + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-ttmlparse + * + * Parses timed text subtitle files described using Timed Text Markup Language + * (TTML). Currently, only the EBU-TT-D profile of TTML, designed for + * distribution of subtitles over IP, is supported. + * + * The parser outputs a #GstBuffer for each scene in the input TTML file, a + * scene being a period of time during which a static set of subtitles should + * be visible. The parser places each text element within a scene into its own + * #GstMemory within the scene's buffer, and attaches metadata to the buffer + * describing the styling and layout associated with all the contained text + * elements. A downstream renderer element uses this information to correctly + * render the text on top of video frames. + * + * + * Example launch lines + * |[ + * gst-launch-1.0 filesrc location= ! video/quicktime ! qtdemux name=q ttmlrender name=r q. ! queue ! h264parse ! avdec_h264 ! autovideoconvert ! r.video_sink filesrc location= blocksize=16777216 ! queue ! ttmlparse ! r.text_sink r. ! ximagesink q. ! queue ! aacparse ! avdec_aac ! audioconvert ! alsasink + * ]| Parse and render TTML subtitles contained in a single XML file over an + * MP4 stream containing H.264 video and AAC audio. + * + */ + +#include +#include +#include +#include +#include + +#include "gstttmlparse.h" +#include "ttmlparse.h" + +GST_DEBUG_CATEGORY_EXTERN (ttmlparse_debug); +#define GST_CAT_DEFAULT ttmlparse_debug + +#define DEFAULT_ENCODING NULL + +static GstStaticPadTemplate sink_templ = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("application/ttml+xml") + ); + +static GstStaticPadTemplate src_templ = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("text/x-raw(meta:GstSubtitleMeta)") + ); + +static gboolean gst_ttml_parse_src_event (GstPad * pad, GstObject * parent, + GstEvent * event); +static gboolean gst_ttml_parse_src_query (GstPad * pad, GstObject * parent, + GstQuery * query); +static gboolean gst_ttml_parse_sink_event (GstPad * pad, GstObject * parent, + GstEvent * event); + +static GstStateChangeReturn gst_ttml_parse_change_state (GstElement * element, + GstStateChange transition); + +static GstFlowReturn gst_ttml_parse_chain (GstPad * sinkpad, GstObject * parent, + GstBuffer * buf); + +#define gst_ttml_parse_parent_class parent_class +G_DEFINE_TYPE (GstTtmlParse, gst_ttml_parse, GST_TYPE_ELEMENT); + +static void +gst_ttml_parse_dispose (GObject * object) +{ + GstTtmlParse *ttmlparse = GST_TTML_PARSE (object); + + GST_DEBUG_OBJECT (ttmlparse, "cleaning up subtitle parser"); + + g_free (ttmlparse->encoding); + ttmlparse->encoding = NULL; + + g_free (ttmlparse->detected_encoding); + ttmlparse->detected_encoding = NULL; + + if (ttmlparse->adapter) { + g_object_unref (ttmlparse->adapter); + ttmlparse->adapter = NULL; + } + + if (ttmlparse->textbuf) { + g_string_free (ttmlparse->textbuf, TRUE); + ttmlparse->textbuf = NULL; + } + + GST_CALL_PARENT (G_OBJECT_CLASS, dispose, (object)); +} + +static void +gst_ttml_parse_class_init (GstTtmlParseClass * klass) +{ + GObjectClass *object_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + + object_class->dispose = gst_ttml_parse_dispose; + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_templ)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_templ)); + gst_element_class_set_static_metadata (element_class, + "TTML subtitle parser", "Codec/Parser/Subtitle", + "Parses TTML subtitle files", + "GStreamer maintainers , " + "Chris Bass "); + + element_class->change_state = gst_ttml_parse_change_state; +} + +static void +gst_ttml_parse_init (GstTtmlParse * ttmlparse) +{ + ttmlparse->sinkpad = gst_pad_new_from_static_template (&sink_templ, "sink"); + gst_pad_set_chain_function (ttmlparse->sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_parse_chain)); + gst_pad_set_event_function (ttmlparse->sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_parse_sink_event)); + gst_element_add_pad (GST_ELEMENT (ttmlparse), ttmlparse->sinkpad); + + ttmlparse->srcpad = gst_pad_new_from_static_template (&src_templ, "src"); + gst_pad_set_event_function (ttmlparse->srcpad, + GST_DEBUG_FUNCPTR (gst_ttml_parse_src_event)); + gst_pad_set_query_function (ttmlparse->srcpad, + GST_DEBUG_FUNCPTR (gst_ttml_parse_src_query)); + gst_element_add_pad (GST_ELEMENT (ttmlparse), ttmlparse->srcpad); + + ttmlparse->textbuf = g_string_new (NULL); + gst_segment_init (&ttmlparse->segment, GST_FORMAT_TIME); + ttmlparse->need_segment = TRUE; + ttmlparse->encoding = g_strdup (DEFAULT_ENCODING); + ttmlparse->detected_encoding = NULL; + ttmlparse->adapter = gst_adapter_new (); +} + +/* + * Source pad functions. + */ +static gboolean +gst_ttml_parse_src_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + GstTtmlParse *self = GST_TTML_PARSE (parent); + gboolean ret = FALSE; + + GST_DEBUG ("Handling %s query", GST_QUERY_TYPE_NAME (query)); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_POSITION:{ + GstFormat fmt; + + gst_query_parse_position (query, &fmt, NULL); + if (fmt != GST_FORMAT_TIME) { + ret = gst_pad_peer_query (self->sinkpad, query); + } else { + ret = TRUE; + gst_query_set_position (query, GST_FORMAT_TIME, self->segment.position); + } + break; + } + case GST_QUERY_SEEKING: + { + GstFormat fmt; + gboolean seekable = FALSE; + + ret = TRUE; + + gst_query_parse_seeking (query, &fmt, NULL, NULL, NULL); + if (fmt == GST_FORMAT_TIME) { + GstQuery *peerquery = gst_query_new_seeking (GST_FORMAT_BYTES); + + seekable = gst_pad_peer_query (self->sinkpad, peerquery); + if (seekable) + gst_query_parse_seeking (peerquery, NULL, &seekable, NULL, NULL); + gst_query_unref (peerquery); + } + + gst_query_set_seeking (query, fmt, seekable, seekable ? 0 : -1, -1); + break; + } + default: + ret = gst_pad_query_default (pad, parent, query); + break; + } + + return ret; +} + +static gboolean +gst_ttml_parse_src_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + GstTtmlParse *self = GST_TTML_PARSE (parent); + gboolean ret = FALSE; + + GST_DEBUG ("Handling %s event", GST_EVENT_TYPE_NAME (event)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_SEEK: + { + GstFormat format; + GstSeekFlags flags; + GstSeekType start_type, stop_type; + gint64 start, stop; + gdouble rate; + gboolean update; + + gst_event_parse_seek (event, &rate, &format, &flags, + &start_type, &start, &stop_type, &stop); + + if (format != GST_FORMAT_TIME) { + GST_WARNING_OBJECT (self, "we only support seeking in TIME format"); + gst_event_unref (event); + goto beach; + } + + /* Convert that seek to a seeking in bytes at position 0, + FIXME: could use an index */ + ret = gst_pad_push_event (self->sinkpad, + gst_event_new_seek (rate, GST_FORMAT_BYTES, flags, + GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, 0)); + + if (ret) { + /* Apply the seek to our segment */ + gst_segment_do_seek (&self->segment, rate, format, flags, + start_type, start, stop_type, stop, &update); + + GST_DEBUG_OBJECT (self, "segment after seek: %" GST_SEGMENT_FORMAT, + &self->segment); + + self->need_segment = TRUE; + } else { + GST_WARNING_OBJECT (self, "seek to 0 bytes failed"); + } + + gst_event_unref (event); + break; + } + default: + ret = gst_pad_event_default (pad, parent, event); + break; + } + +beach: + return ret; +} + +static gchar * +gst_convert_to_utf8 (const gchar * str, gsize len, const gchar * encoding, + gsize * consumed, GError ** err) +{ + gchar *ret = NULL; + + *consumed = 0; + /* The char cast is necessary in glib < 2.24 */ + ret = + g_convert_with_fallback (str, len, "UTF-8", encoding, (char *) "*", + consumed, NULL, err); + if (ret == NULL) + return ret; + + /* + 3 to skip UTF-8 BOM if it was added */ + len = strlen (ret); + if (len >= 3 && (guint8) ret[0] == 0xEF && (guint8) ret[1] == 0xBB + && (guint8) ret[2] == 0xBF) + memmove (ret, ret + 3, len + 1 - 3); + + return ret; +} + +static gchar * +detect_encoding (const gchar * str, gsize len) +{ + if (len >= 3 && (guint8) str[0] == 0xEF && (guint8) str[1] == 0xBB + && (guint8) str[2] == 0xBF) + return g_strdup ("UTF-8"); + + if (len >= 2 && (guint8) str[0] == 0xFE && (guint8) str[1] == 0xFF) + return g_strdup ("UTF-16BE"); + + if (len >= 2 && (guint8) str[0] == 0xFF && (guint8) str[1] == 0xFE) + return g_strdup ("UTF-16LE"); + + if (len >= 4 && (guint8) str[0] == 0x00 && (guint8) str[1] == 0x00 + && (guint8) str[2] == 0xFE && (guint8) str[3] == 0xFF) + return g_strdup ("UTF-32BE"); + + if (len >= 4 && (guint8) str[0] == 0xFF && (guint8) str[1] == 0xFE + && (guint8) str[2] == 0x00 && (guint8) str[3] == 0x00) + return g_strdup ("UTF-32LE"); + + return NULL; +} + +static gchar * +convert_encoding (GstTtmlParse * self, const gchar * str, gsize len, + gsize * consumed) +{ + const gchar *encoding; + GError *err = NULL; + gchar *ret = NULL; + + *consumed = 0; + + /* First try any detected encoding */ + if (self->detected_encoding) { + ret = + gst_convert_to_utf8 (str, len, self->detected_encoding, consumed, &err); + + if (!err) + return ret; + + GST_WARNING_OBJECT (self, "could not convert string from '%s' to UTF-8: %s", + self->detected_encoding, err->message); + g_free (self->detected_encoding); + self->detected_encoding = NULL; + g_error_free (err); + } + + /* Otherwise check if it's UTF8 */ + if (self->valid_utf8) { + if (g_utf8_validate (str, len, NULL)) { + GST_LOG_OBJECT (self, "valid UTF-8, no conversion needed"); + *consumed = len; + return g_strndup (str, len); + } + GST_INFO_OBJECT (self, "invalid UTF-8!"); + self->valid_utf8 = FALSE; + } + + /* Else try fallback */ + encoding = self->encoding; + if (encoding == NULL || *encoding == '\0') { + /* if local encoding is UTF-8 and no encoding specified + * via the environment variable, assume ISO-8859-15 */ + if (g_get_charset (&encoding)) { + encoding = "ISO-8859-15"; + } + } + + ret = gst_convert_to_utf8 (str, len, encoding, consumed, &err); + + if (err) { + GST_WARNING_OBJECT (self, "could not convert string from '%s' to UTF-8: %s", + encoding, err->message); + g_error_free (err); + + /* invalid input encoding, fall back to ISO-8859-15 (always succeeds) */ + ret = gst_convert_to_utf8 (str, len, "ISO-8859-15", consumed, NULL); + } + + GST_LOG_OBJECT (self, + "successfully converted %" G_GSIZE_FORMAT " characters from %s to UTF-8" + "%s", len, encoding, (err) ? " , using ISO-8859-15 as fallback" : ""); + + return ret; +} + +static GstCaps * +gst_ttml_parse_get_src_caps (GstTtmlParse * self) +{ + GstCaps *caps; + GstCapsFeatures *features = gst_caps_features_new ("meta:GstSubtitleMeta", + NULL); + + caps = gst_caps_new_empty_simple ("text/x-raw"); + gst_caps_set_features (caps, 0, features); + return caps; +} + +static void +feed_textbuf (GstTtmlParse * self, GstBuffer * buf) +{ + gboolean discont; + gsize consumed; + gchar *input = NULL; + const guint8 *data; + gsize avail; + + discont = GST_BUFFER_IS_DISCONT (buf); + + if (GST_BUFFER_OFFSET_IS_VALID (buf) && + GST_BUFFER_OFFSET (buf) != self->offset) { + self->offset = GST_BUFFER_OFFSET (buf); + discont = TRUE; + } + + if (discont) { + GST_INFO ("discontinuity"); + /* flush the parser state */ + g_string_truncate (self->textbuf, 0); + gst_adapter_clear (self->adapter); + /* we could set a flag to make sure that the next buffer we push out also + * has the DISCONT flag set, but there's no point really given that it's + * subtitles which are discontinuous by nature. */ + } + + self->offset += gst_buffer_get_size (buf); + + gst_adapter_push (self->adapter, buf); + + avail = gst_adapter_available (self->adapter); + data = gst_adapter_map (self->adapter, avail); + input = convert_encoding (self, (const gchar *) data, avail, &consumed); + + if (input && consumed > 0) { + if (self->textbuf) { + g_string_free (self->textbuf, TRUE); + self->textbuf = NULL; + } + self->textbuf = g_string_new (input); + gst_adapter_unmap (self->adapter); + gst_adapter_flush (self->adapter, consumed); + } else { + gst_adapter_unmap (self->adapter); + } + + g_free (input); +} + +static GstFlowReturn +handle_buffer (GstTtmlParse * self, GstBuffer * buf) +{ + GstFlowReturn ret = GST_FLOW_OK; + GstCaps *caps = NULL; + GList *subtitle_list, *subtitle; + GstClockTime begin = GST_BUFFER_PTS (buf); + GstClockTime duration = GST_BUFFER_DURATION (buf); + + if (self->first_buffer) { + GstMapInfo map; + + gst_buffer_map (buf, &map, GST_MAP_READ); + self->detected_encoding = detect_encoding ((gchar *) map.data, map.size); + gst_buffer_unmap (buf, &map); + self->first_buffer = FALSE; + } + + feed_textbuf (self, buf); + + if (!(caps = gst_ttml_parse_get_src_caps (self))) + return GST_FLOW_EOS; + + /* Push newsegment if needed */ + if (self->need_segment) { + GST_LOG_OBJECT (self, "pushing newsegment event with %" GST_SEGMENT_FORMAT, + &self->segment); + + gst_pad_push_event (self->srcpad, gst_event_new_segment (&self->segment)); + self->need_segment = FALSE; + } + + subtitle_list = ttml_parse (self->textbuf->str, begin, duration); + + for (subtitle = subtitle_list; subtitle; subtitle = subtitle->next) { + GstBuffer *op_buffer = subtitle->data; + self->segment.position = GST_BUFFER_PTS (op_buffer); + + ret = gst_pad_push (self->srcpad, op_buffer); + + if (ret != GST_FLOW_OK) + GST_DEBUG_OBJECT (self, "flow: %s", gst_flow_get_name (ret)); + } + + g_list_free (subtitle_list); + return ret; +} + +static GstFlowReturn +gst_ttml_parse_chain (GstPad * sinkpad, GstObject * parent, GstBuffer * buf) +{ + GstTtmlParse *self = GST_TTML_PARSE (parent); + return handle_buffer (self, buf); +} + +static gboolean +gst_ttml_parse_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + GstTtmlParse *self = GST_TTML_PARSE (parent); + gboolean ret = FALSE; + + GST_DEBUG ("Handling %s event", GST_EVENT_TYPE_NAME (event)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_SEGMENT: + { + const GstSegment *s; + gst_event_parse_segment (event, &s); + if (s->format == GST_FORMAT_TIME) + gst_event_copy_segment (event, &self->segment); + GST_DEBUG_OBJECT (self, "newsegment (%s)", + gst_format_get_name (self->segment.format)); + + /* if not time format, we'll either start with a 0 timestamp anyway or + * it's following a seek in which case we'll have saved the requested + * seek segment and don't want to overwrite it (remember that on a seek + * we always just seek back to the start in BYTES format and just throw + * away all text that's before the requested position; if the subtitles + * come from an upstream demuxer, it won't be able to handle our BYTES + * seek request and instead send us a newsegment from the seek request + * it received via its video pads instead, so all is fine then too) */ + ret = TRUE; + gst_event_unref (event); + break; + } + default: + ret = gst_pad_event_default (pad, parent, event); + break; + } + + return ret; +} + +static GstStateChangeReturn +gst_ttml_parse_change_state (GstElement * element, GstStateChange transition) +{ + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + GstTtmlParse *self = GST_TTML_PARSE (element); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + /* format detection will init the parser state */ + self->offset = 0; + self->valid_utf8 = TRUE; + self->first_buffer = TRUE; + g_free (self->detected_encoding); + self->detected_encoding = NULL; + g_string_truncate (self->textbuf, 0); + gst_adapter_clear (self->adapter); + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + break; + default: + break; + } + + return ret; +} diff --git a/ext/ttml/gstttmlparse.h b/ext/ttml/gstttmlparse.h new file mode 100644 index 0000000..f81fd5f --- /dev/null +++ b/ext/ttml/gstttmlparse.h @@ -0,0 +1,76 @@ +/* GStreamer + * Copyright (C) <2002> David A. Schleef + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) <2015> British Broadcasting Corporation + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_TTML_PARSE_H__ +#define __GST_TTML_PARSE_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_TTML_PARSE \ + (gst_ttml_parse_get_type ()) +#define GST_TTML_PARSE(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_TTML_PARSE, GstTtmlParse)) +#define GST_TTML_PARSE_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_TTML_PARSE, GstTtmlParseClass)) +#define GST_IS_TTML_PARSE(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_TTML_PARSE)) +#define GST_IS_TTML_PARSE_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_TTML_PARSE)) + +typedef struct _GstTtmlParse GstTtmlParse; +typedef struct _GstTtmlParseClass GstTtmlParseClass; + +struct _GstTtmlParse { + GstElement element; + + GstPad *sinkpad, *srcpad; + + /* contains the input in the input encoding */ + GstAdapter *adapter; + /* contains the UTF-8 decoded input */ + GString *textbuf; + + /* seek */ + guint64 offset; + + /* Segment */ + GstSegment segment; + gboolean need_segment; + + gboolean valid_utf8; + gchar *detected_encoding; + gchar *encoding; + + gboolean first_buffer; +}; + +struct _GstTtmlParseClass { + GstElementClass parent_class; +}; + +GType gst_ttml_parse_get_type (void); + +G_END_DECLS + +#endif /* __GST_TTML_PARSE_H__ */ diff --git a/ext/ttml/gstttmlplugin.c b/ext/ttml/gstttmlplugin.c new file mode 100644 index 0000000..cc64bcc --- /dev/null +++ b/ext/ttml/gstttmlplugin.c @@ -0,0 +1,53 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) 2004 Ronald S. Bultje + * Copyright (C) 2006 Tim-Philipp MÃ¼ller + * Copyright (C) <2015> British Broadcasting Corporation + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "gstttmlparse.h" +#include "gstttmlrender.h" + +GST_DEBUG_CATEGORY (ttmlparse_debug); +GST_DEBUG_CATEGORY (ttmlrender_debug); + +static gboolean +plugin_init (GstPlugin * plugin) +{ + if (!gst_element_register (plugin, "ttmlparse", GST_RANK_PRIMARY, + GST_TYPE_TTML_PARSE)) + return FALSE; + if (!gst_element_register (plugin, "ttmlrender", GST_RANK_PRIMARY, + GST_TYPE_TTML_RENDER)) + return FALSE; + + GST_DEBUG_CATEGORY_INIT (ttmlparse_debug, "ttmlparse", 0, "TTML parser"); + GST_DEBUG_CATEGORY_INIT (ttmlrender_debug, "ttmlrender", 0, "TTML renderer"); + + return TRUE; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + ttmlsubs, + "TTML subtitle handling", + plugin_init, VERSION, "LGPL", "gst-ttml", "http://www.bbc.co.uk/rd") diff --git a/ext/ttml/gstttmlrender.c b/ext/ttml/gstttmlrender.c new file mode 100644 index 0000000..91acfb4 --- /dev/null +++ b/ext/ttml/gstttmlrender.c @@ -0,0 +1,2456 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) <2003> David Schleef + * Copyright (C) <2006> Julien Moutte + * Copyright (C) <2006> Zeeshan Ali + * Copyright (C) <2006-2008> Tim-Philipp MÃ¼ller + * Copyright (C) <2009> Young-Ho Cha + * Copyright (C) <2015> British Broadcasting Corporation + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-ttmlrender + * + * Renders timed text on top of a video stream. It receives text in buffers + * from a ttmlparse element; each text string is in its own #GstMemory within + * the GstBuffer, and the styling and layout associated with each text string + * is in metadata attached to the #GstBuffer. + * + * + * Example launch lines + * |[ + * gst-launch-1.0 filesrc location= ! video/quicktime ! qtdemux name=q ttmlrender name=r q. ! queue ! h264parse ! avdec_h264 ! autovideoconvert ! r.video_sink filesrc location= blocksize=16777216 ! queue ! ttmlparse ! r.text_sink r. ! ximagesink q. ! queue ! aacparse ! avdec_aac ! audioconvert ! alsasink + * ]| Parse and render TTML subtitles contained in a single XML file over an + * MP4 stream containing H.264 video and AAC audio: + * + */ + +#include +#include +#include +#include + +#include +#include + +#include "gstttmlrender.h" +#include "subtitle.h" +#include "subtitlemeta.h" + +#define VIDEO_FORMATS GST_VIDEO_OVERLAY_COMPOSITION_BLEND_FORMATS + +#define TTML_RENDER_CAPS GST_VIDEO_CAPS_MAKE (VIDEO_FORMATS) + +#define TTML_RENDER_ALL_CAPS TTML_RENDER_CAPS ";" \ + GST_VIDEO_CAPS_MAKE_WITH_FEATURES ("ANY", GST_VIDEO_FORMATS_ALL) + +GST_DEBUG_CATEGORY_EXTERN (ttmlrender_debug); +#define GST_CAT_DEFAULT ttmlrender_debug + +static GstStaticCaps sw_template_caps = GST_STATIC_CAPS (TTML_RENDER_CAPS); + +static GstStaticPadTemplate src_template_factory = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (TTML_RENDER_ALL_CAPS) + ); + +static GstStaticPadTemplate video_sink_template_factory = +GST_STATIC_PAD_TEMPLATE ("video_sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (TTML_RENDER_ALL_CAPS) + ); + +static GstStaticPadTemplate text_sink_template_factory = +GST_STATIC_PAD_TEMPLATE ("text_sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("text/x-raw(meta:GstSubtitleMeta)") + ); + +#define GST_TTML_RENDER_GET_LOCK(ov) (&GST_TTML_RENDER (ov)->lock) +#define GST_TTML_RENDER_GET_COND(ov) (&GST_TTML_RENDER (ov)->cond) +#define GST_TTML_RENDER_LOCK(ov) (g_mutex_lock (GST_TTML_RENDER_GET_LOCK (ov))) +#define GST_TTML_RENDER_UNLOCK(ov) (g_mutex_unlock (GST_TTML_RENDER_GET_LOCK (ov))) +#define GST_TTML_RENDER_WAIT(ov) (g_cond_wait (GST_TTML_RENDER_GET_COND (ov), GST_TTML_RENDER_GET_LOCK (ov))) +#define GST_TTML_RENDER_SIGNAL(ov) (g_cond_signal (GST_TTML_RENDER_GET_COND (ov))) +#define GST_TTML_RENDER_BROADCAST(ov)(g_cond_broadcast (GST_TTML_RENDER_GET_COND (ov))) + +static GstElementClass *parent_class = NULL; +static void gst_ttml_render_base_init (gpointer g_class); +static void gst_ttml_render_class_init (GstTtmlRenderClass * klass); +static void gst_ttml_render_init (GstTtmlRender * render, + GstTtmlRenderClass * klass); + +static GstStateChangeReturn gst_ttml_render_change_state (GstElement * + element, GstStateChange transition); + +static GstCaps *gst_ttml_render_get_videosink_caps (GstPad * pad, + GstTtmlRender * render, GstCaps * filter); +static GstCaps *gst_ttml_render_get_src_caps (GstPad * pad, + GstTtmlRender * render, GstCaps * filter); +static gboolean gst_ttml_render_setcaps (GstTtmlRender * render, + GstCaps * caps); +static gboolean gst_ttml_render_src_event (GstPad * pad, + GstObject * parent, GstEvent * event); +static gboolean gst_ttml_render_src_query (GstPad * pad, + GstObject * parent, GstQuery * query); + +static gboolean gst_ttml_render_video_event (GstPad * pad, + GstObject * parent, GstEvent * event); +static gboolean gst_ttml_render_video_query (GstPad * pad, + GstObject * parent, GstQuery * query); +static GstFlowReturn gst_ttml_render_video_chain (GstPad * pad, + GstObject * parent, GstBuffer * buffer); + +static gboolean gst_ttml_render_text_event (GstPad * pad, + GstObject * parent, GstEvent * event); +static GstFlowReturn gst_ttml_render_text_chain (GstPad * pad, + GstObject * parent, GstBuffer * buffer); +static GstPadLinkReturn gst_ttml_render_text_pad_link (GstPad * pad, + GstObject * parent, GstPad * peer); +static void gst_ttml_render_text_pad_unlink (GstPad * pad, GstObject * parent); +static void gst_ttml_render_pop_text (GstTtmlRender * render); + +static void gst_ttml_render_finalize (GObject * object); + +static gboolean gst_ttml_render_can_handle_caps (GstCaps * incaps); + +static GstTtmlRenderRenderedImage *gst_ttml_render_rendered_image_new + (GstBuffer * image, gint x, gint y, guint width, guint height); +static GstTtmlRenderRenderedImage *gst_ttml_render_rendered_image_new_empty + (void); +static GstTtmlRenderRenderedImage *gst_ttml_render_rendered_image_copy + (GstTtmlRenderRenderedImage * image); +static void gst_ttml_render_rendered_image_free + (GstTtmlRenderRenderedImage * image); + +GType +gst_ttml_render_get_type (void) +{ + static GType type = 0; + + if (g_once_init_enter ((gsize *) & type)) { + static const GTypeInfo info = { + sizeof (GstTtmlRenderClass), + (GBaseInitFunc) gst_ttml_render_base_init, + NULL, + (GClassInitFunc) gst_ttml_render_class_init, + NULL, + NULL, + sizeof (GstTtmlRender), + 0, + (GInstanceInitFunc) gst_ttml_render_init, + }; + + g_once_init_leave ((gsize *) & type, + g_type_register_static (GST_TYPE_ELEMENT, "GstTtmlRender", &info, 0)); + } + + return type; +} + +static void +gst_ttml_render_base_init (gpointer g_class) +{ + GstTtmlRenderClass *klass = GST_TTML_RENDER_CLASS (g_class); + PangoFontMap *fontmap; + + /* Only lock for the subclasses here, the base class + * doesn't have this mutex yet and it's not necessary + * here */ + if (klass->pango_lock) + g_mutex_lock (klass->pango_lock); + fontmap = pango_cairo_font_map_get_default (); + klass->pango_context = + pango_font_map_create_context (PANGO_FONT_MAP (fontmap)); + if (klass->pango_lock) + g_mutex_unlock (klass->pango_lock); +} + +static void +gst_ttml_render_class_init (GstTtmlRenderClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + + parent_class = g_type_class_peek_parent (klass); + + gobject_class->finalize = gst_ttml_render_finalize; + + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&src_template_factory)); + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&video_sink_template_factory)); + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&text_sink_template_factory)); + + gst_element_class_set_static_metadata (gstelement_class, + "TTML subtitle renderer", "Overlay/Subtitle", + "Renders timed-text subtitles on top of video buffers", + "David Schleef , Zeeshan Ali , " + "Chris Bass "); + + gstelement_class->change_state = + GST_DEBUG_FUNCPTR (gst_ttml_render_change_state); + + klass->pango_lock = g_slice_new (GMutex); + g_mutex_init (klass->pango_lock); +} + +static void +gst_ttml_render_finalize (GObject * object) +{ + GstTtmlRender *render = GST_TTML_RENDER (object); + + if (render->compositions) { + g_list_free_full (render->compositions, + (GDestroyNotify) gst_video_overlay_composition_unref); + render->compositions = NULL; + } + + if (render->text_buffer) { + gst_buffer_unref (render->text_buffer); + render->text_buffer = NULL; + } + + g_mutex_clear (&render->lock); + g_cond_clear (&render->cond); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_ttml_render_init (GstTtmlRender * render, GstTtmlRenderClass * klass) +{ + GstPadTemplate *template; + + /* video sink */ + template = gst_static_pad_template_get (&video_sink_template_factory); + render->video_sinkpad = gst_pad_new_from_template (template, "video_sink"); + gst_object_unref (template); + gst_pad_set_event_function (render->video_sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_video_event)); + gst_pad_set_chain_function (render->video_sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_video_chain)); + gst_pad_set_query_function (render->video_sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_video_query)); + GST_PAD_SET_PROXY_ALLOCATION (render->video_sinkpad); + gst_element_add_pad (GST_ELEMENT (render), render->video_sinkpad); + + template = + gst_element_class_get_pad_template (GST_ELEMENT_CLASS (klass), + "text_sink"); + if (template) { + /* text sink */ + render->text_sinkpad = gst_pad_new_from_template (template, "text_sink"); + + gst_pad_set_event_function (render->text_sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_text_event)); + gst_pad_set_chain_function (render->text_sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_text_chain)); + gst_pad_set_link_function (render->text_sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_text_pad_link)); + gst_pad_set_unlink_function (render->text_sinkpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_text_pad_unlink)); + gst_element_add_pad (GST_ELEMENT (render), render->text_sinkpad); + } + + /* (video) source */ + template = gst_static_pad_template_get (&src_template_factory); + render->srcpad = gst_pad_new_from_template (template, "src"); + gst_object_unref (template); + gst_pad_set_event_function (render->srcpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_src_event)); + gst_pad_set_query_function (render->srcpad, + GST_DEBUG_FUNCPTR (gst_ttml_render_src_query)); + gst_element_add_pad (GST_ELEMENT (render), render->srcpad); + + g_mutex_lock (GST_TTML_RENDER_GET_CLASS (render)->pango_lock); + + render->wait_text = TRUE; + render->need_render = TRUE; + render->text_buffer = NULL; + render->text_linked = FALSE; + + render->compositions = NULL; + + g_mutex_init (&render->lock); + g_cond_init (&render->cond); + gst_segment_init (&render->segment, GST_FORMAT_TIME); + g_mutex_unlock (GST_TTML_RENDER_GET_CLASS (render)->pango_lock); +} + + +/* only negotiate/query video render composition support for now */ +static gboolean +gst_ttml_render_negotiate (GstTtmlRender * render, GstCaps * caps) +{ + GstQuery *query; + gboolean attach = FALSE; + gboolean caps_has_meta = TRUE; + gboolean ret; + GstCapsFeatures *f; + GstCaps *original_caps; + gboolean original_has_meta = FALSE; + gboolean allocation_ret = TRUE; + + GST_DEBUG_OBJECT (render, "performing negotiation"); + + if (!caps) + caps = gst_pad_get_current_caps (render->video_sinkpad); + else + gst_caps_ref (caps); + + if (!caps || gst_caps_is_empty (caps)) + goto no_format; + + original_caps = caps; + + /* Try to use the render meta if possible */ + f = gst_caps_get_features (caps, 0); + + /* if the caps doesn't have the render meta, we query if downstream + * accepts it before trying the version without the meta + * If upstream already is using the meta then we can only use it */ + if (!f + || !gst_caps_features_contains (f, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION)) { + GstCaps *overlay_caps; + + /* In this case we added the meta, but we can work without it + * so preserve the original caps so we can use it as a fallback */ + overlay_caps = gst_caps_copy (caps); + + f = gst_caps_get_features (overlay_caps, 0); + gst_caps_features_add (f, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION); + + ret = gst_pad_peer_query_accept_caps (render->srcpad, overlay_caps); + GST_DEBUG_OBJECT (render, "Downstream accepts the render meta: %d", ret); + if (ret) { + gst_caps_unref (caps); + caps = overlay_caps; + + } else { + /* fallback to the original */ + gst_caps_unref (overlay_caps); + caps_has_meta = FALSE; + } + } else { + original_has_meta = TRUE; + } + GST_DEBUG_OBJECT (render, "Using caps %" GST_PTR_FORMAT, caps); + ret = gst_pad_set_caps (render->srcpad, caps); + + if (ret) { + /* find supported meta */ + query = gst_query_new_allocation (caps, FALSE); + + if (!gst_pad_peer_query (render->srcpad, query)) { + /* no problem, we use the query defaults */ + GST_DEBUG_OBJECT (render, "ALLOCATION query failed"); + allocation_ret = FALSE; + } + + if (caps_has_meta && gst_query_find_allocation_meta (query, + GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, NULL)) + attach = TRUE; + + gst_query_unref (query); + } + + if (!allocation_ret && render->video_flushing) { + ret = FALSE; + } else if (original_caps && !original_has_meta && !attach) { + if (caps_has_meta) { + /* Some elements (fakesink) claim to accept the meta on caps but won't + put it in the allocation query result, this leads below + check to fail. Prevent this by removing the meta from caps */ + gst_caps_unref (caps); + caps = gst_caps_ref (original_caps); + ret = gst_pad_set_caps (render->srcpad, caps); + if (ret && !gst_ttml_render_can_handle_caps (caps)) + ret = FALSE; + } + } + + if (!ret) { + GST_DEBUG_OBJECT (render, "negotiation failed, schedule reconfigure"); + gst_pad_mark_reconfigure (render->srcpad); + } + + gst_caps_unref (caps); + + return ret; + +no_format: + { + if (caps) + gst_caps_unref (caps); + return FALSE; + } +} + +static gboolean +gst_ttml_render_can_handle_caps (GstCaps * incaps) +{ + gboolean ret; + GstCaps *caps; + static GstStaticCaps static_caps = GST_STATIC_CAPS (TTML_RENDER_CAPS); + + caps = gst_static_caps_get (&static_caps); + ret = gst_caps_is_subset (incaps, caps); + gst_caps_unref (caps); + + return ret; +} + +static gboolean +gst_ttml_render_setcaps (GstTtmlRender * render, GstCaps * caps) +{ + GstVideoInfo info; + gboolean ret = FALSE; + + if (!gst_video_info_from_caps (&info, caps)) + goto invalid_caps; + + render->info = info; + render->format = GST_VIDEO_INFO_FORMAT (&info); + render->width = GST_VIDEO_INFO_WIDTH (&info); + render->height = GST_VIDEO_INFO_HEIGHT (&info); + + ret = gst_ttml_render_negotiate (render, caps); + + GST_TTML_RENDER_LOCK (render); + g_mutex_lock (GST_TTML_RENDER_GET_CLASS (render)->pango_lock); + if (!gst_ttml_render_can_handle_caps (caps)) { + GST_DEBUG_OBJECT (render, "unsupported caps %" GST_PTR_FORMAT, caps); + ret = FALSE; + } + + g_mutex_unlock (GST_TTML_RENDER_GET_CLASS (render)->pango_lock); + GST_TTML_RENDER_UNLOCK (render); + + return ret; + + /* ERRORS */ +invalid_caps: + { + GST_DEBUG_OBJECT (render, "could not parse caps"); + return FALSE; + } +} + + +static gboolean +gst_ttml_render_src_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + gboolean ret = FALSE; + GstTtmlRender *render; + + render = GST_TTML_RENDER (parent); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_ttml_render_get_src_caps (pad, render, filter); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + ret = TRUE; + break; + } + default: + ret = gst_pad_query_default (pad, parent, query); + break; + } + + return ret; +} + +static gboolean +gst_ttml_render_src_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + GstTtmlRender *render; + gboolean ret; + + render = GST_TTML_RENDER (parent); + + if (render->text_linked) { + gst_event_ref (event); + ret = gst_pad_push_event (render->video_sinkpad, event); + gst_pad_push_event (render->text_sinkpad, event); + } else { + ret = gst_pad_push_event (render->video_sinkpad, event); + } + + return ret; +} + +/** + * gst_ttml_render_add_feature_and_intersect: + * + * Creates a new #GstCaps containing the (given caps + + * given caps feature) + (given caps intersected by the + * given filter). + * + * Returns: the new #GstCaps + */ +static GstCaps * +gst_ttml_render_add_feature_and_intersect (GstCaps * caps, + const gchar * feature, GstCaps * filter) +{ + int i, caps_size; + GstCaps *new_caps; + + new_caps = gst_caps_copy (caps); + + caps_size = gst_caps_get_size (new_caps); + for (i = 0; i < caps_size; i++) { + GstCapsFeatures *features = gst_caps_get_features (new_caps, i); + + if (!gst_caps_features_is_any (features)) { + gst_caps_features_add (features, feature); + } + } + + gst_caps_append (new_caps, gst_caps_intersect_full (caps, + filter, GST_CAPS_INTERSECT_FIRST)); + + return new_caps; +} + +/** + * gst_ttml_render_intersect_by_feature: + * + * Creates a new #GstCaps based on the following filtering rule. + * + * For each individual caps contained in given caps, if the + * caps uses the given caps feature, keep a version of the caps + * with the feature and an another one without. Otherwise, intersect + * the caps with the given filter. + * + * Returns: the new #GstCaps + */ +static GstCaps * +gst_ttml_render_intersect_by_feature (GstCaps * caps, + const gchar * feature, GstCaps * filter) +{ + int i, caps_size; + GstCaps *new_caps; + + new_caps = gst_caps_new_empty (); + + caps_size = gst_caps_get_size (caps); + for (i = 0; i < caps_size; i++) { + GstStructure *caps_structure = gst_caps_get_structure (caps, i); + GstCapsFeatures *caps_features = + gst_caps_features_copy (gst_caps_get_features (caps, i)); + GstCaps *filtered_caps; + GstCaps *simple_caps = + gst_caps_new_full (gst_structure_copy (caps_structure), NULL); + gst_caps_set_features (simple_caps, 0, caps_features); + + if (gst_caps_features_contains (caps_features, feature)) { + gst_caps_append (new_caps, gst_caps_copy (simple_caps)); + + gst_caps_features_remove (caps_features, feature); + filtered_caps = gst_caps_ref (simple_caps); + } else { + filtered_caps = gst_caps_intersect_full (simple_caps, filter, + GST_CAPS_INTERSECT_FIRST); + } + + gst_caps_unref (simple_caps); + gst_caps_append (new_caps, filtered_caps); + } + + return new_caps; +} + +static GstCaps * +gst_ttml_render_get_videosink_caps (GstPad * pad, + GstTtmlRender * render, GstCaps * filter) +{ + GstPad *srcpad = render->srcpad; + GstCaps *peer_caps = NULL, *caps = NULL, *overlay_filter = NULL; + + if (G_UNLIKELY (!render)) + return gst_pad_get_pad_template_caps (pad); + + if (filter) { + /* filter caps + composition feature + filter caps + * filtered by the software caps. */ + GstCaps *sw_caps = gst_static_caps_get (&sw_template_caps); + overlay_filter = gst_ttml_render_add_feature_and_intersect (filter, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, sw_caps); + gst_caps_unref (sw_caps); + + GST_DEBUG_OBJECT (render, "render filter %" GST_PTR_FORMAT, overlay_filter); + } + + peer_caps = gst_pad_peer_query_caps (srcpad, overlay_filter); + + if (overlay_filter) + gst_caps_unref (overlay_filter); + + if (peer_caps) { + + GST_DEBUG_OBJECT (pad, "peer caps %" GST_PTR_FORMAT, peer_caps); + + if (gst_caps_is_any (peer_caps)) { + /* if peer returns ANY caps, return filtered src pad template caps */ + caps = gst_caps_copy (gst_pad_get_pad_template_caps (srcpad)); + } else { + + /* duplicate caps which contains the composition into one version with + * the meta and one without. Filter the other caps by the software caps */ + GstCaps *sw_caps = gst_static_caps_get (&sw_template_caps); + caps = gst_ttml_render_intersect_by_feature (peer_caps, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, sw_caps); + gst_caps_unref (sw_caps); + } + + gst_caps_unref (peer_caps); + + } else { + /* no peer, our padtemplate is enough then */ + caps = gst_pad_get_pad_template_caps (pad); + } + + if (filter) { + GstCaps *intersection = gst_caps_intersect_full (filter, caps, + GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (caps); + caps = intersection; + } + + GST_DEBUG_OBJECT (render, "returning %" GST_PTR_FORMAT, caps); + + return caps; +} + +static GstCaps * +gst_ttml_render_get_src_caps (GstPad * pad, GstTtmlRender * render, + GstCaps * filter) +{ + GstPad *sinkpad = render->video_sinkpad; + GstCaps *peer_caps = NULL, *caps = NULL, *overlay_filter = NULL; + + if (G_UNLIKELY (!render)) + return gst_pad_get_pad_template_caps (pad); + + if (filter) { + /* duplicate filter caps which contains the composition into one version + * with the meta and one without. Filter the other caps by the software + * caps */ + GstCaps *sw_caps = gst_static_caps_get (&sw_template_caps); + overlay_filter = + gst_ttml_render_intersect_by_feature (filter, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, sw_caps); + gst_caps_unref (sw_caps); + } + + peer_caps = gst_pad_peer_query_caps (sinkpad, overlay_filter); + + if (overlay_filter) + gst_caps_unref (overlay_filter); + + if (peer_caps) { + + GST_DEBUG_OBJECT (pad, "peer caps %" GST_PTR_FORMAT, peer_caps); + + if (gst_caps_is_any (peer_caps)) { + + /* if peer returns ANY caps, return filtered sink pad template caps */ + caps = gst_caps_copy (gst_pad_get_pad_template_caps (sinkpad)); + + } else { + + /* return upstream caps + composition feature + upstream caps + * filtered by the software caps. */ + GstCaps *sw_caps = gst_static_caps_get (&sw_template_caps); + caps = gst_ttml_render_add_feature_and_intersect (peer_caps, + GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, sw_caps); + gst_caps_unref (sw_caps); + } + + gst_caps_unref (peer_caps); + + } else { + /* no peer, our padtemplate is enough then */ + caps = gst_pad_get_pad_template_caps (pad); + } + + if (filter) { + GstCaps *intersection; + + intersection = + gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (caps); + caps = intersection; + } + GST_DEBUG_OBJECT (render, "returning %" GST_PTR_FORMAT, caps); + + return caps; +} + + +static GstFlowReturn +gst_ttml_render_push_frame (GstTtmlRender * render, GstBuffer * video_frame) +{ + GstVideoFrame frame; + GList *compositions = render->compositions; + + if (compositions == NULL) { + GST_CAT_DEBUG (ttmlrender_debug, "No compositions."); + goto done; + } + + if (gst_pad_check_reconfigure (render->srcpad)) + gst_ttml_render_negotiate (render, NULL); + + video_frame = gst_buffer_make_writable (video_frame); + + if (!gst_video_frame_map (&frame, &render->info, video_frame, + GST_MAP_READWRITE)) + goto invalid_frame; + + while (compositions) { + GstVideoOverlayComposition *composition = compositions->data; + gst_video_overlay_composition_blend (composition, &frame); + compositions = compositions->next; + } + + gst_video_frame_unmap (&frame); + +done: + + return gst_pad_push (render->srcpad, video_frame); + + /* ERRORS */ +invalid_frame: + { + gst_buffer_unref (video_frame); + GST_DEBUG_OBJECT (render, "received invalid buffer"); + return GST_FLOW_OK; + } +} + +static GstPadLinkReturn +gst_ttml_render_text_pad_link (GstPad * pad, GstObject * parent, GstPad * peer) +{ + GstTtmlRender *render; + + render = GST_TTML_RENDER (parent); + if (G_UNLIKELY (!render)) + return GST_PAD_LINK_REFUSED; + + GST_DEBUG_OBJECT (render, "Text pad linked"); + + render->text_linked = TRUE; + + return GST_PAD_LINK_OK; +} + +static void +gst_ttml_render_text_pad_unlink (GstPad * pad, GstObject * parent) +{ + GstTtmlRender *render; + + /* don't use gst_pad_get_parent() here, will deadlock */ + render = GST_TTML_RENDER (parent); + + GST_DEBUG_OBJECT (render, "Text pad unlinked"); + + render->text_linked = FALSE; + + gst_segment_init (&render->text_segment, GST_FORMAT_UNDEFINED); +} + +static gboolean +gst_ttml_render_text_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + gboolean ret = FALSE; + GstTtmlRender *render = NULL; + + render = GST_TTML_RENDER (parent); + + GST_LOG_OBJECT (pad, "received event %s", GST_EVENT_TYPE_NAME (event)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_SEGMENT: + { + const GstSegment *segment; + + render->text_eos = FALSE; + + gst_event_parse_segment (event, &segment); + + if (segment->format == GST_FORMAT_TIME) { + GST_TTML_RENDER_LOCK (render); + gst_segment_copy_into (segment, &render->text_segment); + GST_DEBUG_OBJECT (render, "TEXT SEGMENT now: %" GST_SEGMENT_FORMAT, + &render->text_segment); + GST_TTML_RENDER_UNLOCK (render); + } else { + GST_ELEMENT_WARNING (render, STREAM, MUX, (NULL), + ("received non-TIME newsegment event on text input")); + } + + gst_event_unref (event); + ret = TRUE; + + /* wake up the video chain, it might be waiting for a text buffer or + * a text segment update */ + GST_TTML_RENDER_LOCK (render); + GST_TTML_RENDER_BROADCAST (render); + GST_TTML_RENDER_UNLOCK (render); + break; + } + case GST_EVENT_GAP: + { + GstClockTime start, duration; + + gst_event_parse_gap (event, &start, &duration); + if (GST_CLOCK_TIME_IS_VALID (duration)) + start += duration; + /* we do not expect another buffer until after gap, + * so that is our position now */ + render->text_segment.position = start; + + /* wake up the video chain, it might be waiting for a text buffer or + * a text segment update */ + GST_TTML_RENDER_LOCK (render); + GST_TTML_RENDER_BROADCAST (render); + GST_TTML_RENDER_UNLOCK (render); + + gst_event_unref (event); + ret = TRUE; + break; + } + case GST_EVENT_FLUSH_STOP: + GST_TTML_RENDER_LOCK (render); + GST_INFO_OBJECT (render, "text flush stop"); + render->text_flushing = FALSE; + render->text_eos = FALSE; + gst_ttml_render_pop_text (render); + gst_segment_init (&render->text_segment, GST_FORMAT_TIME); + GST_TTML_RENDER_UNLOCK (render); + gst_event_unref (event); + ret = TRUE; + break; + case GST_EVENT_FLUSH_START: + GST_TTML_RENDER_LOCK (render); + GST_INFO_OBJECT (render, "text flush start"); + render->text_flushing = TRUE; + GST_TTML_RENDER_BROADCAST (render); + GST_TTML_RENDER_UNLOCK (render); + gst_event_unref (event); + ret = TRUE; + break; + case GST_EVENT_EOS: + GST_TTML_RENDER_LOCK (render); + render->text_eos = TRUE; + GST_INFO_OBJECT (render, "text EOS"); + /* wake up the video chain, it might be waiting for a text buffer or + * a text segment update */ + GST_TTML_RENDER_BROADCAST (render); + GST_TTML_RENDER_UNLOCK (render); + gst_event_unref (event); + ret = TRUE; + break; + default: + ret = gst_pad_event_default (pad, parent, event); + break; + } + + return ret; +} + +static gboolean +gst_ttml_render_video_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + gboolean ret = FALSE; + GstTtmlRender *render = NULL; + + render = GST_TTML_RENDER (parent); + + GST_DEBUG_OBJECT (pad, "received event %s", GST_EVENT_TYPE_NAME (event)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_CAPS: + { + GstCaps *caps; + gint prev_width = render->width; + gint prev_height = render->height; + + gst_event_parse_caps (event, &caps); + ret = gst_ttml_render_setcaps (render, caps); + if (render->width != prev_width || render->height != prev_height) + render->need_render = TRUE; + gst_event_unref (event); + break; + } + case GST_EVENT_SEGMENT: + { + const GstSegment *segment; + + GST_DEBUG_OBJECT (render, "received new segment"); + + gst_event_parse_segment (event, &segment); + + if (segment->format == GST_FORMAT_TIME) { + GST_DEBUG_OBJECT (render, "VIDEO SEGMENT now: %" GST_SEGMENT_FORMAT, + &render->segment); + + gst_segment_copy_into (segment, &render->segment); + } else { + GST_ELEMENT_WARNING (render, STREAM, MUX, (NULL), + ("received non-TIME newsegment event on video input")); + } + + ret = gst_pad_event_default (pad, parent, event); + break; + } + case GST_EVENT_EOS: + GST_TTML_RENDER_LOCK (render); + GST_INFO_OBJECT (render, "video EOS"); + render->video_eos = TRUE; + GST_TTML_RENDER_UNLOCK (render); + ret = gst_pad_event_default (pad, parent, event); + break; + case GST_EVENT_FLUSH_START: + GST_TTML_RENDER_LOCK (render); + GST_INFO_OBJECT (render, "video flush start"); + render->video_flushing = TRUE; + GST_TTML_RENDER_BROADCAST (render); + GST_TTML_RENDER_UNLOCK (render); + ret = gst_pad_event_default (pad, parent, event); + break; + case GST_EVENT_FLUSH_STOP: + GST_TTML_RENDER_LOCK (render); + GST_INFO_OBJECT (render, "video flush stop"); + render->video_flushing = FALSE; + render->video_eos = FALSE; + gst_segment_init (&render->segment, GST_FORMAT_TIME); + GST_TTML_RENDER_UNLOCK (render); + ret = gst_pad_event_default (pad, parent, event); + break; + default: + ret = gst_pad_event_default (pad, parent, event); + break; + } + + return ret; +} + +static gboolean +gst_ttml_render_video_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + gboolean ret = FALSE; + GstTtmlRender *render; + + render = GST_TTML_RENDER (parent); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_ttml_render_get_videosink_caps (pad, render, filter); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + ret = TRUE; + break; + } + default: + ret = gst_pad_query_default (pad, parent, query); + break; + } + + return ret; +} + +/* Called with lock held */ +static void +gst_ttml_render_pop_text (GstTtmlRender * render) +{ + g_return_if_fail (GST_IS_TTML_RENDER (render)); + + if (render->text_buffer) { + GST_DEBUG_OBJECT (render, "releasing text buffer %p", render->text_buffer); + gst_buffer_unref (render->text_buffer); + render->text_buffer = NULL; + } + + /* Let the text task know we used that buffer */ + GST_TTML_RENDER_BROADCAST (render); +} + +/* We receive text buffers here. If they are out of segment we just ignore them. + If the buffer is in our segment we keep it internally except if another one + is already waiting here, in that case we wait that it gets kicked out */ +static GstFlowReturn +gst_ttml_render_text_chain (GstPad * pad, GstObject * parent, + GstBuffer * buffer) +{ + GstFlowReturn ret = GST_FLOW_OK; + GstTtmlRender *render = NULL; + gboolean in_seg = FALSE; + guint64 clip_start = 0, clip_stop = 0; + + render = GST_TTML_RENDER (parent); + + GST_TTML_RENDER_LOCK (render); + + if (render->text_flushing) { + GST_TTML_RENDER_UNLOCK (render); + ret = GST_FLOW_FLUSHING; + GST_LOG_OBJECT (render, "text flushing"); + goto beach; + } + + if (render->text_eos) { + GST_TTML_RENDER_UNLOCK (render); + ret = GST_FLOW_EOS; + GST_LOG_OBJECT (render, "text EOS"); + goto beach; + } + + GST_LOG_OBJECT (render, "%" GST_SEGMENT_FORMAT " BUFFER: ts=%" + GST_TIME_FORMAT ", end=%" GST_TIME_FORMAT, &render->segment, + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer)), + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer) + + GST_BUFFER_DURATION (buffer))); + + if (G_LIKELY (GST_BUFFER_TIMESTAMP_IS_VALID (buffer))) { + GstClockTime stop; + + if (G_LIKELY (GST_BUFFER_DURATION_IS_VALID (buffer))) + stop = GST_BUFFER_TIMESTAMP (buffer) + GST_BUFFER_DURATION (buffer); + else + stop = GST_CLOCK_TIME_NONE; + + in_seg = gst_segment_clip (&render->text_segment, GST_FORMAT_TIME, + GST_BUFFER_TIMESTAMP (buffer), stop, &clip_start, &clip_stop); + } else { + in_seg = TRUE; + } + + if (in_seg) { + if (GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) + GST_BUFFER_TIMESTAMP (buffer) = clip_start; + else if (GST_BUFFER_DURATION_IS_VALID (buffer)) + GST_BUFFER_DURATION (buffer) = clip_stop - clip_start; + + /* Wait for the previous buffer to go away */ + while (render->text_buffer != NULL) { + GST_DEBUG ("Pad %s:%s has a buffer queued, waiting", + GST_DEBUG_PAD_NAME (pad)); + GST_TTML_RENDER_WAIT (render); + GST_DEBUG ("Pad %s:%s resuming", GST_DEBUG_PAD_NAME (pad)); + if (render->text_flushing) { + GST_TTML_RENDER_UNLOCK (render); + ret = GST_FLOW_FLUSHING; + goto beach; + } + } + + if (GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) + render->text_segment.position = clip_start; + + render->text_buffer = buffer; + /* That's a new text buffer we need to render */ + render->need_render = TRUE; + + /* in case the video chain is waiting for a text buffer, wake it up */ + GST_TTML_RENDER_BROADCAST (render); + } + + GST_TTML_RENDER_UNLOCK (render); + +beach: + + return ret; +} + + +/* Free returned string after use. */ +static gchar * +gst_ttml_render_color_to_string (GstSubtitleColor color) +{ +#if PANGO_VERSION_CHECK (1,38,0) + return g_strdup_printf ("#%02x%02x%02x%02x", + color.r, color.g, color.b, color.a); +#else + return g_strdup_printf ("#%02x%02x%02x", color.r, color.g, color.b); +#endif +} + + +static GstBuffer * +gst_ttml_render_draw_rectangle (guint width, guint height, + GstSubtitleColor color) +{ + GstMapInfo map; + cairo_surface_t *surface; + cairo_t *cairo_state; + GstBuffer *buffer = gst_buffer_new_allocate (NULL, 4 * width * height, NULL); + + gst_buffer_map (buffer, &map, GST_MAP_READWRITE); + surface = cairo_image_surface_create_for_data (map.data, + CAIRO_FORMAT_ARGB32, width, height, width * 4); + cairo_state = cairo_create (surface); + + /* clear surface */ + cairo_set_operator (cairo_state, CAIRO_OPERATOR_CLEAR); + cairo_paint (cairo_state); + cairo_set_operator (cairo_state, CAIRO_OPERATOR_OVER); + + cairo_save (cairo_state); + cairo_set_source_rgba (cairo_state, color.r / 255.0, color.g / 255.0, + color.b / 255.0, color.a / 255.0); + cairo_paint (cairo_state); + cairo_restore (cairo_state); + cairo_destroy (cairo_state); + cairo_surface_destroy (surface); + gst_buffer_unmap (buffer, &map); + + return buffer; +} + + +typedef struct +{ + guint first_char; + guint last_char; +} TextRange; + +static void +_text_range_free (TextRange * range) +{ + g_slice_free (TextRange, range); +} + + +/* Choose fonts for generic fontnames based upon IMSC1 and HbbTV specs. */ +static gchar * +gst_ttml_render_resolve_generic_fontname (const gchar * name) +{ + if ((g_strcmp0 (name, "default") == 0)) { + return + g_strdup ("TiresiasScreenfont,Liberation Mono,Courier New,monospace"); + } else if ((g_strcmp0 (name, "monospace") == 0)) { + return g_strdup ("Letter Gothic,Liberation Mono,Courier New,monospace"); + } else if ((g_strcmp0 (name, "sansSerif") == 0)) { + return g_strdup ("TiresiasScreenfont,sans"); + } else if ((g_strcmp0 (name, "serif") == 0)) { + return g_strdup ("serif"); + } else if ((g_strcmp0 (name, "monospaceSansSerif") == 0)) { + return g_strdup ("Letter Gothic,monospace"); + } else if ((g_strcmp0 (name, "monospaceSerif") == 0)) { + return g_strdup ("Courier New,Liberation Mono,monospace"); + } else if ((g_strcmp0 (name, "proportionalSansSerif") == 0)) { + return g_strdup ("TiresiasScreenfont,Arial,Helvetica,Liberation Sans,sans"); + } else if ((g_strcmp0 (name, "proportionalSerif") == 0)) { + return g_strdup ("serif"); + } else { + return NULL; + } +} + + +static gchar * +gst_ttml_render_get_text_from_buffer (GstBuffer * buf, guint index) +{ + GstMapInfo map; + GstMemory *mem; + gchar *buf_text = NULL; + + mem = gst_buffer_get_memory (buf, index); + if (!mem) { + GST_CAT_ERROR (ttmlrender_debug, "Failed to access memory at index %u.", + index); + return NULL; + } + + if (!gst_memory_map (mem, &map, GST_MAP_READ)) { + GST_CAT_ERROR (ttmlrender_debug, "Failed to map memory at index %u.", + index); + goto map_fail; + } + + buf_text = g_strndup ((const gchar *) map.data, map.size); + if (!g_utf8_validate (buf_text, -1, NULL)) { + GST_CAT_ERROR (ttmlrender_debug, "Text in buffer us not valid UTF-8"); + g_free (buf_text); + buf_text = NULL; + } + + gst_memory_unmap (mem, &map); +map_fail: + gst_memory_unref (mem); + return buf_text; +} + + +typedef struct +{ + const GstSubtitleElement *element; + gchar *text; +} UnifiedElement; + + +static void +_unified_element_free (UnifiedElement * unified_element) +{ + g_free (unified_element->text); + g_slice_free (UnifiedElement, unified_element); +} + + +typedef struct +{ + GPtrArray *unified_elements; +} UnifiedBlock; + + +static void +_unified_block_free (UnifiedBlock * unified_block) +{ + g_ptr_array_unref (unified_block->unified_elements); + g_slice_free (UnifiedBlock, unified_block); +} + + +static UnifiedElement * +_unified_block_get_element (UnifiedBlock * block, guint index) +{ + if (index >= block->unified_elements->len) + return NULL; + else + return g_ptr_array_index (block->unified_elements, index); +} + + +static void +gst_ttml_render_handle_whitespace (UnifiedBlock * block) +{ + UnifiedElement *last = NULL; + UnifiedElement *cur = _unified_block_get_element (block, 0); + UnifiedElement *next = _unified_block_get_element (block, 1); + guint i; + + for (i = 2; cur; ++i) { + if (cur->element->suppress_whitespace) { + if (!last || (g_strcmp0 (last->text, "\n") == 0)) { + /* Strip leading whitespace. */ + if (cur->text[0] == 0x20) { + gchar *tmp = cur->text; + GST_CAT_LOG (ttmlrender_debug, "Stripping leading whitespace."); + cur->text = g_strdup (cur->text + 1); + g_free (tmp); + } + } + if (!next || (g_strcmp0 (next->text, "\n") == 0)) { + /* Strip trailing whitespace. */ + if (cur->text[strlen (cur->text) - 1] == 0x20) { + gchar *tmp = cur->text; + GST_CAT_LOG (ttmlrender_debug, "Stripping trailing whitespace."); + cur->text = g_strndup (cur->text, strlen (cur->text) - 1); + g_free (tmp); + } + } + } + last = cur; + cur = next; + next = _unified_block_get_element (block, i); + } +} + + +static UnifiedBlock * +gst_ttml_render_unify_block (const GstSubtitleBlock * block, GstBuffer * buf) +{ + guint i; + UnifiedBlock *ret = g_slice_new0 (UnifiedBlock); + ret->unified_elements = + g_ptr_array_new_with_free_func ((GDestroyNotify) _unified_element_free); + + for (i = 0; i < gst_subtitle_block_get_element_count (block); ++i) { + UnifiedElement *ue = g_slice_new0 (UnifiedElement); + ue->element = gst_subtitle_block_get_element (block, i); + ue->text = + gst_ttml_render_get_text_from_buffer (buf, ue->element->text_index); + g_ptr_array_add (ret->unified_elements, ue); + } + return ret; +} + + +/* From the elements within @block, generate a string of the subtitle text + * marked-up using pango-markup. Also, store the ranges of characters belonging + * to the text of each element in @text_ranges. */ +static gchar * +gst_ttml_render_generate_marked_up_string (GstTtmlRender * render, + const GstSubtitleBlock * block, GstBuffer * text_buf, + GPtrArray * text_ranges) +{ + gchar *escaped_text, *joined_text, *old_text, *font_family, *font_size, + *fgcolor; + const gchar *font_style, *font_weight, *underline; + guint total_text_length = 0U; + guint element_count = gst_subtitle_block_get_element_count (block); + UnifiedBlock *unified_block; + guint i; + + joined_text = g_strdup (""); + unified_block = gst_ttml_render_unify_block (block, text_buf); + gst_ttml_render_handle_whitespace (unified_block); + + for (i = 0; i < element_count; ++i) { + TextRange *range = g_slice_new0 (TextRange); + UnifiedElement *unified_element = + _unified_block_get_element (unified_block, i); + + escaped_text = g_markup_escape_text (unified_element->text, -1); + GST_CAT_DEBUG (ttmlrender_debug, "Escaped text is: \"%s\"", escaped_text); + range->first_char = total_text_length; + + fgcolor = + gst_ttml_render_color_to_string (unified_element->element-> + style_set->color); + font_size = + g_strdup_printf ("%u", + (guint) (round (unified_element->element->style_set->font_size * + render->height))); + font_family = + gst_ttml_render_resolve_generic_fontname (unified_element-> + element->style_set->font_family); + if (!font_family) + font_family = g_strdup (unified_element->element->style_set->font_family); + font_style = + (unified_element->element->style_set->font_style == + GST_SUBTITLE_FONT_STYLE_NORMAL) ? "normal" : "italic"; + font_weight = + (unified_element->element->style_set->font_weight == + GST_SUBTITLE_FONT_WEIGHT_NORMAL) ? "normal" : "bold"; + underline = + (unified_element->element->style_set->text_decoration == + GST_SUBTITLE_TEXT_DECORATION_UNDERLINE) ? "single" : "none"; + + old_text = joined_text; + joined_text = g_strconcat (joined_text, + "", escaped_text, "", NULL); + GST_CAT_DEBUG (ttmlrender_debug, "Joined text is now: %s", joined_text); + + total_text_length += strlen (unified_element->text); + range->last_char = total_text_length - 1; + GST_CAT_DEBUG (ttmlrender_debug, + "First character index: %u; last character " "index: %u", + range->first_char, range->last_char); + g_ptr_array_insert (text_ranges, i, range); + + g_free (old_text); + g_free (escaped_text); + g_free (fgcolor); + g_free (font_family); + g_free (font_size); + } + + _unified_block_free (unified_block); + return joined_text; +} + + +/* Render the text in a pango-markup string. */ +static GstTtmlRenderRenderedText * +gst_ttml_render_draw_text (GstTtmlRender * render, const gchar * text, + guint max_width, PangoAlignment alignment, guint line_height, + guint max_font_size, gboolean wrap) +{ + GstTtmlRenderClass *class; + GstTtmlRenderRenderedText *ret; + cairo_surface_t *surface, *cropped_surface; + cairo_t *cairo_state, *cropped_state; + GstMapInfo map; + PangoRectangle logical_rect, ink_rect; + gint spacing = 0; + guint buf_width, buf_height; + gint stride; + PangoLayoutLine *line; + PangoRectangle line_extents; + gint bounding_box_x1, bounding_box_x2, bounding_box_y1, bounding_box_y2; + + ret = g_slice_new0 (GstTtmlRenderRenderedText); + ret->text_image = gst_ttml_render_rendered_image_new_empty (); + + class = GST_TTML_RENDER_GET_CLASS (render); + ret->layout = pango_layout_new (class->pango_context); + + pango_layout_set_markup (ret->layout, text, strlen (text)); + GST_CAT_DEBUG (ttmlrender_debug, "Layout text: %s", + pango_layout_get_text (ret->layout)); + if (wrap) { + pango_layout_set_width (ret->layout, max_width * PANGO_SCALE); + pango_layout_set_wrap (ret->layout, PANGO_WRAP_WORD_CHAR); + } else { + pango_layout_set_width (ret->layout, -1); + } + + pango_layout_set_alignment (ret->layout, alignment); + line = pango_layout_get_line_readonly (ret->layout, 0); + pango_layout_line_get_pixel_extents (line, NULL, &line_extents); + + GST_CAT_LOG (ttmlrender_debug, "Requested line_height: %u", line_height); + spacing = line_height - line_extents.height; + pango_layout_set_spacing (ret->layout, PANGO_SCALE * spacing); + GST_CAT_LOG (ttmlrender_debug, "Line spacing set to %d", + pango_layout_get_spacing (ret->layout) / PANGO_SCALE); + + pango_layout_get_pixel_extents (ret->layout, &ink_rect, &logical_rect); + GST_CAT_DEBUG (ttmlrender_debug, "logical_rect.x: %d logical_rect.y: %d " + "logical_rect.width: %d logical_rect.height: %d", logical_rect.x, + logical_rect.y, logical_rect.width, logical_rect.height); + + bounding_box_x1 = MIN (logical_rect.x, ink_rect.x); + bounding_box_x2 = MAX (logical_rect.x + logical_rect.width, + ink_rect.x + ink_rect.width); + bounding_box_y1 = MIN (logical_rect.y, ink_rect.y); + bounding_box_y2 = MAX (logical_rect.y + logical_rect.height, + ink_rect.y + ink_rect.height); + + surface = cairo_image_surface_create (CAIRO_FORMAT_ARGB32, bounding_box_x2, + bounding_box_y2); + cairo_state = cairo_create (surface); + cairo_set_operator (cairo_state, CAIRO_OPERATOR_CLEAR); + cairo_paint (cairo_state); + cairo_set_operator (cairo_state, CAIRO_OPERATOR_OVER); + + /* Render layout. */ + cairo_save (cairo_state); + pango_cairo_show_layout (cairo_state, ret->layout); + cairo_restore (cairo_state); + + buf_width = bounding_box_x2 - bounding_box_x1; + buf_height = (bounding_box_y2 - bounding_box_y1) + spacing; + GST_CAT_DEBUG (ttmlrender_debug, "Output buffer width: %u height: %u", + buf_width, buf_height); + + /* Depending on whether the text is wrapped and its alignment, the image + * created by rendering a PangoLayout will contain more than just the + * rendered text: it may also contain blankspace around the rendered text. + * The following code crops blankspace from around the rendered text, + * returning only the rendered text itself in a GstBuffer. */ + ret->text_image->image = + gst_buffer_new_allocate (NULL, 4 * buf_width * buf_height, NULL); + gst_buffer_memset (ret->text_image->image, 0, 0U, 4 * buf_width * buf_height); + gst_buffer_map (ret->text_image->image, &map, GST_MAP_READWRITE); + + stride = cairo_format_stride_for_width (CAIRO_FORMAT_ARGB32, buf_width); + cropped_surface = + cairo_image_surface_create_for_data (map.data, CAIRO_FORMAT_ARGB32, + buf_width, buf_height, stride); + cropped_state = cairo_create (cropped_surface); + cairo_set_source_surface (cropped_state, surface, -bounding_box_x1, + -(bounding_box_y1 - spacing / 2.0)); + cairo_rectangle (cropped_state, 0, 0, buf_width, buf_height); + cairo_fill (cropped_state); + + cairo_destroy (cairo_state); + cairo_surface_destroy (surface); + cairo_destroy (cropped_state); + cairo_surface_destroy (cropped_surface); + gst_buffer_unmap (ret->text_image->image, &map); + + ret->text_image->width = buf_width; + ret->text_image->height = buf_height; + ret->horiz_offset = bounding_box_x1; + + return ret; +} + + +/* If any of an array of elements has line wrapping enabled, return TRUE. */ +static gboolean +gst_ttml_render_elements_are_wrapped (GPtrArray * elements) +{ + GstSubtitleElement *element; + guint i; + + for (i = 0; i < elements->len; ++i) { + element = g_ptr_array_index (elements, i); + if (element->style_set->wrap_option == GST_SUBTITLE_WRAPPING_ON) + return TRUE; + } + + return FALSE; +} + + +/* Return the maximum font size used in an array of elements. */ +static gdouble +gst_ttml_render_get_max_font_size (GPtrArray * elements) +{ + GstSubtitleElement *element; + guint i; + gdouble max_size = 0.0; + + for (i = 0; i < elements->len; ++i) { + element = g_ptr_array_index (elements, i); + if (element->style_set->font_size > max_size) + max_size = element->style_set->font_size; + } + + return max_size; +} + + +static GstTtmlRenderRenderedImage * +gst_ttml_render_rendered_image_new (GstBuffer * image, gint x, gint y, + guint width, guint height) +{ + GstTtmlRenderRenderedImage *ret; + + ret = g_slice_new0 (GstTtmlRenderRenderedImage); + + ret->image = image; + ret->x = x; + ret->y = y; + ret->width = width; + ret->height = height; + + return ret; +} + + +static GstTtmlRenderRenderedImage * +gst_ttml_render_rendered_image_new_empty (void) +{ + return gst_ttml_render_rendered_image_new (NULL, 0, 0, 0, 0); +} + + +static inline GstTtmlRenderRenderedImage * +gst_ttml_render_rendered_image_copy (GstTtmlRenderRenderedImage * image) +{ + GstTtmlRenderRenderedImage *ret = g_slice_new0 (GstTtmlRenderRenderedImage); + + ret->image = gst_buffer_ref (image->image); + ret->x = image->x; + ret->y = image->y; + ret->width = image->width; + ret->height = image->height; + + return ret; +} + + +static void +gst_ttml_render_rendered_image_free (GstTtmlRenderRenderedImage * image) +{ + if (!image) + return; + gst_buffer_unref (image->image); + g_slice_free (GstTtmlRenderRenderedImage, image); +} + + +/* The order of arguments is significant: @image2 will be rendered on top of + * @image1. */ +static GstTtmlRenderRenderedImage * +gst_ttml_render_rendered_image_combine (GstTtmlRenderRenderedImage * image1, + GstTtmlRenderRenderedImage * image2) +{ + GstTtmlRenderRenderedImage *ret; + GstMapInfo map1, map2, map_dest; + cairo_surface_t *sfc1, *sfc2, *sfc_dest; + cairo_t *state_dest; + + if (image1 && !image2) + return gst_ttml_render_rendered_image_copy (image1); + if (image2 && !image1) + return gst_ttml_render_rendered_image_copy (image2); + + ret = g_slice_new0 (GstTtmlRenderRenderedImage); + + /* Work out dimensions of combined image. */ + ret->x = MIN (image1->x, image2->x); + ret->y = MIN (image1->y, image2->y); + ret->width = MAX (image1->x + image1->width, image2->x + image2->width) + - ret->x; + ret->height = MAX (image1->y + image1->height, image2->y + image2->height) + - ret->y; + + GST_CAT_LOG (ttmlrender_debug, "Dimensions of combined image: x:%u y:%u " + "width:%u height:%u", ret->x, ret->y, ret->width, ret->height); + + /* Create cairo_surface from src images. */ + gst_buffer_map (image1->image, &map1, GST_MAP_READ); + sfc1 = + cairo_image_surface_create_for_data (map1.data, CAIRO_FORMAT_ARGB32, + image1->width, image1->height, + cairo_format_stride_for_width (CAIRO_FORMAT_ARGB32, image1->width)); + + gst_buffer_map (image2->image, &map2, GST_MAP_READ); + sfc2 = + cairo_image_surface_create_for_data (map2.data, CAIRO_FORMAT_ARGB32, + image2->width, image2->height, + cairo_format_stride_for_width (CAIRO_FORMAT_ARGB32, image2->width)); + + /* Create cairo_surface for resultant image. */ + ret->image = gst_buffer_new_allocate (NULL, 4 * ret->width * ret->height, + NULL); + gst_buffer_memset (ret->image, 0, 0U, 4 * ret->width * ret->height); + gst_buffer_map (ret->image, &map_dest, GST_MAP_READWRITE); + sfc_dest = + cairo_image_surface_create_for_data (map_dest.data, CAIRO_FORMAT_ARGB32, + ret->width, ret->height, + cairo_format_stride_for_width (CAIRO_FORMAT_ARGB32, ret->width)); + state_dest = cairo_create (sfc_dest); + + /* Blend image1 into destination surface. */ + cairo_set_source_surface (state_dest, sfc1, image1->x - ret->x, + image1->y - ret->y); + cairo_rectangle (state_dest, image1->x - ret->x, image1->y - ret->y, + image1->width, image1->height); + cairo_fill (state_dest); + + /* Blend image2 into destination surface. */ + cairo_set_source_surface (state_dest, sfc2, image2->x - ret->x, + image2->y - ret->y); + cairo_rectangle (state_dest, image2->x - ret->x, image2->y - ret->y, + image2->width, image2->height); + cairo_fill (state_dest); + + /* Return destination image. */ + cairo_destroy (state_dest); + cairo_surface_destroy (sfc1); + cairo_surface_destroy (sfc2); + cairo_surface_destroy (sfc_dest); + gst_buffer_unmap (image1->image, &map1); + gst_buffer_unmap (image2->image, &map2); + gst_buffer_unmap (ret->image, &map_dest); + + return ret; +} + + +static GstTtmlRenderRenderedImage * +gst_ttml_render_rendered_image_crop (GstTtmlRenderRenderedImage * image, + gint x, gint y, guint width, guint height) +{ + GstTtmlRenderRenderedImage *ret; + GstMapInfo map_src, map_dest; + cairo_surface_t *sfc_src, *sfc_dest; + cairo_t *state_dest; + + if ((x <= image->x) && (y <= image->y) && (width >= image->width) + && (height >= image->height)) + return gst_ttml_render_rendered_image_copy (image); + + if (image->x >= (x + (gint) width) + || (image->x + (gint) image->width) <= x + || image->y >= (y + (gint) height) + || (image->y + (gint) image->height) <= y) { + GST_CAT_WARNING (ttmlrender_debug, + "Crop rectangle doesn't intersect image."); + return NULL; + } + + ret = g_slice_new0 (GstTtmlRenderRenderedImage); + + ret->x = MAX (image->x, x); + ret->y = MAX (image->y, y); + ret->width = MIN ((image->x + image->width) - ret->x, (x + width) - ret->x); + ret->height = MIN ((image->y + image->height) - ret->y, + (y + height) - ret->y); + + GST_CAT_LOG (ttmlrender_debug, "Dimensions of cropped image: x:%u y:%u " + "width:%u height:%u", ret->x, ret->y, ret->width, ret->height); + + /* Create cairo_surface from src image. */ + gst_buffer_map (image->image, &map_src, GST_MAP_READ); + sfc_src = + cairo_image_surface_create_for_data (map_src.data, CAIRO_FORMAT_ARGB32, + image->width, image->height, + cairo_format_stride_for_width (CAIRO_FORMAT_ARGB32, image->width)); + + /* Create cairo_surface for cropped image. */ + ret->image = gst_buffer_new_allocate (NULL, 4 * ret->width * ret->height, + NULL); + gst_buffer_memset (ret->image, 0, 0U, 4 * ret->width * ret->height); + gst_buffer_map (ret->image, &map_dest, GST_MAP_READWRITE); + sfc_dest = + cairo_image_surface_create_for_data (map_dest.data, CAIRO_FORMAT_ARGB32, + ret->width, ret->height, + cairo_format_stride_for_width (CAIRO_FORMAT_ARGB32, ret->width)); + state_dest = cairo_create (sfc_dest); + + /* Copy section of image1 into destination surface. */ + cairo_set_source_surface (state_dest, sfc_src, (image->x - ret->x), + (image->y - ret->y)); + cairo_rectangle (state_dest, 0, 0, ret->width, ret->height); + cairo_fill (state_dest); + + cairo_destroy (state_dest); + cairo_surface_destroy (sfc_src); + cairo_surface_destroy (sfc_dest); + gst_buffer_unmap (image->image, &map_src); + gst_buffer_unmap (ret->image, &map_dest); + + return ret; +} + + +static gboolean +gst_ttml_render_color_is_transparent (GstSubtitleColor * color) +{ + return (color->a == 0); +} + + +/* Render the background rectangles to be placed behind each element. */ +static GstTtmlRenderRenderedImage * +gst_ttml_render_render_element_backgrounds (const GstSubtitleBlock * block, + GPtrArray * char_ranges, PangoLayout * layout, guint origin_x, + guint origin_y, guint line_height, guint line_padding, guint horiz_offset) +{ + gint first_line, last_line, cur_line; + guint padding; + PangoLayoutLine *line; + PangoRectangle first_char_pos, last_char_pos, line_extents; + TextRange *range; + const GstSubtitleElement *element; + guint rect_width; + GstBuffer *rectangle; + guint first_char_start, last_char_end; + guint i; + GstTtmlRenderRenderedImage *ret = NULL; + + for (i = 0; i < char_ranges->len; ++i) { + range = g_ptr_array_index (char_ranges, i); + element = gst_subtitle_block_get_element (block, i); + + GST_CAT_LOG (ttmlrender_debug, "First char index: %u Last char index: %u", + range->first_char, range->last_char); + pango_layout_index_to_pos (layout, range->first_char, &first_char_pos); + pango_layout_index_to_pos (layout, range->last_char, &last_char_pos); + pango_layout_index_to_line_x (layout, range->first_char, 1, + &first_line, NULL); + pango_layout_index_to_line_x (layout, range->last_char, 0, + &last_line, NULL); + + first_char_start = PANGO_PIXELS (first_char_pos.x) - horiz_offset; + last_char_end = PANGO_PIXELS (last_char_pos.x + last_char_pos.width) + - horiz_offset; + + GST_CAT_LOG (ttmlrender_debug, "First char start: %u Last char end: %u", + first_char_start, last_char_end); + GST_CAT_LOG (ttmlrender_debug, "First line: %u Last line: %u", first_line, + last_line); + + for (cur_line = first_line; cur_line <= last_line; ++cur_line) { + guint line_start, line_end; + guint area_start, area_end; + gint first_char_index; + PangoRectangle line_pos; + padding = 0; + + line = pango_layout_get_line (layout, cur_line); + pango_layout_line_get_pixel_extents (line, NULL, &line_extents); + + pango_layout_line_x_to_index (line, 0, &first_char_index, NULL); + pango_layout_index_to_pos (layout, first_char_index, &line_pos); + GST_CAT_LOG (ttmlrender_debug, "First char index:%d position_X:%d " + "position_Y:%d", first_char_index, PANGO_PIXELS (line_pos.x), + PANGO_PIXELS (line_pos.y)); + + line_start = PANGO_PIXELS (line_pos.x) - horiz_offset; + line_end = (PANGO_PIXELS (line_pos.x) + line_extents.width) + - horiz_offset; + + GST_CAT_LOG (ttmlrender_debug, "line_extents.x:%d line_extents.y:%d " + "line_extents.width:%d line_extents.height:%d", line_extents.x, + line_extents.y, line_extents.width, line_extents.height); + GST_CAT_LOG (ttmlrender_debug, "cur_line:%u line start:%u line end:%u " + "first_char_start: %u last_char_end: %u", cur_line, line_start, + line_end, first_char_start, last_char_end); + + if ((cur_line == first_line) && (first_char_start != line_start)) { + area_start = first_char_start + line_padding; + GST_CAT_LOG (ttmlrender_debug, + "First line, but there is preceding text in line."); + } else { + GST_CAT_LOG (ttmlrender_debug, + "Area contains first text on the line; adding padding..."); + ++padding; + area_start = line_start; + } + + if ((cur_line == last_line) && (last_char_end != line_end)) { + GST_CAT_LOG (ttmlrender_debug, + "Last line, but there is following text in line."); + area_end = last_char_end + line_padding; + } else { + GST_CAT_LOG (ttmlrender_debug, + "Area contains last text on the line; adding padding..."); + ++padding; + area_end = line_end + (2 * line_padding); + } + + rect_width = (area_end - area_start); + + if (rect_width > 0) { /*
s will result in zero-width rectangle */ + GstTtmlRenderRenderedImage *image, *tmp; + rectangle = gst_ttml_render_draw_rectangle (rect_width, line_height, + element->style_set->background_color); + image = gst_ttml_render_rendered_image_new (rectangle, + origin_x + area_start, + origin_y + (cur_line * line_height), rect_width, line_height); + tmp = ret; + ret = gst_ttml_render_rendered_image_combine (ret, image); + if (tmp) + gst_ttml_render_rendered_image_free (tmp); + gst_ttml_render_rendered_image_free (image); + } + } + } + + return ret; +} + + +static PangoAlignment +gst_ttml_render_get_alignment (GstSubtitleStyleSet * style_set) +{ + PangoAlignment align = PANGO_ALIGN_LEFT; + + switch (style_set->multi_row_align) { + case GST_SUBTITLE_MULTI_ROW_ALIGN_START: + align = PANGO_ALIGN_LEFT; + break; + case GST_SUBTITLE_MULTI_ROW_ALIGN_CENTER: + align = PANGO_ALIGN_CENTER; + break; + case GST_SUBTITLE_MULTI_ROW_ALIGN_END: + align = PANGO_ALIGN_RIGHT; + break; + case GST_SUBTITLE_MULTI_ROW_ALIGN_AUTO: + switch (style_set->text_align) { + case GST_SUBTITLE_TEXT_ALIGN_START: + case GST_SUBTITLE_TEXT_ALIGN_LEFT: + align = PANGO_ALIGN_LEFT; + break; + case GST_SUBTITLE_TEXT_ALIGN_CENTER: + align = PANGO_ALIGN_CENTER; + break; + case GST_SUBTITLE_TEXT_ALIGN_END: + case GST_SUBTITLE_TEXT_ALIGN_RIGHT: + align = PANGO_ALIGN_RIGHT; + break; + default: + GST_CAT_ERROR (ttmlrender_debug, "Illegal textAlign value (%d)", + style_set->text_align); + break; + } + break; + default: + GST_CAT_ERROR (ttmlrender_debug, "Illegal multiRowAlign value (%d)", + style_set->multi_row_align); + break; + } + return align; +} + + +static GstTtmlRenderRenderedImage * +gst_ttml_render_stitch_blocks (GList * blocks) +{ + guint vert_offset = 0; + GList *block_entry; + GstTtmlRenderRenderedImage *ret = NULL; + + for (block_entry = g_list_first (blocks); block_entry; + block_entry = block_entry->next) { + GstTtmlRenderRenderedImage *block, *tmp; + block = (GstTtmlRenderRenderedImage *) block_entry->data; + tmp = ret; + + block->y += vert_offset; + GST_CAT_LOG (ttmlrender_debug, "Rendering block at vertical offset %u", + vert_offset); + vert_offset = block->y + block->height; + ret = gst_ttml_render_rendered_image_combine (ret, block); + if (tmp) + gst_ttml_render_rendered_image_free (tmp); + } + + if (ret) { + GST_CAT_LOG (ttmlrender_debug, "Height of stitched image: %u", ret->height); + ret->image = gst_buffer_make_writable (ret->image); + } + return ret; +} + + +static void +gst_ttml_render_rendered_text_free (GstTtmlRenderRenderedText * text) +{ + if (text->text_image) + gst_ttml_render_rendered_image_free (text->text_image); + if (text->layout) + g_object_unref (text->layout); + g_slice_free (GstTtmlRenderRenderedText, text); +} + + +static GstTtmlRenderRenderedImage * +gst_ttml_render_render_text_block (GstTtmlRender * render, + const GstSubtitleBlock * block, GstBuffer * text_buf, guint width, + gboolean overflow) +{ + GPtrArray *char_ranges = + g_ptr_array_new_with_free_func ((GDestroyNotify) _text_range_free); + gchar *marked_up_string; + PangoAlignment alignment; + guint max_font_size; + guint line_height; + guint line_padding; + gint text_offset = 0; + GstTtmlRenderRenderedText *rendered_text; + GstTtmlRenderRenderedImage *backgrounds = NULL; + GstTtmlRenderRenderedImage *ret; + + /* Join text from elements to form a single marked-up string. */ + marked_up_string = gst_ttml_render_generate_marked_up_string (render, block, + text_buf, char_ranges); + + max_font_size = (guint) (gst_ttml_render_get_max_font_size (block->elements) + * render->height); + GST_CAT_DEBUG (ttmlrender_debug, "Max font size: %u", max_font_size); + line_height = (guint) round (block->style_set->line_height * max_font_size); + + line_padding = (guint) (block->style_set->line_padding * render->width); + alignment = gst_ttml_render_get_alignment (block->style_set); + + /* Render text to buffer. */ + rendered_text = gst_ttml_render_draw_text (render, marked_up_string, + (width - (2 * line_padding)), alignment, line_height, max_font_size, + gst_ttml_render_elements_are_wrapped (block->elements)); + + switch (block->style_set->text_align) { + case GST_SUBTITLE_TEXT_ALIGN_START: + case GST_SUBTITLE_TEXT_ALIGN_LEFT: + text_offset = line_padding; + break; + case GST_SUBTITLE_TEXT_ALIGN_CENTER: + text_offset = ((gint) width - rendered_text->text_image->width); + text_offset /= 2; + break; + case GST_SUBTITLE_TEXT_ALIGN_END: + case GST_SUBTITLE_TEXT_ALIGN_RIGHT: + text_offset = (gint) width + - (rendered_text->text_image->width + line_padding); + break; + } + + rendered_text->text_image->x = text_offset; + + /* Render background rectangles, if any. */ + backgrounds = gst_ttml_render_render_element_backgrounds (block, char_ranges, + rendered_text->layout, text_offset - line_padding, 0, + (guint) round (block->style_set->line_height * max_font_size), + line_padding, rendered_text->horiz_offset); + + /* Render block background, if non-transparent. */ + if (!gst_ttml_render_color_is_transparent (&block->style_set-> + background_color)) { + GstTtmlRenderRenderedImage *block_background; + GstTtmlRenderRenderedImage *tmp = backgrounds; + + GstBuffer *block_bg_image = gst_ttml_render_draw_rectangle (width, + backgrounds->height, block->style_set->background_color); + block_background = gst_ttml_render_rendered_image_new (block_bg_image, 0, + 0, width, backgrounds->height); + backgrounds = gst_ttml_render_rendered_image_combine (block_background, + backgrounds); + gst_ttml_render_rendered_image_free (tmp); + gst_ttml_render_rendered_image_free (block_background); + } + + /* Combine text and background images. */ + ret = gst_ttml_render_rendered_image_combine (backgrounds, + rendered_text->text_image); + gst_ttml_render_rendered_image_free (backgrounds); + gst_ttml_render_rendered_text_free (rendered_text); + + g_free (marked_up_string); + g_ptr_array_unref (char_ranges); + GST_CAT_DEBUG (ttmlrender_debug, "block width: %u block height: %u", + ret->width, ret->height); + return ret; +} + + +static GstVideoOverlayComposition * +gst_ttml_render_compose_overlay (GstTtmlRenderRenderedImage * image) +{ + GstVideoOverlayRectangle *rectangle; + GstVideoOverlayComposition *ret = NULL; + + gst_buffer_add_video_meta (image->image, GST_VIDEO_FRAME_FLAG_NONE, + GST_VIDEO_OVERLAY_COMPOSITION_FORMAT_RGB, image->width, image->height); + + rectangle = gst_video_overlay_rectangle_new_raw (image->image, image->x, + image->y, image->width, image->height, + GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA); + + ret = gst_video_overlay_composition_new (rectangle); + gst_video_overlay_rectangle_unref (rectangle); + return ret; +} + + +static GstVideoOverlayComposition * +gst_ttml_render_render_text_region (GstTtmlRender * render, + GstSubtitleRegion * region, GstBuffer * text_buf) +{ + GList *blocks = NULL; + guint region_x, region_y, region_width, region_height; + guint window_x, window_y, window_width, window_height; + guint padding_start, padding_end, padding_before, padding_after; + GstTtmlRenderRenderedImage *region_image = NULL; + GstTtmlRenderRenderedImage *blocks_image; + GstVideoOverlayComposition *ret = NULL; + guint i; + + region_width = (guint) (round (region->style_set->extent_w * render->width)); + region_height = + (guint) (round (region->style_set->extent_h * render->height)); + region_x = (guint) (round (region->style_set->origin_x * render->width)); + region_y = (guint) (round (region->style_set->origin_y * render->height)); + + padding_start = + (guint) (round (region->style_set->padding_start * render->width)); + padding_end = + (guint) (round (region->style_set->padding_end * render->width)); + padding_before = + (guint) (round (region->style_set->padding_before * render->height)); + padding_after = + (guint) (round (region->style_set->padding_after * render->height)); + + /* "window" here refers to the section of the region that we're allowed to + * render into, i.e., the region minus padding. */ + window_x = region_x + padding_start; + window_y = region_y + padding_before; + window_width = region_width - (padding_start + padding_end); + window_height = region_height - (padding_before + padding_after); + + GST_CAT_DEBUG (ttmlrender_debug, + "Padding: start: %u end: %u before: %u after: %u", + padding_start, padding_end, padding_before, padding_after); + + /* Render region background, if non-transparent. */ + if (!gst_ttml_render_color_is_transparent (®ion->style_set-> + background_color)) { + GstBuffer *bg_rect; + + bg_rect = gst_ttml_render_draw_rectangle (region_width, region_height, + region->style_set->background_color); + region_image = gst_ttml_render_rendered_image_new (bg_rect, region_x, + region_y, region_width, region_height); + } + + /* Render each block and append to list. */ + for (i = 0; i < gst_subtitle_region_get_block_count (region); ++i) { + const GstSubtitleBlock *block; + GstTtmlRenderRenderedImage *rendered_block; + + block = gst_subtitle_region_get_block (region, i); + rendered_block = gst_ttml_render_render_text_block (render, block, text_buf, + window_width, TRUE); + + blocks = g_list_append (blocks, rendered_block); + } + + if (blocks) { + GstTtmlRenderRenderedImage *tmp; + + blocks_image = gst_ttml_render_stitch_blocks (blocks); + g_list_free_full (blocks, + (GDestroyNotify) gst_ttml_render_rendered_image_free); + blocks_image->x += window_x; + + switch (region->style_set->display_align) { + case GST_SUBTITLE_DISPLAY_ALIGN_BEFORE: + blocks_image->y = window_y; + break; + case GST_SUBTITLE_DISPLAY_ALIGN_CENTER: + blocks_image->y = region_y + ((gint) ((region_height + padding_before) + - (padding_after + blocks_image->height))) / 2; + break; + case GST_SUBTITLE_DISPLAY_ALIGN_AFTER: + blocks_image->y = (region_y + region_height) + - (padding_after + blocks_image->height); + break; + } + + if ((region->style_set->overflow == GST_SUBTITLE_OVERFLOW_MODE_HIDDEN) + && ((blocks_image->height > window_height) + || (blocks_image->width > window_width))) { + GstTtmlRenderRenderedImage *tmp = blocks_image; + blocks_image = gst_ttml_render_rendered_image_crop (blocks_image, + window_x, window_y, window_width, window_height); + gst_ttml_render_rendered_image_free (tmp); + } + + tmp = region_image; + region_image = gst_ttml_render_rendered_image_combine (region_image, + blocks_image); + if (tmp) + gst_ttml_render_rendered_image_free (tmp); + gst_ttml_render_rendered_image_free (blocks_image); + } + + GST_CAT_DEBUG (ttmlrender_debug, "Height of rendered region: %u", + region_image->height); + + ret = gst_ttml_render_compose_overlay (region_image); + gst_ttml_render_rendered_image_free (region_image); + return ret; +} + + +static GstFlowReturn +gst_ttml_render_video_chain (GstPad * pad, GstObject * parent, + GstBuffer * buffer) +{ + GstTtmlRender *render; + GstFlowReturn ret = GST_FLOW_OK; + gboolean in_seg = FALSE; + guint64 start, stop, clip_start = 0, clip_stop = 0; + gchar *text = NULL; + + render = GST_TTML_RENDER (parent); + + if (!GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) + goto missing_timestamp; + + /* ignore buffers that are outside of the current segment */ + start = GST_BUFFER_TIMESTAMP (buffer); + + if (!GST_BUFFER_DURATION_IS_VALID (buffer)) { + stop = GST_CLOCK_TIME_NONE; + } else { + stop = start + GST_BUFFER_DURATION (buffer); + } + + GST_LOG_OBJECT (render, "%" GST_SEGMENT_FORMAT " BUFFER: ts=%" + GST_TIME_FORMAT ", end=%" GST_TIME_FORMAT, &render->segment, + GST_TIME_ARGS (start), GST_TIME_ARGS (stop)); + + /* segment_clip() will adjust start unconditionally to segment_start if + * no stop time is provided, so handle this ourselves */ + if (stop == GST_CLOCK_TIME_NONE && start < render->segment.start) + goto out_of_segment; + + in_seg = gst_segment_clip (&render->segment, GST_FORMAT_TIME, start, stop, + &clip_start, &clip_stop); + + if (!in_seg) + goto out_of_segment; + + /* if the buffer is only partially in the segment, fix up stamps */ + if (clip_start != start || (stop != -1 && clip_stop != stop)) { + GST_DEBUG_OBJECT (render, "clipping buffer timestamp/duration to segment"); + buffer = gst_buffer_make_writable (buffer); + GST_BUFFER_TIMESTAMP (buffer) = clip_start; + if (stop != -1) + GST_BUFFER_DURATION (buffer) = clip_stop - clip_start; + } + + /* now, after we've done the clipping, fix up end time if there's no + * duration (we only use those estimated values internally though, we + * don't want to set bogus values on the buffer itself) */ + if (stop == -1) { + if (render->info.fps_n && render->info.fps_d) { + GST_DEBUG_OBJECT (render, "estimating duration based on framerate"); + stop = start + gst_util_uint64_scale_int (GST_SECOND, + render->info.fps_d, render->info.fps_n); + } else { + GST_LOG_OBJECT (render, "no duration, assuming minimal duration"); + stop = start + 1; /* we need to assume some interval */ + } + } + + gst_object_sync_values (GST_OBJECT (render), GST_BUFFER_TIMESTAMP (buffer)); + +wait_for_text_buf: + + GST_TTML_RENDER_LOCK (render); + + if (render->video_flushing) + goto flushing; + + if (render->video_eos) + goto have_eos; + + /* Text pad not linked; push input video frame */ + if (!render->text_linked) { + GST_LOG_OBJECT (render, "Text pad not linked"); + GST_TTML_RENDER_UNLOCK (render); + ret = gst_pad_push (render->srcpad, buffer); + goto not_linked; + } + + /* Text pad linked, check if we have a text buffer queued */ + if (render->text_buffer) { + gboolean pop_text = FALSE, valid_text_time = TRUE; + GstClockTime text_start = GST_CLOCK_TIME_NONE; + GstClockTime text_end = GST_CLOCK_TIME_NONE; + GstClockTime text_running_time = GST_CLOCK_TIME_NONE; + GstClockTime text_running_time_end = GST_CLOCK_TIME_NONE; + GstClockTime vid_running_time, vid_running_time_end; + + /* if the text buffer isn't stamped right, pop it off the + * queue and display it for the current video frame only */ + if (!GST_BUFFER_TIMESTAMP_IS_VALID (render->text_buffer) || + !GST_BUFFER_DURATION_IS_VALID (render->text_buffer)) { + GST_WARNING_OBJECT (render, + "Got text buffer with invalid timestamp or duration"); + pop_text = TRUE; + valid_text_time = FALSE; + } else { + text_start = GST_BUFFER_TIMESTAMP (render->text_buffer); + text_end = text_start + GST_BUFFER_DURATION (render->text_buffer); + } + + vid_running_time = + gst_segment_to_running_time (&render->segment, GST_FORMAT_TIME, start); + vid_running_time_end = + gst_segment_to_running_time (&render->segment, GST_FORMAT_TIME, stop); + + /* If timestamp and duration are valid */ + if (valid_text_time) { + text_running_time = + gst_segment_to_running_time (&render->text_segment, + GST_FORMAT_TIME, text_start); + text_running_time_end = + gst_segment_to_running_time (&render->text_segment, + GST_FORMAT_TIME, text_end); + } + + GST_LOG_OBJECT (render, "T: %" GST_TIME_FORMAT " - %" GST_TIME_FORMAT, + GST_TIME_ARGS (text_running_time), + GST_TIME_ARGS (text_running_time_end)); + GST_LOG_OBJECT (render, "V: %" GST_TIME_FORMAT " - %" GST_TIME_FORMAT, + GST_TIME_ARGS (vid_running_time), GST_TIME_ARGS (vid_running_time_end)); + + /* Text too old or in the future */ + if (valid_text_time && text_running_time_end <= vid_running_time) { + /* text buffer too old, get rid of it and do nothing */ + GST_LOG_OBJECT (render, "text buffer too old, popping"); + pop_text = FALSE; + gst_ttml_render_pop_text (render); + GST_TTML_RENDER_UNLOCK (render); + goto wait_for_text_buf; + } else if (valid_text_time && vid_running_time_end <= text_running_time) { + GST_LOG_OBJECT (render, "text in future, pushing video buf"); + GST_TTML_RENDER_UNLOCK (render); + /* Push the video frame */ + ret = gst_pad_push (render->srcpad, buffer); + } else { + if (render->need_render) { + GstSubtitleRegion *region = NULL; + GstSubtitleMeta *subtitle_meta = NULL; + guint i; + + if (render->compositions) { + g_list_free_full (render->compositions, + (GDestroyNotify) gst_video_overlay_composition_unref); + render->compositions = NULL; + } + + subtitle_meta = gst_buffer_get_subtitle_meta (render->text_buffer); + g_assert (subtitle_meta != NULL); + + for (i = 0; i < subtitle_meta->regions->len; ++i) { + GstVideoOverlayComposition *composition; + region = g_ptr_array_index (subtitle_meta->regions, i); + g_assert (region != NULL); + composition = gst_ttml_render_render_text_region (render, region, + render->text_buffer); + render->compositions = g_list_append (render->compositions, + composition); + } + render->need_render = FALSE; + } + + GST_TTML_RENDER_UNLOCK (render); + ret = gst_ttml_render_push_frame (render, buffer); + + if (valid_text_time && text_running_time_end <= vid_running_time_end) { + GST_LOG_OBJECT (render, "text buffer not needed any longer"); + pop_text = TRUE; + } + } + if (pop_text) { + GST_TTML_RENDER_LOCK (render); + gst_ttml_render_pop_text (render); + GST_TTML_RENDER_UNLOCK (render); + } + } else { + gboolean wait_for_text_buf = TRUE; + + if (render->text_eos) + wait_for_text_buf = FALSE; + + if (!render->wait_text) + wait_for_text_buf = FALSE; + + /* Text pad linked, but no text buffer available - what now? */ + if (render->text_segment.format == GST_FORMAT_TIME) { + GstClockTime text_start_running_time, text_position_running_time; + GstClockTime vid_running_time; + + vid_running_time = + gst_segment_to_running_time (&render->segment, GST_FORMAT_TIME, + GST_BUFFER_TIMESTAMP (buffer)); + text_start_running_time = + gst_segment_to_running_time (&render->text_segment, + GST_FORMAT_TIME, render->text_segment.start); + text_position_running_time = + gst_segment_to_running_time (&render->text_segment, + GST_FORMAT_TIME, render->text_segment.position); + + if ((GST_CLOCK_TIME_IS_VALID (text_start_running_time) && + vid_running_time < text_start_running_time) || + (GST_CLOCK_TIME_IS_VALID (text_position_running_time) && + vid_running_time < text_position_running_time)) { + wait_for_text_buf = FALSE; + } + } + + if (wait_for_text_buf) { + GST_DEBUG_OBJECT (render, "no text buffer, need to wait for one"); + GST_TTML_RENDER_WAIT (render); + GST_DEBUG_OBJECT (render, "resuming"); + GST_TTML_RENDER_UNLOCK (render); + goto wait_for_text_buf; + } else { + GST_TTML_RENDER_UNLOCK (render); + GST_LOG_OBJECT (render, "no need to wait for a text buffer"); + ret = gst_pad_push (render->srcpad, buffer); + } + } + +not_linked: + g_free (text); + + /* Update position */ + render->segment.position = clip_start; + + return ret; + +missing_timestamp: + { + GST_WARNING_OBJECT (render, "buffer without timestamp, discarding"); + gst_buffer_unref (buffer); + return GST_FLOW_OK; + } + +flushing: + { + GST_TTML_RENDER_UNLOCK (render); + GST_DEBUG_OBJECT (render, "flushing, discarding buffer"); + gst_buffer_unref (buffer); + return GST_FLOW_FLUSHING; + } +have_eos: + { + GST_TTML_RENDER_UNLOCK (render); + GST_DEBUG_OBJECT (render, "eos, discarding buffer"); + gst_buffer_unref (buffer); + return GST_FLOW_EOS; + } +out_of_segment: + { + GST_DEBUG_OBJECT (render, "buffer out of segment, discarding"); + gst_buffer_unref (buffer); + return GST_FLOW_OK; + } +} + +static GstStateChangeReturn +gst_ttml_render_change_state (GstElement * element, GstStateChange transition) +{ + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + GstTtmlRender *render = GST_TTML_RENDER (element); + + switch (transition) { + case GST_STATE_CHANGE_PAUSED_TO_READY: + GST_TTML_RENDER_LOCK (render); + render->text_flushing = TRUE; + render->video_flushing = TRUE; + /* pop_text will broadcast on the GCond and thus also make the video + * chain exit if it's waiting for a text buffer */ + gst_ttml_render_pop_text (render); + GST_TTML_RENDER_UNLOCK (render); + break; + default: + break; + } + + ret = parent_class->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + GST_TTML_RENDER_LOCK (render); + render->text_flushing = FALSE; + render->video_flushing = FALSE; + render->video_eos = FALSE; + render->text_eos = FALSE; + gst_segment_init (&render->segment, GST_FORMAT_TIME); + gst_segment_init (&render->text_segment, GST_FORMAT_TIME); + GST_TTML_RENDER_UNLOCK (render); + break; + default: + break; + } + + return ret; +} diff --git a/ext/ttml/gstttmlrender.h b/ext/ttml/gstttmlrender.h new file mode 100644 index 0000000..9a1fba1 --- /dev/null +++ b/ext/ttml/gstttmlrender.h @@ -0,0 +1,124 @@ +/* GStreamer + * Copyright (C) <1999> Erik Walthinsen + * Copyright (C) <2003> David Schleef + * Copyright (C) <2006> Julien Moutte + * Copyright (C) <2006> Zeeshan Ali + * Copyright (C) <2006-2008> Tim-Philipp MÃ¼ller + * Copyright (C) <2009> Young-Ho Cha + * Copyright (C) <2015> British Broadcasting Corporation + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_TTML_RENDER_H__ +#define __GST_TTML_RENDER_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_TTML_RENDER (gst_ttml_render_get_type()) +#define GST_TTML_RENDER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),\ + GST_TYPE_TTML_RENDER, GstTtmlRender)) +#define GST_TTML_RENDER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),\ + GST_TYPE_TTML_RENDER, \ + GstTtmlRenderClass)) +#define GST_TTML_RENDER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj),\ + GST_TYPE_TTML_RENDER, \ + GstTtmlRenderClass)) +#define GST_IS_TTML_RENDER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),\ + GST_TYPE_TTML_RENDER)) +#define GST_IS_TTML_RENDER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),\ + GST_TYPE_TTML_RENDER)) + +typedef struct _GstTtmlRender GstTtmlRender; +typedef struct _GstTtmlRenderClass GstTtmlRenderClass; +typedef struct _GstTtmlRenderRenderedImage GstTtmlRenderRenderedImage; +typedef struct _GstTtmlRenderRenderedText GstTtmlRenderRenderedText; + +struct _GstTtmlRenderRenderedImage { + GstBuffer *image; + gint x; + gint y; + guint width; + guint height; +}; + +struct _GstTtmlRenderRenderedText { + GstTtmlRenderRenderedImage *text_image; + + /* In order to get the positions of characters within a paragraph rendered by + * pango we need to retain a reference to the PangoLayout object that was + * used to render that paragraph. */ + PangoLayout *layout; + + /* The coordinates in @layout will be offset horizontally with respect to the + * position of those characters in @text_image. Store that offset here so + * that the information in @layout can be used to locate the position and + * extent of text areas in @text_image. */ + guint horiz_offset; +}; + + +struct _GstTtmlRender { + GstElement element; + + GstPad *video_sinkpad; + GstPad *text_sinkpad; + GstPad *srcpad; + + GstSegment segment; + GstSegment text_segment; + GstBuffer *text_buffer; + gboolean text_linked; + gboolean video_flushing; + gboolean video_eos; + gboolean text_flushing; + gboolean text_eos; + + GMutex lock; + GCond cond; /* to signal removal of a queued text + * buffer, arrival of a text buffer, + * a text segment update, or a change + * in status (e.g. shutdown, flushing) */ + + GstVideoInfo info; + GstVideoFormat format; + gint width; + gint height; + + gboolean want_background; + gboolean wait_text; + + gboolean need_render; + + GList * compositions; +}; + +struct _GstTtmlRenderClass { + GstElementClass parent_class; + + PangoContext *pango_context; + GMutex *pango_lock; +}; + +GType gst_ttml_render_get_type(void) G_GNUC_CONST; + +G_END_DECLS + +#endif /* __GST_TTML_RENDER_H */ diff --git a/ext/ttml/subtitle.c b/ext/ttml/subtitle.c new file mode 100644 index 0000000..2beac79 --- /dev/null +++ b/ext/ttml/subtitle.c @@ -0,0 +1,312 @@ +/* GStreamer + * Copyright (C) <2015> British Broadcasting Corporation + * Author: Chris Bass + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstsubtitle + * @short_description: Library for describing sets of static subtitles. + * + * This library enables the description of static text scenes made up of a + * number of regions, which may contain a number of block and inline text + * elements. It is derived from the concepts and features defined in the Timed + * Text Markup Language 1 (TTML1), Second Edition + * (http://www.w3.org/TR/ttaf1-dfxp), and the EBU-TT-D profile of TTML1 + * (https://tech.ebu.ch/files/live/sites/tech/files/shared/tech/tech3380.pdf). + */ + +#include "subtitle.h" + +/** + * gst_subtitle_style_set_new: + * + * Create a new #GstSubtitleStyleSet with default values for all properties. + * + * Returns: (transfer full): A newly-allocated #GstSubtitleStyleSet. + */ +GstSubtitleStyleSet * +gst_subtitle_style_set_new (void) +{ + GstSubtitleStyleSet *ret = g_slice_new0 (GstSubtitleStyleSet); + GstSubtitleColor white = { 255, 255, 255, 255 }; + GstSubtitleColor transparent = { 0, 0, 0, 0 }; + + ret->font_family = g_strdup ("default"); + ret->font_size = 1.0; + ret->line_height = 1.25; + ret->color = white; + ret->background_color = transparent; + ret->line_padding = 0.0; + ret->origin_x = ret->origin_y = 0.0; + ret->extent_w = ret->extent_h = 0.0; + ret->padding_start = ret->padding_end + = ret->padding_before = ret->padding_after = 0.0; + + return ret; +} + +/** + * gst_subtitle_style_set_free: + * @style_set: A #GstSubtitleStyleSet. + * + * Free @style_set and its associated memory. + */ +void +gst_subtitle_style_set_free (GstSubtitleStyleSet * style_set) +{ + g_return_if_fail (style_set != NULL); + g_free (style_set->font_family); + g_slice_free (GstSubtitleStyleSet, style_set); +} + + +static void +_gst_subtitle_element_free (GstSubtitleElement * element) +{ + g_return_if_fail (element != NULL); + gst_subtitle_style_set_free (element->style_set); + g_slice_free (GstSubtitleElement, element); +} + +GST_DEFINE_MINI_OBJECT_TYPE (GstSubtitleElement, gst_subtitle_element); + +/** + * gst_subtitle_element_new: + * @style_set: (transfer full): A #GstSubtitleStyleSet that defines the styling + * and layout associated with this inline text element. + * @text_index: The index within a #GstBuffer of the #GstMemory that contains + * the text of this inline text element. + * + * Allocates a new #GstSubtitleElement. + * + * Returns: (transfer full): A newly-allocated #GstSubtitleElement. Unref + * with gst_subtitle_element_unref() when no longer needed. + */ +GstSubtitleElement * +gst_subtitle_element_new (GstSubtitleStyleSet * style_set, + guint text_index, gboolean suppress_whitespace) +{ + GstSubtitleElement *element; + + g_return_val_if_fail (style_set != NULL, NULL); + + element = g_slice_new0 (GstSubtitleElement); + gst_mini_object_init (GST_MINI_OBJECT_CAST (element), 0, + gst_subtitle_element_get_type (), NULL, NULL, + (GstMiniObjectFreeFunction) _gst_subtitle_element_free); + + element->style_set = style_set; + element->text_index = text_index; + element->suppress_whitespace = suppress_whitespace; + + return element; +} + +static void +_gst_subtitle_block_free (GstSubtitleBlock * block) +{ + g_return_if_fail (block != NULL); + gst_subtitle_style_set_free (block->style_set); + g_ptr_array_unref (block->elements); + g_slice_free (GstSubtitleBlock, block); +} + +GST_DEFINE_MINI_OBJECT_TYPE (GstSubtitleBlock, gst_subtitle_block); + + +/** + * gst_subtitle_block_new: + * @style_set: (transfer full): A #GstSubtitleStyleSet that defines the styling + * and layout associated with this block of text elements. + * + * Allocates a new #GstSubtitleBlock. + * + * Returns: (transfer full): A newly-allocated #GstSubtitleBlock. Unref + * with gst_subtitle_block_unref() when no longer needed. + */ +GstSubtitleBlock * +gst_subtitle_block_new (GstSubtitleStyleSet * style_set) +{ + GstSubtitleBlock *block; + + g_return_val_if_fail (style_set != NULL, NULL); + + block = g_slice_new0 (GstSubtitleBlock); + gst_mini_object_init (GST_MINI_OBJECT_CAST (block), 0, + gst_subtitle_block_get_type (), NULL, NULL, + (GstMiniObjectFreeFunction) _gst_subtitle_block_free); + + block->style_set = style_set; + block->elements = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_subtitle_element_unref); + + return block; +} + +/** + * gst_subtitle_block_add_element: + * @block: A #GstSubtitleBlock. + * @element: (transfer full): A #GstSubtitleElement to add. + * + * Adds a #GstSubtitleElement to @block. + */ +void +gst_subtitle_block_add_element (GstSubtitleBlock * block, + GstSubtitleElement * element) +{ + g_return_if_fail (block != NULL); + g_return_if_fail (element != NULL); + + g_ptr_array_add (block->elements, element); +} + +/** + * gst_subtitle_block_get_element_count: + * @block: A #GstSubtitleBlock. + * + * Returns: The number of #GstSubtitleElements in @block. + */ +guint +gst_subtitle_block_get_element_count (const GstSubtitleBlock * block) +{ + g_return_val_if_fail (block != NULL, 0); + + return block->elements->len; +} + +/** + * gst_subtitle_block_get_element: + * @block: A #GstSubtitleBlock. + * @index: Index of the element to get. + * + * Gets the #GstSubtitleElement at @index in the array of elements held by + * @block. + * + * Returns: (transfer none): The #GstSubtitleElement at @index in the array of + * elements held by @block, or %NULL if @index is out-of-bounds. The + * function does not return a reference; the caller should obtain a reference + * using gst_subtitle_element_ref(), if needed. + */ +const GstSubtitleElement * +gst_subtitle_block_get_element (const GstSubtitleBlock * block, guint index) +{ + g_return_val_if_fail (block != NULL, NULL); + + if (index >= block->elements->len) + return NULL; + else + return g_ptr_array_index (block->elements, index); +} + +static void +_gst_subtitle_region_free (GstSubtitleRegion * region) +{ + g_return_if_fail (region != NULL); + gst_subtitle_style_set_free (region->style_set); + g_ptr_array_unref (region->blocks); + g_slice_free (GstSubtitleRegion, region); +} + +GST_DEFINE_MINI_OBJECT_TYPE (GstSubtitleRegion, gst_subtitle_region); + + +/** + * gst_subtitle_region_new: + * @style_set: (transfer full): A #GstSubtitleStyleSet that defines the styling + * and layout associated with this region. + * + * Allocates a new #GstSubtitleRegion. + * + * Returns: (transfer full): A newly-allocated #GstSubtitleRegion. Unref + * with gst_subtitle_region_unref() when no longer needed. + */ +GstSubtitleRegion * +gst_subtitle_region_new (GstSubtitleStyleSet * style_set) +{ + GstSubtitleRegion *region; + + g_return_val_if_fail (style_set != NULL, NULL); + + region = g_slice_new0 (GstSubtitleRegion); + gst_mini_object_init (GST_MINI_OBJECT_CAST (region), 0, + gst_subtitle_region_get_type (), NULL, NULL, + (GstMiniObjectFreeFunction) _gst_subtitle_region_free); + + region->style_set = style_set; + region->blocks = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_subtitle_block_unref); + + return region; +} + +/** + * gst_subtitle_region_add_block: + * @region: A #GstSubtitleRegion. + * @block: (transfer full): A #GstSubtitleBlock which should be added + * to @region's array of blocks. + * + * Adds a #GstSubtitleBlock to the end of the array of blocks held by @region. + * @region will take ownership of @block, and will unref it when @region + * is freed. + */ +void +gst_subtitle_region_add_block (GstSubtitleRegion * region, + GstSubtitleBlock * block) +{ + g_return_if_fail (region != NULL); + g_return_if_fail (block != NULL); + + g_ptr_array_add (region->blocks, block); +} + +/** + * gst_subtitle_region_get_block_count: + * @region: A #GstSubtitleRegion. + * + * Returns: The number of blocks in @region. + */ +guint +gst_subtitle_region_get_block_count (const GstSubtitleRegion * region) +{ + g_return_val_if_fail (region != NULL, 0); + + return region->blocks->len; +} + +/** + * gst_subtitle_region_get_block: + * @region: A #GstSubtitleRegion. + * @index: Index of the block to get. + * + * Gets the block at @index in the array of blocks held by @region. + * + * Returns: (transfer none): The #GstSubtitleBlock at @index in the array of + * blocks held by @region, or %NULL if @index is out-of-bounds. The + * function does not return a reference; the caller should obtain a reference + * using gst_subtitle_block_ref(), if needed. + */ +const GstSubtitleBlock * +gst_subtitle_region_get_block (const GstSubtitleRegion * region, guint index) +{ + g_return_val_if_fail (region != NULL, NULL); + + if (index >= region->blocks->len) + return NULL; + else + return g_ptr_array_index (region->blocks, index); +} diff --git a/ext/ttml/subtitle.h b/ext/ttml/subtitle.h new file mode 100644 index 0000000..95333d3 --- /dev/null +++ b/ext/ttml/subtitle.h @@ -0,0 +1,592 @@ +/* GStreamer + * Copyright (C) <2015> British Broadcasting Corporation + * Author: Chris Bass + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_SUBTITLE_H__ +#define __GST_SUBTITLE_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +typedef struct _GstSubtitleColor GstSubtitleColor; +typedef struct _GstSubtitleStyleSet GstSubtitleStyleSet; +typedef struct _GstSubtitleElement GstSubtitleElement; +typedef struct _GstSubtitleBlock GstSubtitleBlock; +typedef struct _GstSubtitleRegion GstSubtitleRegion; + +/** + * GstSubtitleWritingMode: + * @GST_SUBTITLE_WRITING_MODE_LRTB: Text progression is left-to-right, + * top-to-bottom. + * @GST_SUBTITLE_WRITING_MODE_RLTB: Text progression is right-to-left, + * top-to-bottom. + * @GST_SUBTITLE_WRITING_MODE_TBRL: Text progression is top-to-bottom, + * right-to-left. + * @GST_SUBTITLE_WRITING_MODE_TBLR: Text progression is top-to-bottom, + * left-to-right. + * + * Writing mode of text content. The values define the direction of progression + * of both inline text (#GstSubtitleElements) and blocks of text + * (#GstSubtitleBlocks). + */ +typedef enum { + GST_SUBTITLE_WRITING_MODE_LRTB, + GST_SUBTITLE_WRITING_MODE_RLTB, + GST_SUBTITLE_WRITING_MODE_TBRL, + GST_SUBTITLE_WRITING_MODE_TBLR +} GstSubtitleWritingMode; + +/** + * GstSubtitleDisplayAlign: + * @GST_SUBTITLE_DISPLAY_ALIGN_BEFORE: Blocks should be aligned at the start of + * the containing region. + * @GST_SUBTITLE_DISPLAY_ALIGN_CENTER: Blocks should be aligned in the center + * of the containing region. + * @GST_SUBTITLE_DISPLAY_ALIGN_AFTER: Blocks should be aligned to the end of + * the containing region. + * + * Defines the alignment of text blocks within a region in the direction in + * which blocks are being stacked. For text that is written left-to-right and + * top-to-bottom, this corresponds to the vertical alignment of text blocks. + */ +typedef enum { + GST_SUBTITLE_DISPLAY_ALIGN_BEFORE, + GST_SUBTITLE_DISPLAY_ALIGN_CENTER, + GST_SUBTITLE_DISPLAY_ALIGN_AFTER +} GstSubtitleDisplayAlign; + +/** + * GstSubtitleBackgroundMode: + * @GST_SUBTITLE_BACKGROUND_MODE_ALWAYS: Background rectangle should be visible + * at all times. + * @GST_SUBTITLE_BACKGROUND_MODE_WHEN_ACTIVE: Background rectangle should be + * visible only when text is rendered into the corresponding region. + * + * Defines whether the background rectangle of a region should be visible at + * all times or only when text is rendered within it. + */ +typedef enum { + GST_SUBTITLE_BACKGROUND_MODE_ALWAYS, + GST_SUBTITLE_BACKGROUND_MODE_WHEN_ACTIVE +} GstSubtitleBackgroundMode; + +/** + * GstSubtitleOverflowMode: + * @GST_SUBTITLE_OVERFLOW_MODE_HIDDEN: If text and/or background rectangles + * flowed into the region overflow the bounds of that region, they should + * be clipped at the region boundary. + * @GST_SUBTITLE_OVERFLOW_MODE_VISIBLE: If text and/or background rectangles + * flowed into the region overflow the bounds of that region, they should be + * allowed to overflow the region boundary. + * + * Defines what should happen to text that overflows its containing region. + */ +typedef enum { + GST_SUBTITLE_OVERFLOW_MODE_HIDDEN, + GST_SUBTITLE_OVERFLOW_MODE_VISIBLE +} GstSubtitleOverflowMode; + +/** + * GstSubtitleColor: + * @r: Red value. + * @g: Green value. + * @b: Blue value. + * @a: Alpha value (0 = totally transparent; 255 = totally opaque). + * + * Describes an RGBA color. + */ +struct _GstSubtitleColor { + guint8 r; + guint8 g; + guint8 b; + guint8 a; +}; + +/** + * GstSubtitleTextDirection: + * @GST_SUBTITLE_TEXT_DIRECTION_LTR: Text direction is left-to-right. + * @GST_SUBTITLE_TEXT_DIRECTION_RTL: Text direction is right-to-left. + * + * Defines the progression direction of unicode text that is being treated by + * the unicode bidirectional algorithm as embedded or overidden (see + * http://unicode.org/reports/tr9/ for more details of the unicode + * bidirectional algorithm). + */ +typedef enum { + GST_SUBTITLE_TEXT_DIRECTION_LTR, + GST_SUBTITLE_TEXT_DIRECTION_RTL +} GstSubtitleTextDirection; + +/** + * GstSubtitleTextAlign: + * @GST_SUBTITLE_TEXT_ALIGN_START: Text areas should be rendered at the + * start of the block area, with respect to the direction in which text is + * being rendered. For text that is rendered left-to-right this corresponds to + * the left of the block area; for text that is rendered right-to-left this + * corresponds to the right of the block area. + * @GST_SUBTITLE_TEXT_ALIGN_LEFT: Text areas should be rendered at the left of + * the block area. + * @GST_SUBTITLE_TEXT_ALIGN_CENTER: Text areas should be rendered at the center + * of the block area. + * @GST_SUBTITLE_TEXT_ALIGN_RIGHT: Text areas should be rendered at the right + * of the block area. + * @GST_SUBTITLE_TEXT_ALIGN_END: Text areas should be rendered at the end of + * the block area, with respect to the direction in which text is being + * rendered. For text that is rendered left-to-right this corresponds to the + * right of the block area; for text that is rendered right-to-left this + * corresponds to end of the block area. + * + * Defines how inline text areas within a block should be aligned within the + * block area. + */ +typedef enum { + GST_SUBTITLE_TEXT_ALIGN_START, + GST_SUBTITLE_TEXT_ALIGN_LEFT, + GST_SUBTITLE_TEXT_ALIGN_CENTER, + GST_SUBTITLE_TEXT_ALIGN_RIGHT, + GST_SUBTITLE_TEXT_ALIGN_END +} GstSubtitleTextAlign; + +/** + * GstSubtitleFontStyle: + * @GST_SUBTITLE_FONT_STYLE_NORMAL: Normal font style. + * @GST_SUBTITLE_FONT_STYLE_ITALIC: Italic font style. + * + * Defines styling that should be applied to the glyphs of a font used to + * render text within an inline text element. + */ +typedef enum { + GST_SUBTITLE_FONT_STYLE_NORMAL, + GST_SUBTITLE_FONT_STYLE_ITALIC +} GstSubtitleFontStyle; + +/** + * GstSubtitleFontWeight: + * @GST_SUBTITLE_FONT_WEIGHT_NORMAL: Normal weight. + * @GST_SUBTITLE_FONT_WEIGHT_BOLD: Bold weight. + * + * Defines the font weight that should be applied to the glyphs of a font used + * to render text within an inline text element. + */ +typedef enum { + GST_SUBTITLE_FONT_WEIGHT_NORMAL, + GST_SUBTITLE_FONT_WEIGHT_BOLD +} GstSubtitleFontWeight; + +/** + * GstSubtitleTextDecoration: + * @GST_SUBTITLE_TEXT_DECORATION_NONE: Text should not be decorated. + * @GST_SUBTITLE_TEXT_DECORATION_UNDERLINE: Text should be underlined. + * + * Defines the decoration that should be applied to the glyphs of a font used + * to render text within an inline text element. + */ +typedef enum { + GST_SUBTITLE_TEXT_DECORATION_NONE, + GST_SUBTITLE_TEXT_DECORATION_UNDERLINE +} GstSubtitleTextDecoration; + +/** + * GstSubtitleUnicodeBidi: + * @GST_SUBTITLE_UNICODE_BIDI_NORMAL: Text should progress according the the + * default behaviour of the Unicode bidirectional algorithm. + * @GST_SUBTITLE_UNICODE_BIDI_EMBED: Text should be treated as being embedded + * with a specific direction (given by a #GstSubtitleTextDecoration value + * defined elsewhere). + * @GST_SUBTITLE_UNICODE_BIDI_OVERRIDE: Text should be forced to have a + * specific direction (given by a #GstSubtitleTextDecoration value defined + * elsewhere). + * + * Defines directional embedding or override according to the Unicode + * bidirectional algorithm. See http://unicode.org/reports/tr9/ for more + * details of the Unicode bidirectional algorithm. + */ +typedef enum { + GST_SUBTITLE_UNICODE_BIDI_NORMAL, + GST_SUBTITLE_UNICODE_BIDI_EMBED, + GST_SUBTITLE_UNICODE_BIDI_OVERRIDE +} GstSubtitleUnicodeBidi; + +/** + * GstSubtitleWrapping: + * @GST_SUBTITLE_WRAPPING_ON: Lines that overflow the region boundary should be + * wrapped. + * @GST_SUBTITLE_WRAPPING_OFF: Lines that overflow the region boundary should + * not be wrapped. + * + * Defines how a renderer should treat lines of text that overflow the boundary + * of the region into which they are being rendered. + */ +typedef enum { + GST_SUBTITLE_WRAPPING_ON, + GST_SUBTITLE_WRAPPING_OFF +} GstSubtitleWrapping; + +/** + * GstSubtitleMultiRowAlign: + * @GST_SUBTITLE_MULTI_ROW_ALIGN_AUTO: Lines should be aligned according to the + * value of #GstSubtitleTextAlign associated with that text. + * @GST_SUBTITLE_MULTI_ROW_ALIGN_START: Lines should be aligned at their + * starting edge. The edge that is considered the starting edge depends upon + * the direction of that text. + * @GST_SUBTITLE_MULTI_ROW_ALIGN_CENTER: Lines should be center-aligned. + * @GST_SUBTITLE_MULTI_ROW_ALIGN_END: Lines should be aligned at their trailing + * edge. The edge that is considered the trailing edge depends upon the + * direction of that text. + * + * Defines how multiple 'rows' (i.e, lines) in a block should be aligned + * relative to each other. + * + * This is based upon the ebutts:multiRowAlign attribute defined in the + * EBU-TT-D specification. + */ +typedef enum { + GST_SUBTITLE_MULTI_ROW_ALIGN_AUTO, + GST_SUBTITLE_MULTI_ROW_ALIGN_START, + GST_SUBTITLE_MULTI_ROW_ALIGN_CENTER, + GST_SUBTITLE_MULTI_ROW_ALIGN_END +} GstSubtitleMultiRowAlign; + +/** + * GstSubtitleStyleSet: + * @text_direction: Defines the direction of text that has been declared by the + * #GstSubtitleStyleSet:unicode_bidi attribute to be embbedded or overridden. + * Applies to both #GstSubtitleBlocks and #GstSubtitleElements. + * @font_family: The name of the font family that should be used to render the + * text of an inline element. Applies only to #GstSubtitleElements. + * @font_size: The size of the font that should be used to render the text + * of an inline element. The size is given as a multiple of the display height, + * where 1.0 equals the height of the display. Applies only to + * #GstSubtitleElements. + * @line_height: The inter-baseline separation between lines generated when + * rendering inline text elements within a block area. The height is given as a + * multiple of the the overall display height, where 1.0 equals the height of + * the display. Applies only to #GstSubtitleBlocks. + * @text_align: Controls the alignent of lines of text within a block area. + * Note that this attribute does not control the alignment of lines relative to + * each other within a block area: that is determined by + * #GstSubtitleStyleSet:multi_row_align. Applies only to #GstSubtitleBlocks. + * @color: The color that should be used when rendering the text of an inline + * element. Applies only to #GstSubtitleElements. + * @background_color: The color of the rectangle that should be rendered behind + * the contents of a #GstSubtitleRegion, #GstSubtitleBlock or + * #GstSubtitleElement. + * @font_style: The style of the font that should be used to render the text + * of an inline element. Applies only to #GstSubtitleElements. + * @font_weight: The weight of the font that should be used to render the text + * of an inline element. Applies only to #GstSubtitleElements. + * @text_decoration: The decoration that should be applied to the text of an + * inline element. Applies only to #GstSubtitleElements. + * @unicode_bidi: Controls how unicode text within a block or inline element + * should be treated by the unicode bidirectional algorithm. Applies to both + * #GstSubtitleBlocks and #GstSubtitleElements. + * @wrap_option: Defines whether or not automatic line breaking should apply to + * the lines generated when rendering a block of text elements. Applies only to + * #GstSubtitleBlocks. + * @multi_row_align: Defines how 'rows' (i.e., lines) within a text block + * should be aligned relative to each other. Note that this attribute does not + * determine how a block of text is aligned within that block area: that is + * determined by @text_align. Applies only to #GstSubtitleBlocks. + * @line_padding: Defines how much horizontal padding should be added on the + * start and end of each rendered line; this allows the insertion of space + * between the start/end of text lines and their background rectangles for + * better-looking subtitles. This is based upon the ebutts:linePadding + * attribute defined in the EBU-TT-D specification. Applies only to + * #GstSubtitleBlocks. + * @origin_x: The horizontal origin of a region into which text blocks may be + * rendered. Given as a multiple of the overall display width, where 1.0 equals + * the width of the display. Applies only to #GstSubtitleRegions. + * @origin_y: The vertical origin of a region into which text blocks may be + * rendered. Given as a multiple of the overall display height, where 1.0 + * equals the height of the display. Applies only to #GstSubtitleRegions. + * @extent_w: The horizontal extent of a region into which text blocks may be + * rendered. Given as a multiple of the overall display width, where 1.0 equals + * the width of the display. Applies only to #GstSubtitleRegions. + * @extent_h: The vertical extent of a region into which text blocks may be + * rendered. Given as a multiple of the overall display height, where 1.0 + * equals the height of the display. Applies only to #GstSubtitleRegions. + * @display_align: The alignment of generated text blocks in the direction in + * which blocks are being stacked. For text that flows left-to-right and + * top-to-bottom, for example, this corresponds to the vertical alignment of + * text blocks. Applies only to #GstSubtitleRegions. + * @padding_start: The horizontal indent of text from the leading edge of a + * region into which blocks may be rendered. Given as a multiple of the overall + * display width, where 1.0 equals the width of the display. Applies only to + * #GstSubtitleRegions. + * @padding_end: The horizontal indent of text from the trailing edge of a + * region into which blocks may be rendered. Given as a multiple of the overall + * display width, where 1.0 equals the width of the display. Applies only to + * #GstSubtitleRegions. + * @padding_before: The vertical indent of text from the top edge of a region + * into which blocks may be rendered. Given as a multiple of the overall + * display height, where 1.0 equals the height of the display. Applies only to + * #GstSubtitleRegions. + * @padding_after: The vertical indent of text from the bottom edge of a + * region into which blocks may be rendered. Given as a multiple of the overall + * display height, where 1.0 equals the height of the display. Applies only to + * #GstSubtitleRegions. + * @writing_mode: Defines the direction in which both inline elements and + * blocks should be stacked when rendered into an on-screen region. Applies + * only to #GstSubtitleRegions. + * @show_background: Defines whether the background of a region should be + * displayed at all times or only when it has text rendered into it. Applies + * only to #GstSubtitleRegions. + * @overflow: Defines what should happen if text and background rectangles + * generated by rendering text blocks overflow the size of their containing + * region. Applies only to #GstSubtitleRegions. + * + * Holds a set of attributes that describes the styling and layout that apply + * to #GstSubtitleRegion, #GstSubtitleBlock and/or #GstSubtitleElement objects. + * + * Note that, though each of the above object types have an associated + * #GstSubtitleStyleSet, not all attributes in a #GstSubtitleStyleSet type + * apply to all object types: #GstSubtitleStyleSet:overflow applies only to + * #GstSubtitleRegions, for example, while #GstSubtitleStyleSet:font_style + * applies only to #GstSubtitleElements. Some attributes apply to multiple + * object types: #GstSubtitleStyleSet:background_color, for example, applies to + * all object types. The types to which each attribute applies is given in the + * description of that attribute below. + */ +struct _GstSubtitleStyleSet { + GstSubtitleTextDirection text_direction; + gchar *font_family; + gdouble font_size; + gdouble line_height; + GstSubtitleTextAlign text_align; + GstSubtitleColor color; + GstSubtitleColor background_color; + GstSubtitleFontStyle font_style; + GstSubtitleFontWeight font_weight; + GstSubtitleTextDecoration text_decoration; + GstSubtitleUnicodeBidi unicode_bidi; + GstSubtitleWrapping wrap_option; + GstSubtitleMultiRowAlign multi_row_align; + gdouble line_padding; + gdouble origin_x, origin_y; + gdouble extent_w, extent_h; + GstSubtitleDisplayAlign display_align; + gdouble padding_start, padding_end, padding_before, padding_after; + GstSubtitleWritingMode writing_mode; + GstSubtitleBackgroundMode show_background; + GstSubtitleOverflowMode overflow; +}; + +GstSubtitleStyleSet * gst_subtitle_style_set_new (void); + +void gst_subtitle_style_set_free (GstSubtitleStyleSet * style_set); + + +/** + * GstSubtitleElement: + * @mini_object: The parent #GstMiniObject. + * @style_set: Styling associated with this element. + * @text_index: Index into the #GstBuffer associated with this + * #GstSubtitleElement; the index identifies the #GstMemory within the + * #GstBuffer that holds the #GstSubtitleElement's text. + * @suppress_whitespace: Indicates whether or not a renderer should suppress + * whitespace in the element's text. + * + * Represents an inline text element. + * + * In TTML this would correspond to inline text resulting from a + * element, an anonymous span (e.g., text within a tag), or a + * element. + */ +struct _GstSubtitleElement +{ + GstMiniObject mini_object; + + GstSubtitleStyleSet *style_set; + guint text_index; + gboolean suppress_whitespace; + + /*< private >*/ + gpointer _gst_reserved[GST_PADDING]; +}; + +GType gst_subtitle_element_get_type (void); + +GstSubtitleElement * gst_subtitle_element_new (GstSubtitleStyleSet * style_set, + guint text_index, gboolean suppress_whitespace); + +/** + * gst_subtitle_element_ref: + * @element: A #GstSubtitleElement. + * + * Increments the refcount of @element. + * + * Returns: (transfer full): @element. + */ +static inline GstSubtitleElement * +gst_subtitle_element_ref (GstSubtitleElement * element) +{ + return (GstSubtitleElement *) + gst_mini_object_ref (GST_MINI_OBJECT_CAST (element)); +} + +/** + * gst_subtitle_element_unref: + * @element: (transfer full): A #GstSubtitleElement. + * + * Decrements the refcount of @element. If the refcount reaches 0, @element + * will be freed. + */ +static inline void +gst_subtitle_element_unref (GstSubtitleElement * element) +{ + gst_mini_object_unref (GST_MINI_OBJECT_CAST (element)); +} + + +/** + * GstSubtitleBlock: + * @mini_object: The parent #GstMiniObject. + * @style_set: Styling associated with this block. + * + * Represents a text block made up of one or more inline text elements (i.e., + * one or more #GstSubtitleElements). + * + * In TTML this would correspond to the block of text resulting from the inline + * elements within a single . + */ +struct _GstSubtitleBlock +{ + GstMiniObject mini_object; + + GstSubtitleStyleSet *style_set; + + /*< private >*/ + GPtrArray *elements; + gpointer _gst_reserved[GST_PADDING]; +}; + +GType gst_subtitle_block_get_type (void); + +GstSubtitleBlock * gst_subtitle_block_new (GstSubtitleStyleSet * style_set); + +void gst_subtitle_block_add_element ( + GstSubtitleBlock * block, + GstSubtitleElement * element); + +guint gst_subtitle_block_get_element_count (const GstSubtitleBlock * block); + +const GstSubtitleElement * gst_subtitle_block_get_element ( + const GstSubtitleBlock * block, guint index); + +/** + * gst_subtitle_block_ref: + * @block: A #GstSubtitleBlock. + * + * Increments the refcount of @block. + * + * Returns: (transfer full): @block. + */ +static inline GstSubtitleBlock * +gst_subtitle_block_ref (GstSubtitleBlock * block) +{ + return (GstSubtitleBlock *) + gst_mini_object_ref (GST_MINI_OBJECT_CAST (block)); +} + +/** + * gst_subtitle_block_unref: + * @block: (transfer full): A #GstSubtitleBlock. + * + * Decrements the refcount of @block. If the refcount reaches 0, @block will + * be freed. + */ +static inline void +gst_subtitle_block_unref (GstSubtitleBlock * block) +{ + gst_mini_object_unref (GST_MINI_OBJECT_CAST (block)); +} + + +/** + * GstSubtitleRegion: + * @mini_object: The parent #GstMiniObject. + * @style_set: Styling associated with this region. + * + * Represents an on-screen region in which is displayed zero or more + * #GstSubtitleBlocks. + * + * In TTML this corresponds to a <region> into which zero or more + * s may be rendered. A #GstSubtitleRegion allows a background + * rectangle to be displayed in a region area even if no text blocks are + * rendered into it, as per the behaviour allowed by TTML regions whose + * tts:showBackground style attribute is set to "always". + */ +struct _GstSubtitleRegion +{ + GstMiniObject mini_object; + + GstSubtitleStyleSet *style_set; + + /*< private >*/ + GPtrArray *blocks; + gpointer _gst_reserved[GST_PADDING]; +}; + +GType gst_subtitle_region_get_type (void); + +GstSubtitleRegion * gst_subtitle_region_new (GstSubtitleStyleSet * style_set); + +void gst_subtitle_region_add_block ( + GstSubtitleRegion * region, + GstSubtitleBlock * block); + +guint gst_subtitle_region_get_block_count (const GstSubtitleRegion * region); + +const GstSubtitleBlock * gst_subtitle_region_get_block ( + const GstSubtitleRegion * region, guint index); + +/** + * gst_subtitle_region_ref: + * @region: A #GstSubtitleRegion. + * + * Increments the refcount of @region. + * + * Returns: (transfer full): @region. + */ +static inline GstSubtitleRegion * +gst_subtitle_region_ref (GstSubtitleRegion * region) +{ + return (GstSubtitleRegion *) + gst_mini_object_ref (GST_MINI_OBJECT_CAST (region)); +} + +/** + * gst_subtitle_region_unref: + * @region: (transfer full): A #GstSubtitleRegion. + * + * Decrements the refcount of @region. If the refcount reaches 0, @region will + * be freed. + */ +static inline void +gst_subtitle_region_unref (GstSubtitleRegion * region) +{ + gst_mini_object_unref (GST_MINI_OBJECT_CAST (region)); +} + +G_END_DECLS + +#endif /* __GST_SUBTITLE_H__ */ diff --git a/ext/ttml/subtitlemeta.c b/ext/ttml/subtitlemeta.c new file mode 100644 index 0000000..69da5f5 --- /dev/null +++ b/ext/ttml/subtitlemeta.c @@ -0,0 +1,101 @@ +/* GStreamer + * Copyright (C) <2015> British Broadcasting Corporation + * Author: Chris Bass + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstsubtitlemeta + * @short_description: Metadata class for timed-text subtitles. + * + * The GstSubtitleMeta class enables the layout and styling information needed + * to render subtitle text to be attached to a #GstBuffer containing that text. + */ + +#include "subtitlemeta.h" + +GType +gst_subtitle_meta_api_get_type (void) +{ + static volatile GType type; + static const gchar *tags[] = { "memory", NULL }; + + if (g_once_init_enter (&type)) { + GType _type = gst_meta_api_type_register ("GstSubtitleMetaAPI", tags); + g_once_init_leave (&type, _type); + } + return type; +} + +gboolean +gst_subtitle_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) +{ + GstSubtitleMeta *subtitle_meta = (GstSubtitleMeta *) meta; + + subtitle_meta->regions = NULL; + return TRUE; +} + +void +gst_subtitle_meta_free (GstMeta * meta, GstBuffer * buffer) +{ + GstSubtitleMeta *subtitle_meta = (GstSubtitleMeta *) meta; + + if (subtitle_meta->regions) + g_ptr_array_unref (subtitle_meta->regions); +} + +const GstMetaInfo * +gst_subtitle_meta_get_info (void) +{ + static const GstMetaInfo *subtitle_meta_info = NULL; + + if (g_once_init_enter (&subtitle_meta_info)) { + const GstMetaInfo *meta = + gst_meta_register (GST_SUBTITLE_META_API_TYPE, "GstSubtitleMeta", + sizeof (GstSubtitleMeta), gst_subtitle_meta_init, + gst_subtitle_meta_free, (GstMetaTransformFunction) NULL); + g_once_init_leave (&subtitle_meta_info, meta); + } + return subtitle_meta_info; +} + +/** + * gst_buffer_add_subtitle_meta: + * @buffer: (transfer none): #GstBuffer holding subtitle text, to which + * subtitle metadata should be added. + * @regions: (transfer full): A #GPtrArray of #GstSubtitleRegions. + * + * Attaches subtitle metadata to a #GstBuffer. + * + * Returns: A pointer to the added #GstSubtitleMeta if successful; %NULL if + * unsuccessful. + */ +GstSubtitleMeta * +gst_buffer_add_subtitle_meta (GstBuffer * buffer, GPtrArray * regions) +{ + GstSubtitleMeta *meta; + + g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL); + g_return_val_if_fail (regions != NULL, NULL); + + meta = (GstSubtitleMeta *) gst_buffer_add_meta (buffer, + GST_SUBTITLE_META_INFO, NULL); + + meta->regions = regions; + return meta; +} diff --git a/ext/ttml/subtitlemeta.h b/ext/ttml/subtitlemeta.h new file mode 100644 index 0000000..e533451 --- /dev/null +++ b/ext/ttml/subtitlemeta.h @@ -0,0 +1,66 @@ +/* GStreamer + * Copyright (C) <2015> British Broadcasting Corporation + * Author: Chris Bass + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_SUBTITLE_META_H__ +#define __GST_SUBTITLE_META_H__ + +#include +#include "subtitle.h" + +G_BEGIN_DECLS + +typedef struct _GstSubtitleMeta GstSubtitleMeta; + +/** + * GstSubtitleMeta: + * @meta: The parent #GstMeta. + * @regions: The #GstSubtitleRegions containing layout and styling information + * needed to render the subtitle text contained in the associated #GstBuffer. + * + * Metadata type that describes the layout and styling of subtitle text + * contained in a #GstBuffer. + */ +struct _GstSubtitleMeta { + GstMeta meta; + + GPtrArray *regions; +}; + +GType gst_subtitle_meta_api_get_type (void); +#define GST_SUBTITLE_META_API_TYPE (gst_subtitle_meta_api_get_type()) + +#define gst_buffer_get_subtitle_meta(b) \ + ((GstSubtitleMeta*)gst_buffer_get_meta ((b), GST_SUBTITLE_META_API_TYPE)) + +#define GST_SUBTITLE_META_INFO (gst_subtitle_meta_get_info()) + +gboolean gst_subtitle_meta_init (GstMeta * meta, gpointer params, + GstBuffer * buffer); + +void gst_subtitle_meta_free (GstMeta * meta, GstBuffer * buffer); + +const GstMetaInfo * gst_subtitle_meta_get_info (void); + +GstSubtitleMeta * gst_buffer_add_subtitle_meta (GstBuffer * buffer, + GPtrArray * regions); + +G_END_DECLS + +#endif /* __GST_SUBTITLE_META_H__ */ diff --git a/ext/ttml/ttmlparse.c b/ext/ttml/ttmlparse.c new file mode 100644 index 0000000..51bc5b6 --- /dev/null +++ b/ext/ttml/ttmlparse.c @@ -0,0 +1,1813 @@ +/* GStreamer TTML subtitle parser + * Copyright (C) <2015> British Broadcasting Corporation + * Authors: + * Chris Bass + * Peter Taylour + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/* + * Parses subtitle files encoded using the EBU-TT-D profile of TTML, as defined + * in https://tech.ebu.ch/files/live/sites/tech/files/shared/tech/tech3380.pdf + * and http://www.w3.org/TR/ttaf1-dfxp/, respectively. + */ + +#include + +#include +#include +#include +#include +#include +#include + +#include "ttmlparse.h" +#include "subtitle.h" +#include "subtitlemeta.h" + +#define DEFAULT_CELLRES_X 32 +#define DEFAULT_CELLRES_Y 15 +#define MAX_FONT_FAMILY_NAME_LENGTH 128 + +GST_DEBUG_CATEGORY_EXTERN (ttmlparse_debug); +#define GST_CAT_DEFAULT ttmlparse_debug + +static gchar *ttml_get_xml_property (const xmlNode * node, const char *name); + +typedef struct _TtmlStyleSet TtmlStyleSet; +typedef struct _TtmlElement TtmlElement; +typedef struct _TtmlScene TtmlScene; + +typedef enum +{ + TTML_ELEMENT_TYPE_STYLE, + TTML_ELEMENT_TYPE_REGION, + TTML_ELEMENT_TYPE_BODY, + TTML_ELEMENT_TYPE_DIV, + TTML_ELEMENT_TYPE_P, + TTML_ELEMENT_TYPE_SPAN, + TTML_ELEMENT_TYPE_ANON_SPAN, + TTML_ELEMENT_TYPE_BR +} TtmlElementType; + +typedef enum +{ + TTML_WHITESPACE_MODE_NONE, + TTML_WHITESPACE_MODE_DEFAULT, + TTML_WHITESPACE_MODE_PRESERVE, +} TtmlWhitespaceMode; + +struct _TtmlElement +{ + TtmlElementType type; + gchar *id; + TtmlWhitespaceMode whitespace_mode; + gchar **styles; + gchar *region; + GstClockTime begin; + GstClockTime end; + TtmlStyleSet *style_set; + gchar *text; + guint text_index; +}; + +/* Represents a static scene consisting of one or more trees of elements that + * should be visible over a specific period of time. */ +struct _TtmlScene +{ + GstClockTime begin; + GstClockTime end; + GList *trees; + GstBuffer *buf; +}; + +struct _TtmlStyleSet +{ + GHashTable *table; +}; + + +static TtmlStyleSet * +ttml_style_set_new (void) +{ + TtmlStyleSet *ret = g_slice_new0 (TtmlStyleSet); + ret->table = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + return ret; +} + + +static void +ttml_style_set_delete (TtmlStyleSet * style_set) +{ + if (style_set) { + g_hash_table_unref (style_set->table); + g_slice_free (TtmlStyleSet, style_set); + } +} + + +/* If attribute with name @attr_name already exists in @style_set, its value + * will be replaced by @attr_value. */ +static gboolean +ttml_style_set_add_attr (TtmlStyleSet * style_set, const gchar * attr_name, + const gchar * attr_value) +{ + return g_hash_table_insert (style_set->table, g_strdup (attr_name), + g_strdup (attr_value)); +} + + +static gboolean +ttml_style_set_contains_attr (TtmlStyleSet * style_set, const gchar * attr_name) +{ + return g_hash_table_contains (style_set->table, attr_name); +} + + +static const gchar * +ttml_style_set_get_attr (TtmlStyleSet * style_set, const gchar * attr_name) +{ + return g_hash_table_lookup (style_set->table, attr_name); +} + + +static guint8 +ttml_hex_pair_to_byte (const gchar * hex_pair) +{ + gint hi_digit, lo_digit; + + hi_digit = g_ascii_xdigit_value (*hex_pair); + lo_digit = g_ascii_xdigit_value (*(hex_pair + 1)); + return (hi_digit << 4) + lo_digit; +} + + +/* Color strings in EBU-TT-D can have the form "#RRBBGG" or "#RRBBGGAA". */ +static GstSubtitleColor +ttml_parse_colorstring (const gchar * color) +{ + guint length; + const gchar *c = NULL; + GstSubtitleColor ret = { 0, 0, 0, 0 }; + + if (!color) + return ret; + + length = strlen (color); + if (((length == 7) || (length == 9)) && *color == '#') { + c = color + 1; + + ret.r = ttml_hex_pair_to_byte (c); + ret.g = ttml_hex_pair_to_byte (c + 2); + ret.b = ttml_hex_pair_to_byte (c + 4); + + if (length == 7) + ret.a = G_MAXUINT8; + else + ret.a = ttml_hex_pair_to_byte (c + 6); + + GST_CAT_LOG (ttmlparse_debug, "Returning color - r:%u b:%u g:%u a:%u", + ret.r, ret.b, ret.g, ret.a); + } else { + GST_CAT_ERROR (ttmlparse_debug, "Invalid color string: %s", color); + } + + return ret; +} + + +static void +ttml_style_set_print (TtmlStyleSet * style_set) +{ + GHashTableIter iter; + gpointer attr_name, attr_value; + + if (!style_set) { + GST_CAT_LOG (ttmlparse_debug, "\t\t[NULL]"); + return; + } + + g_hash_table_iter_init (&iter, style_set->table); + while (g_hash_table_iter_next (&iter, &attr_name, &attr_value)) { + GST_CAT_LOG (ttmlparse_debug, "\t\t%s: %s", (const gchar *) attr_name, + (const gchar *) attr_value); + } +} + + +static TtmlStyleSet * +ttml_parse_style_set (const xmlNode * node) +{ + TtmlStyleSet *s; + gchar *value = NULL; + xmlAttrPtr attr; + + value = ttml_get_xml_property (node, "id"); + if (!value) { + GST_CAT_ERROR (ttmlparse_debug, "styles must have an ID."); + return NULL; + } + g_free (value); + + s = ttml_style_set_new (); + + for (attr = node->properties; attr != NULL; attr = attr->next) { + if (attr->ns && ((g_strcmp0 ((const gchar *) attr->ns->prefix, "tts") == 0) + || (g_strcmp0 ((const gchar *) attr->ns->prefix, "ebutts") == 0))) { + ttml_style_set_add_attr (s, (const gchar *) attr->name, + (const gchar *) attr->children->content); + } + } + + return s; +} + + +static void +ttml_delete_element (TtmlElement * element) +{ + g_free ((gpointer) element->id); + if (element->styles) + g_strfreev (element->styles); + g_free ((gpointer) element->region); + ttml_style_set_delete (element->style_set); + g_free ((gpointer) element->text); + g_slice_free (TtmlElement, element); +} + + +static gchar * +ttml_get_xml_property (const xmlNode * node, const char *name) +{ + xmlChar *xml_string = NULL; + gchar *gst_string = NULL; + + g_return_val_if_fail (strlen (name) < 128, NULL); + + xml_string = xmlGetProp (node, (xmlChar *) name); + if (!xml_string) + return NULL; + gst_string = g_strdup ((gchar *) xml_string); + xmlFree (xml_string); + return gst_string; +} + + +/* EBU-TT-D timecodes have format hours:minutes:seconds[.fraction] */ +static GstClockTime +ttml_parse_timecode (const gchar * timestring) +{ + gchar **strings; + guint64 hours = 0, minutes = 0, seconds = 0, milliseconds = 0; + GstClockTime time = GST_CLOCK_TIME_NONE; + + GST_CAT_LOG (ttmlparse_debug, "time string: %s", timestring); + + strings = g_strsplit (timestring, ":", 3); + if (g_strv_length (strings) != 3U) { + GST_CAT_ERROR (ttmlparse_debug, "badly formatted time string: %s", + timestring); + return time; + } + + hours = g_ascii_strtoull (strings[0], NULL, 10U); + minutes = g_ascii_strtoull (strings[1], NULL, 10U); + if (g_strstr_len (strings[2], -1, ".")) { + guint n_digits; + gchar **substrings = g_strsplit (strings[2], ".", 2); + seconds = g_ascii_strtoull (substrings[0], NULL, 10U); + n_digits = strlen (substrings[1]); + milliseconds = g_ascii_strtoull (substrings[1], NULL, 10U); + milliseconds = + (guint64) (milliseconds * pow (10.0, (3 - (double) n_digits))); + g_strfreev (substrings); + } else { + seconds = g_ascii_strtoull (strings[2], NULL, 10U); + } + + if (minutes > 59 || seconds > 60) { + GST_CAT_ERROR (ttmlparse_debug, "invalid time string " + "(minutes or seconds out-of-bounds): %s\n", timestring); + } + + g_strfreev (strings); + GST_CAT_LOG (ttmlparse_debug, + "hours: %" G_GUINT64_FORMAT " minutes: %" G_GUINT64_FORMAT + " seconds: %" G_GUINT64_FORMAT " milliseconds: %" G_GUINT64_FORMAT "", + hours, minutes, seconds, milliseconds); + + time = hours * GST_SECOND * 3600 + + minutes * GST_SECOND * 60 + + seconds * GST_SECOND + milliseconds * GST_MSECOND; + + return time; +} + + +static TtmlElement * +ttml_parse_element (const xmlNode * node) +{ + TtmlElement *element; + TtmlElementType type; + gchar *value; + + GST_CAT_DEBUG (ttmlparse_debug, "Element name: %s", + (const char *) node->name); + if ((g_strcmp0 ((const char *) node->name, "style") == 0)) { + type = TTML_ELEMENT_TYPE_STYLE; + } else if ((g_strcmp0 ((const char *) node->name, "region") == 0)) { + type = TTML_ELEMENT_TYPE_REGION; + } else if ((g_strcmp0 ((const char *) node->name, "body") == 0)) { + type = TTML_ELEMENT_TYPE_BODY; + } else if ((g_strcmp0 ((const char *) node->name, "div") == 0)) { + type = TTML_ELEMENT_TYPE_DIV; + } else if ((g_strcmp0 ((const char *) node->name, "p") == 0)) { + type = TTML_ELEMENT_TYPE_P; + } else if ((g_strcmp0 ((const char *) node->name, "span") == 0)) { + type = TTML_ELEMENT_TYPE_SPAN; + } else if ((g_strcmp0 ((const char *) node->name, "text") == 0)) { + type = TTML_ELEMENT_TYPE_ANON_SPAN; + } else if ((g_strcmp0 ((const char *) node->name, "br") == 0)) { + type = TTML_ELEMENT_TYPE_BR; + } else { + return NULL; + } + + element = g_slice_new0 (TtmlElement); + element->type = type; + + if ((value = ttml_get_xml_property (node, "id"))) { + element->id = g_strdup (value); + g_free (value); + } + + if ((value = ttml_get_xml_property (node, "style"))) { + element->styles = g_strsplit (value, " ", 0); + GST_CAT_DEBUG (ttmlparse_debug, "%u style(s) referenced in element.", + g_strv_length (element->styles)); + g_free (value); + } + + if (element->type == TTML_ELEMENT_TYPE_STYLE + || element->type == TTML_ELEMENT_TYPE_REGION) { + TtmlStyleSet *ss; + ss = ttml_parse_style_set (node); + if (ss) + element->style_set = ss; + else + GST_CAT_WARNING (ttmlparse_debug, + "Style or Region contains no styling attributes."); + } + + if ((value = ttml_get_xml_property (node, "region"))) { + element->region = g_strdup (value); + g_free (value); + } + + if ((value = ttml_get_xml_property (node, "begin"))) { + element->begin = ttml_parse_timecode (value); + g_free (value); + } else { + element->begin = GST_CLOCK_TIME_NONE; + } + + if ((value = ttml_get_xml_property (node, "end"))) { + element->end = ttml_parse_timecode (value); + g_free (value); + } else { + element->end = GST_CLOCK_TIME_NONE; + } + + if (node->content) { + GST_CAT_LOG (ttmlparse_debug, "Node content: %s", node->content); + element->text = g_strdup ((const gchar *) node->content); + } + + if ((value = ttml_get_xml_property (node, "space"))) { + if (g_strcmp0 (value, "preserve") == 0) + element->whitespace_mode = TTML_WHITESPACE_MODE_PRESERVE; + else if (g_strcmp0 (value, "default") == 0) + element->whitespace_mode = TTML_WHITESPACE_MODE_DEFAULT; + g_free (value); + } + + return element; +} + + +static GNode * +ttml_parse_body (const xmlNode * node) +{ + GNode *ret; + TtmlElement *element; + + GST_CAT_LOG (ttmlparse_debug, "parsing node %s", node->name); + element = ttml_parse_element (node); + if (element) + ret = g_node_new (element); + else + return NULL; + + for (node = node->children; node != NULL; node = node->next) { + GNode *descendants = NULL; + if (!xmlIsBlankNode (node) && (descendants = ttml_parse_body (node))) + g_node_append (ret, descendants); + } + + return ret; +} + + +/* Update the fields of a GstSubtitleStyleSet, @style_set, according to the + * values defined in a TtmlStyleSet, @tss, and a given cell resolution. */ +static void +ttml_update_style_set (GstSubtitleStyleSet * style_set, TtmlStyleSet * tss, + guint cellres_x, guint cellres_y) +{ + const gchar *attr; + + if ((attr = ttml_style_set_get_attr (tss, "textDirection"))) { + if (g_strcmp0 (attr, "rtl") == 0) + style_set->text_direction = GST_SUBTITLE_TEXT_DIRECTION_RTL; + else + style_set->text_direction = GST_SUBTITLE_TEXT_DIRECTION_LTR; + } + + if ((attr = ttml_style_set_get_attr (tss, "fontFamily"))) { + if (strlen (attr) <= MAX_FONT_FAMILY_NAME_LENGTH) { + g_free (style_set->font_family); + style_set->font_family = g_strdup (attr); + } else { + GST_CAT_WARNING (ttmlparse_debug, + "Ignoring font family name as it's overly long."); + } + } + + if ((attr = ttml_style_set_get_attr (tss, "fontSize"))) { + style_set->font_size = g_ascii_strtod (attr, NULL) / 100.0; + } + style_set->font_size *= (1.0 / cellres_y); + + if ((attr = ttml_style_set_get_attr (tss, "lineHeight"))) { + /* The TTML spec (section 8.2.12) recommends using a line height of 125% + * when "normal" is specified. */ + if (g_strcmp0 (attr, "normal") == 0) + style_set->line_height = 1.25; + else + style_set->line_height = g_ascii_strtod (attr, NULL) / 100.0; + } + + if ((attr = ttml_style_set_get_attr (tss, "textAlign"))) { + if (g_strcmp0 (attr, "left") == 0) + style_set->text_align = GST_SUBTITLE_TEXT_ALIGN_LEFT; + else if (g_strcmp0 (attr, "center") == 0) + style_set->text_align = GST_SUBTITLE_TEXT_ALIGN_CENTER; + else if (g_strcmp0 (attr, "right") == 0) + style_set->text_align = GST_SUBTITLE_TEXT_ALIGN_RIGHT; + else if (g_strcmp0 (attr, "end") == 0) + style_set->text_align = GST_SUBTITLE_TEXT_ALIGN_END; + else + style_set->text_align = GST_SUBTITLE_TEXT_ALIGN_START; + } + + if ((attr = ttml_style_set_get_attr (tss, "color"))) { + style_set->color = ttml_parse_colorstring (attr); + } + + if ((attr = ttml_style_set_get_attr (tss, "backgroundColor"))) { + style_set->background_color = ttml_parse_colorstring (attr); + } + + if ((attr = ttml_style_set_get_attr (tss, "fontStyle"))) { + if (g_strcmp0 (attr, "italic") == 0) + style_set->font_style = GST_SUBTITLE_FONT_STYLE_ITALIC; + else + style_set->font_style = GST_SUBTITLE_FONT_STYLE_NORMAL; + } + + if ((attr = ttml_style_set_get_attr (tss, "fontWeight"))) { + if (g_strcmp0 (attr, "bold") == 0) + style_set->font_weight = GST_SUBTITLE_FONT_WEIGHT_BOLD; + else + style_set->font_weight = GST_SUBTITLE_FONT_WEIGHT_NORMAL; + } + + if ((attr = ttml_style_set_get_attr (tss, "textDecoration"))) { + if (g_strcmp0 (attr, "underline") == 0) + style_set->text_decoration = GST_SUBTITLE_TEXT_DECORATION_UNDERLINE; + else + style_set->text_decoration = GST_SUBTITLE_TEXT_DECORATION_NONE; + } + + if ((attr = ttml_style_set_get_attr (tss, "unicodeBidi"))) { + if (g_strcmp0 (attr, "embed") == 0) + style_set->unicode_bidi = GST_SUBTITLE_UNICODE_BIDI_EMBED; + else if (g_strcmp0 (attr, "bidiOverride") == 0) + style_set->unicode_bidi = GST_SUBTITLE_UNICODE_BIDI_OVERRIDE; + else + style_set->unicode_bidi = GST_SUBTITLE_UNICODE_BIDI_NORMAL; + } + + if ((attr = ttml_style_set_get_attr (tss, "wrapOption"))) { + if (g_strcmp0 (attr, "noWrap") == 0) + style_set->wrap_option = GST_SUBTITLE_WRAPPING_OFF; + else + style_set->wrap_option = GST_SUBTITLE_WRAPPING_ON; + } + + if ((attr = ttml_style_set_get_attr (tss, "multiRowAlign"))) { + if (g_strcmp0 (attr, "start") == 0) + style_set->multi_row_align = GST_SUBTITLE_MULTI_ROW_ALIGN_START; + else if (g_strcmp0 (attr, "center") == 0) + style_set->multi_row_align = GST_SUBTITLE_MULTI_ROW_ALIGN_CENTER; + else if (g_strcmp0 (attr, "end") == 0) + style_set->multi_row_align = GST_SUBTITLE_MULTI_ROW_ALIGN_END; + else + style_set->multi_row_align = GST_SUBTITLE_MULTI_ROW_ALIGN_AUTO; + } + + if ((attr = ttml_style_set_get_attr (tss, "linePadding"))) { + style_set->line_padding = g_ascii_strtod (attr, NULL); + style_set->line_padding *= (1.0 / cellres_x); + } + + if ((attr = ttml_style_set_get_attr (tss, "origin"))) { + gchar *c; + style_set->origin_x = g_ascii_strtod (attr, &c) / 100.0; + while (!g_ascii_isdigit (*c) && *c != '+' && *c != '-') + ++c; + style_set->origin_y = g_ascii_strtod (c, NULL) / 100.0; + } + + if ((attr = ttml_style_set_get_attr (tss, "extent"))) { + gchar *c; + style_set->extent_w = g_ascii_strtod (attr, &c) / 100.0; + if ((style_set->origin_x + style_set->extent_w) > 1.0) { + style_set->extent_w = 1.0 - style_set->origin_x; + } + while (!g_ascii_isdigit (*c) && *c != '+' && *c != '-') + ++c; + style_set->extent_h = g_ascii_strtod (c, NULL) / 100.0; + if ((style_set->origin_y + style_set->extent_h) > 1.0) { + style_set->extent_h = 1.0 - style_set->origin_y; + } + } + + if ((attr = ttml_style_set_get_attr (tss, "displayAlign"))) { + if (g_strcmp0 (attr, "center") == 0) + style_set->display_align = GST_SUBTITLE_DISPLAY_ALIGN_CENTER; + else if (g_strcmp0 (attr, "after") == 0) + style_set->display_align = GST_SUBTITLE_DISPLAY_ALIGN_AFTER; + else + style_set->display_align = GST_SUBTITLE_DISPLAY_ALIGN_BEFORE; + } + + if ((attr = ttml_style_set_get_attr (tss, "padding"))) { + gchar **decimals; + guint n_decimals; + guint i; + + decimals = g_strsplit (attr, "%", 0); + n_decimals = g_strv_length (decimals) - 1; + for (i = 0; i < n_decimals; ++i) + g_strstrip (decimals[i]); + + switch (n_decimals) { + case 1: + style_set->padding_start = style_set->padding_end = + style_set->padding_before = style_set->padding_after = + g_ascii_strtod (decimals[0], NULL) / 100.0; + break; + + case 2: + style_set->padding_before = style_set->padding_after = + g_ascii_strtod (decimals[0], NULL) / 100.0; + style_set->padding_start = style_set->padding_end = + g_ascii_strtod (decimals[1], NULL) / 100.0; + break; + + case 3: + style_set->padding_before = g_ascii_strtod (decimals[0], NULL) / 100.0; + style_set->padding_start = style_set->padding_end = + g_ascii_strtod (decimals[1], NULL) / 100.0; + style_set->padding_after = g_ascii_strtod (decimals[2], NULL) / 100.0; + break; + + case 4: + style_set->padding_before = g_ascii_strtod (decimals[0], NULL) / 100.0; + style_set->padding_end = g_ascii_strtod (decimals[1], NULL) / 100.0; + style_set->padding_after = g_ascii_strtod (decimals[2], NULL) / 100.0; + style_set->padding_start = g_ascii_strtod (decimals[3], NULL) / 100.0; + break; + } + g_strfreev (decimals); + + /* Padding values in TTML files are relative to the region width & height; + * make them relative to the overall display width & height like all other + * dimensions. */ + style_set->padding_before *= style_set->extent_h; + style_set->padding_after *= style_set->extent_h; + style_set->padding_end *= style_set->extent_w; + style_set->padding_start *= style_set->extent_w; + } + + if ((attr = ttml_style_set_get_attr (tss, "writingMode"))) { + if (g_str_has_prefix (attr, "rl")) + style_set->writing_mode = GST_SUBTITLE_WRITING_MODE_RLTB; + else if ((g_strcmp0 (attr, "tbrl") == 0) + || (g_strcmp0 (attr, "tb") == 0)) + style_set->writing_mode = GST_SUBTITLE_WRITING_MODE_TBRL; + else if (g_strcmp0 (attr, "tblr") == 0) + style_set->writing_mode = GST_SUBTITLE_WRITING_MODE_TBLR; + else + style_set->writing_mode = GST_SUBTITLE_WRITING_MODE_LRTB; + } + + if ((attr = ttml_style_set_get_attr (tss, "showBackground"))) { + if (g_strcmp0 (attr, "whenActive") == 0) + style_set->show_background = GST_SUBTITLE_BACKGROUND_MODE_WHEN_ACTIVE; + else + style_set->show_background = GST_SUBTITLE_BACKGROUND_MODE_ALWAYS; + } + + if ((attr = ttml_style_set_get_attr (tss, "overflow"))) { + if (g_strcmp0 (attr, "visible") == 0) + style_set->overflow = GST_SUBTITLE_OVERFLOW_MODE_VISIBLE; + else + style_set->overflow = GST_SUBTITLE_OVERFLOW_MODE_HIDDEN; + } +} + + +static TtmlStyleSet * +ttml_style_set_copy (TtmlStyleSet * style_set) +{ + GHashTableIter iter; + gpointer attr_name, attr_value; + TtmlStyleSet *ret = ttml_style_set_new (); + + g_hash_table_iter_init (&iter, style_set->table); + while (g_hash_table_iter_next (&iter, &attr_name, &attr_value)) { + ttml_style_set_add_attr (ret, (const gchar *) attr_name, + (const gchar *) attr_value); + } + + return ret; +} + + +/* set2 overrides set1. Unlike style inheritance, merging will result in all + * values from set1 being merged into set2. */ +static TtmlStyleSet * +ttml_style_set_merge (TtmlStyleSet * set1, TtmlStyleSet * set2) +{ + TtmlStyleSet *ret = NULL; + + if (set1) { + ret = ttml_style_set_copy (set1); + + if (set2) { + GHashTableIter iter; + gpointer attr_name, attr_value; + + g_hash_table_iter_init (&iter, set2->table); + while (g_hash_table_iter_next (&iter, &attr_name, &attr_value)) { + ttml_style_set_add_attr (ret, (const gchar *) attr_name, + (const gchar *) attr_value); + } + } + } else if (set2) { + ret = ttml_style_set_copy (set2); + } + + return ret; +} + + +static gchar * +ttml_get_relative_font_size (const gchar * parent_size, + const gchar * child_size) +{ + guint psize = (guint) g_ascii_strtoull (parent_size, NULL, 10U); + guint csize = (guint) g_ascii_strtoull (child_size, NULL, 10U); + csize = (csize * psize) / 100U; + return g_strdup_printf ("%u%%", csize); +} + + +static TtmlStyleSet * +ttml_style_set_inherit (TtmlStyleSet * parent, TtmlStyleSet * child) +{ + TtmlStyleSet *ret = NULL; + GHashTableIter iter; + gpointer attr_name, attr_value; + + if (child) { + ret = ttml_style_set_copy (child); + } else { + ret = ttml_style_set_new (); + } + + if (!parent) + return ret; + + g_hash_table_iter_init (&iter, parent->table); + while (g_hash_table_iter_next (&iter, &attr_name, &attr_value)) { + /* In TTML, if an element which has a defined fontSize is the child of an + * element that also has a defined fontSize, the child's font size is + * relative to that of its parent. If its parent doesn't have a defined + * fontSize, then the child's fontSize is relative to the document's cell + * size. Therefore, if the former is true, we calculate the value of + * fontSize based on the parent's fontSize; otherwise, we simply keep + * the value defined in the child's style set. */ + if (g_strcmp0 ((const gchar *) attr_name, "fontSize") == 0 + && ttml_style_set_contains_attr (ret, (const gchar *) attr_name)) { + const gchar *original_child_font_size = + ttml_style_set_get_attr (child, "fontSize"); + gchar *scaled_child_font_size = + ttml_get_relative_font_size ((const gchar *) attr_value, + original_child_font_size); + GST_CAT_LOG (ttmlparse_debug, "Calculated font size: %s", + scaled_child_font_size); + ttml_style_set_add_attr (ret, (const gchar *) attr_name, + scaled_child_font_size); + g_free (scaled_child_font_size); + } + + /* Not all styling attributes are inherited in TTML. */ + if (g_strcmp0 ((const gchar *) attr_name, "backgroundColor") != 0 + && g_strcmp0 ((const gchar *) attr_name, "origin") != 0 + && g_strcmp0 ((const gchar *) attr_name, "extent") != 0 + && g_strcmp0 ((const gchar *) attr_name, "displayAlign") != 0 + && g_strcmp0 ((const gchar *) attr_name, "overflow") != 0 + && g_strcmp0 ((const gchar *) attr_name, "padding") != 0 + && g_strcmp0 ((const gchar *) attr_name, "writingMode") != 0 + && g_strcmp0 ((const gchar *) attr_name, "showBackground") != 0 + && g_strcmp0 ((const gchar *) attr_name, "unicodeBidi") != 0) { + if (!ttml_style_set_contains_attr (ret, (const gchar *) attr_name)) { + ttml_style_set_add_attr (ret, (const gchar *) attr_name, + (const gchar *) attr_value); + } + } + } + + return ret; +} + + +static gchar * +ttml_get_element_type_string (TtmlElement * element) +{ + switch (element->type) { + case TTML_ELEMENT_TYPE_STYLE: + return g_strdup ("