2 * Copyright (C) 2012 Smart TV Alliance
3 * Author: Thiago Sousa Santos <thiago.sousa.santos@collabora.com>, Collabora Ltd.
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Library General Public
9 * License as published by the Free Software Foundation; either
10 * version 2 of the License, or (at your option) any later version.
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Library General Public License for more details.
17 * You should have received a copy of the GNU Library General Public
18 * License along with this library; if not, write to the
19 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
20 * Boston, MA 02111-1307, USA.
24 * SECTION:element-mssdemux
27 * Demuxes a Microsoft's Smooth Streaming manifest into its audio and/or video streams.
34 * = Smooth streaming in a few lines
35 * A SS stream is defined by a xml manifest file. This file has a list of
36 * tracks (StreamIndex), each one can have multiple QualityLevels, that define
37 * different encoding/bitrates. When playing a track, only one of those
38 * QualityLevels can be active at a time (per stream).
40 * The StreamIndex defines a URL with {time} and {bitrate} tags that are
41 * replaced by values indicated by the fragment start times and the selected
42 * QualityLevel, that generates the fragments URLs.
44 * Another relevant detail is that the Isomedia fragments for smoothstreaming
45 * won't contains a 'moov' atom, nor a 'stsd', so there is no information
46 * about the media type/configuration on the fragments, it must be extracted
47 * from the Manifest and passed downstream. mssdemux does this via GstCaps.
49 * = How mssdemux works
50 * There is a gstmssmanifest.c utility that holds the manifest and parses
51 * and has functions to extract information from it. mssdemux received the
52 * manifest from its sink pad and starts processing it when it gets EOS.
54 * The Manifest is parsed and the streams are exposed, 1 pad for each, with
55 * a initially selected QualityLevel. Each stream starts its own GstTaks that
56 * is responsible for downloading fragments and pushing them downstream.
58 * When a new connection-speed is set, mssdemux evaluates the available
59 * QualityLevels and might decide to switch to another one. In this case it
60 * pushes a new GstCaps event indicating the new caps on the pads.
62 * All operations that intend to update the GstTasks state should be protected
63 * with the GST_OBJECT_LOCK.
70 #include <glib/gi18n-lib.h>
76 #include "gstmssdemux.h"
78 GST_DEBUG_CATEGORY (mssdemux2_debug);
80 static GstStaticPadTemplate gst_mss_demux_sink_template =
81 GST_STATIC_PAD_TEMPLATE ("sink",
84 GST_STATIC_CAPS ("application/vnd.ms-sstr+xml")
87 static GstStaticPadTemplate gst_mss_demux_videosrc_template =
88 GST_STATIC_PAD_TEMPLATE ("video_%02u",
93 static GstStaticPadTemplate gst_mss_demux_audiosrc_template =
94 GST_STATIC_PAD_TEMPLATE ("audio_%02u",
99 typedef struct _GstMssDemux2 GstMssDemux2;
100 typedef struct _GstMssDemux2Class GstMssDemux2Class;
102 #define gst_mss_demux2_parent_class parent_class
103 G_DEFINE_TYPE (GstMssDemux2, gst_mss_demux2, GST_TYPE_ADAPTIVE_DEMUX);
105 #define gst_hls_demux_stream_parent_class stream_parent_class
106 G_DEFINE_TYPE (GstMssDemuxStream, gst_mss_demux_stream,
107 GST_TYPE_ADAPTIVE_DEMUX2_STREAM);
109 GST_ELEMENT_REGISTER_DEFINE (mssdemux2, "mssdemux2",
110 GST_RANK_PRIMARY + 1, GST_TYPE_MSS_DEMUX2);
112 static void gst_mss_demux_dispose (GObject * object);
114 static gboolean gst_mss_demux_is_live (GstAdaptiveDemux * demux);
115 static gboolean gst_mss_demux_process_manifest (GstAdaptiveDemux * demux,
117 static GstClockTime gst_mss_demux_get_duration (GstAdaptiveDemux * demux);
118 static void gst_mss_demux_reset (GstAdaptiveDemux * demux);
119 static GstFlowReturn gst_mss_demux_stream_seek (GstAdaptiveDemux2Stream *
120 stream, gboolean forward, GstSeekFlags flags, GstClockTimeDiff ts,
121 GstClockTimeDiff * final_ts);
122 static gboolean gst_mss_demux_stream_has_next_fragment (GstAdaptiveDemux2Stream
125 gst_mss_demux_stream_advance_fragment (GstAdaptiveDemux2Stream * stream);
126 static gboolean gst_mss_demux_stream_select_bitrate (GstAdaptiveDemux2Stream *
127 stream, guint64 bitrate);
129 gst_mss_demux_stream_update_fragment_info (GstAdaptiveDemux2Stream * stream);
130 static gboolean gst_mss_demux_seek (GstAdaptiveDemux * demux, GstEvent * seek);
132 gst_mss_demux_get_manifest_update_interval (GstAdaptiveDemux * demux);
134 gst_mss_demux_stream_get_fragment_waiting_time (GstAdaptiveDemux2Stream *
137 gst_mss_demux_update_manifest_data (GstAdaptiveDemux * demux,
139 static gboolean gst_mss_demux_get_live_seek_range (GstAdaptiveDemux * demux,
140 gint64 * start, gint64 * stop);
141 static GstFlowReturn gst_mss_demux_data_received (GstAdaptiveDemux * demux,
142 GstAdaptiveDemux2Stream * stream, GstBuffer * buffer);
144 gst_mss_demux_requires_periodical_playlist_update (GstAdaptiveDemux * demux);
145 GstStreamType gst_stream_type_from_mss_type (GstMssStreamType mtype);
148 gst_mss_demux_stream_class_init (GstMssDemuxStreamClass * klass)
153 gst_mss_demux_stream_init (GstMssDemuxStream * stream)
158 gst_mss_demux2_class_init (GstMssDemuxClass * klass)
160 GObjectClass *gobject_class;
161 GstElementClass *gstelement_class;
162 GstAdaptiveDemuxClass *gstadaptivedemux_class;
164 gobject_class = (GObjectClass *) klass;
165 gstelement_class = (GstElementClass *) klass;
166 gstadaptivedemux_class = (GstAdaptiveDemuxClass *) klass;
168 gst_element_class_add_static_pad_template (gstelement_class,
169 &gst_mss_demux_sink_template);
170 gst_element_class_add_static_pad_template (gstelement_class,
171 &gst_mss_demux_videosrc_template);
172 gst_element_class_add_static_pad_template (gstelement_class,
173 &gst_mss_demux_audiosrc_template);
174 gst_element_class_set_static_metadata (gstelement_class,
175 "Smooth Streaming demuxer (v2)", "Codec/Demuxer/Adaptive",
176 "Parse and demultiplex a Smooth Streaming manifest into audio and video "
177 "streams", "Thiago Santos <thiago.sousa.santos@collabora.com>");
179 gobject_class->dispose = gst_mss_demux_dispose;
181 gstadaptivedemux_class->process_manifest = gst_mss_demux_process_manifest;
182 gstadaptivedemux_class->is_live = gst_mss_demux_is_live;
183 gstadaptivedemux_class->get_duration = gst_mss_demux_get_duration;
184 gstadaptivedemux_class->get_manifest_update_interval =
185 gst_mss_demux_get_manifest_update_interval;
186 gstadaptivedemux_class->reset = gst_mss_demux_reset;
187 gstadaptivedemux_class->seek = gst_mss_demux_seek;
188 gstadaptivedemux_class->stream_seek = gst_mss_demux_stream_seek;
189 gstadaptivedemux_class->stream_advance_fragment =
190 gst_mss_demux_stream_advance_fragment;
191 gstadaptivedemux_class->stream_has_next_fragment =
192 gst_mss_demux_stream_has_next_fragment;
193 gstadaptivedemux_class->stream_select_bitrate =
194 gst_mss_demux_stream_select_bitrate;
195 gstadaptivedemux_class->stream_update_fragment_info =
196 gst_mss_demux_stream_update_fragment_info;
197 gstadaptivedemux_class->stream_get_fragment_waiting_time =
198 gst_mss_demux_stream_get_fragment_waiting_time;
199 gstadaptivedemux_class->update_manifest_data =
200 gst_mss_demux_update_manifest_data;
201 gstadaptivedemux_class->get_live_seek_range =
202 gst_mss_demux_get_live_seek_range;
203 gstadaptivedemux_class->data_received = gst_mss_demux_data_received;
204 gstadaptivedemux_class->requires_periodical_playlist_update =
205 gst_mss_demux_requires_periodical_playlist_update;
207 GST_DEBUG_CATEGORY_INIT (mssdemux2_debug, "mssdemux2", 0,
208 "mssdemux2 element");
212 gst_mss_demux2_init (GstMssDemux * mssdemux)
217 gst_mss_demux_reset (GstAdaptiveDemux * demux)
219 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
221 if (mssdemux->manifest) {
222 gst_mss_manifest_free (mssdemux->manifest);
223 mssdemux->manifest = NULL;
225 g_free (mssdemux->base_url);
226 mssdemux->base_url = NULL;
230 gst_mss_demux_dispose (GObject * object)
232 gst_mss_demux_reset (GST_ADAPTIVE_DEMUX_CAST (object));
234 G_OBJECT_CLASS (parent_class)->dispose (object);
238 gst_mss_demux_is_live (GstAdaptiveDemux * demux)
240 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
242 g_return_val_if_fail (mssdemux->manifest != NULL, FALSE);
244 return gst_mss_manifest_is_live (mssdemux->manifest);
248 gst_mss_demux_get_duration (GstAdaptiveDemux * demux)
250 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
252 g_return_val_if_fail (mssdemux->manifest != NULL, FALSE);
254 return gst_mss_manifest_get_gst_duration (mssdemux->manifest);
258 gst_mss_demux_stream_update_fragment_info (GstAdaptiveDemux2Stream * stream)
260 GstMssDemuxStream *mssstream = (GstMssDemuxStream *) stream;
261 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (stream->demux);
265 gst_adaptive_demux2_stream_fragment_clear (&stream->fragment);
266 ret = gst_mss_stream_get_fragment_url (mssstream->manifest_stream, &path);
268 if (ret == GST_FLOW_OK) {
269 stream->fragment.uri = g_strdup_printf ("%s/%s", mssdemux->base_url, path);
270 stream->fragment.stream_time =
271 gst_mss_stream_get_fragment_gst_timestamp (mssstream->manifest_stream);
272 stream->fragment.duration =
273 gst_mss_stream_get_fragment_gst_duration (mssstream->manifest_stream);
281 gst_mss_demux_stream_seek (GstAdaptiveDemux2Stream * stream, gboolean forward,
282 GstSeekFlags flags, GstClockTimeDiff ts, GstClockTimeDiff * final_ts)
284 GstMssDemuxStream *mssstream = (GstMssDemuxStream *) stream;
286 gst_mss_stream_seek (mssstream->manifest_stream, forward, flags, ts,
292 gst_mss_demux_stream_advance_fragment (GstAdaptiveDemux2Stream * stream)
294 GstMssDemuxStream *mssstream = (GstMssDemuxStream *) stream;
296 if (stream->demux->segment.rate >= 0)
297 return gst_mss_stream_advance_fragment (mssstream->manifest_stream);
299 return gst_mss_stream_regress_fragment (mssstream->manifest_stream);
303 create_mss_caps (GstMssDemuxStream * stream, GstCaps * caps)
305 return gst_caps_new_simple ("video/quicktime", "variant", G_TYPE_STRING,
306 "mss-fragmented", "timescale", G_TYPE_UINT64,
307 gst_mss_stream_get_timescale (stream->manifest_stream), "media-caps",
308 GST_TYPE_CAPS, caps, NULL);
312 gst_mss_demux_apply_protection_system (GstCaps * caps,
313 const gchar * selected_system)
317 g_return_if_fail (selected_system);
318 s = gst_caps_get_structure (caps, 0);
319 gst_structure_set (s,
320 "original-media-type", G_TYPE_STRING, gst_structure_get_name (s),
321 GST_PROTECTION_SYSTEM_ID_CAPS_FIELD, G_TYPE_STRING, selected_system,
323 gst_structure_set_name (s, "application/x-cenc");
328 gst_mss_demux_setup_streams (GstAdaptiveDemux * demux)
330 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
331 GSList *streams = gst_mss_manifest_get_streams (mssdemux->manifest);
332 GSList *active_streams = NULL;
334 const gchar *protection_system_id =
335 gst_mss_manifest_get_protection_system_id (mssdemux->manifest);
336 const gchar *protection_data =
337 gst_mss_manifest_get_protection_data (mssdemux->manifest);
338 gboolean protected = protection_system_id && protection_data;
339 const gchar *selected_system = NULL;
340 guint64 max_bitrate = G_MAXUINT64;
342 if (streams == NULL) {
343 GST_INFO_OBJECT (mssdemux, "No streams found in the manifest");
344 GST_ELEMENT_ERROR (mssdemux, STREAM, DEMUX,
345 (_("This file contains no playable streams.")),
346 ("no streams found at the Manifest"));
351 const gchar *sys_ids[2] = { protection_system_id, NULL };
353 selected_system = gst_protection_select_system (sys_ids);
354 if (!selected_system) {
355 GST_ERROR_OBJECT (mssdemux, "stream is protected, but no "
356 "suitable decryptor element has been found");
361 if (demux->connection_speed != 0)
362 max_bitrate = demux->connection_speed;
364 for (iter = streams; iter; iter = g_slist_next (iter)) {
365 GstAdaptiveDemux2Stream *stream = NULL;
366 GstMssDemuxStream *mss_stream;
367 GstMssStream *manifeststream = iter->data;
368 GstAdaptiveDemuxTrack *track;
369 GstStreamType stream_type =
370 gst_stream_type_from_mss_type (gst_mss_stream_get_type
372 const gchar *lang, *stream_id = gst_stream_type_get_name (stream_type);
375 GstTagList *tags = NULL;
377 name = g_strdup_printf ("mss-stream-%s", stream_id);
378 mss_stream = g_object_new (GST_TYPE_MSS_DEMUX_STREAM, "name", name, NULL);
381 stream = GST_ADAPTIVE_DEMUX2_STREAM_CAST (mss_stream);
382 stream->stream_type = stream_type;
384 mss_stream->manifest_stream = manifeststream;
385 gst_mss_stream_set_active (manifeststream, TRUE);
387 /* Set the maximum bitrate now that the underlying stream is active. This
388 * ensures that we get the proper caps and information. */
389 gst_mss_stream_select_bitrate (manifeststream, max_bitrate);
391 caps = gst_mss_stream_get_caps (mss_stream->manifest_stream);
392 gst_adaptive_demux2_stream_set_caps (stream, create_mss_caps (mss_stream,
394 lang = gst_mss_stream_get_lang (mss_stream->manifest_stream);
396 tags = gst_tag_list_new (GST_TAG_LANGUAGE_CODE, lang, NULL);
398 track = gst_adaptive_demux_track_new (demux, stream_type,
399 GST_STREAM_FLAG_NONE, (gchar *) stream_id, create_mss_caps (mss_stream,
402 gst_adaptive_demux2_add_stream (demux, stream);
403 gst_adaptive_demux2_stream_add_track (stream, track);
404 gst_adaptive_demux_track_unref (track);
406 GST_DEBUG_OBJECT (stream, "Current quality bitrate %" G_GUINT64_FORMAT,
407 gst_mss_stream_get_current_bitrate (manifeststream));
410 gst_adaptive_demux2_stream_set_tags (stream, tags);
412 active_streams = g_slist_prepend (active_streams, mss_stream);
415 for (iter = active_streams; iter; iter = g_slist_next (iter)) {
416 GstMssDemuxStream *stream = iter->data;
419 GstBuffer *protection_buffer =
420 gst_buffer_new_wrapped (g_strdup (protection_data),
421 strlen (protection_data));
423 gst_event_new_protection (protection_system_id, protection_buffer,
426 GST_LOG_OBJECT (stream, "Queueing Protection event on source pad");
427 gst_adaptive_demux2_stream_queue_event ((GstAdaptiveDemux2Stream *)
429 gst_buffer_unref (protection_buffer);
433 g_slist_free (active_streams);
438 gst_mss_demux_update_base_url (GstMssDemux * mssdemux)
440 GstAdaptiveDemux *demux = GST_ADAPTIVE_DEMUX_CAST (mssdemux);
443 g_free (mssdemux->base_url);
446 g_strdup (demux->manifest_base_uri ? demux->manifest_base_uri : demux->
448 baseurl_end = g_strrstr (mssdemux->base_url, "/Manifest");
449 if (baseurl_end == NULL) {
451 baseurl_end = g_strrstr (mssdemux->base_url, "/manifest");
454 /* set the new end of the string */
455 baseurl_end[0] = '\0';
457 GST_WARNING_OBJECT (mssdemux, "Stream's URI didn't end with /manifest");
463 gst_mss_demux_process_manifest (GstAdaptiveDemux * demux, GstBuffer * buf)
465 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
467 gst_mss_demux_update_base_url (mssdemux);
469 mssdemux->manifest = gst_mss_manifest_new (buf);
470 if (!mssdemux->manifest) {
471 GST_ELEMENT_ERROR (mssdemux, STREAM, FORMAT, ("Bad manifest file"),
472 ("Xml manifest file couldn't be parsed"));
475 return gst_mss_demux_setup_streams (demux);
479 gst_mss_demux_stream_select_bitrate (GstAdaptiveDemux2Stream * stream,
482 GstMssDemuxStream *mssstream = (GstMssDemuxStream *) stream;
483 gboolean ret = FALSE;
485 GST_DEBUG_OBJECT (stream,
486 "Using stream download bitrate %" G_GUINT64_FORMAT, bitrate);
488 if (gst_mss_stream_select_bitrate (mssstream->manifest_stream,
489 bitrate / MAX (1.0, ABS (stream->demux->segment.rate)))) {
492 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (stream->demux);
493 const gchar *protection_system_id =
494 gst_mss_manifest_get_protection_system_id (mssdemux->manifest);
495 const gchar *protection_data =
496 gst_mss_manifest_get_protection_data (mssdemux->manifest);
497 gboolean protected = protection_system_id && protection_data;
499 caps = gst_mss_stream_get_caps (mssstream->manifest_stream);
501 GST_DEBUG_OBJECT (stream,
502 "Starting streams reconfiguration due to bitrate changes");
505 const gchar *sys_ids[2] = { protection_system_id, NULL };
506 const gchar *selected_system = gst_protection_select_system (sys_ids);
508 if (!selected_system) {
509 GST_ERROR_OBJECT (mssdemux, "stream is protected, but no "
510 "suitable decryptor element has been found");
511 gst_caps_unref (caps);
515 gst_mss_demux_apply_protection_system (caps, selected_system);
518 msscaps = create_mss_caps (mssstream, caps);
520 GST_DEBUG_OBJECT (stream,
521 "Stream changed bitrate to %" G_GUINT64_FORMAT " caps: %"
523 gst_mss_stream_get_current_bitrate (mssstream->manifest_stream), caps);
525 gst_caps_unref (caps);
527 gst_adaptive_demux2_stream_set_caps (stream, msscaps);
529 GST_DEBUG_OBJECT (stream, "Finished streams reconfiguration");
534 #define SEEK_UPDATES_PLAY_POSITION(r, start_type, stop_type) \
535 ((r >= 0 && start_type != GST_SEEK_TYPE_NONE) || \
536 (r < 0 && stop_type != GST_SEEK_TYPE_NONE))
539 gst_mss_demux_seek (GstAdaptiveDemux * demux, GstEvent * seek)
544 GstSeekType start_type, stop_type;
546 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
548 gst_event_parse_seek (seek, &rate, &format, &flags, &start_type, &start,
551 GST_DEBUG_OBJECT (mssdemux,
552 "seek event, rate: %f start: %" GST_TIME_FORMAT " stop: %"
553 GST_TIME_FORMAT, rate, GST_TIME_ARGS (start), GST_TIME_ARGS (stop));
555 if (SEEK_UPDATES_PLAY_POSITION (rate, start_type, stop_type)) {
557 gst_mss_manifest_seek (mssdemux->manifest, rate >= 0, start);
559 gst_mss_manifest_seek (mssdemux->manifest, rate >= 0, stop);
566 gst_mss_demux_stream_has_next_fragment (GstAdaptiveDemux2Stream * stream)
568 GstMssDemuxStream *mssstream = (GstMssDemuxStream *) stream;
570 return gst_mss_stream_has_next_fragment (mssstream->manifest_stream);
574 gst_mss_demux_get_manifest_update_interval (GstAdaptiveDemux * demux)
576 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
577 GstClockTime interval;
579 /* Not much information about this in the MSS spec. It seems that
580 * the fragments contain an UUID box that should tell the next
581 * fragments time and duration so one wouldn't need to fetch
582 * the Manifest again, but we need a fallback here. So use 2 times
583 * the current fragment duration */
585 interval = gst_mss_manifest_get_min_fragment_duration (mssdemux->manifest);
586 if (!GST_CLOCK_TIME_IS_VALID (interval))
587 interval = 2 * GST_SECOND; /* default to 2 seconds */
589 interval = 2 * (interval / GST_USECOND);
595 gst_mss_demux_stream_get_fragment_waiting_time (GstAdaptiveDemux2Stream *
598 /* Wait a second for live streams so we don't try premature fragments downloading */
603 gst_mss_demux_update_manifest_data (GstAdaptiveDemux * demux,
606 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
608 gst_mss_demux_update_base_url (mssdemux);
610 gst_mss_manifest_reload_fragments (mssdemux->manifest, buffer);
615 gst_mss_demux_get_live_seek_range (GstAdaptiveDemux * demux, gint64 * start,
618 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
620 return gst_mss_manifest_get_live_seek_range (mssdemux->manifest, start, stop);
624 gst_mss_demux_data_received (GstAdaptiveDemux * demux,
625 GstAdaptiveDemux2Stream * stream, GstBuffer * buffer)
627 GstMssDemux *mssdemux = GST_MSS_DEMUX_CAST (demux);
628 GstMssDemuxStream *mssstream = (GstMssDemuxStream *) stream;
631 if (!gst_mss_manifest_is_live (mssdemux->manifest)) {
632 return gst_adaptive_demux2_stream_push_buffer (stream, buffer);
635 if (gst_mss_stream_fragment_parsing_needed (mssstream->manifest_stream)) {
636 gst_mss_manifest_live_adapter_push (mssstream->manifest_stream, buffer);
638 gst_mss_manifest_live_adapter_available (mssstream->manifest_stream);
639 // FIXME: try to reduce this minimal size.
640 if (available < 4096) {
643 GST_LOG_OBJECT (stream, "enough data, parsing fragment.");
645 gst_mss_manifest_live_adapter_take_buffer (mssstream->manifest_stream,
647 gst_mss_stream_parse_fragment (mssstream->manifest_stream, buffer);
651 return gst_adaptive_demux2_stream_push_buffer (stream, buffer);
655 gst_mss_demux_requires_periodical_playlist_update (GstAdaptiveDemux * demux)
661 gst_stream_type_from_mss_type (GstMssStreamType mtype)
664 case MSS_STREAM_TYPE_AUDIO:
665 return GST_STREAM_TYPE_AUDIO;
666 case MSS_STREAM_TYPE_VIDEO:
667 return GST_STREAM_TYPE_VIDEO;
669 return GST_STREAM_TYPE_UNKNOWN;