--- /dev/null
+/* Copyright (C) 2015 Centricular Ltd
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <gst/gst.h>
+#include <gst/video/gstvideosink.h>
+
+#define STR_HELPER(x) #x
+#define STR(x) STR_HELPER(x)
+
+/* Change this to set the output resolution */
+#define OUTPUT_VIDEO_WIDTH 1280
+#define OUTPUT_VIDEO_HEIGHT 720
+
+/* Video and audio caps outputted by the mixers */
+#define RAW_AUDIO_CAPS_STR "audio/x-raw, format=(string)S16LE, " \
+"layout=(string)interleaved, rate=(int)44100, channels=(int)2, " \
+"channel-mask=(bitmask)0x03"
+
+#define RAW_VIDEO_CAPS_STR "video/x-raw, width=(int)" STR(OUTPUT_VIDEO_WIDTH) \
+", height=(int)" STR(OUTPUT_VIDEO_HEIGHT) ", framerate=(fraction)25/1, " \
+"format=I420, pixel-aspect-ratio=(fraction)1/1, " \
+"interlace-mode=(string)progressive"
+
+GST_DEBUG_CATEGORY_STATIC (playout);
+#define GST_CAT_DEFAULT playout
+
+typedef enum
+{
+ PLAYOUT_APP_STATE_READY, /* Newly created */
+ PLAYOUT_APP_STATE_PLAYING, /* Playing an item */
+ PLAYOUT_APP_STATE_EOS /* Finished playing, all items EOS */
+} PlayoutAppState;
+
+typedef struct
+{
+ /* Application state */
+ PlayoutAppState state;
+
+ /* An array of PlayoutItems that will be played in sequence */
+ GPtrArray *play_queue;
+ /* Index of the currently-playing item */
+ gint play_queue_current;
+ /* Lock access to the play queue */
+ GMutex play_queue_lock;
+
+ GMainLoop *main_loop;
+
+ /* Pipeline */
+ GstElement *pipeline;
+
+ /* Output */
+ GstElement *video_mixer;
+ GstElement *video_sink;
+ GstVideoRectangle video_orect; /* w/h/x/y of the output */
+
+ GstElement *audio_mixer;
+ GstElement *audio_sink;
+
+ /* The duration of all items that have been played in ns.
+ * Only updates when a new item is activated. */
+ guint64 elapsed_duration;
+} PlayoutApp;
+
+typedef enum
+{
+ PLAYOUT_ITEM_STATE_NEW, /* Newly created */
+ PLAYOUT_ITEM_STATE_PREPARED, /* Prepared and ready to activate */
+ PLAYOUT_ITEM_STATE_ACTIVATED, /* Activated */
+ PLAYOUT_ITEM_STATE_FIRST_VBUFFER, /* First video buffer has gone through */
+ PLAYOUT_ITEM_STATE_AGGREGATING, /* Audio & video buffers are aggregating */
+ PLAYOUT_ITEM_STATE_EOS /* At least one pad is EOS */
+} PlayoutItemState;
+
+typedef struct
+{
+ PlayoutApp *app;
+ PlayoutItemState state;
+
+ gchar *fn;
+
+ GstElement *decoder; /* bin with uridecodebin + converters */
+
+ /* We just use the first audio stream and ignore the rest (if there is audio) */
+ GstPad *audio_pad; /* decoder bin audio src ghostpad */
+ GstPad *video_pad; /* decoder bin video src ghostpad */
+ GstVideoRectangle video_irect; /* input w/h/x/y of the item */
+ GstVideoRectangle video_orect; /* output w/h/x/y of the item */
+
+ /* When this item has finished preparing and all pads have been connected to
+ * mixers, the pads will be blocked till it's this item's turn to be played */
+ gulong audio_pad_probe_block_id;
+ gulong video_pad_probe_block_id;
+
+ /* The current running time of this item; updated with every audio buffer if
+ * this item has audio; otherwise it's updated withe very video buffer */
+ guint64 running_time;
+} PlayoutItem;
+
+static PlayoutApp *playout_app_new (void);
+static void playout_app_free (PlayoutApp * app);
+static PlayoutItem *playout_item_new (PlayoutApp * app, const gchar * fn);
+static void playout_item_free (PlayoutItem * item);
+
+static void playout_app_add_item (PlayoutApp * app, const gchar * fn);
+static gboolean playout_app_prepare_item (PlayoutItem * item);
+static gboolean playout_app_activate_item (PlayoutItem * item);
+static gboolean playout_app_activate_next_item (PlayoutApp * app);
+static gboolean playout_app_activate_next_item_early (PlayoutApp * app);
+static PlayoutItem *playout_app_get_current_item (PlayoutApp * app);
+static gboolean playout_app_remove_item (PlayoutItem * item);
+
+static void
+playout_app_add_audio_sink (PlayoutApp * app)
+{
+ GstElement *audio_resample, *audio_conv, *queue;
+
+ /* audiomixer doesn't do conversion yet, so we don't need an output capsfilter
+ * for this branch. The output format is decided by the sink pads, which all
+ * have to have the same format. */
+ app->audio_mixer = gst_element_factory_make ("audiomixer", "audio_mixer");
+ audio_conv = gst_element_factory_make ("audioconvert", "mixer_audioconvert");
+ audio_resample = gst_element_factory_make ("audioresample",
+ "audio_mixer_audioresample");
+ queue = gst_element_factory_make ("queue", "asink_queue");
+ app->audio_sink = gst_element_factory_make ("autoaudiosink", NULL);
+ g_object_set (app->audio_sink, "async-handling", TRUE, NULL);
+ gst_bin_add_many (GST_BIN (app->pipeline), app->audio_mixer, audio_conv,
+ audio_resample, queue, app->audio_sink, NULL);
+ gst_element_link_many (app->audio_mixer, audio_conv, audio_resample,
+ queue, app->audio_sink, NULL);
+
+ if (!gst_element_sync_state_with_parent (app->audio_mixer) ||
+ !gst_element_sync_state_with_parent (audio_conv) ||
+ !gst_element_sync_state_with_parent (audio_resample) ||
+ !gst_element_sync_state_with_parent (queue) ||
+ !gst_element_sync_state_with_parent (app->audio_sink))
+ GST_ERROR ("app: unable to sync audio mixer + sink state with pipeline");
+}
+
+static PlayoutApp *
+playout_app_new (void)
+{
+ GstElement *video_capsfilter, *queue;
+ GstCaps *caps;
+ PlayoutApp *app;
+
+ app = g_new0 (PlayoutApp, 1);
+
+ app->state = PLAYOUT_APP_STATE_READY;
+
+ app->play_queue =
+ g_ptr_array_new_with_free_func ((GDestroyNotify) playout_item_free);
+ app->play_queue_current = -1;
+ g_mutex_init (&app->play_queue_lock);
+
+ app->main_loop = g_main_loop_new (NULL, FALSE);
+
+ app->pipeline = gst_pipeline_new ("pipeline");
+
+ /* It's best to set a caps filter for the video output format */
+ app->video_orect.w = OUTPUT_VIDEO_WIDTH;
+ app->video_orect.h = OUTPUT_VIDEO_HEIGHT;
+ app->video_orect.x = 0;
+ app->video_orect.y = 0;
+ app->video_mixer = gst_element_factory_make ("compositor", "video_mixer");
+ /* Set the background as black; faster while compositing, and allows us to
+ * rescale videos with a different aspect ratio than the output in a way that
+ * adds black borders automatically */
+ g_object_set (app->video_mixer, "background", 1, NULL);
+ queue = gst_element_factory_make ("queue", "vsink_queue");
+ app->video_sink = gst_element_factory_make ("autovideosink", NULL);
+ g_object_set (app->video_sink, "async-handling", TRUE, NULL);
+ video_capsfilter = gst_element_factory_make ("capsfilter",
+ "video_mixer_capsfilter");
+ caps = gst_caps_from_string (RAW_VIDEO_CAPS_STR);
+ g_object_set (video_capsfilter, "caps", caps, NULL);
+ gst_caps_unref (caps);
+ gst_bin_add_many (GST_BIN (app->pipeline), app->video_mixer, video_capsfilter,
+ queue, app->video_sink, NULL);
+ gst_element_link_many (app->video_mixer, video_capsfilter, queue,
+ app->video_sink, NULL);
+
+ return app;
+}
+
+static void
+playout_app_free (PlayoutApp * app)
+{
+ GST_DEBUG ("Freeing app");
+ g_ptr_array_unref (app->play_queue);
+ g_main_loop_unref (app->main_loop);
+ gst_element_set_state (app->pipeline, GST_STATE_NULL);
+ gst_object_unref (app->pipeline);
+ g_free (app);
+}
+
+static void
+playout_app_eos (GstBus * bus, GstMessage * msg, PlayoutApp * app)
+{
+ g_print ("All streams EOS, exiting...\n");
+ g_main_loop_quit (app->main_loop);
+}
+
+static PlayoutItem *
+playout_item_new (PlayoutApp * app, const gchar * fn)
+{
+ PlayoutItem *item = g_new0 (PlayoutItem, 1);
+
+ item->app = app;
+ item->state = PLAYOUT_ITEM_STATE_NEW;
+ item->fn = g_strdup (fn);
+
+ return item;
+}
+
+/* Unlink and release the pad */
+static gboolean
+playout_remove_pad (GstPad * srcpad)
+{
+ GstPad *sinkpad;
+ GstElement *mixer;
+
+ sinkpad = gst_pad_get_peer (srcpad);
+ if (!sinkpad)
+ return FALSE;
+ if (!gst_pad_unlink (srcpad, sinkpad))
+ return FALSE;
+ mixer = gst_pad_get_parent_element (sinkpad);
+ gst_element_release_request_pad (mixer, sinkpad);
+ GST_DEBUG ("Released some pad");
+
+ gst_object_unref (sinkpad);
+ gst_object_unref (mixer);
+ return FALSE;
+}
+
+static GstPadProbeReturn
+playout_item_pad_probe_blocked (GstPad * srcpad, GstPadProbeInfo * info,
+ PlayoutItem * item)
+{
+ if (srcpad == item->audio_pad) {
+ item->audio_pad_probe_block_id = GST_PAD_PROBE_INFO_ID (info);
+ } else if (srcpad == item->video_pad) {
+ item->video_pad_probe_block_id = GST_PAD_PROBE_INFO_ID (info);
+ } else {
+ g_assert_not_reached ();
+ }
+
+ return GST_PAD_PROBE_OK;
+}
+
+static GstPadProbeReturn
+playout_item_pad_probe_pad_running_time (GstPad * srcpad,
+ GstPadProbeInfo * info, PlayoutItem * item)
+{
+ GstEvent *event;
+ GstBuffer *buffer;
+ guint64 running_time;
+ const GstSegment *segment;
+
+ buffer = GST_PAD_PROBE_INFO_BUFFER (info);
+ event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0);
+ GST_TRACE ("%s: pad sticky event: %" GST_PTR_FORMAT, item->fn, event);
+
+ if (event) {
+ gst_event_parse_segment (event, &segment);
+ gst_event_unref (event);
+ running_time = gst_segment_to_running_time (segment, GST_FORMAT_TIME,
+ GST_BUFFER_PTS (buffer));
+ } else {
+ GST_WARNING ("%s: unable to parse event for segment; falling back to pts. "
+ "Output will probably have glitches.", item->fn);
+ running_time = GST_BUFFER_PTS (buffer);
+ }
+
+ item->running_time = running_time + GST_BUFFER_DURATION (buffer);
+ GST_TRACE ("%s: running time is %" G_GUINT64_FORMAT ", duration is %"
+ G_GUINT64_FORMAT, item->fn, item->running_time,
+ GST_BUFFER_DURATION (buffer));
+
+ return GST_PAD_PROBE_PASS;
+}
+
+static GstPadProbeReturn
+playout_item_pad_probe_video_pad_eos_on_buffer (GstPad * srcpad,
+ GstPadProbeInfo * info, PlayoutItem * prev_item)
+{
+ PlayoutItem *current_item;
+
+ current_item = playout_app_get_current_item (prev_item->app);
+
+ if (!current_item)
+ return GST_PAD_PROBE_REMOVE;
+
+ /* Step through the item's states as buffers pass through. The first buffer
+ * will be taken by the video_mixer, and kept till the audio running time
+ * matches the video buffer running time. When the second buffer gets through,
+ * we know that this pad has begun aggregating. */
+ switch (current_item->state) {
+ case PLAYOUT_ITEM_STATE_NEW:
+ case PLAYOUT_ITEM_STATE_PREPARED:
+ GST_DEBUG ("%s: new/prepared", current_item->fn);
+ break;
+ case PLAYOUT_ITEM_STATE_ACTIVATED:
+ GST_DEBUG ("%s: activated -> first vbuffer", current_item->fn);
+ current_item->state = PLAYOUT_ITEM_STATE_FIRST_VBUFFER;
+ break;
+ case PLAYOUT_ITEM_STATE_FIRST_VBUFFER:
+ GST_DEBUG ("%s: first vbuffer -> aggregating", current_item->fn);
+ current_item->state = PLAYOUT_ITEM_STATE_AGGREGATING;
+ gst_pad_remove_probe (srcpad, GST_PAD_PROBE_INFO_ID (info));
+ /* Item is aggregating, release the previous item's video pad */
+ goto release;
+ break;
+ case PLAYOUT_ITEM_STATE_EOS:
+ return GST_PAD_PROBE_REMOVE;
+ default:
+ g_assert_not_reached ();
+ }
+
+ return GST_PAD_PROBE_PASS;
+
+release:
+ {
+ playout_remove_pad (prev_item->video_pad);
+ GST_DEBUG ("%s: released video pad", prev_item->fn);
+ prev_item->video_pad = NULL;
+
+ /* If there's no audio pad, or if the audio pad is already EOS, we can
+ * remove this item from the queue which will free it. Need to free the
+ * item from the main thread because it causes the item's decoder bin
+ * to be removed from the pipeline, which cannot be done in the
+ * streaming thread */
+ if (prev_item->audio_pad == NULL) {
+ GST_DEBUG ("%s: queued item removal (last pad is video)", prev_item->fn);
+ g_main_context_invoke (NULL, (GSourceFunc) playout_app_remove_item,
+ prev_item);
+ }
+
+ /* Pad probe has already been removed above */
+ return GST_PAD_PROBE_PASS;
+ }
+}
+
+/* This is called on EOS for both item->audio_pad and item->video_pad
+ *
+ * FIXME: Add locking. Both pads could go EOS at the exact same time. */
+static GstPadProbeReturn
+playout_item_pad_probe_event (GstPad * srcpad, GstPadProbeInfo * info,
+ PlayoutItem * item)
+{
+ GstEventType type;
+ gboolean ret = TRUE;
+ GstPadProbeReturn probe_ret = GST_PAD_PROBE_DROP;
+
+ type = GST_EVENT_TYPE (GST_PAD_PROBE_INFO_DATA (info));
+ if (type != GST_EVENT_EOS)
+ return GST_PAD_PROBE_PASS;
+
+ /* We might get two EOSes on this pad if we send an artificial EOS. Remove
+ * the probe so this is only called once for each pad */
+ gst_pad_remove_probe (srcpad, GST_PAD_PROBE_INFO_ID (info));
+
+ GST_DEBUG ("%s: recvd some EOS", item->fn);
+
+ if (item->state != PLAYOUT_ITEM_STATE_EOS) {
+ /* We have more than one pad per item (video + audio item), and this is the
+ * first pad to go EOS or we have only one pad per item, and that pad has
+ * gone EOS. For the first case, the other pad might still have some buffers
+ * to output before going EOS, but we need to activate the next item and
+ * start outputting buffers from that immediately. */
+
+ /* Update the total elapsed duration from the item's current running time */
+ item->app->elapsed_duration += item->running_time;
+
+ GST_DEBUG ("%s: activating next item", item->fn);
+ /* Activate the next item if and only if this is the first pad to go EOS */
+ ret = playout_app_activate_next_item (item->app);
+ if (!ret) {
+ GST_DEBUG ("%s: App is going EOS", item->fn);
+ item->state = PLAYOUT_ITEM_STATE_EOS;
+ item->app->state = PLAYOUT_APP_STATE_EOS;
+ /* If we couldn't activate the next item, pass the EOS event onward,
+ * ending the stream */
+ probe_ret = GST_PAD_PROBE_PASS;
+ }
+ }
+
+ g_assert (srcpad != NULL);
+
+ if (srcpad == item->audio_pad) {
+ GST_DEBUG ("%s: audio pad was EOS", item->fn);
+
+ if (item->app->state != PLAYOUT_APP_STATE_EOS) {
+ /* While activating the next item, we ensure that there's no offset mismatch
+ * which would cause audiomixer to output silence, so we can release the
+ * previous item's audio request pad here. We also unlink the audio pad;
+ * nothing else is needed from it */
+ playout_remove_pad (srcpad);
+ GST_DEBUG ("%s: released audio pad", item->fn);
+
+ /* If there's no video pad, or if the video pad is already EOS, we can
+ * remove this item from the queue which will free it. Need to free the
+ * item from the main thread because it causes the item's decoder bin
+ * to be removed from the pipeline, which cannot be done in the
+ * streaming thread */
+ if (item->video_pad == NULL) {
+ GST_DEBUG ("%s: queued item removal (last pad is audio)", item->fn);
+ g_main_context_invoke (NULL, (GSourceFunc) playout_app_remove_item,
+ item);
+ }
+ } else {
+ /* If this is the last pad on audio_mixer, let the EOS go through
+ * before unlinking/releasing the pad. This should happen within 500ms. */
+ g_timeout_add (500, (GSourceFunc) playout_remove_pad, srcpad);
+ GST_DEBUG ("%s: queued audio pad release", item->fn);
+
+ if (item->video_pad == NULL) {
+ /* Unlike above, we need to wait till the pad is removed before removing
+ * the item from the app, so we queue it for 100ms afterwards */
+ GST_DEBUG ("%s: queued last item removal (last pad is audio)",
+ item->fn);
+ g_timeout_add (600, (GSourceFunc) playout_app_remove_item, item);
+ }
+ }
+ item->audio_pad = NULL;
+ } else if (srcpad == item->video_pad) {
+
+ GST_DEBUG ("%s: video pad was EOS", item->fn);
+
+ if (item->audio_pad != NULL)
+ GST_WARNING ("%s: video pad went EOS before audio pad! "
+ "There will be audio/video glitches while switching.", item->fn);
+
+ if (item->app->state != PLAYOUT_APP_STATE_EOS) {
+ PlayoutItem *next_item;
+
+ next_item = playout_app_get_current_item (item->app);
+ GST_DEBUG ("%s: next item is %s, %i/%i", item->fn, next_item->fn,
+ next_item->state, PLAYOUT_ITEM_STATE_ACTIVATED);
+
+ g_assert (next_item != NULL);
+ /* If there's another item being activated, release this video pad only
+ * when the next item's video pad starts being aggregated; that happens
+ * when this probe receives its 2nd buffer from the next item */
+ gst_pad_add_probe (next_item->video_pad, GST_PAD_PROBE_TYPE_BUFFER,
+ (GstPadProbeCallback) playout_item_pad_probe_video_pad_eos_on_buffer,
+ item, NULL);
+ } else {
+ /* If this is the last pad on video_mixer, let the EOS go through
+ * before unlinking/releasing the pad. This should happen within 500ms. */
+ g_timeout_add (500, (GSourceFunc) playout_remove_pad, srcpad);
+ GST_DEBUG ("%s: queued video pad release", item->fn);
+ item->video_pad = NULL;
+ }
+ probe_ret = GST_PAD_PROBE_PASS;
+ } else {
+ g_assert_not_reached ();
+ }
+
+ item->state = PLAYOUT_ITEM_STATE_EOS;
+
+ /* NOTE: If the srcpad has been unlinked, the return value is useless */
+ return probe_ret;
+}
+
+/* On the "pad-added" signal of uridecodebin, add converters and connect to
+ * audio/video mixers */
+static void
+playout_item_new_pad (GstElement * uridecodebin, GstPad * pad,
+ PlayoutItem * item)
+{
+ GstStructure *s;
+ GstCaps *caps;
+ GstPad *sinkpad, *srcpad;
+ GstElement *queue;
+ GstPadProbeType block_probe_type;
+
+ caps = gst_pad_get_current_caps (pad);
+ s = gst_caps_get_structure (caps, 0);
+ GST_DEBUG ("%s: new pad: %p, caps: %s", item->fn, pad,
+ gst_structure_get_name (s));
+
+ if (gst_structure_has_name (s, "audio/x-raw")) {
+ if (item->audio_pad != NULL)
+ /* Ignore all audio pads after the first one */
+ goto out;
+ goto audio;
+ } else if (gst_structure_has_name (s, "video/x-raw")) {
+ if (item->video_pad != NULL)
+ /* Ignore all video pads after the first one */
+ goto out;
+ goto video;
+ } else {
+ goto out;
+ }
+
+audio:
+ {
+ GstCaps *wanted_caps;
+ GstElement *audioconvert, *audioresample, *capsfilter;
+
+ /* Audio pad found; add audio mixer and audio sink to the pipeline.
+ * NOTE: If any items after this do not have an audio pad, the pipeline will
+ * mess up because the audio sink will not receive any data. */
+ if (item->app->audio_sink == NULL)
+ playout_app_add_audio_sink (item->app);
+
+ wanted_caps = gst_caps_from_string (RAW_AUDIO_CAPS_STR);
+
+ if (!gst_caps_is_equal (caps, wanted_caps)) {
+ GST_DEBUG ("%s: converting audio caps", item->fn);
+ /* We need to convert the audio to the wanted format because
+ * audiomixer doesn't do format conversion */
+ audioresample = gst_element_factory_make ("audioresample", NULL);
+ audioconvert = gst_element_factory_make ("audioconvert", NULL);
+ capsfilter = gst_element_factory_make ("capsfilter", NULL);
+ g_object_set (capsfilter, "caps", wanted_caps, NULL);
+ queue = gst_element_factory_make ("queue", NULL);
+ gst_bin_add_many (GST_BIN (item->decoder), audioresample, audioconvert,
+ capsfilter, queue, NULL);
+
+ sinkpad = gst_element_get_static_pad (audioresample, "sink");
+ gst_pad_link (pad, sinkpad);
+ gst_object_unref (sinkpad);
+ gst_element_link_many (audioresample, audioconvert, capsfilter, queue,
+ NULL);
+ srcpad = gst_element_get_static_pad (queue, "src");
+
+ if (!gst_element_sync_state_with_parent (audioresample) ||
+ !gst_element_sync_state_with_parent (audioconvert) ||
+ !gst_element_sync_state_with_parent (capsfilter) ||
+ !gst_element_sync_state_with_parent (queue)) {
+ GST_ERROR ("%s: unable to sync audio converter state with decoder",
+ item->fn);
+ goto out;
+ }
+ } else {
+ queue = gst_element_factory_make ("queue", NULL);
+ gst_bin_add (GST_BIN (item->decoder), queue);
+ sinkpad = gst_element_get_static_pad (queue, "sink");
+ gst_pad_link (pad, sinkpad);
+ gst_object_unref (sinkpad);
+
+ srcpad = gst_element_get_static_pad (queue, "src");
+
+ if (!gst_element_sync_state_with_parent (queue)) {
+ GST_ERROR ("%s: unable to sync audio queue state with decoder",
+ item->fn);
+ goto out;
+ }
+ }
+ gst_caps_unref (wanted_caps);
+
+ /* Convert the audioconvert src pad to a ghostpad on the bin */
+ item->audio_pad = gst_ghost_pad_new (NULL, srcpad);
+ gst_pad_set_active (item->audio_pad, TRUE);
+ gst_element_add_pad (item->decoder, item->audio_pad);
+ gst_object_unref (srcpad);
+
+ srcpad = item->audio_pad;
+ GST_DEBUG ("%s: created audio pad", item->fn);
+ goto done;
+ }
+
+video:
+ {
+ if (!gst_structure_get_int (s, "width", &item->video_irect.w) ||
+ !gst_structure_get_int (s, "height", &item->video_irect.h))
+ GST_WARNING ("%s: unable to set width/height from caps", item->fn);
+ item->video_irect.x = item->video_irect.y = 0;
+
+ queue = gst_element_factory_make ("queue", "video-decoder-queue-%u");
+ gst_bin_add (GST_BIN (item->decoder), queue);
+
+ if (!gst_element_sync_state_with_parent (queue)) {
+ GST_ERROR ("%s: unable to sync video queue state with decoder", item->fn);
+ goto out;
+ }
+
+ sinkpad = gst_element_get_static_pad (queue, "sink");
+ gst_pad_link (pad, sinkpad);
+ gst_object_unref (sinkpad);
+
+ /* Convert the queue src pad to a ghostpad on the bin */
+ srcpad = gst_element_get_static_pad (queue, "src");
+ item->video_pad = gst_ghost_pad_new (NULL, srcpad);
+ gst_pad_set_active (item->video_pad, TRUE);
+ gst_element_add_pad (item->decoder, item->video_pad);
+ gst_object_unref (srcpad);
+
+ srcpad = item->video_pad;
+ GST_DEBUG ("%s: created video pad", item->fn);
+ goto done;
+ }
+
+done:
+ /* We let events and queries through */
+ block_probe_type = GST_PAD_PROBE_TYPE_BLOCK |
+ GST_PAD_PROBE_TYPE_BUFFER | GST_PAD_PROBE_TYPE_BUFFER_LIST;
+ /* If the app is already playing an item, block everything except queries
+ * till we need to play this item */
+ if (item->app->state != PLAYOUT_APP_STATE_READY)
+ gst_pad_add_probe (srcpad, block_probe_type,
+ (GstPadProbeCallback) playout_item_pad_probe_blocked, item, NULL);
+ /* Probe events for EOS */
+ gst_pad_add_probe (srcpad, GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM,
+ (GstPadProbeCallback) playout_item_pad_probe_event, item, NULL);
+
+out:
+ gst_caps_unref (caps);
+}
+
+/* All pads on uridecodebin have finished being populated; the item has been
+ * prepared and is ready to be activated */
+static void
+playout_item_no_more_pads (GstElement * uridecodebin, PlayoutItem * item)
+{
+ /* Set a buffer pad probe that constantly updates the item's
+ * elapsed_duration using the duration of each audio buffer */
+ if (item->audio_pad) {
+ gst_pad_add_probe (item->audio_pad, GST_PAD_PROBE_TYPE_BUFFER,
+ (GstPadProbeCallback) playout_item_pad_probe_pad_running_time,
+ item, NULL);
+ } else if (item->video_pad) {
+ gst_pad_add_probe (item->video_pad, GST_PAD_PROBE_TYPE_BUFFER,
+ (GstPadProbeCallback) playout_item_pad_probe_pad_running_time,
+ item, NULL);
+ } else {
+ GST_ERROR ("%s: no pads were generated! Can't continue playing!", item->fn);
+ return;
+ }
+
+ item->state = PLAYOUT_ITEM_STATE_PREPARED;
+ GST_DEBUG ("%s: prepared", item->fn);
+
+ if (item->app->state != PLAYOUT_APP_STATE_READY)
+ /* This item will be activated when the previous one is EOS */
+ return;
+
+ GST_DEBUG ("Application isn't already playing; activate the item and prepare"
+ " the next one");
+
+ playout_app_activate_item (item);
+ item->state = PLAYOUT_ITEM_STATE_ACTIVATED;
+ item->app->state = PLAYOUT_APP_STATE_PLAYING;
+
+ if (item->app->play_queue->len > 1)
+ playout_app_prepare_item (g_ptr_array_index (item->app->play_queue, 1));
+}
+
+static GstElement *
+playout_item_create_decoder (PlayoutItem * item)
+{
+ GstElement *bin, *dec;
+ GError *err = NULL;
+ gchar *uri;
+
+ uri = gst_filename_to_uri (item->fn, &err);
+ if (err != NULL) {
+ GST_WARNING ("Could not convert '%s' to uri: %s", item->fn, err->message);
+ g_error_free (err);
+ return NULL;
+ }
+
+ bin = gst_bin_new (NULL);
+ dec = gst_element_factory_make ("uridecodebin", NULL);
+ g_object_set (dec, "uri", uri, NULL);
+ g_free (uri);
+
+ gst_bin_add (GST_BIN (bin), dec);
+
+ g_signal_connect (dec, "pad-added", G_CALLBACK (playout_item_new_pad), item);
+ g_signal_connect (dec, "no-more-pads", G_CALLBACK (playout_item_no_more_pads),
+ item);
+
+ return bin;
+}
+
+static void
+playout_item_free (PlayoutItem * item)
+{
+ GST_DEBUG ("Entering free");
+ switch (gst_element_set_state (item->decoder, GST_STATE_NULL)) {
+ case GST_STATE_CHANGE_FAILURE:
+ GST_ERROR ("%s: Unable to change state to NULL", item->fn);
+ break;
+ case GST_STATE_CHANGE_SUCCESS:
+ GST_DEBUG ("%s: State change success", item->fn);
+ break;
+ default:
+ GST_DEBUG ("%s: Some async/no-preroll", item->fn);
+ }
+
+ gst_bin_remove (GST_BIN (item->app->pipeline), item->decoder);
+ GST_DEBUG ("%s: bin removed", item->fn);
+
+ g_free (item->fn);
+ g_free (item);
+ GST_DEBUG ("item freed");
+}
+
+static guint64
+playout_item_pad_get_segment_time (GstPad * srcpad)
+{
+ GstEvent *event;
+ const GstSegment *segment;
+
+ event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0);
+ if (!event)
+ return 0;
+ gst_event_parse_segment (event, &segment);
+ gst_event_unref (event);
+ return segment->time;
+}
+
+static void
+playout_app_add_item (PlayoutApp * app, const gchar * fn)
+{
+ PlayoutItem *item;
+
+ item = playout_item_new (app, fn);
+
+ g_mutex_lock (&app->play_queue_lock);
+ g_ptr_array_add (app->play_queue, item);
+ g_mutex_unlock (&app->play_queue_lock);
+}
+
+static gboolean
+playout_app_remove_item (PlayoutItem * item)
+{
+ PlayoutApp *app;
+ GST_DEBUG ("%s: removing and freeing", item->fn);
+
+ app = item->app;
+
+ g_mutex_lock (&app->play_queue_lock);
+ g_ptr_array_remove (app->play_queue, item);
+ /* This item has been removed from the array, decrement the index */
+ app->play_queue_current--;
+ g_mutex_unlock (&app->play_queue_lock);
+
+ /* Don't call this again */
+ return FALSE;
+}
+
+static PlayoutItem *
+playout_app_get_current_item (PlayoutApp * app)
+{
+ if (app->play_queue_current < 0 ||
+ app->play_queue->len < (app->play_queue_current + 1))
+ return NULL;
+
+ return g_ptr_array_index (app->play_queue, app->play_queue_current);
+}
+
+static gboolean
+playout_app_prepare_item (PlayoutItem * item)
+{
+ PlayoutApp *app = item->app;
+
+ if (item->decoder != NULL)
+ return TRUE;
+
+ item->decoder = playout_item_create_decoder (item);
+
+ if (item->decoder == NULL)
+ return FALSE;
+
+ gst_bin_add (GST_BIN (app->pipeline), item->decoder);
+
+ if (!gst_element_sync_state_with_parent (item->decoder)) {
+ GST_ERROR ("%s: unable to sync state with parent", item->fn);
+ return FALSE;
+ }
+
+ GST_DEBUG ("%s: preparing", item->fn);
+
+ /* All further processing is done in the "no-more-pads" callback of
+ * uridecodebin */
+ return TRUE;
+}
+
+/* Called exactly once for each item */
+static gboolean
+playout_app_activate_item (PlayoutItem * item)
+{
+ GstPad *sinkpad;
+ guint64 segment_time;
+ PlayoutApp *app = item->app;
+
+ if (item->state != PLAYOUT_ITEM_STATE_PREPARED) {
+ GST_ERROR ("Item %s is not ready to be activated!", item->fn);
+ return FALSE;
+ }
+
+ if (!item->audio_pad && !item->video_pad) {
+ GST_ERROR ("Item %s has no pads! Can't activate it!", item->fn);
+ return FALSE;
+ }
+
+ /* Hook up to mixers and remove the probes blocking the pads */
+ if (item->audio_pad) {
+ GST_DEBUG ("%s: hooking up audio pad to mixer", item->fn);
+ sinkpad = gst_element_get_request_pad (app->audio_mixer, "sink_%u");
+ gst_pad_link (item->audio_pad, sinkpad);
+
+ segment_time = playout_item_pad_get_segment_time (item->audio_pad);
+ if (segment_time > 0) {
+ /* If the segment time is > 0, the new pad wants audiomixer to output audio
+ * silence for that duration. This will cause audio glitches, so we move
+ * the pad offset back by that amount and tell audiomixer to start mixing
+ * our buffers immediately. */
+ GST_DEBUG ("%s: subtracting segment time %" G_GUINT64_FORMAT " from "
+ "elapsed duration before setting it as the pad offset", item->fn,
+ segment_time);
+ if (app->elapsed_duration > segment_time)
+ app->elapsed_duration -= segment_time;
+ else
+ app->elapsed_duration = 0;
+ }
+
+ if (app->elapsed_duration > 0) {
+ GST_DEBUG ("%s: set audio pad offset to %" G_GUINT64_FORMAT "ms",
+ item->fn, app->elapsed_duration / GST_MSECOND);
+ gst_pad_set_offset (item->audio_pad, app->elapsed_duration);
+ }
+
+ if (item->audio_pad_probe_block_id > 0) {
+ GST_DEBUG ("%s: removing audio pad block probe", item->fn);
+ gst_pad_remove_probe (item->audio_pad, item->audio_pad_probe_block_id);
+ }
+ gst_object_unref (sinkpad);
+ }
+
+ if (item->video_pad) {
+ GST_DEBUG ("%s: hooking up video pad to mixer", item->fn);
+ sinkpad = gst_element_get_request_pad (app->video_mixer, "sink_%u");
+
+ /* Get new height/width/xpos/ypos such that the video scales up or down to
+ * fit within the output video size without any cropping */
+ gst_video_sink_center_rect (item->video_irect, item->app->video_orect,
+ &item->video_orect, TRUE);
+ GST_DEBUG ("%s: w: %i, h: %i, x: %i, y: %i\n", item->fn,
+ item->video_orect.w, item->video_orect.h, item->video_orect.x,
+ item->video_orect.y);
+ g_object_set (sinkpad, "width", item->video_orect.w, "height",
+ item->video_orect.h, "xpos", item->video_orect.x, "ypos",
+ item->video_orect.y, NULL);
+
+ /* If this is not the last item, on EOS, continue to aggregate using the
+ * last buffer till the pad is released */
+ if (item->app->play_queue->len != (item->app->play_queue_current + 2))
+ g_object_set (sinkpad, "ignore-eos", TRUE, NULL);
+ else
+ GST_DEBUG ("%s: last item, not setting ignore-eos", item->fn);
+ gst_pad_link (item->video_pad, sinkpad);
+
+ if (app->elapsed_duration > 0) {
+ GST_DEBUG ("%s: set video pad offset to %" G_GUINT64_FORMAT "ms",
+ item->fn, app->elapsed_duration / GST_MSECOND);
+ gst_pad_set_offset (item->video_pad, app->elapsed_duration);
+ }
+
+ if (item->video_pad_probe_block_id > 0) {
+ GST_DEBUG ("%s: removing video pad block probe", item->fn);
+ gst_pad_remove_probe (item->video_pad, item->video_pad_probe_block_id);
+ }
+ gst_object_unref (sinkpad);
+ }
+
+ item->state = PLAYOUT_ITEM_STATE_ACTIVATED;
+ g_mutex_lock (&item->app->play_queue_lock);
+ item->app->play_queue_current++;
+ g_mutex_unlock (&item->app->play_queue_lock);
+
+ GST_DEBUG ("%s: activated", item->fn);
+
+ return TRUE;
+}
+
+/* Activate the next item, and prepare the one after that for later activation */
+static gboolean
+playout_app_activate_next_item (PlayoutApp * app)
+{
+ PlayoutItem *item;
+ gboolean ret;
+
+ if (app->play_queue->len < (app->play_queue_current + 2)) {
+ g_print ("No more items to play\n");
+ return FALSE;
+ }
+
+ item = g_ptr_array_index (app->play_queue, app->play_queue_current + 1);
+ ret = playout_app_activate_item (item);
+ if (!ret) {
+ /* Tell caller, who can then decide whether to skip or error out */
+ GST_ERROR ("%s: unable to activate", item->fn);
+ return FALSE;
+ }
+ if (app->play_queue->len > (app->play_queue_current + 1)) {
+ item = g_ptr_array_index (app->play_queue, app->play_queue_current + 1);
+ /* FIXME: What if this fails? Prepare the next one in the queue? */
+ ret = playout_app_prepare_item (item);
+ if (!ret)
+ GST_ERROR ("%s: unable to prepare", item->fn);
+ }
+ return ret;
+}
+
+static GstPadProbeReturn
+playout_item_pad_probe_video_pad_running_time (GstPad * srcpad,
+ GstPadProbeInfo * info, PlayoutItem * item)
+{
+ GstEvent *event;
+ GstBuffer *buffer;
+ guint64 running_time;
+ const GstSegment *segment;
+
+ buffer = GST_PAD_PROBE_INFO_BUFFER (info);
+ event = gst_pad_get_sticky_event (srcpad, GST_EVENT_SEGMENT, 0);
+ GST_TRACE ("%s: video sticky event: %" GST_PTR_FORMAT, item->fn, event);
+
+ if (event) {
+ gst_event_parse_segment (event, &segment);
+ gst_event_unref (event);
+ running_time = gst_segment_to_running_time (segment, GST_FORMAT_TIME,
+ GST_BUFFER_PTS (buffer));
+ } else {
+ GST_WARNING ("%s: unable to parse video event for segment; falling back to "
+ "pts", item->fn);
+ running_time = GST_BUFFER_PTS (buffer);
+ }
+
+ if (running_time >= item->running_time) {
+ /* The video buffer passing through video_mixer now matches the audio buffer
+ * that passed through audio_mixer when the early switch was requested, so
+ * this is the time to send an EOS to video_pad, which will complete the
+ * switch */
+ GST_DEBUG ("Sending video EOS to %s", item->fn);
+ gst_pad_push_event (item->video_pad, gst_event_new_eos ());
+ return GST_PAD_PROBE_DROP;
+ } else {
+ return GST_PAD_PROBE_PASS;
+ }
+}
+
+static gboolean
+playout_app_activate_next_item_early (PlayoutApp * app)
+{
+ PlayoutItem *item;
+
+ item = playout_app_get_current_item (app);
+ if (!item) {
+ GST_WARNING ("Unable to switch early, no current item");
+ return FALSE;
+ }
+
+ if (item->audio_pad) {
+ /* If we have an audio pad, EOS audio first, always */
+ GST_DEBUG ("Sending audio EOS to %s", item->fn);
+ gst_pad_push_event (item->audio_pad, gst_event_new_eos ());
+ /* We can't send the EOS to the video_pad yet because the running times for
+ * both mixers are different due to buffering at the audio sink. So we wait
+ * till the running time of the video_pad matches that of the audio_pad at
+ * the time the audio EOS was sent, and then EOS video as well. */
+ gst_pad_add_probe (item->video_pad, GST_PAD_PROBE_TYPE_BUFFER,
+ (GstPadProbeCallback) playout_item_pad_probe_video_pad_running_time,
+ item, NULL);
+ } else if (item->video_pad) {
+ /* If we have a video pad, EOS audio first, always */
+ GST_DEBUG ("Sending video EOS to %s", item->fn);
+ gst_pad_push_event (item->video_pad, gst_event_new_eos ());
+ } else {
+ g_assert_not_reached ();
+ }
+
+ /* Return FALSE so this function is called only once */
+ return FALSE;
+}
+
+static gboolean
+playout_app_play (PlayoutApp * app)
+{
+ PlayoutItem *item;
+
+ item = app->play_queue->len ? g_ptr_array_index (app->play_queue, 0) : NULL;
+ if (!item) {
+ g_printerr ("Nothing to play\n");
+ return FALSE;
+ }
+
+ playout_app_prepare_item (item);
+ return TRUE;
+}
+
+/*
+ * playout: An example application to sequentially and seamlessly play a list of
+ * audio-video or video-only files.
+ *
+ * This example application uses the compositor and audiomixer elements combined
+ * with pad probes to stitch together a list of A/V or V-only files in such
+ * a way that audio and video glitching is minimised. Mixing A/V and V-only
+ * files is not supported because it complicates the architecture quite a bit.
+ *
+ * Due to the fundamental difference in the representation of audio and video
+ * data, unless constructed specifically for the purpose of being stitched back,
+ * the audio and video tracks of files will rarely end at the same PTS. There is
+ * usually a sync difference of a few frames. This application tries to stitch
+ * together the audio tracks as perfectly as possible, and duplicates/drops
+ * video frames if there is an underrun/overrun. Even when audio samples are
+ * played back-to-back, there might be glitches due to quirks in the decoder.
+ *
+ * The list of PlayoutItems can be edited and added to dynamically; except the
+ * currently-playing item and the next one (which has been prepared already).
+ */
+int
+main (int argc, char **argv)
+{
+ GstBus *bus;
+ gint switch_after_ms = 0;
+ gchar **f, **filenames = NULL;
+ GOptionEntry options[] = {
+ {"switch-after", 's', 0, G_OPTION_ARG_INT, &switch_after_ms, "Time after "
+ "which the next item will be forcibly activated", "MILLISECONDS"},
+ {G_OPTION_REMAINING, 0, 0, G_OPTION_ARG_FILENAME_ARRAY, &filenames, NULL},
+ {NULL}
+ };
+ GOptionContext *ctx;
+ PlayoutApp *app;
+ GError *err = NULL;
+
+ ctx = g_option_context_new ("FILENAME1 [FILENAME2] [FILENAME3] ...");
+ g_option_context_add_main_entries (ctx, options, NULL);
+ g_option_context_add_group (ctx, gst_init_get_option_group ());
+
+ if (!g_option_context_parse (ctx, &argc, &argv, &err)) {
+ if (err)
+ g_printerr ("Error initializing: %s\n", err->message);
+ else
+ g_printerr ("Error initializing: Unknown error!\n");
+ return 1;
+ }
+ g_option_context_free (ctx);
+
+ GST_DEBUG_CATEGORY_INIT (playout, "playout", 0, "Playout example app");
+
+ app = playout_app_new ();
+
+ if (filenames == NULL || *filenames == NULL) {
+ g_printerr ("Usage: %s FILENAME1 FILENAME2\n", argv[0]);
+ return 1;
+ }
+
+ for (f = filenames; f != NULL && *f != NULL; ++f)
+ playout_app_add_item (app, *f);
+
+ g_strfreev (filenames);
+
+ if (!playout_app_play (app))
+ return 1;
+
+ GST_DEBUG ("Setting pipeline to PLAYING");
+
+ bus = gst_pipeline_get_bus (GST_PIPELINE (app->pipeline));
+ gst_bus_add_signal_watch (bus);
+ g_signal_connect (bus, "message::eos", G_CALLBACK (playout_app_eos), app);
+ gst_object_unref (bus);
+
+ gst_element_set_state (app->pipeline, GST_STATE_PLAYING);
+
+ if (switch_after_ms)
+ g_timeout_add (switch_after_ms,
+ (GSourceFunc) playout_app_activate_next_item_early, app);
+
+ GST_DEBUG ("Running mainloop");
+ g_main_loop_run (app->main_loop);
+
+ playout_app_free (app);
+
+ return 0;
+}
projects / platform / upstream / gstreamer.git / commitdiff