X-Git-Url: http://review.tizen.org/git/?a=blobdiff_plain;f=gst%2Fisomp4%2Fgstqtmux.c;h=94a131481e28ff8c4a109eec379cd14ea842625a;hb=f05e14ba316d67b9cdf1440d9b2129da03625a30;hp=c4b9ff2c0f542351cc67d897a2ae6f4bfbf8607b;hpb=6f3737f0c5d63d79e880fd8ba904f3df608d7d40;p=platform%2Fupstream%2Fgst-plugins-good.git diff --git a/gst/isomp4/gstqtmux.c b/gst/isomp4/gstqtmux.c index c4b9ff2..94a1314 100644 --- a/gst/isomp4/gstqtmux.c +++ b/gst/isomp4/gstqtmux.c @@ -65,9 +65,10 @@ * The fragmented file features defined (only) in ISO Base Media are used by * ISMV files making up (a.o.) Smooth Streaming (ismlmux). * - * A few properties (#GstQTMux:movie-timescale, #GstQTMux:trak-timescale) allow - * adjusting some technical parameters, which might be useful in (rare) cases to - * resolve compatibility issues in some situations. + * A few properties (#GstQTMux:movie-timescale, #GstQTMux:trak-timescale, + * #GstQTMuxPad:trak-timescale) allow adjusting some technical parameters, + * which might be useful in (rare) cases to resolve compatibility issues in + * some situations. * * Some other properties influence the result more fundamentally. * A typical mov/mp4 file's metadata (aka moov) is located at the end of the @@ -96,6 +97,17 @@ * #GstQTMux::reserved-duration-remaining property to see how close to full * the reserved space is becoming. * + * Applications that wish to be able to use/edit a file while it is being + * written to by live content, can use the "Robust Prefill Muxing" mode. That + * mode is a variant of the "Robust Muxing" mode in that it will pre-allocate a + * completely valid header from the start for all tracks (i.e. it appears as + * though the file is "reserved-max-duration" long with all samples + * present). This mode can be enabled by setting the + * #GstQTMux::reserved-moov-update-period and #GstQTMux::reserved-prefill + * properties. Note that this mode is only possible with input streams that have + * a fixed sample size (such as raw audio and Prores Video) and that don't + * have reordered samples. + * * * Example pipelines * |[ @@ -146,6 +158,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_qt_mux_debug); #define GST_CAT_DEFAULT gst_qt_mux_debug +#ifndef ABSDIFF +#define ABSDIFF(a, b) ((a) > (b) ? (a) - (b) : (b) - (a)) +#endif + /* Hacker notes. * * The basic building blocks of MP4 files are: @@ -243,6 +259,115 @@ gst_qt_mux_dts_method_get_type (void) (gst_qt_mux_dts_method_get_type ()) #endif +enum +{ + PROP_PAD_0, + PROP_PAD_TRAK_TIMESCALE, +}; + +#define DEFAULT_PAD_TRAK_TIMESCALE 0 + +GType gst_qt_mux_pad_get_type (void); + +#define GST_TYPE_QT_MUX_PAD \ + (gst_qt_mux_pad_get_type()) +#define GST_QT_MUX_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_QT_MUX_PAD, GstQTMuxPad)) +#define GST_QT_MUX_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_QT_MUX_PAD, GstQTMuxPadClass)) +#define GST_IS_QT_MUX_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_QT_MUX_PAD)) +#define GST_IS_QT_MUX_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_QT_MUX_PAD)) +#define GST_QT_MUX_PAD_CAST(obj) \ + ((GstQTMuxPad *)(obj)) + +typedef struct _GstQTMuxPad GstQTMuxPad; +typedef struct _GstQTMuxPadClass GstQTMuxPadClass; + +struct _GstQTMuxPad +{ + GstPad parent; + + guint32 trak_timescale; +}; + +struct _GstQTMuxPadClass +{ + GstPadClass parent; +}; + +G_DEFINE_TYPE (GstQTMuxPad, gst_qt_mux_pad, GST_TYPE_PAD); + +static void +gst_qt_mux_pad_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstQTMuxPad *pad = GST_QT_MUX_PAD_CAST (object); + + GST_OBJECT_LOCK (pad); + switch (prop_id) { + case PROP_PAD_TRAK_TIMESCALE: + pad->trak_timescale = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_UNLOCK (pad); +} + +static void +gst_qt_mux_pad_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstQTMuxPad *pad = GST_QT_MUX_PAD_CAST (object); + + GST_OBJECT_LOCK (pad); + switch (prop_id) { + case PROP_PAD_TRAK_TIMESCALE: + g_value_set_uint (value, pad->trak_timescale); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_UNLOCK (pad); +} + +static void +gst_qt_mux_pad_class_init (GstQTMuxPadClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + + gobject_class->get_property = gst_qt_mux_pad_get_property; + gobject_class->set_property = gst_qt_mux_pad_set_property; + + g_object_class_install_property (gobject_class, PROP_PAD_TRAK_TIMESCALE, + g_param_spec_uint ("trak-timescale", "Track timescale", + "Timescale to use for this pad's trak (units per second, 0 is automatic)", + 0, G_MAXUINT32, DEFAULT_PAD_TRAK_TIMESCALE, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_qt_mux_pad_init (GstQTMuxPad * pad) +{ + pad->trak_timescale = DEFAULT_PAD_TRAK_TIMESCALE; +} + +static guint32 +gst_qt_mux_pad_get_timescale (GstQTMuxPad * pad) +{ + guint32 timescale; + + GST_OBJECT_LOCK (pad); + timescale = pad->trak_timescale; + GST_OBJECT_UNLOCK (pad); + + return timescale; +} + /* QTMux signals and args */ enum { @@ -264,12 +389,18 @@ enum PROP_RESERVED_DURATION_REMAINING, PROP_RESERVED_MOOV_UPDATE_PERIOD, PROP_RESERVED_BYTES_PER_SEC, + PROP_RESERVED_PREFILL, #ifndef GST_REMOVE_DEPRECATED PROP_DTS_METHOD, #endif PROP_DO_CTTS, PROP_INTERLEAVE_BYTES, PROP_INTERLEAVE_TIME, + PROP_MAX_RAW_AUDIO_DRIFT, + PROP_START_GAP_THRESHOLD, +#ifdef TIZEN_FEATURE_GST_MUX_ENHANCEMENT + PROP_EXPECTED_TRAILER_SIZE, +#endif /* TIZEN_FEATURE_GST_MUX_ENHANCEMENT */ }; /* some spare for header size as well */ @@ -289,8 +420,11 @@ enum #define DEFAULT_RESERVED_MAX_DURATION GST_CLOCK_TIME_NONE #define DEFAULT_RESERVED_MOOV_UPDATE_PERIOD GST_CLOCK_TIME_NONE #define DEFAULT_RESERVED_BYTES_PER_SEC_PER_TRAK 550 +#define DEFAULT_RESERVED_PREFILL FALSE #define DEFAULT_INTERLEAVE_BYTES 0 #define DEFAULT_INTERLEAVE_TIME 250*GST_MSECOND +#define DEFAULT_MAX_RAW_AUDIO_DRIFT 40 * GST_MSECOND +#define DEFAULT_START_GAP_THRESHOLD 0 static void gst_qt_mux_finalize (GObject * object); @@ -320,15 +454,240 @@ static GstFlowReturn gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, static GstFlowReturn gst_qt_mux_robust_recording_rewrite_moov (GstQTMux * qtmux); +static void gst_qt_mux_update_global_statistics (GstQTMux * qtmux); +static void gst_qt_mux_update_edit_lists (GstQTMux * qtmux); + static GstElementClass *parent_class = NULL; +#ifdef TIZEN_FEATURE_GST_MUX_ENHANCEMENT +/* + [[ Metadata Size ]] + 1. Common + free = 8 + moov = 8 + mvhd = 108 + ------------- + total : 124 + + 2. Video + i. Video common + trak = 8 + tkhd = 92 + mdia = 8 + mdhd = 32 + hdlr = 45 + minf = 8 + vmhd = 20 + dinf = 36 (8, dref : 16 , url : 12) + stbl = 8 + --------------- + total : 257 + + ii. Variation in file format + - MP4 + ftyp = 32 + udta = 61 + - 3GP + ftyp = 28 + udta = 8 + + iii. Variation in codec + - MPEG4 + stsd = 137(16, mp4v : 86, esds : 35) + + - H.264 = 487(or 489) + (8*stts_count) + (8*frame) + (4*I-frame) + stsd = 134 (SPS 9, PPS 4) or 136 (SPS 111, PPS 4) + + - H.263 = 470 + + (8*stts_count) + (8*frame) + (4*I-frame) + stsd = 102 -> different from H.264 + + iv. Variation in frame + stts = 16 + (8*stts_count) + stss = 16 + (4*I-frame) + stsc = 28 + stsz = 20 + (4*frame) + stco = 16 + (4*frame) + + 3. Audio + i. Audio common + trak = 8 + tkhd = 92 + mdia = 8 + mdhd = 32 + hdlr = 45 + minf = 8 + smhd = 16 + dinf = 36 (8, dref : 16, url : 12) + stbl = 8 + --------------- + total : 253 + + stts = 16 + stsz = 20 + stco = 16 + ------------ + total : 52 + + ii. Variation in file format + - MP4 + udta = 61 + - 3GP + udta = 8 + + iii. Variation in codec + - Common + stts = 16 + (8*stts_count) + stsc = 28 + stsz = 20 + (4*frame) + stco = 16 + (4*frame) + + - AAC + stsd = 94 (16, mp4a : 78(36 ,esds : 42)) + + - AMR + stsd = 69 (16, samr : 53(36, damr : 17)) +*/ + +/* trailer entry size */ +#define ENTRY_SIZE_VIDEO_STTS 8 +#define ENTRY_SIZE_VIDEO_STSS 4 +#define ENTRY_SIZE_VIDEO_STSZ 4 +#define ENTRY_SIZE_VIDEO_STCO 4 +#define ENTRY_SIZE_AUDIO_STTS 8 +#define ENTRY_SIZE_AUDIO_STSZ 4 +#define ENTRY_SIZE_AUDIO_STCO 4 + +#define ENTRY_SIZE_VIDEO_MPEG4_STSD 137 +#define ENTRY_SIZE_VIDEO_H263P_STSD 102 +#define ENTRY_SIZE_AUDIO_AAC_STSD 94 +#define ENTRY_SIZE_AUDIO_AMR_STSD 69 + +#define ENTRY_SIZE_STSC 28 +#define ENTRY_SIZE_VIDEO_ST 68 /*atom size (stss + stts + stsc + stsz + stco ) * (size + atom + version + flags + sample count)+stsz(sample size) */ +#define ENTRY_SIZE_AUDIO_ST 52 /*atom size (stss + stsc + stsz + stco ) * (size + atom + version + flags + sample count)+stsz(sample size) */ + +/* common */ +#define MUX_COMMON_SIZE_HEADER 124 /* free + moov + moov.mvhd*/ + +#define MUX_COMMON_SIZE_VIDEO_HEADER 257 +#define MUX_COMMON_SIZE_AUDIO_HEADER 253 + +#define MUX_COMMON_SIZE_MP4_FTYP 32 +#define MUX_COMMON_SIZE_3GP_FTYP 28 + +#define MUX_COMMON_SIZE_MP4_UDTA 61 +#define MUX_COMMON_SIZE_3GP_UDTA 8 + +static void +gst_qt_mux_update_expected_trailer_size (GstQTMux *qtmux, GstQTPad *pad) +{ + guint nb_video_frames = 0; + guint nb_video_i_frames = 0; + guint nb_video_stts_entry = 0; + guint nb_audio_frames = 0; + guint nb_audio_stts_entry = 0; + gboolean video_stream = FALSE; + gboolean audio_stream = FALSE; + guint exp_size = 0; + GstQTMuxClass *qtmux_klass = NULL; + + if (qtmux == NULL || pad == NULL) { + GST_ERROR_OBJECT (qtmux, "Invalid parameter"); + return; + } + + qtmux_klass = (GstQTMuxClass *)(G_OBJECT_GET_CLASS(qtmux)); + + if (!strncmp(GST_PAD_NAME(pad->collect.pad), "video", 5)) { + nb_video_frames += pad->trak->mdia.minf.stbl.stsz.table_size; + nb_video_i_frames += pad->trak->mdia.minf.stbl.stss.entries.len; + nb_video_stts_entry += pad->trak->mdia.minf.stbl.stts.entries.len; + + video_stream = TRUE; + } else if (!strncmp(GST_PAD_NAME(pad->collect.pad), "audio", 5)) { + nb_audio_frames += pad->trak->mdia.minf.stbl.stsz.table_size; + nb_audio_stts_entry += pad->trak->mdia.minf.stbl.stts.entries.len; + + audio_stream = TRUE; + } + + /* free + moov + mvhd */ + qtmux->expected_trailer_size = MUX_COMMON_SIZE_HEADER; + + /* ftyp + udta * 3 (There is 3 udta fields and it's same size) */ + switch (qtmux_klass->format) { + case GST_QT_MUX_FORMAT_MP4: + qtmux->expected_trailer_size += MUX_COMMON_SIZE_MP4_FTYP + MUX_COMMON_SIZE_MP4_UDTA * 3; + break; + case GST_QT_MUX_FORMAT_3GP: + qtmux->expected_trailer_size += MUX_COMMON_SIZE_3GP_FTYP + MUX_COMMON_SIZE_3GP_UDTA * 3; + break; + default: + break; + } + + /* Calculate trailer size for video stream */ + if (video_stream) { + switch (pad->fourcc) { + case FOURCC_h263: + case FOURCC_s263: + exp_size += MUX_COMMON_SIZE_VIDEO_HEADER + ENTRY_SIZE_VIDEO_H263P_STSD; + break; + case FOURCC_mp4v: + case FOURCC_MP4V: + case FOURCC_fmp4: + case FOURCC_FMP4: + case FOURCC_3gp4: + case FOURCC_3gp6: + case FOURCC_3gg6: + exp_size += MUX_COMMON_SIZE_VIDEO_HEADER + ENTRY_SIZE_VIDEO_MPEG4_STSD; + break; + default: + break; + } + + /* frame related */ + exp_size += ENTRY_SIZE_VIDEO_ST + (ENTRY_SIZE_VIDEO_STTS * nb_video_stts_entry) + + (ENTRY_SIZE_VIDEO_STSS * nb_video_i_frames) + (ENTRY_SIZE_STSC) + + ((ENTRY_SIZE_VIDEO_STSZ + ENTRY_SIZE_VIDEO_STCO) * nb_video_frames); + + qtmux->video_expected_trailer_size = exp_size; + } + + /* Calculate trailer size for audio stream */ + if (audio_stream) { + exp_size += MUX_COMMON_SIZE_AUDIO_HEADER + ENTRY_SIZE_AUDIO_ST + (ENTRY_SIZE_AUDIO_STTS * nb_audio_stts_entry) + + (ENTRY_SIZE_STSC) + ((ENTRY_SIZE_AUDIO_STSZ + ENTRY_SIZE_AUDIO_STCO) * nb_audio_frames); + + if (pad->fourcc == FOURCC_samr) + exp_size += ENTRY_SIZE_AUDIO_AMR_STSD; + else + exp_size += ENTRY_SIZE_AUDIO_AAC_STSD; + + qtmux->audio_expected_trailer_size = exp_size; + } + + qtmux->expected_trailer_size += qtmux->video_expected_trailer_size + qtmux->audio_expected_trailer_size; + + /* + GST_INFO_OBJECT (qtmux, "pad type %s", GST_PAD_NAME(pad->collect.pad)); + GST_INFO_OBJECT (qtmux, "VIDEO : stts-entry=[%d], i-frame=[%d], video-sample=[%d]", nb_video_stts_entry, nb_video_i_frames, nb_video_frames); + GST_INFO_OBJECT (qtmux, "AUDIO : stts-entry=[%d], audio-sample=[%d]", nb_audio_stts_entry, nb_audio_frames); + GST_INFO_OBJECT (qtmux, "expected trailer size %d", qtmux->expected_trailer_size); + */ + + return; +} +#endif /* TIZEN_FEATURE_GST_MUX_ENHANCEMENT */ + static void gst_qt_mux_base_init (gpointer g_class) { GstElementClass *element_class = GST_ELEMENT_CLASS (g_class); GstQTMuxClass *klass = (GstQTMuxClass *) g_class; GstQTMuxClassParams *params; - GstPadTemplate *videosinktempl, *audiosinktempl, *subtitlesinktempl; + GstPadTemplate *videosinktempl, *audiosinktempl, *subtitlesinktempl, + *captionsinktempl; GstPadTemplate *srctempl; gchar *longname, *description; @@ -353,23 +712,33 @@ gst_qt_mux_base_init (gpointer g_class) gst_element_class_add_pad_template (element_class, srctempl); if (params->audio_sink_caps) { - audiosinktempl = gst_pad_template_new ("audio_%u", - GST_PAD_SINK, GST_PAD_REQUEST, params->audio_sink_caps); + audiosinktempl = gst_pad_template_new_with_gtype ("audio_%u", + GST_PAD_SINK, GST_PAD_REQUEST, params->audio_sink_caps, + GST_TYPE_QT_MUX_PAD); gst_element_class_add_pad_template (element_class, audiosinktempl); } if (params->video_sink_caps) { - videosinktempl = gst_pad_template_new ("video_%u", - GST_PAD_SINK, GST_PAD_REQUEST, params->video_sink_caps); + videosinktempl = gst_pad_template_new_with_gtype ("video_%u", + GST_PAD_SINK, GST_PAD_REQUEST, params->video_sink_caps, + GST_TYPE_QT_MUX_PAD); gst_element_class_add_pad_template (element_class, videosinktempl); } if (params->subtitle_sink_caps) { - subtitlesinktempl = gst_pad_template_new ("subtitle_%u", - GST_PAD_SINK, GST_PAD_REQUEST, params->subtitle_sink_caps); + subtitlesinktempl = gst_pad_template_new_with_gtype ("subtitle_%u", + GST_PAD_SINK, GST_PAD_REQUEST, params->subtitle_sink_caps, + GST_TYPE_QT_MUX_PAD); gst_element_class_add_pad_template (element_class, subtitlesinktempl); } + if (params->caption_sink_caps) { + captionsinktempl = gst_pad_template_new_with_gtype ("caption_%u", + GST_PAD_SINK, GST_PAD_REQUEST, params->caption_sink_caps, + GST_TYPE_QT_MUX_PAD); + gst_element_class_add_pad_template (element_class, captionsinktempl); + } + klass->format = params->prop->format; } @@ -381,6 +750,9 @@ gst_qt_mux_class_init (GstQTMuxClass * klass) GParamFlags streamable_flags; const gchar *streamable_desc; gboolean streamable; +#ifdef TIZEN_FEATURE_GST_MUX_ENHANCEMENT + GParamSpec *tspec = NULL; +#endif /* TIZEN_FEATURE_GST_MUX_ENHANCEMENT */ #define STREAMABLE_DESC "If set to true, the output should be as if it is to "\ "be streamed and hence no indexes written or duration written." @@ -483,6 +855,12 @@ gst_qt_mux_class_init (GstQTMuxClass * klass) "Multiplier for converting reserved-max-duration into bytes of header to reserve, per second, per track", 0, 10000, DEFAULT_RESERVED_BYTES_PER_SEC_PER_TRAK, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_RESERVED_PREFILL, + g_param_spec_boolean ("reserved-prefill", + "Reserved Prefill Samples Table", + "Prefill samples table of reserved duration", + DEFAULT_RESERVED_PREFILL, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_INTERLEAVE_BYTES, g_param_spec_uint64 ("interleave-bytes", "Interleave (bytes)", "Interleave between streams in bytes", @@ -493,6 +871,26 @@ gst_qt_mux_class_init (GstQTMuxClass * klass) "Interleave between streams in nanoseconds", 0, G_MAXUINT64, DEFAULT_INTERLEAVE_TIME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_MAX_RAW_AUDIO_DRIFT, + g_param_spec_uint64 ("max-raw-audio-drift", "Max Raw Audio Drift", + "Maximum allowed drift of raw audio samples vs. timestamps in nanoseconds", + 0, G_MAXUINT64, DEFAULT_MAX_RAW_AUDIO_DRIFT, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_START_GAP_THRESHOLD, + g_param_spec_uint64 ("start-gap-threshold", "Start Gap Threshold", + "Threshold for creating an edit list for gaps at the start in nanoseconds", + 0, G_MAXUINT64, DEFAULT_START_GAP_THRESHOLD, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + +#ifdef TIZEN_FEATURE_GST_MUX_ENHANCEMENT + tspec = g_param_spec_uint("expected-trailer-size", "Expected Trailer Size", + "Expected trailer size (bytes)", + 0, G_MAXUINT, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS); + if (tspec) + g_object_class_install_property(gobject_class, PROP_EXPECTED_TRAILER_SIZE, tspec); + else + GST_ERROR("g_param_spec failed for \"expected-trailer-size\""); +#endif /* TIZEN_FEATURE_GST_MUX_ENHANCEMENT */ gstelement_class->request_new_pad = GST_DEBUG_FUNCPTR (gst_qt_mux_request_new_pad); @@ -508,6 +906,7 @@ gst_qt_mux_pad_reset (GstQTPad * qtpad) qtpad->sample_size = 0; qtpad->sync = FALSE; qtpad->last_dts = 0; + qtpad->sample_offset = 0; qtpad->dts_adjustment = GST_CLOCK_TIME_NONE; qtpad->first_ts = GST_CLOCK_TIME_NONE; qtpad->first_dts = GST_CLOCK_TIME_NONE; @@ -535,6 +934,9 @@ gst_qt_mux_pad_reset (GstQTPad * qtpad) qtpad->traf = NULL; } atom_array_clear (&qtpad->fragment_buffers); + if (qtpad->samples) + g_array_unref (qtpad->samples); + qtpad->samples = NULL; /* reference owned elsewhere */ qtpad->tfra = NULL; @@ -544,6 +946,10 @@ gst_qt_mux_pad_reset (GstQTPad * qtpad) if (qtpad->first_tc) gst_video_time_code_free (qtpad->first_tc); qtpad->first_tc = NULL; + + if (qtpad->raw_audio_adapter) + gst_object_unref (qtpad->raw_audio_adapter); + qtpad->raw_audio_adapter = NULL; } /* @@ -607,6 +1013,7 @@ gst_qt_mux_reset (GstQTMux * qtmux, gboolean alloc) if (alloc) { qtmux->moov = atom_moov_new (qtmux->context); +#ifndef TIZEN_FEATURE_GST_MUX_ENHANCEMENT /* ensure all is as nice and fresh as request_new_pad would provide it */ for (walk = qtmux->sinkpads; walk; walk = g_slist_next (walk)) { GstQTPad *qtpad = (GstQTPad *) walk->data; @@ -614,6 +1021,7 @@ gst_qt_mux_reset (GstQTMux * qtmux, gboolean alloc) qtpad->trak = atom_trak_new (qtmux->context); atom_moov_add_trak (qtmux->moov, qtpad->trak); } +#endif } qtmux->current_pad = NULL; @@ -625,6 +1033,12 @@ gst_qt_mux_reset (GstQTMux * qtmux, gboolean alloc) qtmux->last_moov_update = GST_CLOCK_TIME_NONE; qtmux->muxed_since_last_update = 0; qtmux->reserved_duration_remaining = GST_CLOCK_TIME_NONE; + +#ifdef TIZEN_FEATURE_GST_MUX_ENHANCEMENT + qtmux->expected_trailer_size = 0; + qtmux->video_expected_trailer_size = 0; + qtmux->audio_expected_trailer_size = 0; +#endif /* TIZEN_FEATURE_GST_MUX_ENHANCEMENT */ } static void @@ -655,6 +1069,8 @@ gst_qt_mux_init (GstQTMux * qtmux, GstQTMuxClass * qtmux_klass) DEFAULT_RESERVED_BYTES_PER_SEC_PER_TRAK; qtmux->interleave_bytes = DEFAULT_INTERLEAVE_BYTES; qtmux->interleave_time = DEFAULT_INTERLEAVE_TIME; + qtmux->max_raw_audio_drift = DEFAULT_MAX_RAW_AUDIO_DRIFT; + qtmux->start_gap_threshold = DEFAULT_START_GAP_THRESHOLD; /* always need this */ qtmux->context = @@ -710,6 +1126,159 @@ gst_qt_mux_prepare_jpc_buffer (GstQTPad * qtpad, GstBuffer * buf, return newbuf; } +static gsize +extract_608_field_from_s334_1a (const guint8 * ccdata, gsize ccdata_size, + guint field, guint8 ** res) +{ + guint8 *storage; + gsize storage_size = 128; + gsize i, res_size = 0; + + storage = g_malloc0 (storage_size); + + /* Iterate over the ccdata and put the corresponding tuples for the given field + * in the storage */ + for (i = 0; i < ccdata_size; i += 3) { + if ((field == 1 && (ccdata[i * 3] & 0x80)) || + (field == 2 && !(ccdata[i * 3] & 0x80))) { + GST_DEBUG ("Storing matching cc for field %d : 0x%02x 0x%02x", field, + ccdata[i * 3 + 1], ccdata[i * 3 + 2]); + if (res_size >= storage_size) { + storage_size += 128; + storage = g_realloc (storage, storage_size); + } + storage[res_size] = ccdata[i * 3 + 1]; + storage[res_size + 1] = ccdata[i * 3 + 2]; + res_size += 2; + } + } + + if (res_size == 0) { + g_free (storage); + *res = NULL; + return 0; + } + + *res = storage; + return res_size; +} + + +static GstBuffer * +gst_qt_mux_prepare_caption_buffer (GstQTPad * qtpad, GstBuffer * buf, + GstQTMux * qtmux) +{ + GstBuffer *newbuf = NULL; + GstMapInfo map, inmap; + gsize size; + gboolean in_prefill; + + if (buf == NULL) + return NULL; + + in_prefill = (qtmux->mux_mode == GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL); + + size = gst_buffer_get_size (buf); + gst_buffer_map (buf, &inmap, GST_MAP_READ); + + GST_LOG_OBJECT (qtmux, + "Preparing caption buffer %" GST_FOURCC_FORMAT " size:%" G_GSIZE_FORMAT, + GST_FOURCC_ARGS (qtpad->fourcc), size); + + switch (qtpad->fourcc) { + case FOURCC_c608: + { + guint8 *cdat, *cdt2; + gsize cdat_size, cdt2_size, total_size = 0; + gsize write_offs = 0; + + cdat_size = + extract_608_field_from_s334_1a (inmap.data, inmap.size, 1, &cdat); + cdt2_size = + extract_608_field_from_s334_1a (inmap.data, inmap.size, 2, &cdt2); + + if (cdat_size) + total_size += cdat_size + 8; + if (cdt2_size) + total_size += cdt2_size + 8; + if (total_size == 0) { + GST_DEBUG_OBJECT (qtmux, "No 608 data ?"); + /* FIXME : We might want to *always* store something, even if + * it's "empty" CC (i.e. 0x80 0x80) */ + break; + } + + newbuf = gst_buffer_new_and_alloc (in_prefill ? 20 : total_size); + /* Let's copy over all metadata and not the memory */ + gst_buffer_copy_into (newbuf, buf, GST_BUFFER_COPY_METADATA, 0, size); + + gst_buffer_map (newbuf, &map, GST_MAP_WRITE); + if (cdat_size || in_prefill) { + GST_WRITE_UINT32_BE (map.data, in_prefill ? 10 : cdat_size + 8); + GST_WRITE_UINT32_LE (map.data + 4, FOURCC_cdat); + if (cdat_size) + memcpy (map.data + 8, cdat, in_prefill ? 2 : cdat_size); + else { + /* Write 'empty' CC */ + map.data[8] = 0x80; + map.data[9] = 0x80; + } + write_offs = in_prefill ? 10 : cdat_size + 8; + if (cdat_size) + g_free (cdat); + } + + if (cdt2_size || in_prefill) { + GST_WRITE_UINT32_BE (map.data + write_offs, + in_prefill ? 10 : cdt2_size + 8); + GST_WRITE_UINT32_LE (map.data + write_offs + 4, FOURCC_cdt2); + if (cdt2_size) + memcpy (map.data + write_offs + 8, cdt2, in_prefill ? 2 : cdt2_size); + else { + /* Write 'empty' CC */ + map.data[write_offs + 8] = 0x80; + map.data[write_offs + 9] = 0x80; + } + if (cdt2_size) + g_free (cdt2); + } + gst_buffer_unmap (newbuf, &map); + break; + } + break; + case FOURCC_c708: + { + /* Take the whole CDP */ + if (in_prefill && size > 256) { + GST_ERROR_OBJECT (qtmux, "Input C708 CDP too big for prefill mode !"); + break; + } + newbuf = gst_buffer_new_and_alloc (in_prefill ? 256 + 8 : size + 8); + + /* Let's copy over all metadata and not the memory */ + gst_buffer_copy_into (newbuf, buf, GST_BUFFER_COPY_METADATA, 0, size); + + gst_buffer_map (newbuf, &map, GST_MAP_WRITE); + + GST_WRITE_UINT32_BE (map.data, size + 8); + GST_WRITE_UINT32_LE (map.data + 4, FOURCC_ccdp); + memcpy (map.data + 8, inmap.data, inmap.size); + + gst_buffer_unmap (newbuf, &map); + break; + } + default: + /* theoretically this should never happen, but let's keep this here in case */ + GST_WARNING_OBJECT (qtmux, "Unknown caption format"); + break; + } + + gst_buffer_unmap (buf, &inmap); + gst_buffer_unref (buf); + + return newbuf; +} + static GstBuffer * gst_qt_mux_prepare_tx3g_buffer (GstQTPad * qtpad, GstBuffer * buf, GstQTMux * qtmux) @@ -2100,6 +2669,493 @@ fail: qtmux->moov_recov_file = NULL; } +static guint64 +prefill_get_block_index (GstQTMux * qtmux, GstQTPad * qpad) +{ + switch (qpad->fourcc) { + case FOURCC_apch: + case FOURCC_apcn: + case FOURCC_apcs: + case FOURCC_apco: + case FOURCC_ap4h: + case FOURCC_ap4x: + case FOURCC_c608: + case FOURCC_c708: + return qpad->sample_offset; + case FOURCC_sowt: + case FOURCC_twos: + return gst_util_uint64_scale_ceil (qpad->sample_offset, + qpad->expected_sample_duration_n, + qpad->expected_sample_duration_d * + atom_trak_get_timescale (qpad->trak)); + default: + return -1; + } +} + +static guint +prefill_get_sample_size (GstQTMux * qtmux, GstQTPad * qpad) +{ + switch (qpad->fourcc) { + case FOURCC_apch: + if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 480) { + return 300000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 576) { + return 350000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 720) { + return 525000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 1080) { + return 1050000; + } else { + return 4150000; + } + break; + case FOURCC_apcn: + if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 480) { + return 200000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 576) { + return 250000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 720) { + return 350000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 1080) { + return 700000; + } else { + return 2800000; + } + break; + case FOURCC_apcs: + if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 480) { + return 150000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 576) { + return 200000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 720) { + return 250000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 1080) { + return 500000; + } else { + return 2800000; + } + break; + case FOURCC_apco: + if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 480) { + return 80000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 576) { + return 100000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 720) { + return 150000; + } else if (((SampleTableEntryMP4V *) qpad->trak_ste)->height <= 1080) { + return 250000; + } else { + return 900000; + } + break; + case FOURCC_c608: + /* We always write both cdat and cdt2 atom in prefill mode */ + return 20; + case FOURCC_c708: + /* We're cheating a bit by always allocating 256 bytes plus 8 bytes for the atom header + * even if we use less */ + return 256 + 8; + case FOURCC_sowt: + case FOURCC_twos:{ + guint64 block_idx; + guint64 next_sample_offset; + + block_idx = prefill_get_block_index (qtmux, qpad); + next_sample_offset = + gst_util_uint64_scale (block_idx + 1, + qpad->expected_sample_duration_d * + atom_trak_get_timescale (qpad->trak), + qpad->expected_sample_duration_n); + + return (next_sample_offset - qpad->sample_offset) * qpad->sample_size; + } + case FOURCC_ap4h: + case FOURCC_ap4x: + default: + GST_ERROR_OBJECT (qtmux, "unsupported codec for pre-filling"); + return -1; + } + + return -1; +} + +static GstClockTime +prefill_get_next_timestamp (GstQTMux * qtmux, GstQTPad * qpad) +{ + switch (qpad->fourcc) { + case FOURCC_apch: + case FOURCC_apcn: + case FOURCC_apcs: + case FOURCC_apco: + case FOURCC_ap4h: + case FOURCC_ap4x: + case FOURCC_c608: + case FOURCC_c708: + return gst_util_uint64_scale (qpad->sample_offset + 1, + qpad->expected_sample_duration_d * GST_SECOND, + qpad->expected_sample_duration_n); + case FOURCC_sowt: + case FOURCC_twos:{ + guint64 block_idx; + guint64 next_sample_offset; + + block_idx = prefill_get_block_index (qtmux, qpad); + next_sample_offset = + gst_util_uint64_scale (block_idx + 1, + qpad->expected_sample_duration_d * + atom_trak_get_timescale (qpad->trak), + qpad->expected_sample_duration_n); + + return gst_util_uint64_scale (next_sample_offset, GST_SECOND, + atom_trak_get_timescale (qpad->trak)); + } + default: + GST_ERROR_OBJECT (qtmux, "unsupported codec for pre-filling"); + return -1; + } + + return -1; +} + +static GstBuffer * +prefill_raw_audio_prepare_buf_func (GstQTPad * qtpad, GstBuffer * buf, + GstQTMux * qtmux) +{ + guint64 block_idx; + guint64 nsamples; + GstClockTime input_timestamp; + guint64 input_timestamp_distance; + + if (buf) + gst_adapter_push (qtpad->raw_audio_adapter, buf); + + block_idx = gst_util_uint64_scale_ceil (qtpad->raw_audio_adapter_offset, + qtpad->expected_sample_duration_n, + qtpad->expected_sample_duration_d * + atom_trak_get_timescale (qtpad->trak)); + nsamples = + gst_util_uint64_scale (block_idx + 1, + qtpad->expected_sample_duration_d * atom_trak_get_timescale (qtpad->trak), + qtpad->expected_sample_duration_n) - qtpad->raw_audio_adapter_offset; + + if ((!GST_COLLECT_PADS_STATE_IS_SET (&qtpad->collect, + GST_COLLECT_PADS_STATE_EOS) + && gst_adapter_available (qtpad->raw_audio_adapter) < + nsamples * qtpad->sample_size) + || gst_adapter_available (qtpad->raw_audio_adapter) == 0) { + return NULL; + } + + input_timestamp = + gst_adapter_prev_pts (qtpad->raw_audio_adapter, + &input_timestamp_distance); + if (input_timestamp != GST_CLOCK_TIME_NONE) + input_timestamp += + gst_util_uint64_scale (input_timestamp_distance, GST_SECOND, + qtpad->sample_size * atom_trak_get_timescale (qtpad->trak)); + + buf = + gst_adapter_take_buffer (qtpad->raw_audio_adapter, + !GST_COLLECT_PADS_STATE_IS_SET (&qtpad->collect, + GST_COLLECT_PADS_STATE_EOS) ? nsamples * + qtpad->sample_size : gst_adapter_available (qtpad->raw_audio_adapter)); + GST_BUFFER_PTS (buf) = input_timestamp; + GST_BUFFER_DTS (buf) = GST_CLOCK_TIME_NONE; + GST_BUFFER_DURATION (buf) = GST_CLOCK_TIME_NONE; + + qtpad->raw_audio_adapter_offset += nsamples; + + /* Check if we have yet another block of raw audio in the adapter */ + nsamples = + gst_util_uint64_scale (block_idx + 2, + qtpad->expected_sample_duration_d * atom_trak_get_timescale (qtpad->trak), + qtpad->expected_sample_duration_n) - qtpad->raw_audio_adapter_offset; + if (gst_adapter_available (qtpad->raw_audio_adapter) >= + nsamples * qtpad->sample_size) { + input_timestamp = + gst_adapter_prev_pts (qtpad->raw_audio_adapter, + &input_timestamp_distance); + if (input_timestamp != GST_CLOCK_TIME_NONE) + input_timestamp += + gst_util_uint64_scale (input_timestamp_distance, GST_SECOND, + qtpad->sample_size * atom_trak_get_timescale (qtpad->trak)); + qtpad->raw_audio_adapter_pts = input_timestamp; + } else { + qtpad->raw_audio_adapter_pts = GST_CLOCK_TIME_NONE; + } + + return buf; +} + +static void +find_video_sample_duration (GstQTMux * qtmux, guint * dur_n, guint * dur_d) +{ + GSList *walk; + + /* Find the (first) video track and assume that we have to output + * in that size */ + for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *tmp_qpad = (GstQTPad *) cdata; + + if (tmp_qpad->trak->is_video) { + *dur_n = tmp_qpad->expected_sample_duration_n; + *dur_d = tmp_qpad->expected_sample_duration_d; + break; + } + } + + if (walk == NULL) { + GST_INFO_OBJECT (qtmux, + "Found no video framerate, using 40ms audio buffers"); + *dur_n = 25; + *dur_d = 1; + } +} + +/* Called when all pads are prerolled to adjust and */ +static gboolean +prefill_update_sample_size (GstQTMux * qtmux, GstQTPad * qpad) +{ + switch (qpad->fourcc) { + case FOURCC_apch: + case FOURCC_apcn: + case FOURCC_apcs: + case FOURCC_apco: + case FOURCC_ap4h: + case FOURCC_ap4x: + { + guint sample_size = prefill_get_sample_size (qtmux, qpad); + atom_trak_set_constant_size_samples (qpad->trak, sample_size); + return TRUE; + } + case FOURCC_c608: + case FOURCC_c708: + { + guint sample_size = prefill_get_sample_size (qtmux, qpad); + /* We need a "valid" duration */ + find_video_sample_duration (qtmux, &qpad->expected_sample_duration_n, + &qpad->expected_sample_duration_d); + atom_trak_set_constant_size_samples (qpad->trak, sample_size); + return TRUE; + } + case FOURCC_sowt: + case FOURCC_twos:{ + find_video_sample_duration (qtmux, &qpad->expected_sample_duration_n, + &qpad->expected_sample_duration_d); + /* Set a prepare_buf_func that ensures this */ + qpad->prepare_buf_func = prefill_raw_audio_prepare_buf_func; + qpad->raw_audio_adapter = gst_adapter_new (); + qpad->raw_audio_adapter_offset = 0; + qpad->raw_audio_adapter_pts = GST_CLOCK_TIME_NONE; + + return TRUE; + } + default: + return TRUE; + } +} + +/* Only called at startup when doing the "fake" iteration of all tracks in order + * to prefill the sample tables in the header. */ +static GstQTPad * +find_best_pad_prefill_start (GstQTMux * qtmux) +{ + GSList *walk; + GstQTPad *best_pad = NULL; + + /* If interleave limits have been specified and the current pad is within + * those interleave limits, pick that one, otherwise let's try to figure out + * the next best one. */ + if (qtmux->current_pad && + (qtmux->interleave_bytes != 0 || qtmux->interleave_time != 0) && + (qtmux->interleave_bytes == 0 + || qtmux->current_chunk_size <= qtmux->interleave_bytes) + && (qtmux->interleave_time == 0 + || qtmux->current_chunk_duration <= qtmux->interleave_time) + && qtmux->mux_mode != GST_QT_MUX_MODE_FRAGMENTED + && qtmux->mux_mode != GST_QT_MUX_MODE_FRAGMENTED_STREAMABLE) { + + if (qtmux->current_pad->total_duration < qtmux->reserved_max_duration) { + best_pad = qtmux->current_pad; + } + } else if (qtmux->collect->data->next) { + /* Attempt to try another pad if we have one. Otherwise use the only pad + * present */ + best_pad = qtmux->current_pad = NULL; + } + + /* The next best pad is the one which has the lowest timestamp and hasn't + * exceeded the reserved max duration */ + if (!best_pad) { + GstClockTime best_time = GST_CLOCK_TIME_NONE; + + for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *qtpad = (GstQTPad *) cdata; + GstClockTime timestamp; + + if (qtpad->total_duration >= qtmux->reserved_max_duration) + continue; + + timestamp = qtpad->total_duration; + + if (best_pad == NULL || + !GST_CLOCK_TIME_IS_VALID (best_time) || timestamp < best_time) { + best_pad = qtpad; + best_time = timestamp; + } + } + } + + return best_pad; +} + +/* Called when starting the file in prefill_mode to figure out all the entries + * of the header based on the input stream and reserved maximum duration. + * + * The _actual_ header (i.e. with the proper duration and trimmed sample tables) + * will be updated and written on EOS. */ +static gboolean +gst_qt_mux_prefill_samples (GstQTMux * qtmux) +{ + GstQTPad *qpad; + GSList *walk; + GstQTMuxClass *qtmux_klass = (GstQTMuxClass *) (G_OBJECT_GET_CLASS (qtmux)); + + /* Update expected sample sizes/durations as needed, this is for raw + * audio where samples are actual audio samples. */ + for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *qpad = (GstQTPad *) cdata; + + if (!prefill_update_sample_size (qtmux, qpad)) + return FALSE; + } + + if (qtmux_klass->format == GST_QT_MUX_FORMAT_QT) { + /* For the first sample check/update timecode as needed. We do that before + * all actual samples as the code in gst_qt_mux_add_buffer() does it with + * initial buffer directly, not with last_buf */ + for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *qpad = (GstQTPad *) cdata; + GstBuffer *buffer = + gst_collect_pads_peek (qtmux->collect, (GstCollectData *) qpad); + GstVideoTimeCodeMeta *tc_meta; + + if (buffer && (tc_meta = gst_buffer_get_video_time_code_meta (buffer)) + && qpad->trak->is_video) { + GstVideoTimeCode *tc = &tc_meta->tc; + + qpad->tc_trak = atom_trak_new (qtmux->context); + atom_moov_add_trak (qtmux->moov, qpad->tc_trak); + + qpad->trak->tref = atom_tref_new (FOURCC_tmcd); + atom_tref_add_entry (qpad->trak->tref, qpad->tc_trak->tkhd.track_ID); + + atom_trak_set_timecode_type (qpad->tc_trak, qtmux->context, + qpad->trak->mdia.mdhd.time_info.timescale, tc); + + atom_trak_add_samples (qpad->tc_trak, 1, 1, 4, + qtmux->mdat_size, FALSE, 0); + + qpad->tc_pos = qtmux->mdat_size; + qpad->first_tc = gst_video_time_code_copy (tc); + qpad->first_pts = GST_BUFFER_PTS (buffer); + + qtmux->current_chunk_offset = -1; + qtmux->current_chunk_size = 0; + qtmux->current_chunk_duration = 0; + qtmux->mdat_size += 4; + } + if (buffer) + gst_buffer_unref (buffer); + } + } + + while ((qpad = find_best_pad_prefill_start (qtmux))) { + GstClockTime timestamp, next_timestamp, duration; + guint nsamples, sample_size; + guint64 chunk_offset; + gint64 scaled_duration; + gint64 pts_offset = 0; + gboolean sync = FALSE; + TrakBufferEntryInfo sample_entry; + + sample_size = prefill_get_sample_size (qtmux, qpad); + + if (sample_size == -1) { + return FALSE; + } + + if (!qpad->samples) + qpad->samples = g_array_new (FALSE, FALSE, sizeof (TrakBufferEntryInfo)); + + timestamp = qpad->total_duration; + next_timestamp = prefill_get_next_timestamp (qtmux, qpad); + duration = next_timestamp - timestamp; + + if (qpad->first_ts == GST_CLOCK_TIME_NONE) + qpad->first_ts = timestamp; + if (qpad->first_dts == GST_CLOCK_TIME_NONE) + qpad->first_dts = timestamp; + + if (qtmux->current_pad != qpad || qtmux->current_chunk_offset == -1) { + qtmux->current_pad = qpad; + if (qtmux->current_chunk_offset == -1) + qtmux->current_chunk_offset = qtmux->mdat_size; + else + qtmux->current_chunk_offset += qtmux->current_chunk_size; + qtmux->current_chunk_size = 0; + qtmux->current_chunk_duration = 0; + } + if (qpad->sample_size) + nsamples = sample_size / qpad->sample_size; + else + nsamples = 1; + qpad->last_dts = timestamp; + scaled_duration = gst_util_uint64_scale_round (timestamp + duration, + atom_trak_get_timescale (qpad->trak), + GST_SECOND) - gst_util_uint64_scale_round (timestamp, + atom_trak_get_timescale (qpad->trak), GST_SECOND); + + qtmux->current_chunk_size += sample_size; + qtmux->current_chunk_duration += duration; + qpad->total_bytes += sample_size; + + chunk_offset = qtmux->current_chunk_offset; + + /* I-frame only, no frame reordering */ + sync = FALSE; + pts_offset = 0; + + if (qtmux->current_chunk_duration > qtmux->longest_chunk + || !GST_CLOCK_TIME_IS_VALID (qtmux->longest_chunk)) { + qtmux->longest_chunk = qtmux->current_chunk_duration; + } + + sample_entry.track_id = qpad->trak->tkhd.track_ID; + sample_entry.nsamples = nsamples; + sample_entry.delta = scaled_duration / nsamples; + sample_entry.size = sample_size / nsamples; + sample_entry.chunk_offset = chunk_offset; + sample_entry.pts_offset = pts_offset; + sample_entry.sync = sync; + sample_entry.do_pts = TRUE; + g_array_append_val (qpad->samples, sample_entry); + atom_trak_add_samples (qpad->trak, nsamples, scaled_duration / nsamples, + sample_size / nsamples, chunk_offset, sync, pts_offset); + + qpad->total_duration = next_timestamp; + qtmux->mdat_size += sample_size; + qpad->sample_offset += nsamples; + } + + return TRUE; +} + static GstFlowReturn gst_qt_mux_start_file (GstQTMux * qtmux) { @@ -2110,6 +3166,7 @@ gst_qt_mux_start_file (GstQTMux * qtmux) gchar s_id[32]; GstClockTime reserved_max_duration; guint reserved_bytes_per_sec_per_trak; + GSList *walk; GST_DEBUG_OBJECT (qtmux, "starting file"); @@ -2146,7 +3203,10 @@ gst_qt_mux_start_file (GstQTMux * qtmux) } else if (qtmux->fast_start) { qtmux->mux_mode = GST_QT_MUX_MODE_FAST_START; } else if (reserved_max_duration != GST_CLOCK_TIME_NONE) { - qtmux->mux_mode = GST_QT_MUX_MODE_ROBUST_RECORDING; + if (qtmux->reserved_prefill) + qtmux->mux_mode = GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL; + else + qtmux->mux_mode = GST_QT_MUX_MODE_ROBUST_RECORDING; } switch (qtmux->mux_mode) { @@ -2162,6 +3222,10 @@ gst_qt_mux_start_file (GstQTMux * qtmux) (NULL)); return GST_FLOW_ERROR; } + if (qtmux->reserved_moov_update_period == GST_CLOCK_TIME_NONE) { + GST_WARNING_OBJECT (qtmux, + "Robust muxing requires reserved-moov-update-period to be set"); + } break; case GST_QT_MUX_MODE_FAST_START: case GST_QT_MUX_MODE_FRAGMENTED_STREAMABLE: @@ -2175,6 +3239,14 @@ gst_qt_mux_start_file (GstQTMux * qtmux) g_object_notify (G_OBJECT (qtmux), "streamable"); } break; + case GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL: + if (!gst_qt_mux_downstream_is_seekable (qtmux)) { + GST_WARNING_OBJECT (qtmux, + "downstream is not seekable, will not be able " + "to trim samples table at the end if less than reserved-duration is " + "recorded"); + } + break; } /* let downstream know we think in BYTES and expect to do seeking later on */ @@ -2215,6 +3287,50 @@ gst_qt_mux_start_file (GstQTMux * qtmux) qtmux->timescale = suggested_timescale; } + /* Set width/height/timescale of any closed caption tracks to that of the + * first video track */ + { + guint video_width = 0, video_height = 0; + guint32 video_timescale = 0; + GSList *walk; + + for (walk = qtmux->sinkpads; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *qpad = (GstQTPad *) cdata; + + if (!qpad->trak) + continue; + + /* Not closed caption */ + if (qpad->trak->mdia.hdlr.handler_type != FOURCC_clcp) + continue; + + if (video_width == 0 || video_height == 0 || video_timescale == 0) { + GSList *walk2; + + for (walk2 = qtmux->sinkpads; walk2; walk2 = g_slist_next (walk2)) { + GstCollectData *cdata2 = (GstCollectData *) walk2->data; + GstQTPad *qpad2 = (GstQTPad *) cdata2; + + if (!qpad2->trak) + continue; + + /* not video */ + if (!qpad2->trak->mdia.minf.vmhd) + continue; + + video_width = qpad2->trak->tkhd.width; + video_height = qpad2->trak->tkhd.height; + video_timescale = qpad2->trak->mdia.mdhd.time_info.timescale; + } + } + + qpad->trak->tkhd.width = video_width << 16; + qpad->trak->tkhd.height = video_height << 16; + qpad->trak->mdia.mdhd.time_info.timescale = video_timescale; + } + } + /* initialize our moov recovery file */ if (qtmux->moov_recov_file_path) { gst_qt_mux_prepare_moov_recovery (qtmux); @@ -2248,7 +3364,6 @@ gst_qt_mux_start_file (GstQTMux * qtmux) FALSE); break; case GST_QT_MUX_MODE_ROBUST_RECORDING: - ret = gst_qt_mux_prepare_and_send_ftyp (qtmux); if (ret != GST_FLOW_OK) break; @@ -2342,6 +3457,97 @@ gst_qt_mux_start_file (GstQTMux * qtmux) gst_qt_mux_send_mdat_header (qtmux, &qtmux->header_size, 0, TRUE, FALSE); break; + case GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL: + ret = gst_qt_mux_prepare_and_send_ftyp (qtmux); + if (ret != GST_FLOW_OK) + break; + + /* Store this as the moov offset for later updating. + * We record mdat position below */ + qtmux->moov_pos = qtmux->header_size; + + if (!gst_qt_mux_prefill_samples (qtmux)) { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, + ("Unsupported codecs or configuration for prefill mode"), (NULL)); + + return GST_FLOW_ERROR; + } + + gst_qt_mux_update_global_statistics (qtmux); + gst_qt_mux_configure_moov (qtmux); + gst_qt_mux_update_edit_lists (qtmux); + gst_qt_mux_setup_metadata (qtmux); + + /* Moov header with pre-filled samples */ + ret = gst_qt_mux_send_moov (qtmux, &qtmux->header_size, 0, FALSE, FALSE); + if (ret != GST_FLOW_OK) + return ret; + + /* last_moov_size now contains the full size of the moov, moov_pos the + * position. This allows us to rewrite it in the very end as needed */ + qtmux->reserved_moov_size = + qtmux->last_moov_size + 12 * g_slist_length (qtmux->sinkpads) + 8; + + /* Send an additional free atom at the end so we definitely have space + * to rewrite the moov header at the end and remove the samples that + * were not actually written */ + ret = + gst_qt_mux_send_free_atom (qtmux, &qtmux->header_size, + 12 * g_slist_length (qtmux->sinkpads) + 8, FALSE); + if (ret != GST_FLOW_OK) + return ret; + + /* extra atoms go after the free/moov(s), before the mdat */ + ret = + gst_qt_mux_send_extra_atoms (qtmux, TRUE, &qtmux->header_size, FALSE); + if (ret != GST_FLOW_OK) + return ret; + + qtmux->mdat_pos = qtmux->header_size; + + /* And now send the mdat header */ + ret = + gst_qt_mux_send_mdat_header (qtmux, &qtmux->header_size, + qtmux->mdat_size, TRUE, FALSE); + + /* chunks position is set relative to the first byte of the + * MDAT atom payload. Set the overall offset into the file */ + atom_moov_chunks_set_offset (qtmux->moov, qtmux->header_size); + + { + GstSegment segment; + + gst_segment_init (&segment, GST_FORMAT_BYTES); + segment.start = qtmux->moov_pos; + gst_pad_push_event (qtmux->srcpad, gst_event_new_segment (&segment)); + + ret = gst_qt_mux_send_moov (qtmux, NULL, 0, FALSE, FALSE); + if (ret != GST_FLOW_OK) + return ret; + + segment.start = qtmux->header_size; + gst_pad_push_event (qtmux->srcpad, gst_event_new_segment (&segment)); + } + + qtmux->current_chunk_size = 0; + qtmux->current_chunk_duration = 0; + qtmux->current_chunk_offset = -1; + qtmux->mdat_size = 0; + qtmux->current_pad = NULL; + qtmux->longest_chunk = GST_CLOCK_TIME_NONE; + + for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *qtpad = (GstQTPad *) cdata; + + qtpad->total_bytes = 0; + qtpad->total_duration = 0; + qtpad->first_dts = qtpad->first_ts = GST_CLOCK_TIME_NONE; + qtpad->last_dts = GST_CLOCK_TIME_NONE; + qtpad->sample_offset = 0; + } + + break; case GST_QT_MUX_MODE_FAST_START: GST_OBJECT_LOCK (qtmux); qtmux->fast_start_file = g_fopen (qtmux->fast_start_file_path, "wb+"); @@ -2460,14 +3666,24 @@ gst_qt_mux_update_global_statistics (GstQTMux * qtmux) /* having flushed above, can check for buffers now */ if (GST_CLOCK_TIME_IS_VALID (qtpad->first_ts)) { + GstClockTime first_pts_in = qtpad->first_ts; + /* it should be, since we got first_ts by adding adjustment + * to a positive incoming PTS */ + if (qtpad->dts_adjustment <= first_pts_in) + first_pts_in -= qtpad->dts_adjustment; /* determine max stream duration */ if (!GST_CLOCK_TIME_IS_VALID (qtmux->last_dts) || qtpad->last_dts > qtmux->last_dts) { qtmux->last_dts = qtpad->last_dts; } if (!GST_CLOCK_TIME_IS_VALID (qtmux->first_ts) - || qtpad->first_ts < qtmux->first_ts) { - qtmux->first_ts = qtpad->first_ts; + || first_pts_in < qtmux->first_ts) { + /* we need the original incoming PTS here, as this first_ts + * is used in update_edit_lists to construct the edit list that arrange + * for sync'ed streams. The first_ts is most likely obtained from + * some (audio) stream with 0 dts_adjustment and initial 0 PTS, + * so it makes no difference, though it matters in other cases */ + qtmux->first_ts = first_pts_in; } } @@ -2536,16 +3752,22 @@ gst_qt_mux_update_edit_lists (GstQTMux * qtmux) has_gap = (qtpad->first_ts > (qtmux->first_ts + qtpad->dts_adjustment)); if (has_gap) { - GstClockTime diff; + GstClockTime diff, trak_lateness; diff = qtpad->first_ts - (qtmux->first_ts + qtpad->dts_adjustment); lateness = gst_util_uint64_scale_round (diff, qtmux->timescale, GST_SECOND); - if (lateness > 0) { + /* Allow up to 1 trak timescale unit of lateness, Such a small + * timestamp/duration can't be represented by the trak-specific parts + * of the headers anyway, so it's irrelevantly small */ + trak_lateness = gst_util_uint64_scale (diff, + atom_trak_get_timescale (qtpad->trak), GST_SECOND); + + if (trak_lateness > 0 && diff > qtmux->start_gap_threshold) { GST_DEBUG_OBJECT (qtmux, "Pad %s is a late stream by %" GST_TIME_FORMAT, - GST_PAD_NAME (qtpad->collect.pad), GST_TIME_ARGS (lateness)); + GST_PAD_NAME (qtpad->collect.pad), GST_TIME_ARGS (diff)); atom_trak_set_elst_entry (qtpad->trak, 0, lateness, (guint32) - 1, (guint32) (1 * 65536.0)); @@ -2599,6 +3821,10 @@ gst_qt_mux_update_timecode (GstQTMux * qtmux, GstQTPad * qtpad) GstBuffer *buf; GstMapInfo map; guint64 offset = qtpad->tc_pos; + GstQTMuxClass *qtmux_klass = (GstQTMuxClass *) (G_OBJECT_GET_CLASS (qtmux)); + + if (qtmux_klass->format != GST_QT_MUX_FORMAT_QT) + return GST_FLOW_OK; g_assert (qtpad->tc_pos != -1); @@ -2671,9 +3897,11 @@ gst_qt_mux_stop_file (GstQTMux * qtmux) * mvhd should be consistent with empty moov * (but TODO maybe some clients do not handle that well ?) */ qtmux->moov->mvex.mehd.fragment_duration = - gst_util_uint64_scale (qtmux->last_dts, qtmux->timescale, GST_SECOND); - GST_DEBUG_OBJECT (qtmux, "rewriting moov with mvex duration %" - GST_TIME_FORMAT, GST_TIME_ARGS (qtmux->last_dts)); + gst_util_uint64_scale_round (qtmux->last_dts, qtmux->timescale, + GST_SECOND); + GST_DEBUG_OBJECT (qtmux, + "rewriting moov with mvex duration %" GST_TIME_FORMAT, + GST_TIME_ARGS (qtmux->last_dts)); /* seek and rewrite the header */ gst_segment_init (&segment, GST_FORMAT_BYTES); segment.start = qtmux->moov_pos; @@ -2691,6 +3919,184 @@ gst_qt_mux_stop_file (GstQTMux * qtmux) return gst_qt_mux_update_mdat_size (qtmux, qtmux->mdat_pos, qtmux->mdat_size, NULL, TRUE); } + case GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL:{ + GSList *walk; + guint32 next_track_id = qtmux->moov->mvhd.next_track_id; + + for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *qpad = (GstQTPad *) cdata; + guint64 block_idx; + AtomSTBL *stbl = &qpad->trak->mdia.minf.stbl; + + /* Get the block index of the last sample we wrote, not of the next + * sample we would write */ + block_idx = prefill_get_block_index (qtmux, qpad); + + /* stts */ + if (block_idx > 0) { + STTSEntry *entry; + guint64 nsamples = 0; + gint i, n; + + n = atom_array_get_len (&stbl->stts.entries); + for (i = 0; i < n; i++) { + entry = &atom_array_index (&stbl->stts.entries, i); + if (nsamples + entry->sample_count >= qpad->sample_offset) { + entry->sample_count = qpad->sample_offset - nsamples; + stbl->stts.entries.len = i + 1; + break; + } + nsamples += entry->sample_count; + } + g_assert (i < n); + } else { + stbl->stts.entries.len = 0; + } + + /* stsz */ + { + g_assert (stbl->stsz.entries.len == 0); + stbl->stsz.table_size = qpad->sample_offset; + } + + /* stco/stsc */ + { + gint i, n; + guint64 nsamples = 0; + gint chunk_index = 0; + const TrakBufferEntryInfo *sample_entry; + + if (block_idx > 0) { + sample_entry = + &g_array_index (qpad->samples, TrakBufferEntryInfo, + block_idx - 1); + + n = stbl->stco64.entries.len; + for (i = 0; i < n; i++) { + guint64 *entry = &atom_array_index (&stbl->stco64.entries, i); + + if (*entry == sample_entry->chunk_offset) { + stbl->stco64.entries.len = i + 1; + chunk_index = i + 1; + break; + } + } + g_assert (i < n); + g_assert (chunk_index > 0); + + n = stbl->stsc.entries.len; + for (i = 0; i < n; i++) { + STSCEntry *entry = &atom_array_index (&stbl->stsc.entries, i); + + if (entry->first_chunk >= chunk_index) + break; + + if (i > 0) { + nsamples += + (entry->first_chunk - atom_array_index (&stbl->stsc.entries, + i - + 1).first_chunk) * atom_array_index (&stbl->stsc.entries, + i - 1).samples_per_chunk; + } + } + g_assert (i <= n); + + if (i > 0) { + STSCEntry *prev_entry = + &atom_array_index (&stbl->stsc.entries, i - 1); + nsamples += + (chunk_index - + prev_entry->first_chunk) * prev_entry->samples_per_chunk; + if (qpad->sample_offset - nsamples > 0) { + stbl->stsc.entries.len = i; + atom_stsc_add_new_entry (&stbl->stsc, chunk_index, + qpad->sample_offset - nsamples); + } else { + stbl->stsc.entries.len = i; + stbl->stco64.entries.len--; + } + } else { + /* Everything in a single chunk */ + stbl->stsc.entries.len = 0; + atom_stsc_add_new_entry (&stbl->stsc, chunk_index, + qpad->sample_offset); + } + } else { + stbl->stco64.entries.len = 0; + stbl->stsc.entries.len = 0; + } + } + + { + GList *walk2; + + for (walk2 = qtmux->moov->mvex.trexs; walk2; walk2 = walk2->next) { + AtomTREX *trex = walk2->data; + + if (trex->track_ID == qpad->trak->tkhd.track_ID) { + trex->track_ID = next_track_id; + break; + } + } + + qpad->trak->tkhd.track_ID = next_track_id++; + } + } + qtmux->moov->mvhd.next_track_id = next_track_id; + + gst_qt_mux_update_global_statistics (qtmux); + gst_qt_mux_configure_moov (qtmux); + + gst_qt_mux_update_edit_lists (qtmux); + + /* Check if any gap edit lists were added. We don't have any space + * reserved for this in the moov and the pre-finalized moov would have + * broken A/V synchronization. Error out here now + */ + for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *qpad = (GstQTPad *) cdata; + + if (qpad->trak->edts + && g_slist_length (qpad->trak->edts->elst.entries) > 1) { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("Can't support gaps in prefill mode")); + + return GST_FLOW_ERROR; + } + } + + gst_qt_mux_setup_metadata (qtmux); + atom_moov_chunks_set_offset (qtmux->moov, qtmux->header_size); + + { + GstSegment segment; + + gst_segment_init (&segment, GST_FORMAT_BYTES); + segment.start = qtmux->moov_pos; + gst_pad_push_event (qtmux->srcpad, gst_event_new_segment (&segment)); + + ret = + gst_qt_mux_send_moov (qtmux, NULL, qtmux->reserved_moov_size, FALSE, + FALSE); + if (ret != GST_FLOW_OK) + return ret; + + if (qtmux->reserved_moov_size > qtmux->last_moov_size) { + ret = + gst_qt_mux_send_free_atom (qtmux, NULL, + qtmux->reserved_moov_size - qtmux->last_moov_size, TRUE); + } + + if (ret != GST_FLOW_OK) + return ret; + } + + ret = gst_qt_mux_update_mdat_size (qtmux, qtmux->mdat_pos, + qtmux->mdat_size, NULL, FALSE); + return ret; + } default: break; } @@ -3017,6 +4423,18 @@ gst_qt_mux_robust_recording_update (GstQTMux * qtmux, GstClockTime position) guint64 mdat_offset = qtmux->mdat_pos + 16 + qtmux->mdat_size; GST_OBJECT_LOCK (qtmux); + + /* Update the offset of how much we've muxed, so the + * report of remaining space keeps counting down */ + if (position > qtmux->last_moov_update && + position - qtmux->last_moov_update > qtmux->muxed_since_last_update) { + GST_LOG_OBJECT (qtmux, + "Muxed time %" G_GUINT64_FORMAT " since last moov update", + qtmux->muxed_since_last_update); + qtmux->muxed_since_last_update = position - qtmux->last_moov_update; + } + + /* Next, check if we're supposed to send periodic moov updates downstream */ if (qtmux->reserved_moov_update_period == GST_CLOCK_TIME_NONE) { GST_OBJECT_UNLOCK (qtmux); return GST_FLOW_OK; @@ -3027,15 +4445,6 @@ gst_qt_mux_robust_recording_update (GstQTMux * qtmux, GstClockTime position) (position <= qtmux->last_moov_update || (position - qtmux->last_moov_update) < qtmux->reserved_moov_update_period)) { - /* Update the offset of how much we've muxed, so the - * report of remaining space keeps counting down */ - if (position > qtmux->last_moov_update && - position - qtmux->last_moov_update > qtmux->muxed_since_last_update) { - GST_LOG_OBJECT (qtmux, - "Muxed time %" G_GUINT64_FORMAT " since last moov update", - qtmux->muxed_since_last_update); - qtmux->muxed_since_last_update = position - qtmux->last_moov_update; - } GST_OBJECT_UNLOCK (qtmux); return GST_FLOW_OK; /* No update needed yet */ } @@ -3080,6 +4489,56 @@ gst_qt_mux_register_and_push_sample (GstQTMux * qtmux, GstQTPad * pad, } switch (qtmux->mux_mode) { + case GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL:{ + const TrakBufferEntryInfo *sample_entry; + guint64 block_idx = prefill_get_block_index (qtmux, pad); + + if (block_idx >= pad->samples->len) { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("Unexpected sample %" G_GUINT64_FORMAT ", expected up to %u", + block_idx, pad->samples->len)); + gst_buffer_unref (buffer); + return GST_FLOW_ERROR; + } + + /* Check if all values are as expected */ + sample_entry = + &g_array_index (pad->samples, TrakBufferEntryInfo, block_idx); + + /* Allow +/- 1 difference for the scaled_duration to allow + * for some rounding errors + */ + if (sample_entry->nsamples != nsamples + || ABSDIFF (sample_entry->delta, scaled_duration) > 1 + || sample_entry->size != sample_size + || sample_entry->chunk_offset != chunk_offset + || sample_entry->pts_offset != pts_offset + || sample_entry->sync != sync) { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("Unexpected values in sample %" G_GUINT64_FORMAT, + pad->sample_offset + 1)); + GST_ERROR_OBJECT (qtmux, "Expected: samples %u, delta %u, size %u, " + "chunk offset %" G_GUINT64_FORMAT ", " + "pts offset %" G_GUINT64_FORMAT ", sync %d", + sample_entry->nsamples, + sample_entry->delta, + sample_entry->size, + sample_entry->chunk_offset, + sample_entry->pts_offset, sample_entry->sync); + GST_ERROR_OBJECT (qtmux, "Got: samples %u, delta %u, size %u, " + "chunk offset %" G_GUINT64_FORMAT ", " + "pts offset %" G_GUINT64_FORMAT ", sync %d", + nsamples, + (guint) scaled_duration, + sample_size, chunk_offset, pts_offset, sync); + + gst_buffer_unref (buffer); + return GST_FLOW_ERROR; + } + + ret = gst_qt_mux_send_buffer (qtmux, buffer, &qtmux->mdat_size, TRUE); + break; + } case GST_QT_MUX_MODE_MOOV_AT_END: case GST_QT_MUX_MODE_FAST_START: case GST_QT_MUX_MODE_ROBUST_RECORDING: @@ -3103,6 +4562,23 @@ gst_qt_mux_register_and_push_sample (GstQTMux * qtmux, GstQTPad * pad, return ret; } +static void +gst_qt_mux_register_buffer_in_chunk (GstQTMux * qtmux, GstQTPad * pad, + guint buffer_size, GstClockTime duration) +{ + /* not that much happens here, + * but updating any of this very likely needs to happen all in sync, + * unless there is a very good reason not to */ + + /* for computing the avg bitrate */ + pad->total_bytes += buffer_size; + pad->total_duration += duration; + /* for keeping track of where we are in chunk; + * ensures that data really is located as recorded in atoms */ + qtmux->current_chunk_size += buffer_size; + qtmux->current_chunk_duration += duration; +} + static GstFlowReturn gst_qt_mux_check_and_update_timecode (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf, GstFlowReturn ret) @@ -3112,6 +4588,13 @@ gst_qt_mux_check_and_update_timecode (GstQTMux * qtmux, GstQTPad * pad, GstBuffer *tc_buf; gsize szret; guint32 frames_since_daily_jam; + GstQTMuxClass *qtmux_klass = (GstQTMuxClass *) (G_OBJECT_GET_CLASS (qtmux)); + + if (!pad->trak->is_video) + return ret; + + if (qtmux_klass->format != GST_QT_MUX_FORMAT_QT) + return ret; if (buf == NULL || (pad->tc_trak != NULL && pad->tc_pos == -1)) return ret; @@ -3130,7 +4613,6 @@ gst_qt_mux_check_and_update_timecode (GstQTMux * qtmux, GstQTPad * pad, g_free (tc_str); #endif g_assert (pad->tc_trak == NULL); - tc_buf = gst_buffer_new_allocate (NULL, 4, NULL); pad->first_tc = gst_video_time_code_copy (tc); /* If frames are out of order, the frame we're currently getting might * not be the first one. Just write a 0 timecode for now and wait @@ -3152,8 +4634,10 @@ gst_qt_mux_check_and_update_timecode (GstQTMux * qtmux, GstQTPad * pad, pad->trak->tref = atom_tref_new (FOURCC_tmcd); atom_tref_add_entry (pad->trak->tref, pad->tc_trak->tkhd.track_ID); - atom_trak_set_timecode_type (pad->tc_trak, qtmux->context, pad->first_tc); + atom_trak_set_timecode_type (pad->tc_trak, qtmux->context, + pad->trak->mdia.mdhd.time_info.timescale, pad->first_tc); + tc_buf = gst_buffer_new_allocate (NULL, 4, NULL); szret = gst_buffer_fill (tc_buf, 0, &frames_since_daily_jam, 4); g_assert (szret == 4); @@ -3166,6 +4650,21 @@ gst_qt_mux_check_and_update_timecode (GstQTMux * qtmux, GstQTPad * pad, qtmux->current_chunk_offset = -1; qtmux->current_chunk_size = 0; qtmux->current_chunk_duration = 0; + } else if (qtmux->mux_mode == GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL) { + frames_since_daily_jam = + gst_video_time_code_frames_since_daily_jam (pad->first_tc); + frames_since_daily_jam = GUINT32_TO_BE (frames_since_daily_jam); + + tc_buf = gst_buffer_new_allocate (NULL, 4, NULL); + szret = gst_buffer_fill (tc_buf, 0, &frames_since_daily_jam, 4); + g_assert (szret == 4); + + ret = gst_qt_mux_send_buffer (qtmux, tc_buf, &qtmux->mdat_size, TRUE); + pad->tc_pos = -1; + + qtmux->current_chunk_offset = -1; + qtmux->current_chunk_size = 0; + qtmux->current_chunk_duration = 0; } else if (pad->is_out_of_order) { /* Check for a lower timecode than the one stored */ g_assert (pad->tc_trak != NULL); @@ -3206,13 +4705,19 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) gint64 pts_offset = 0; gboolean sync = FALSE; GstFlowReturn ret = GST_FLOW_OK; + guint buffer_size; if (!pad->fourcc) goto not_negotiated; /* if this pad has a prepare function, call it */ if (pad->prepare_buf_func != NULL) { - buf = pad->prepare_buf_func (pad, buf, qtmux); + GstBuffer *new_buf; + + new_buf = pad->prepare_buf_func (pad, buf, qtmux); + if (buf && !new_buf) + return GST_FLOW_OK; + buf = new_buf; } ret = gst_qt_mux_check_and_update_timecode (qtmux, pad, buf, ret); @@ -3237,7 +4742,6 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) GST_PAD_NAME (pad->collect.pad)); } #endif - qtmux->current_pad = pad; goto exit; } @@ -3276,10 +4780,39 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) GST_ERROR ("decreasing DTS value %" GST_TIME_FORMAT " < %" GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_DTS (buf)), GST_TIME_ARGS (GST_BUFFER_DTS (last_buf))); - buf = gst_buffer_make_writable (buf); + pad->last_buf = buf = gst_buffer_make_writable (buf); GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (last_buf); } + buffer_size = gst_buffer_get_size (last_buf); + + if (qtmux->mux_mode == GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL) { + guint required_buffer_size = prefill_get_sample_size (qtmux, pad); + guint fill_size = required_buffer_size - buffer_size; + GstMemory *mem; + GstMapInfo map; + + if (required_buffer_size < buffer_size) { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("Sample size %u bigger than expected maximum %u", buffer_size, + required_buffer_size)); + goto bail; + } + + if (fill_size > 0) { + GST_DEBUG_OBJECT (qtmux, + "Padding buffer by %u bytes to reach required %u bytes", fill_size, + required_buffer_size); + mem = gst_allocator_alloc (NULL, fill_size, NULL); + gst_memory_map (mem, &map, GST_MAP_WRITE); + memset (map.data, 0, map.size); + gst_memory_unmap (mem, &map); + last_buf = gst_buffer_make_writable (last_buf); + gst_buffer_append_memory (last_buf, mem); + buffer_size = required_buffer_size; + } + } + /* duration actually means time delta between samples, so we calculate * the duration based on the difference in DTS or PTS, falling back * to DURATION if the other two don't exist, such as with the last @@ -3318,20 +4851,33 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) /* fragments only deal with 1 buffer == 1 chunk (== 1 sample) */ if (pad->sample_size && !qtmux->fragment_sequence) { + GstClockTime expected_timestamp; + /* Constant size packets: usually raw audio (with many samples per buffer (= chunk)), but can also be fixed-packet-size codecs like ADPCM */ sample_size = pad->sample_size; - if (gst_buffer_get_size (last_buf) % sample_size != 0) + if (buffer_size % sample_size != 0) goto fragmented_sample; + /* note: qt raw audio storage warps it implicitly into a timewise - * perfect stream, discarding buffer times */ + * perfect stream, discarding buffer times. + * If the difference between the current PTS and the expected one + * becomes too big, we error out: there was a gap and we have no way to + * represent that, causing A/V sync to be off */ + expected_timestamp = + gst_util_uint64_scale (pad->sample_offset, GST_SECOND, + atom_trak_get_timescale (pad->trak)) + pad->first_ts; + if (ABSDIFF (GST_BUFFER_DTS_OR_PTS (last_buf), + expected_timestamp) > qtmux->max_raw_audio_drift) + goto raw_audio_timestamp_drift; + if (GST_BUFFER_DURATION (last_buf) != GST_CLOCK_TIME_NONE) { nsamples = gst_util_uint64_scale_round (GST_BUFFER_DURATION (last_buf), atom_trak_get_timescale (pad->trak), GST_SECOND); duration = GST_BUFFER_DURATION (last_buf); } else { - nsamples = gst_buffer_get_size (last_buf) / sample_size; + nsamples = buffer_size / sample_size; duration = gst_util_uint64_scale_round (nsamples, GST_SECOND, atom_trak_get_timescale (pad->trak)); @@ -3339,12 +4885,14 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) /* timescale = samplerate */ scaled_duration = 1; - pad->last_dts += duration * nsamples; + pad->last_dts = + pad->first_dts + gst_util_uint64_scale_round (pad->sample_offset + + nsamples, GST_SECOND, atom_trak_get_timescale (pad->trak)); } else { nsamples = 1; - sample_size = gst_buffer_get_size (last_buf); - if ((buf && GST_BUFFER_DTS_IS_VALID (buf)) - || GST_BUFFER_DTS_IS_VALID (last_buf)) { + sample_size = buffer_size; + if (!pad->sparse && ((buf && GST_BUFFER_DTS_IS_VALID (buf)) + || GST_BUFFER_DTS_IS_VALID (last_buf))) { gint64 scaled_dts; if (buf && GST_BUFFER_DTS_IS_VALID (buf)) { pad->last_dts = GST_BUFFER_DTS (buf); @@ -3371,11 +4919,7 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) } } - /* for computing the avg bitrate */ - pad->total_bytes += gst_buffer_get_size (last_buf); - pad->total_duration += duration; - qtmux->current_chunk_size += gst_buffer_get_size (last_buf); - qtmux->current_chunk_duration += duration; + gst_qt_mux_register_buffer_in_chunk (qtmux, pad, buffer_size, duration); chunk_offset = qtmux->current_chunk_offset; @@ -3422,10 +4966,47 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) qtmux->longest_chunk = qtmux->current_chunk_duration; } + if (qtmux->mux_mode == GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL) { + const TrakBufferEntryInfo *sample_entry; + guint64 block_idx = prefill_get_block_index (qtmux, pad); + + if (block_idx >= pad->samples->len) { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("Unexpected sample %" G_GUINT64_FORMAT ", expected up to %u", + block_idx, pad->samples->len)); + goto bail; + } + + /* Check if all values are as expected */ + sample_entry = + &g_array_index (pad->samples, TrakBufferEntryInfo, block_idx); + + if (chunk_offset < sample_entry->chunk_offset) { + guint fill_size = sample_entry->chunk_offset - chunk_offset; + GstBuffer *fill_buf; + + fill_buf = gst_buffer_new_allocate (NULL, fill_size, NULL); + gst_buffer_memset (fill_buf, 0, 0, fill_size); + + ret = gst_qt_mux_send_buffer (qtmux, fill_buf, &qtmux->mdat_size, TRUE); + if (ret != GST_FLOW_OK) + goto bail; + qtmux->current_chunk_offset = chunk_offset = sample_entry->chunk_offset; + qtmux->current_chunk_size = buffer_size; + qtmux->current_chunk_duration = duration; + } else if (chunk_offset != sample_entry->chunk_offset) { + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("Unexpected chunk offset %" G_GUINT64_FORMAT ", expected up to %" + G_GUINT64_FORMAT, chunk_offset, sample_entry->chunk_offset)); + goto bail; + } + } + /* now we go and register this buffer/sample all over */ ret = gst_qt_mux_register_and_push_sample (qtmux, pad, last_buf, buf == NULL, nsamples, last_dts, scaled_duration, sample_size, chunk_offset, sync, TRUE, pts_offset); + pad->sample_offset += nsamples; /* if this is sparse and we have a next buffer, check if there is any gap * between them to insert an empty sample */ @@ -3435,21 +5016,27 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) gint64 empty_duration = GST_BUFFER_PTS (buf) - (GST_BUFFER_PTS (last_buf) + duration); gint64 empty_duration_scaled; + guint empty_size; empty_buf = pad->create_empty_buffer (pad, empty_duration); - empty_duration_scaled = gst_util_uint64_scale_round (empty_duration, - atom_trak_get_timescale (pad->trak), GST_SECOND); + pad->last_dts = GST_BUFFER_PTS (buf); + empty_duration_scaled = gst_util_uint64_scale_round (pad->last_dts, + atom_trak_get_timescale (pad->trak), GST_SECOND) + - (last_dts + scaled_duration); + empty_size = gst_buffer_get_size (empty_buf); - pad->total_bytes += gst_buffer_get_size (empty_buf); - pad->total_duration += duration; + gst_qt_mux_register_buffer_in_chunk (qtmux, pad, empty_size, + empty_duration); ret = gst_qt_mux_register_and_push_sample (qtmux, pad, empty_buf, FALSE, 1, last_dts + scaled_duration, empty_duration_scaled, - gst_buffer_get_size (empty_buf), chunk_offset, sync, TRUE, 0); - } else { - /* our only case currently is tx3g subtitles, so there is no reason to fill this yet */ + empty_size, chunk_offset, sync, TRUE, 0); + } else if (pad->fourcc != FOURCC_c608 && pad->fourcc != FOURCC_c708) { + /* This assert is kept here to make sure implementors of new + * sparse input format decide whether there needs to be special + * gap handling or not */ g_assert_not_reached (); GST_WARNING_OBJECT (qtmux, "no empty buffer creation function found for pad %s", @@ -3457,6 +5044,10 @@ gst_qt_mux_add_buffer (GstQTMux * qtmux, GstQTPad * pad, GstBuffer * buf) } } +#ifdef TIZEN_FEATURE_GST_MUX_ENHANCEMENT + gst_qt_mux_update_expected_trailer_size(qtmux, pad); +#endif /* TIZEN_FEATURE_GST_MUX_ENHANCEMENT */ + exit: return ret; @@ -3473,6 +5064,18 @@ fragmented_sample: ("Audio buffer contains fragmented sample.")); goto bail; } +raw_audio_timestamp_drift: + { + /* TODO: Could in theory be implemented with edit lists */ + GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), + ("Audio stream timestamps are drifting (got %" GST_TIME_FORMAT + ", expected %" GST_TIME_FORMAT "). This is not supported yet!", + GST_TIME_ARGS (GST_BUFFER_DTS_OR_PTS (last_buf)), + GST_TIME_ARGS (gst_util_uint64_scale (pad->sample_offset, + GST_SECOND, + atom_trak_get_timescale (pad->trak)) + pad->first_ts))); + goto bail; + } no_pts: { GST_ELEMENT_ERROR (qtmux, STREAM, MUX, (NULL), ("Buffer has no PTS.")); @@ -3546,9 +5149,64 @@ find_best_pad (GstQTMux * qtmux, GstCollectPads * pads) GSList *walk; GstQTPad *best_pad = NULL; - if (qtmux->current_pad && - (qtmux->interleave_bytes != 0 || qtmux->interleave_time != 0) && - (qtmux->interleave_bytes == 0 + if (qtmux->mux_mode == GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL) { + guint64 smallest_offset = G_MAXUINT64; + guint64 chunk_offset = 0; + + for (walk = qtmux->collect->data; walk; walk = g_slist_next (walk)) { + GstCollectData *cdata = (GstCollectData *) walk->data; + GstQTPad *qtpad = (GstQTPad *) cdata; + const TrakBufferEntryInfo *sample_entry; + guint64 block_idx, current_block_idx; + guint64 chunk_offset_offset = 0; + GstBuffer *tmp_buf = + gst_collect_pads_peek (pads, (GstCollectData *) qtpad); + + /* Check for EOS pads and just skip them */ + if (!tmp_buf && !qtpad->last_buf && (!qtpad->raw_audio_adapter + || gst_adapter_available (qtpad->raw_audio_adapter) == 0)) + continue; + if (tmp_buf) + gst_buffer_unref (tmp_buf); + + /* Find the exact offset where the next sample of this track is supposed + * to be written at */ + block_idx = current_block_idx = prefill_get_block_index (qtmux, qtpad); + sample_entry = + &g_array_index (qtpad->samples, TrakBufferEntryInfo, block_idx); + while (block_idx > 0) { + const TrakBufferEntryInfo *tmp = + &g_array_index (qtpad->samples, TrakBufferEntryInfo, block_idx - 1); + + if (tmp->chunk_offset != sample_entry->chunk_offset) + break; + chunk_offset_offset += tmp->size * tmp->nsamples; + block_idx--; + } + + /* Except for the previously selected pad being EOS we always have + * qtmux->current_chunk_offset + qtmux->current_chunk_size + * == + * sample_entry->chunk_offset + chunk_offset_offset + * for the best pad. Instead of checking that, we just return the + * pad that has the smallest offset for the next to-be-written sample. + */ + if (sample_entry->chunk_offset + chunk_offset_offset < smallest_offset) { + smallest_offset = sample_entry->chunk_offset + chunk_offset_offset; + best_pad = qtpad; + chunk_offset = sample_entry->chunk_offset; + } + } + + if (chunk_offset != qtmux->current_chunk_offset) { + qtmux->current_pad = NULL; + } + + return best_pad; + } + + if (qtmux->current_pad && (qtmux->interleave_bytes != 0 + || qtmux->interleave_time != 0) && (qtmux->interleave_bytes == 0 || qtmux->current_chunk_size <= qtmux->interleave_bytes) && (qtmux->interleave_time == 0 || qtmux->current_chunk_duration <= qtmux->interleave_time) @@ -3641,9 +5299,16 @@ gst_qt_mux_collected (GstCollectPads * pads, gpointer user_data) /* clipping already converted to running time */ if (best_pad != NULL) { - GstBuffer *buf = gst_collect_pads_pop (pads, (GstCollectData *) best_pad); + GstBuffer *buf = NULL; + + if (qtmux->mux_mode != GST_QT_MUX_MODE_ROBUST_RECORDING_PREFILL || + best_pad->raw_audio_adapter == NULL || + best_pad->raw_audio_adapter_pts == GST_CLOCK_TIME_NONE) + buf = gst_collect_pads_pop (pads, (GstCollectData *) best_pad); + + g_assert (buf || best_pad->last_buf || (best_pad->raw_audio_adapter + && gst_adapter_available (best_pad->raw_audio_adapter) > 0)); - g_assert (buf || best_pad->last_buf); if (buf) gst_qt_pad_adjust_buffer_dts (qtmux, best_pad, (GstCollectData *) best_pad, &buf); @@ -3685,6 +5350,36 @@ gst_qtmux_caps_is_subset_full (GstQTMux * qtmux, GstCaps * subset, return gst_structure_foreach (sub_s, check_field, sup_s); } +/* will unref @qtmux */ +static gboolean +gst_qt_mux_can_renegotiate (GstQTMux * qtmux, GstPad * pad, GstCaps * caps) +{ + GstCaps *current_caps; + + /* does not go well to renegotiate stream mid-way, unless + * the old caps are a subset of the new one (this means upstream + * added more info to the caps, as both should be 'fixed' caps) */ + current_caps = gst_pad_get_current_caps (pad); + g_assert (caps != NULL); + + if (!gst_qtmux_caps_is_subset_full (qtmux, current_caps, caps)) { + gst_caps_unref (current_caps); + GST_WARNING_OBJECT (qtmux, + "pad %s refused renegotiation to %" GST_PTR_FORMAT, + GST_PAD_NAME (pad), caps); + gst_object_unref (qtmux); + return FALSE; + } + + GST_DEBUG_OBJECT (qtmux, + "pad %s accepted renegotiation to %" GST_PTR_FORMAT " from %" + GST_PTR_FORMAT, GST_PAD_NAME (pad), caps, current_caps); + gst_object_unref (qtmux); + gst_caps_unref (current_caps); + + return TRUE; +} + static gboolean gst_qt_mux_audio_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) { @@ -3701,31 +5396,16 @@ gst_qt_mux_audio_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) AtomInfo *ext_atom = NULL; gint constant_size = 0; const gchar *stream_format; + guint32 timescale; - qtpad->prepare_buf_func = NULL; - - /* does not go well to renegotiate stream mid-way, unless - * the old caps are a subset of the new one (this means upstream - * added more info to the caps, as both should be 'fixed' caps) */ - if (qtpad->fourcc) { - GstCaps *current_caps; - - current_caps = gst_pad_get_current_caps (pad); - g_assert (caps != NULL); - - if (!gst_qtmux_caps_is_subset_full (qtmux, current_caps, caps)) { - gst_caps_unref (current_caps); - goto refuse_renegotiation; - } - GST_DEBUG_OBJECT (qtmux, - "pad %s accepted renegotiation to %" GST_PTR_FORMAT " from %" - GST_PTR_FORMAT, GST_PAD_NAME (pad), caps, current_caps); - gst_caps_unref (current_caps); - } + if (qtpad->fourcc) + return gst_qt_mux_can_renegotiate (qtmux, pad, caps); GST_DEBUG_OBJECT (qtmux, "%s:%s, caps=%" GST_PTR_FORMAT, GST_DEBUG_PAD_NAME (pad), caps); + qtpad->prepare_buf_func = NULL; + format = qtmux_klass->format; structure = gst_caps_get_structure (caps, 0); mimetype = gst_structure_get_name (structure); @@ -3778,10 +5458,12 @@ gst_qt_mux_audio_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) qtpad->max_bitrate); } if (layer == 1) { - g_warn_if_fail (format == GST_QT_MUX_FORMAT_MP4); + g_warn_if_fail (format == GST_QT_MUX_FORMAT_MP4 + || format == GST_QT_MUX_FORMAT_QT); entry.samples_per_packet = 384; } else if (layer == 2) { - g_warn_if_fail (format == GST_QT_MUX_FORMAT_MP4); + g_warn_if_fail (format == GST_QT_MUX_FORMAT_MP4 + || format == GST_QT_MUX_FORMAT_QT); entry.samples_per_packet = 1152; } else { g_warn_if_fail (layer == 3); @@ -4019,14 +5701,18 @@ gst_qt_mux_audio_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) if (!entry.fourcc) goto refuse_caps; + timescale = gst_qt_mux_pad_get_timescale (GST_QT_MUX_PAD_CAST (pad)); + if (!timescale && qtmux->trak_timescale) + timescale = qtmux->trak_timescale; + else if (!timescale) + timescale = entry.sample_rate; + /* ok, set the pad info accordingly */ qtpad->fourcc = entry.fourcc; qtpad->sample_size = constant_size; qtpad->trak_ste = (SampleTableEntry *) atom_trak_set_audio_type (qtpad->trak, - qtmux->context, &entry, - qtmux->trak_timescale ? qtmux->trak_timescale : entry.sample_rate, - ext_atom, constant_size); + qtmux->context, &entry, timescale, ext_atom, constant_size); gst_object_unref (qtmux); return TRUE; @@ -4039,31 +5725,6 @@ refuse_caps: gst_object_unref (qtmux); return FALSE; } -refuse_renegotiation: - { - GST_WARNING_OBJECT (qtmux, - "pad %s refused renegotiation to %" GST_PTR_FORMAT, - GST_PAD_NAME (pad), caps); - gst_object_unref (qtmux); - return FALSE; - } -} - -/* return number of centiframes per second */ -static guint -adjust_rate (gint n, gint d) -{ - if (n == 0) - return 10000; - - if (d != 1 && d != 1001) { - /* otherwise there are probably rounding errors and we should rather guess - * if it's close enough to a well known framerate */ - gst_video_guess_framerate (gst_util_uint64_scale (d, GST_SECOND, n), &n, - &d); - } - - return gst_util_uint64_scale (n, 100, d); } static gboolean @@ -4085,31 +5746,16 @@ gst_qt_mux_video_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) GList *ext_atom_list = NULL; gboolean sync = FALSE; int par_num, par_den; + const gchar *multiview_mode; - qtpad->prepare_buf_func = NULL; - - /* does not go well to renegotiate stream mid-way, unless - * the old caps are a subset of the new one (this means upstream - * added more info to the caps, as both should be 'fixed' caps) */ - if (qtpad->fourcc) { - GstCaps *current_caps; - - current_caps = gst_pad_get_current_caps (pad); - g_assert (caps != NULL); - - if (!gst_qtmux_caps_is_subset_full (qtmux, current_caps, caps)) { - gst_caps_unref (current_caps); - goto refuse_renegotiation; - } - GST_DEBUG_OBJECT (qtmux, - "pad %s accepted renegotiation to %" GST_PTR_FORMAT " from %" - GST_PTR_FORMAT, GST_PAD_NAME (pad), caps, current_caps); - gst_caps_unref (current_caps); - } + if (qtpad->fourcc) + return gst_qt_mux_can_renegotiate (qtmux, pad, caps); GST_DEBUG_OBJECT (qtmux, "%s:%s, caps=%" GST_PTR_FORMAT, GST_DEBUG_PAD_NAME (pad), caps); + qtpad->prepare_buf_func = NULL; + format = qtmux_klass->format; structure = gst_caps_get_structure (caps, 0); mimetype = gst_structure_get_name (structure); @@ -4140,11 +5786,48 @@ gst_qt_mux_video_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) /* bring frame numerator into a range that ensures both reasonable resolution * as well as a fair duration */ - rate = qtmux->trak_timescale ? - qtmux->trak_timescale : adjust_rate (framerate_num, framerate_den); + qtpad->expected_sample_duration_n = framerate_num; + qtpad->expected_sample_duration_d = framerate_den; + + rate = gst_qt_mux_pad_get_timescale (GST_QT_MUX_PAD_CAST (pad)); + if (!rate && qtmux->trak_timescale) + rate = qtmux->trak_timescale; + else if (!rate) + rate = atom_framerate_to_timescale (framerate_num, framerate_den); + GST_DEBUG_OBJECT (qtmux, "Rate of video track selected: %" G_GUINT32_FORMAT, rate); + multiview_mode = gst_structure_get_string (structure, "multiview-mode"); + if (multiview_mode && !qtpad->trak->mdia.minf.stbl.svmi) { + GstVideoMultiviewMode mode; + GstVideoMultiviewFlags flags = 0; + + mode = gst_video_multiview_mode_from_caps_string (multiview_mode); + gst_structure_get_flagset (structure, "multiview-flags", &flags, NULL); + switch (mode) { + case GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE: + qtpad->trak->mdia.minf.stbl.svmi = + atom_svmi_new (0, + flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST); + break; + case GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED: + qtpad->trak->mdia.minf.stbl.svmi = + atom_svmi_new (1, + flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST); + break; + case GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME: + qtpad->trak->mdia.minf.stbl.svmi = + atom_svmi_new (2, + flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST); + break; + default: + GST_DEBUG_OBJECT (qtmux, "Unsupported multiview-mode %s", + multiview_mode); + break; + } + } + /* set common properties */ entry.width = width; entry.height = height; @@ -4323,6 +6006,9 @@ gst_qt_mux_video_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) } else if (strcmp (mimetype, "image/jpeg") == 0) { entry.fourcc = FOURCC_jpeg; sync = FALSE; + } else if (strcmp (mimetype, "image/png") == 0) { + entry.fourcc = FOURCC_png; + sync = FALSE; } else if (strcmp (mimetype, "image/x-j2c") == 0 || strcmp (mimetype, "image/x-jpc") == 0) { const gchar *colorspace; @@ -4357,8 +6043,9 @@ gst_qt_mux_video_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) goto refuse_caps; } } else if (strcmp (mimetype, "video/x-vp8") == 0) { - entry.fourcc = FOURCC_VP80; - sync = FALSE; + entry.fourcc = FOURCC_vp08; + } else if (strcmp (mimetype, "video/x-vp9") == 0) { + entry.fourcc = FOURCC_vp09; } else if (strcmp (mimetype, "video/x-dirac") == 0) { entry.fourcc = FOURCC_drac; } else if (strcmp (mimetype, "video/x-qt-part") == 0) { @@ -4397,6 +6084,35 @@ gst_qt_mux_video_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) } else if (strcmp (mimetype, "video/x-cineform") == 0) { entry.fourcc = FOURCC_cfhd; sync = FALSE; + } else if (strcmp (mimetype, "video/x-av1") == 0) { + gint presentation_delay; + guint8 presentation_delay_byte = 0; + GstBuffer *av1_codec_data; + + if (gst_structure_get_int (structure, "presentation-delay", + &presentation_delay)) { + presentation_delay_byte = 1 << 5; + presentation_delay_byte |= MAX (0xF, presentation_delay & 0xF); + } + + + av1_codec_data = gst_buffer_new_allocate (NULL, 5, NULL); + /* Fill version and 3 bytes of flags to 0 */ + gst_buffer_memset (av1_codec_data, 0, 0, 4); + gst_buffer_fill (av1_codec_data, 4, &presentation_delay_byte, 1); + if (codec_data) + av1_codec_data = gst_buffer_append (av1_codec_data, + gst_buffer_ref ((GstBuffer *) codec_data)); + + entry.fourcc = FOURCC_av01; + + ext_atom = build_btrt_extension (0, qtpad->avg_bitrate, qtpad->max_bitrate); + if (ext_atom != NULL) + ext_atom_list = g_list_prepend (ext_atom_list, ext_atom); + ext_atom = build_codec_data_extension (FOURCC_av1C, av1_codec_data); + if (ext_atom != NULL) + ext_atom_list = g_list_prepend (ext_atom_list, ext_atom); + gst_buffer_unref (av1_codec_data); } if (!entry.fourcc) @@ -4566,14 +6282,6 @@ refuse_caps: gst_object_unref (qtmux); return FALSE; } -refuse_renegotiation: - { - GST_WARNING_OBJECT (qtmux, - "pad %s refused renegotiation to %" GST_PTR_FORMAT, GST_PAD_NAME (pad), - caps); - gst_object_unref (qtmux); - return FALSE; - } } static gboolean @@ -4584,24 +6292,8 @@ gst_qt_mux_subtitle_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) GstStructure *structure; SubtitleSampleEntry entry = { 0, }; - /* does not go well to renegotiate stream mid-way, unless - * the old caps are a subset of the new one (this means upstream - * added more info to the caps, as both should be 'fixed' caps) */ - if (qtpad->fourcc) { - GstCaps *current_caps; - - current_caps = gst_pad_get_current_caps (pad); - g_assert (caps != NULL); - - if (!gst_qtmux_caps_is_subset_full (qtmux, current_caps, caps)) { - gst_caps_unref (current_caps); - goto refuse_renegotiation; - } - GST_DEBUG_OBJECT (qtmux, - "pad %s accepted renegotiation to %" GST_PTR_FORMAT " from %" - GST_PTR_FORMAT, GST_PAD_NAME (pad), caps, current_caps); - gst_caps_unref (current_caps); - } + if (qtpad->fourcc) + return gst_qt_mux_can_renegotiate (qtmux, pad, caps); GST_DEBUG_OBJECT (qtmux, "%s:%s, caps=%" GST_PTR_FORMAT, GST_DEBUG_PAD_NAME (pad), caps); @@ -4643,11 +6335,66 @@ refuse_caps: gst_object_unref (qtmux); return FALSE; } -refuse_renegotiation: +} + +static gboolean +gst_qt_mux_caption_sink_set_caps (GstQTPad * qtpad, GstCaps * caps) +{ + GstPad *pad = qtpad->collect.pad; + GstQTMux *qtmux = GST_QT_MUX_CAST (gst_pad_get_parent (pad)); + GstStructure *structure; + guint32 fourcc_entry; + guint32 timescale; + + if (qtpad->fourcc) + return gst_qt_mux_can_renegotiate (qtmux, pad, caps); + + GST_DEBUG_OBJECT (qtmux, "%s:%s, caps=%" GST_PTR_FORMAT, + GST_DEBUG_PAD_NAME (pad), caps); + + /* captions default */ + qtpad->is_out_of_order = FALSE; + qtpad->sync = FALSE; + qtpad->sparse = TRUE; + /* Closed caption data are within atoms */ + qtpad->prepare_buf_func = gst_qt_mux_prepare_caption_buffer; + + structure = gst_caps_get_structure (caps, 0); + + /* We know we only handle 608,format=s334-1a and 708,format=cdp */ + if (gst_structure_has_name (structure, "closedcaption/x-cea-608")) { + fourcc_entry = FOURCC_c608; + } else if (gst_structure_has_name (structure, "closedcaption/x-cea-708")) { + fourcc_entry = FOURCC_c708; + } else + goto refuse_caps; + + /* We set the real timescale later to the one from the video track when + * writing the headers */ + timescale = gst_qt_mux_pad_get_timescale (GST_QT_MUX_PAD_CAST (pad)); + if (!timescale && qtmux->trak_timescale) + timescale = qtmux->trak_timescale; + else if (!timescale) + timescale = 30000; + + qtpad->fourcc = fourcc_entry; + qtpad->trak_ste = + (SampleTableEntry *) atom_trak_set_caption_type (qtpad->trak, + qtmux->context, timescale, fourcc_entry); + + /* Initialize caption track language code to 0 unless something else is + * specified. Without this, Final Cut considers it "non-standard" + */ + qtpad->trak->mdia.mdhd.language_code = 0; + + gst_object_unref (qtmux); + return TRUE; + + /* ERRORS */ +refuse_caps: { - GST_WARNING_OBJECT (qtmux, - "pad %s refused renegotiation to %" GST_PTR_FORMAT, GST_PAD_NAME (pad), - caps); + GST_WARNING_OBJECT (qtmux, "pad %s refused caps %" GST_PTR_FORMAT, + GST_PAD_NAME (pad), caps); gst_object_unref (qtmux); return FALSE; } @@ -4725,9 +6472,7 @@ gst_qt_mux_sink_event (GstCollectPads * pads, GstCollectData * data, g_assert (qtpad); if (qtpad->trak) { /* https://developer.apple.com/library/mac/#documentation/QuickTime/QTFF/QTFFChap4/qtff4.html */ - qtpad->trak->mdia.mdhd.language_code = - (iso_code[0] - 0x60) * 0x400 + (iso_code[1] - 0x60) * 0x20 + - (iso_code[2] - 0x60); + qtpad->trak->mdia.mdhd.language_code = language_code (iso_code); } } g_free (code); @@ -4824,13 +6569,23 @@ gst_qt_mux_request_new_pad (GstElement * element, name = g_strdup_printf ("subtitle_%u", qtmux->subtitle_pads++); } lock = FALSE; + } else if (templ == gst_element_class_get_pad_template (klass, "caption_%u")) { + setcaps_func = gst_qt_mux_caption_sink_set_caps; + if (req_name != NULL && sscanf (req_name, "caption_%u", &pad_id) == 1) { + name = g_strdup (req_name); + } else { + name = g_strdup_printf ("caption_%u", qtmux->caption_pads++); + } + lock = FALSE; } else goto wrong_template; GST_DEBUG_OBJECT (qtmux, "Requested pad: %s", name); /* create pad and add to collections */ - newpad = gst_pad_new_from_template (templ, name); + newpad = + g_object_new (GST_TYPE_QT_MUX_PAD, "name", name, "direction", + templ->direction, "template", templ, NULL); g_free (name); collect_pad = (GstQTPad *) gst_collect_pads_add_pad (qtmux->collect, newpad, sizeof (GstQTPad), @@ -4933,12 +6688,26 @@ gst_qt_mux_get_property (GObject * object, case PROP_RESERVED_BYTES_PER_SEC: g_value_set_uint (value, qtmux->reserved_bytes_per_sec_per_trak); break; + case PROP_RESERVED_PREFILL: + g_value_set_boolean (value, qtmux->reserved_prefill); + break; case PROP_INTERLEAVE_BYTES: g_value_set_uint64 (value, qtmux->interleave_bytes); break; case PROP_INTERLEAVE_TIME: g_value_set_uint64 (value, qtmux->interleave_time); break; + case PROP_MAX_RAW_AUDIO_DRIFT: + g_value_set_uint64 (value, qtmux->max_raw_audio_drift); + break; + case PROP_START_GAP_THRESHOLD: + g_value_set_uint64 (value, qtmux->start_gap_threshold); + break; +#ifdef TIZEN_FEATURE_GST_MUX_ENHANCEMENT + case PROP_EXPECTED_TRAILER_SIZE: + g_value_set_uint(value, qtmux->expected_trailer_size); + break; +#endif /* TIZEN_FEATURE_GST_MUX_ENHANCEMENT */ default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -5016,6 +6785,9 @@ gst_qt_mux_set_property (GObject * object, case PROP_RESERVED_BYTES_PER_SEC: qtmux->reserved_bytes_per_sec_per_trak = g_value_get_uint (value); break; + case PROP_RESERVED_PREFILL: + qtmux->reserved_prefill = g_value_get_boolean (value); + break; case PROP_INTERLEAVE_BYTES: qtmux->interleave_bytes = g_value_get_uint64 (value); qtmux->interleave_bytes_set = TRUE; @@ -5024,6 +6796,12 @@ gst_qt_mux_set_property (GObject * object, qtmux->interleave_time = g_value_get_uint64 (value); qtmux->interleave_time_set = TRUE; break; + case PROP_MAX_RAW_AUDIO_DRIFT: + qtmux->max_raw_audio_drift = g_value_get_uint64 (value); + break; + case PROP_START_GAP_THRESHOLD: + qtmux->start_gap_threshold = g_value_get_uint64 (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -5104,7 +6882,7 @@ gst_qt_mux_register (GstPlugin * plugin) while (TRUE) { GstQTMuxFormatProp *prop; - GstCaps *subtitle_caps; + GstCaps *subtitle_caps, *caption_caps; prop = &gst_qt_mux_format_list[i]; format = prop->format; @@ -5123,6 +6901,12 @@ gst_qt_mux_register (GstPlugin * plugin) } else { gst_caps_unref (subtitle_caps); } + caption_caps = gst_static_caps_get (&prop->caption_sink_caps); + if (!gst_caps_is_equal (caption_caps, GST_CAPS_NONE)) { + params->caption_sink_caps = caption_caps; + } else { + gst_caps_unref (caption_caps); + } /* create the type now */ type = g_type_register_static (GST_TYPE_ELEMENT, prop->type_name, &typeinfo,