From 23f7739ba48f67fe55e6e068c9185155ae0c6539 Mon Sep 17 00:00:00 2001 From: Patrick Radizi Date: Thu, 14 Sep 2017 11:20:17 +0200 Subject: [PATCH] rtpbin: add option for increasing ts_offset gradually Instant large changes to ts_offset may cause timestamps to move backwards and also cause visible effects in media playback. The new option max-ts-offset-adjustment lets the application control the rate to apply changes to ts_offset. https://bugzilla.gnome.org/show_bug.cgi?id=784002 --- gst/rtpmanager/gstrtpbin.c | 31 +++++++++++- gst/rtpmanager/gstrtpbin.h | 1 + gst/rtpmanager/gstrtpjitterbuffer.c | 94 +++++++++++++++++++++++++++++++++++-- gst/rtpmanager/rtpsource.c | 3 +- gst/rtsp/gstrtspsrc.c | 32 ++++++++++++- gst/rtsp/gstrtspsrc.h | 1 + 6 files changed, 156 insertions(+), 6 deletions(-) diff --git a/gst/rtpmanager/gstrtpbin.c b/gst/rtpmanager/gstrtpbin.c index 0260b9d..b17ddc9 100644 --- a/gst/rtpmanager/gstrtpbin.c +++ b/gst/rtpmanager/gstrtpbin.c @@ -309,6 +309,7 @@ enum #define DEFAULT_MAX_MISORDER_TIME 2000 #define DEFAULT_RFC7273_SYNC FALSE #define DEFAULT_MAX_STREAMS G_MAXUINT +#define DEFAULT_MAX_TS_OFFSET_ADJUSTMENT 0 enum { @@ -333,7 +334,8 @@ enum PROP_MAX_DROPOUT_TIME, PROP_MAX_MISORDER_TIME, PROP_RFC7273_SYNC, - PROP_MAX_STREAMS + PROP_MAX_STREAMS, + PROP_MAX_TS_OFFSET_ADJUSTMENT }; #define GST_RTP_BIN_RTCP_SYNC_TYPE (gst_rtp_bin_rtcp_sync_get_type()) @@ -1759,6 +1761,8 @@ create_stream (GstRtpBinSession * session, guint32 ssrc) g_object_set (buffer, "max-dropout-time", rtpbin->max_dropout_time, "max-misorder-time", rtpbin->max_misorder_time, NULL); g_object_set (buffer, "rfc7273-sync", rtpbin->rfc7273_sync, NULL); + g_object_set (buffer, "max-ts-offset-adjustment", + rtpbin->max_ts_offset_adjustment, NULL); g_signal_emit (rtpbin, gst_rtp_bin_signals[SIGNAL_NEW_JITTERBUFFER], 0, buffer, session->id, ssrc); @@ -2493,6 +2497,22 @@ gst_rtp_bin_class_init (GstRtpBinClass * klass) 0, G_MAXUINT, DEFAULT_MAX_STREAMS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** + * GstRtpBin:max-ts-offset-adjustment: + * + * Syncing time stamps to NTP time adds a time offset. This parameter + * specifies the maximum number of nanoseconds per frame that this time offset + * may be adjusted with. This is used to avoid sudden large changes to time + * stamps. + */ + g_object_class_install_property (gobject_class, PROP_MAX_TS_OFFSET_ADJUSTMENT, + g_param_spec_uint64 ("max-ts-offset-adjustment", + "Max Timestamp Offset Adjustment", + "The maximum number of nanoseconds per frame that time stamp offsets " + "may be adjusted (0 = no limit).", 0, G_MAXUINT64, + DEFAULT_MAX_TS_OFFSET_ADJUSTMENT, G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS)); + gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_rtp_bin_change_state); gstelement_class->request_new_pad = GST_DEBUG_FUNCPTR (gst_rtp_bin_request_new_pad); @@ -2564,6 +2584,7 @@ gst_rtp_bin_init (GstRtpBin * rtpbin) rtpbin->max_misorder_time = DEFAULT_MAX_MISORDER_TIME; rtpbin->rfc7273_sync = DEFAULT_RFC7273_SYNC; rtpbin->max_streams = DEFAULT_MAX_STREAMS; + rtpbin->max_ts_offset_adjustment = DEFAULT_MAX_TS_OFFSET_ADJUSTMENT; /* some default SDES entries */ cname = g_strdup_printf ("user%u@host-%x", g_random_int (), g_random_int ()); @@ -2788,6 +2809,11 @@ gst_rtp_bin_set_property (GObject * object, guint prop_id, case PROP_MAX_STREAMS: rtpbin->max_streams = g_value_get_uint (value); break; + case PROP_MAX_TS_OFFSET_ADJUSTMENT: + rtpbin->max_ts_offset_adjustment = g_value_get_uint64 (value); + gst_rtp_bin_propagate_property_to_jitterbuffer (rtpbin, + "max-ts-offset-adjustment", value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -2876,6 +2902,9 @@ gst_rtp_bin_get_property (GObject * object, guint prop_id, case PROP_MAX_STREAMS: g_value_set_uint (value, rtpbin->max_streams); break; + case PROP_MAX_TS_OFFSET_ADJUSTMENT: + g_value_set_uint64 (value, rtpbin->max_ts_offset_adjustment); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; diff --git a/gst/rtpmanager/gstrtpbin.h b/gst/rtpmanager/gstrtpbin.h index 384b76d..224d3ca 100644 --- a/gst/rtpmanager/gstrtpbin.h +++ b/gst/rtpmanager/gstrtpbin.h @@ -75,6 +75,7 @@ struct _GstRtpBin { guint32 max_misorder_time; gboolean rfc7273_sync; guint max_streams; + guint64 max_ts_offset_adjustment; /* a list of session */ GSList *sessions; diff --git a/gst/rtpmanager/gstrtpjitterbuffer.c b/gst/rtpmanager/gstrtpjitterbuffer.c index 8574f17..1c1d198 100644 --- a/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/gst/rtpmanager/gstrtpjitterbuffer.c @@ -131,6 +131,7 @@ enum #define DEFAULT_LATENCY_MS 200 #define DEFAULT_DROP_ON_LATENCY FALSE #define DEFAULT_TS_OFFSET 0 +#define DEFAULT_MAX_TS_OFFSET_ADJUSTMENT 0 #define DEFAULT_DO_LOST FALSE #define DEFAULT_MODE RTP_JITTER_BUFFER_MODE_SLAVE #define DEFAULT_PERCENT 0 @@ -160,6 +161,7 @@ enum PROP_LATENCY, PROP_DROP_ON_LATENCY, PROP_TS_OFFSET, + PROP_MAX_TS_OFFSET_ADJUSTMENT, PROP_DO_LOST, PROP_MODE, PROP_PERCENT, @@ -281,6 +283,7 @@ struct _GstRtpJitterBufferPrivate guint64 latency_ns; gboolean drop_on_latency; gint64 ts_offset; + guint64 max_ts_offset_adjustment; gboolean do_lost; gboolean do_retransmission; gboolean rtx_next_seqnum; @@ -337,7 +340,7 @@ struct _GstRtpJitterBufferPrivate gint last_pt; gint32 clock_rate; gint64 clock_base; - gint64 prev_ts_offset; + gint64 ts_offset_remainder; /* when we are shutting down */ GstFlowReturn srcresult; @@ -368,6 +371,7 @@ struct _GstRtpJitterBufferPrivate /* for the jitter */ GstClockTime last_dts; + GstClockTime last_pts; guint64 last_rtptime; GstClockTime avg_jitter; }; @@ -550,6 +554,20 @@ gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** + * GstRtpJitterBuffer:max-ts-offset-adjustment: + * + * The maximum number of nanoseconds per frame that time offset may be + * adjusted with. This is used to avoid sudden large changes to time stamps. + */ + g_object_class_install_property (gobject_class, PROP_MAX_TS_OFFSET_ADJUSTMENT, + g_param_spec_uint64 ("max-ts-offset-adjustment", + "Max Timestamp Offset Adjustment", + "The maximum number of nanoseconds per frame that time stamp " + "offsets may be adjusted (0 = no limit).", 0, G_MAXUINT64, + DEFAULT_MAX_TS_OFFSET_ADJUSTMENT, G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS)); + + /** * GstRtpJitterBuffer:do-lost: * * Send out a GstRTPPacketLost event downstream when a packet is considered @@ -980,6 +998,8 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer) priv->latency_ms = DEFAULT_LATENCY_MS; priv->latency_ns = priv->latency_ms * GST_MSECOND; priv->drop_on_latency = DEFAULT_DROP_ON_LATENCY; + priv->ts_offset = DEFAULT_TS_OFFSET; + priv->max_ts_offset_adjustment = DEFAULT_MAX_TS_OFFSET_ADJUSTMENT; priv->do_lost = DEFAULT_DO_LOST; priv->do_retransmission = DEFAULT_DO_RETRANSMISSION; priv->rtx_next_seqnum = DEFAULT_RTX_NEXT_SEQNUM; @@ -997,7 +1017,9 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer) priv->max_misorder_time = DEFAULT_MAX_MISORDER_TIME; priv->faststart_min_packets = DEFAULT_FASTSTART_MIN_PACKETS; + priv->ts_offset_remainder = 0; priv->last_dts = -1; + priv->last_pts = -1; priv->last_rtptime = -1; priv->avg_jitter = 0; priv->timers = g_array_new (FALSE, TRUE, sizeof (TimerData)); @@ -1571,7 +1593,7 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer) priv->srcresult = GST_FLOW_OK; gst_segment_init (&priv->segment, GST_FORMAT_TIME); priv->last_popped_seqnum = -1; - priv->last_out_time = -1; + priv->last_out_time = GST_CLOCK_TIME_NONE; priv->next_seqnum = -1; priv->seqnum_base = -1; priv->ips_rtptime = -1; @@ -1990,6 +2012,32 @@ check_buffering_percent (GstRtpJitterBuffer * jitterbuffer, gint percent) return message; } +static void +update_offset (GstRtpJitterBuffer * jitterbuffer) +{ + GstRtpJitterBufferPrivate *priv; + + priv = jitterbuffer->priv; + + if (priv->ts_offset_remainder != 0) { + GST_DEBUG ("adjustment %" G_GUINT64_FORMAT " remain %" G_GINT64_FORMAT + " off %" G_GINT64_FORMAT, priv->max_ts_offset_adjustment, + priv->ts_offset_remainder, priv->ts_offset); + if (ABS (priv->ts_offset_remainder) > priv->max_ts_offset_adjustment) { + if (priv->ts_offset_remainder > 0) { + priv->ts_offset += priv->max_ts_offset_adjustment; + priv->ts_offset_remainder -= priv->max_ts_offset_adjustment; + } else { + priv->ts_offset -= priv->max_ts_offset_adjustment; + priv->ts_offset_remainder += priv->max_ts_offset_adjustment; + } + } else { + priv->ts_offset += priv->ts_offset_remainder; + priv->ts_offset_remainder = 0; + } + } +} + static GstClockTime apply_offset (GstRtpJitterBuffer * jitterbuffer, GstClockTime timestamp) { @@ -3388,6 +3436,11 @@ pop_and_push_next (GstRtpJitterBuffer * jitterbuffer, guint seqnum) gst_segment_position_from_running_time (&priv->segment, GST_FORMAT_TIME, item->pts); + /* if this is a new frame, check if ts_offset needs to be updated */ + if (pts != priv->last_pts) { + update_offset (jitterbuffer); + } + /* apply timestamp with offset to buffer now */ GST_BUFFER_DTS (outbuf) = apply_offset (jitterbuffer, dts); GST_BUFFER_PTS (outbuf) = apply_offset (jitterbuffer, pts); @@ -3395,6 +3448,20 @@ pop_and_push_next (GstRtpJitterBuffer * jitterbuffer, guint seqnum) /* update the elapsed time when we need to check against the npt stop time. */ update_estimated_eos (jitterbuffer, item); + /* verify that an offset has not caused time stamps to go backwards, if so + * handle by reusing the previous timestamp */ + if (priv->last_out_time != GST_CLOCK_TIME_NONE && + GST_BUFFER_PTS (outbuf) < priv->last_out_time) { + GST_DEBUG_OBJECT (jitterbuffer, "buffer PTS %" GST_TIME_FORMAT + " older than preceding PTS %" GST_TIME_FORMAT + " adjusting to %" GST_TIME_FORMAT, + GST_TIME_ARGS (GST_BUFFER_PTS (outbuf)), + GST_TIME_ARGS (priv->last_out_time), + GST_TIME_ARGS (priv->last_out_time)); + GST_BUFFER_PTS (outbuf) = priv->last_out_time; + } + + priv->last_pts = pts; priv->last_out_time = GST_BUFFER_PTS (outbuf); break; case ITEM_TYPE_LOST: @@ -4480,10 +4547,26 @@ gst_rtp_jitter_buffer_set_property (GObject * object, break; case PROP_TS_OFFSET: JBUF_LOCK (priv); - priv->ts_offset = g_value_get_int64 (value); + if (priv->max_ts_offset_adjustment != 0) { + gint64 new_offset = g_value_get_int64 (value); + + if (new_offset > priv->ts_offset) { + priv->ts_offset_remainder = new_offset - priv->ts_offset; + } else { + priv->ts_offset_remainder = -(priv->ts_offset - new_offset); + } + } else { + priv->ts_offset = g_value_get_int64 (value); + priv->ts_offset_remainder = 0; + } priv->ts_discont = TRUE; JBUF_UNLOCK (priv); break; + case PROP_MAX_TS_OFFSET_ADJUSTMENT: + JBUF_LOCK (priv); + priv->max_ts_offset_adjustment = g_value_get_uint64 (value); + JBUF_UNLOCK (priv); + break; case PROP_DO_LOST: JBUF_LOCK (priv); priv->do_lost = g_value_get_boolean (value); @@ -4607,6 +4690,11 @@ gst_rtp_jitter_buffer_get_property (GObject * object, g_value_set_int64 (value, priv->ts_offset); JBUF_UNLOCK (priv); break; + case PROP_MAX_TS_OFFSET_ADJUSTMENT: + JBUF_LOCK (priv); + g_value_set_uint64 (value, priv->max_ts_offset_adjustment); + JBUF_UNLOCK (priv); + break; case PROP_DO_LOST: JBUF_LOCK (priv); g_value_set_boolean (value, priv->do_lost); diff --git a/gst/rtpmanager/rtpsource.c b/gst/rtpmanager/rtpsource.c index e79fe0c..d726408 100644 --- a/gst/rtpmanager/rtpsource.c +++ b/gst/rtpmanager/rtpsource.c @@ -1280,7 +1280,8 @@ rtp_source_send_rtp (RTPSource * src, RTPPacketInfo * pinfo) return GST_FLOW_OK; if (src->pt_set && src->pt != pinfo->pt) { - GST_WARNING ("Changing pt from %u to %u for SSRC %u", src->pt, pinfo->pt, src->ssrc); + GST_WARNING ("Changing pt from %u to %u for SSRC %u", src->pt, pinfo->pt, + src->ssrc); } src->pt = pinfo->pt; diff --git a/gst/rtsp/gstrtspsrc.c b/gst/rtsp/gstrtspsrc.c index ddf9005..f696b0c 100644 --- a/gst/rtsp/gstrtspsrc.c +++ b/gst/rtsp/gstrtspsrc.c @@ -228,6 +228,7 @@ gst_rtsp_src_ntp_time_source_get_type (void) #define DEFAULT_USER_AGENT "GStreamer/" PACKAGE_VERSION #define DEFAULT_MAX_RTCP_RTP_TIME_DIFF 1000 #define DEFAULT_RFC7273_SYNC FALSE +#define DEFAULT_MAX_TS_OFFSET_ADJUSTMENT 0 enum { @@ -267,7 +268,8 @@ enum PROP_NTP_TIME_SOURCE, PROP_USER_AGENT, PROP_MAX_RTCP_RTP_TIME_DIFF, - PROP_RFC7273_SYNC + PROP_RFC7273_SYNC, + PROP_MAX_TS_OFFSET_ADJUSTMENT }; #define GST_TYPE_RTSP_NAT_METHOD (gst_rtsp_nat_method_get_type()) @@ -749,6 +751,22 @@ gst_rtspsrc_class_init (GstRTSPSrcClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); /** + * GstRTSPSrc:max-ts-offset-adjustment: + * + * Syncing time stamps to NTP time adds a time offset. This parameter + * specifies the maximum number of nanoseconds per frame that this time offset + * may be adjusted with. This is used to avoid sudden large changes to time + * stamps. + */ + g_object_class_install_property (gobject_class, PROP_MAX_TS_OFFSET_ADJUSTMENT, + g_param_spec_uint64 ("max-ts-offset-adjustment", + "Max Timestamp Offset Adjustment", + "The maximum number of nanoseconds per frame that time stamp offsets " + "may be adjusted (0 = no limit).", 0, G_MAXUINT64, + DEFAULT_MAX_TS_OFFSET_ADJUSTMENT, G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS)); + + /** * GstRTSPSrc::handle-request: * @rtspsrc: a #GstRTSPSrc * @request: a #GstRTSPMessage @@ -896,6 +914,7 @@ gst_rtspsrc_init (GstRTSPSrc * src) src->user_agent = g_strdup (DEFAULT_USER_AGENT); src->max_rtcp_rtp_time_diff = DEFAULT_MAX_RTCP_RTP_TIME_DIFF; src->rfc7273_sync = DEFAULT_RFC7273_SYNC; + src->max_ts_offset_adjustment = DEFAULT_MAX_TS_OFFSET_ADJUSTMENT; /* get a list of all extensions */ src->extensions = gst_rtsp_ext_list_get (); @@ -1186,6 +1205,9 @@ gst_rtspsrc_set_property (GObject * object, guint prop_id, const GValue * value, case PROP_RFC7273_SYNC: rtspsrc->rfc7273_sync = g_value_get_boolean (value); break; + case PROP_MAX_TS_OFFSET_ADJUSTMENT: + rtspsrc->max_ts_offset_adjustment = g_value_get_uint64 (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1335,6 +1357,9 @@ gst_rtspsrc_get_property (GObject * object, guint prop_id, GValue * value, case PROP_RFC7273_SYNC: g_value_set_boolean (value, rtspsrc->rfc7273_sync); break; + case PROP_MAX_TS_OFFSET_ADJUSTMENT: + g_value_set_uint64 (value, rtspsrc->max_ts_offset_adjustment); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -3166,6 +3191,11 @@ gst_rtspsrc_stream_configure_manager (GstRTSPSrc * src, GstRTSPStream * stream, src->max_rtcp_rtp_time_diff, NULL); } + if (g_object_class_find_property (klass, "max-ts-offset-adjustment")) { + g_object_set (src->manager, "max-ts-offset-adjustment", + src->max_ts_offset_adjustment, NULL); + } + /* buffer mode pauses are handled by adding offsets to buffer times, * but some depayloaders may have a hard time syncing output times * with such input times, e.g. container ones, most notably ASF */ diff --git a/gst/rtsp/gstrtspsrc.h b/gst/rtsp/gstrtspsrc.h index 87ec1a1..c85995f 100644 --- a/gst/rtsp/gstrtspsrc.h +++ b/gst/rtsp/gstrtspsrc.h @@ -250,6 +250,7 @@ struct _GstRTSPSrc { gchar *user_agent; GstClockTime max_rtcp_rtp_time_diff; gboolean rfc7273_sync; + guint64 max_ts_offset_adjustment; /* state */ GstRTSPState state; -- 2.7.4