1 /* GStreamer plugin for forward error correction
2 * Copyright (C) 2017 Pexip
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 * Author: Mikhail Fludkov <misha@pexip.com>
22 * SECTION:element-rtpreddec
23 * @short_description: RTP Redundant Audio Data (RED) decoder
26 * Decode Redundant Audio Data (RED) as per RFC 2198.
28 * This element is mostly provided for chrome webrtc compatibility:
29 * chrome will wrap ulpfec-protected streams in RED packets, and such
30 * streams need to be unwrapped by this element before being passed on
31 * to #GstRtpUlpFecDec.
33 * The #GstRtpRedDec:pt property should be set to the expected payload
34 * types of the RED packets.
36 * When using #GstRtpBin, this element should be inserted through the
37 * #GstRtpBin::request-aux-receiver signal.
42 * gst-launch-1.0 udpsrc port=8888 caps="application/x-rtp, payload=96, clock-rate=90000" ! rtpreddec pt=122 ! rtpstorage size-time=220000000 ! rtpssrcdemux ! application/x-rtp, payload=96, clock-rate=90000, media=video, encoding-name=H264 ! rtpjitterbuffer do-lost=1 latency=200 ! rtpulpfecdec pt=122 ! rtph264depay ! avdec_h264 ! videoconvert ! autovideosink
43 * ]| This example will receive a stream with RED and ULP FEC and try to reconstruct the packets.
45 * See also: #GstRtpRedEnc, #GstWebRTCBin, #GstRtpBin
49 #include <gst/rtp/gstrtpbuffer.h>
51 #include "rtpredcommon.h"
52 #include "gstrtpreddec.h"
53 #include "rtpulpfeccommon.h"
55 #define RTP_HISTORY_MAX_SIZE (16)
63 #define RTP_HIST_ITEM_TIMESTAMP(p) ((RTPHistItem *)p)->timestamp
64 #define RTP_HIST_ITEM_SEQ(p) ((RTPHistItem *)p)->seq
66 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
69 GST_STATIC_CAPS ("application/x-rtp"));
71 static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
74 GST_STATIC_CAPS ("application/x-rtp"));
77 #define MIN_PT UNDEF_PT
79 #define DEFAULT_PT UNDEF_PT
81 GST_DEBUG_CATEGORY_STATIC (gst_rtp_red_dec_debug);
82 #define GST_CAT_DEFAULT gst_rtp_red_dec_debug
84 G_DEFINE_TYPE (GstRtpRedDec, gst_rtp_red_dec, GST_TYPE_ELEMENT);
94 rtp_hist_item_alloc (void)
96 return g_slice_new (RTPHistItem);
100 rtp_hist_item_free (gpointer item)
102 g_slice_free (RTPHistItem, item);
106 gst_rtp_red_history_find_less_or_equal (gconstpointer item,
107 gconstpointer timestamp)
109 guint32 t = GPOINTER_TO_UINT (timestamp);
110 gint32 diff = t - RTP_HIST_ITEM_TIMESTAMP (item);
115 gst_rtp_red_history_find_less (gconstpointer item, gconstpointer timestamp)
117 guint32 t = GPOINTER_TO_UINT (timestamp);
118 gint32 diff = t - RTP_HIST_ITEM_TIMESTAMP (item);
123 gst_rtp_red_history_update (GstRtpRedDec * self, GstRTPBuffer * rtp)
126 GList *link, *sibling;
128 /* If we have not reached MAX number of elements in the history,
129 * allocate a new link and a new item,
130 * otherwise reuse the tail (the oldest data) without any reallocations
132 if (self->rtp_history->length < RTP_HISTORY_MAX_SIZE) {
133 item = rtp_hist_item_alloc ();
134 link = g_list_alloc ();
137 link = g_queue_pop_tail_link (self->rtp_history);
141 item->timestamp = gst_rtp_buffer_get_timestamp (rtp);
142 item->seq = gst_rtp_buffer_get_seq (rtp);
144 /* Looking for a place to insert new link.
145 * The queue has newest to oldest rtp timestamps, so in 99% cases
146 * it is inserted before the head of the queue */
147 sibling = g_list_find_custom (self->rtp_history->head,
148 GUINT_TO_POINTER (item->timestamp),
149 gst_rtp_red_history_find_less_or_equal);
150 g_queue_push_nth_link (self->rtp_history,
151 g_list_position (self->rtp_history->head, sibling), link);
155 rtp_red_buffer_is_valid (GstRtpRedDec * self, GstRTPBuffer * red_rtp,
156 gsize * dst_first_red_payload_offset)
158 guint8 *payload = gst_rtp_buffer_get_payload (red_rtp);
159 gsize payload_len = gst_rtp_buffer_get_payload_len (red_rtp);
160 gsize red_hdrs_offset = 0;
161 guint red_hdrs_checked = 0;
162 guint redundant_payload_len = 0;
165 gpointer red_hdr = payload + red_hdrs_offset;
167 gboolean is_redundant;
171 /* Can we address the first byte where F bit is located ? */
172 if (red_hdrs_offset + 1 > payload_len)
173 goto red_buffer_invalid;
175 is_redundant = rtp_red_block_is_redundant (red_hdr);
177 /* Is it the last block? */
179 red_hdr_len = rtp_red_block_header_get_length (TRUE);
181 /* Can we address all the other bytes in RED block header? */
182 if (red_hdrs_offset + red_hdr_len > payload_len)
183 goto red_buffer_invalid;
185 redundant_payload_len += rtp_red_block_get_payload_length (red_hdr);
186 red_hdrs_offset += red_hdr_len;
188 red_hdr_len = rtp_red_block_header_get_length (FALSE);
189 red_hdrs_offset += red_hdr_len;
194 /* Do we have enough data to create redundant packets & main packet. Keep in
195 * mind that redundant_payload_len contains the length of redundant packets only.
197 if (red_hdrs_offset + redundant_payload_len >= payload_len)
198 goto red_buffer_invalid;
200 *dst_first_red_payload_offset = red_hdrs_offset;
202 GST_LOG_OBJECT (self, "RED packet has %u blocks", red_hdrs_checked);
206 GST_WARNING_OBJECT (self, "Received invalid RED packet "
207 "ssrc=0x%08x pt=%u tstamp=%u seq=%u size=%u, "
209 gst_rtp_buffer_get_ssrc (red_rtp),
210 gst_rtp_buffer_get_payload_type (red_rtp),
211 gst_rtp_buffer_get_timestamp (red_rtp),
212 gst_rtp_buffer_get_seq (red_rtp),
213 gst_rtp_buffer_get_packet_len (red_rtp), red_hdrs_checked);
218 gst_red_history_lost_seq_num_for_timestamp (GstRtpRedDec * self,
219 guint32 timestamp, guint16 * dst_seq_num)
221 GList *older_sibling = g_list_find_custom (self->rtp_history->head,
222 GUINT_TO_POINTER (timestamp),
223 gst_rtp_red_history_find_less);
226 guint32 timestamp_diff;
227 gint seq_diff, lost_packet_idx;
229 if (NULL == older_sibling) {
230 if (self->rtp_history->length == RTP_HISTORY_MAX_SIZE)
231 GST_WARNING_OBJECT (self, "History is too short. "
232 "Oldest rtp timestamp %u, looking for %u, size %u",
233 RTP_HIST_ITEM_TIMESTAMP (self->rtp_history->tail->data),
234 timestamp, self->rtp_history->length);
238 if (NULL == older_sibling->prev) {
239 GST_WARNING_OBJECT (self, "RED block timestamp offset probably wrong. "
240 "Latest rtp timestamp %u, looking for %u, size %u",
241 RTP_HIST_ITEM_TIMESTAMP (self->rtp_history->head->data),
242 timestamp, self->rtp_history->length);
246 older = older_sibling->data;
247 newer = older_sibling->prev->data;
248 /* We know for sure @older has lower timestamp than we are looking for,
249 * if @newer has the same timestamp, there is no packet loss and we
250 * don't need to use redundant data */
251 if (newer->timestamp == timestamp)
254 seq_diff = gst_rtp_buffer_compare_seqnum (older->seq, newer->seq);
257 GST_WARNING_OBJECT (self, "RED block timestamp offset is wrong: "
258 "#%u,%u #%u,%u looking for %u",
259 older->seq, older->timestamp,
260 newer->seq, newer->timestamp, timestamp);
262 GST_WARNING_OBJECT (self, "RTP timestamps increasing while "
263 "sequence numbers decreasing: #%u,%u #%u,%u",
264 older->seq, older->timestamp, newer->seq, newer->timestamp);
268 timestamp_diff = newer->timestamp - older->timestamp;
269 for (lost_packet_idx = 1; lost_packet_idx < seq_diff; ++lost_packet_idx) {
270 guint32 lost_timestamp = older->timestamp +
271 lost_packet_idx * timestamp_diff / seq_diff;
272 if (lost_timestamp == timestamp) {
273 *dst_seq_num = older->seq + lost_packet_idx;
278 GST_WARNING_OBJECT (self, "Can't find RED block timestamp "
279 "#%u,%u #%u,%u looking for %u",
280 older->seq, older->timestamp, newer->seq, newer->timestamp, timestamp);
285 gst_rtp_red_create_packet (GstRtpRedDec * self, GstRTPBuffer * red_rtp,
286 gboolean marker, guint8 pt, guint16 seq_num, guint32 timestamp,
287 gsize red_payload_subbuffer_start, gsize red_payload_subbuffer_len)
289 guint csrc_count = gst_rtp_buffer_get_csrc_count (red_rtp);
290 GstBuffer *ret = gst_rtp_buffer_new_allocate (0, 0, csrc_count);
291 GstRTPBuffer ret_rtp = GST_RTP_BUFFER_INIT;
293 if (!gst_rtp_buffer_map (ret, GST_MAP_WRITE, &ret_rtp))
294 g_assert_not_reached ();
296 gst_rtp_buffer_set_marker (&ret_rtp, marker);
297 gst_rtp_buffer_set_payload_type (&ret_rtp, pt);
298 gst_rtp_buffer_set_seq (&ret_rtp, seq_num);
299 gst_rtp_buffer_set_timestamp (&ret_rtp, timestamp);
300 gst_rtp_buffer_set_ssrc (&ret_rtp, gst_rtp_buffer_get_ssrc (red_rtp));
301 for (i = 0; i < csrc_count; ++i)
302 gst_rtp_buffer_set_csrc (&ret_rtp, i, gst_rtp_buffer_get_csrc (red_rtp, i));
303 gst_rtp_buffer_unmap (&ret_rtp);
305 ret = gst_buffer_append (ret,
306 gst_rtp_buffer_get_payload_subbuffer (red_rtp,
307 red_payload_subbuffer_start, red_payload_subbuffer_len));
309 /* Timestamps, meta, flags from the RED packet should go to main block packet */
310 gst_buffer_copy_into (ret, red_rtp->buffer, GST_BUFFER_COPY_METADATA, 0, -1);
315 gst_rtp_red_create_from_redundant_block (GstRtpRedDec * self,
316 GstRTPBuffer * red_rtp, gsize * red_hdr_offset, gsize * red_payload_offset)
318 guint8 *payload = gst_rtp_buffer_get_payload (red_rtp);
319 guint8 *red_hdr = payload + *red_hdr_offset;
320 guint32 lost_timestamp = gst_rtp_buffer_get_timestamp (red_rtp) -
321 rtp_red_block_get_timestamp_offset (red_hdr);
323 GstBuffer *ret = NULL;
324 guint16 lost_seq = 0;
325 if (gst_red_history_lost_seq_num_for_timestamp (self, lost_timestamp,
327 GST_LOG_OBJECT (self, "Recovering from RED packet pt=%u ts=%u seq=%u"
328 " len=%u present", rtp_red_block_get_payload_type (red_hdr),
329 lost_timestamp, lost_seq, rtp_red_block_get_payload_length (red_hdr));
331 gst_rtp_red_create_packet (self, red_rtp, FALSE,
332 rtp_red_block_get_payload_type (red_hdr), lost_seq, lost_timestamp,
333 *red_payload_offset, rtp_red_block_get_payload_length (red_hdr));
334 GST_BUFFER_FLAG_SET (ret, GST_RTP_BUFFER_FLAG_REDUNDANT);
336 GST_LOG_OBJECT (self, "Ignore RED packet pt=%u ts=%u len=%u because already"
337 " present", rtp_red_block_get_payload_type (red_hdr), lost_timestamp,
338 rtp_red_block_get_payload_length (red_hdr));
341 *red_hdr_offset += rtp_red_block_header_get_length (TRUE);
342 *red_payload_offset += rtp_red_block_get_payload_length (red_hdr);
347 gst_rtp_red_create_from_main_block (GstRtpRedDec * self,
348 GstRTPBuffer * red_rtp, gsize red_hdr_offset, gsize * red_payload_offset)
350 guint8 *payload = gst_rtp_buffer_get_payload (red_rtp);
351 GstBuffer *ret = gst_rtp_red_create_packet (self, red_rtp,
352 gst_rtp_buffer_get_marker (red_rtp),
353 rtp_red_block_get_payload_type (payload + red_hdr_offset),
354 gst_rtp_buffer_get_seq (red_rtp),
355 gst_rtp_buffer_get_timestamp (red_rtp),
356 *red_payload_offset, -1);
357 *red_payload_offset = gst_rtp_buffer_get_payload_len (red_rtp);
358 GST_LOG_OBJECT (self, "Extracting main payload from RED pt=%u seq=%u ts=%u"
359 " marker=%u", rtp_red_block_get_payload_type (payload + red_hdr_offset),
360 gst_rtp_buffer_get_seq (red_rtp), gst_rtp_buffer_get_timestamp (red_rtp),
361 gst_rtp_buffer_get_marker (red_rtp));
367 gst_rtp_red_create_from_block (GstRtpRedDec * self, GstRTPBuffer * red_rtp,
368 gsize * red_hdr_offset, gsize * red_payload_offset)
370 guint8 *payload = gst_rtp_buffer_get_payload (red_rtp);
372 if (rtp_red_block_is_redundant (payload + (*red_hdr_offset)))
373 return gst_rtp_red_create_from_redundant_block (self, red_rtp,
374 red_hdr_offset, red_payload_offset);
376 return gst_rtp_red_create_from_main_block (self, red_rtp, *red_hdr_offset,
381 gst_rtp_red_process (GstRtpRedDec * self, GstRTPBuffer * red_rtp,
382 gsize first_red_payload_offset)
384 gsize red_hdr_offset = 0;
385 gsize red_payload_offset = first_red_payload_offset;
386 gsize payload_len = gst_rtp_buffer_get_payload_len (red_rtp);
387 GstFlowReturn ret = GST_FLOW_OK;
391 gst_rtp_red_create_from_block (self, red_rtp, &red_hdr_offset,
392 &red_payload_offset);
394 ret = gst_pad_push (self->srcpad, buf);
395 } while (GST_FLOW_OK == ret && red_payload_offset < payload_len);
401 gst_rtp_red_dec_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer)
403 GstRtpRedDec *self = GST_RTP_RED_DEC (parent);
404 GstRTPBuffer irtp = GST_RTP_BUFFER_INIT;
405 GstFlowReturn ret = GST_FLOW_OK;
406 gsize first_red_payload_offset = 0;
408 if (self->pt == UNDEF_PT)
409 return gst_pad_push (self->srcpad, buffer);
411 if (!gst_rtp_buffer_map (buffer, GST_MAP_READ, &irtp))
412 return gst_pad_push (self->srcpad, buffer);
414 gst_rtp_red_history_update (self, &irtp);
416 if (self->pt != gst_rtp_buffer_get_payload_type (&irtp)) {
417 GST_LOG_RTP_PACKET (self, "rtp header (incoming)", &irtp);
419 gst_rtp_buffer_unmap (&irtp);
420 return gst_pad_push (self->srcpad, buffer);
423 self->num_received++;
425 if (rtp_red_buffer_is_valid (self, &irtp, &first_red_payload_offset)) {
426 GST_DEBUG_RTP_PACKET (self, "rtp header (red)", &irtp);
427 ret = gst_rtp_red_process (self, &irtp, first_red_payload_offset);
430 gst_rtp_buffer_unmap (&irtp);
431 gst_buffer_unref (buffer);
436 gst_rtp_red_dec_dispose (GObject * obj)
438 GstRtpRedDec *self = GST_RTP_RED_DEC (obj);
440 g_queue_free_full (self->rtp_history, rtp_hist_item_free);
442 G_OBJECT_CLASS (gst_rtp_red_dec_parent_class)->dispose (obj);
446 gst_rtp_red_dec_init (GstRtpRedDec * self)
448 GstPadTemplate *pad_template;
451 gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (self), "src");
452 self->srcpad = gst_pad_new_from_template (pad_template, "src");
453 gst_element_add_pad (GST_ELEMENT_CAST (self), self->srcpad);
456 gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (self), "sink");
457 self->sinkpad = gst_pad_new_from_template (pad_template, "sink");
458 gst_pad_set_chain_function (self->sinkpad,
459 GST_DEBUG_FUNCPTR (gst_rtp_red_dec_chain));
460 GST_PAD_SET_PROXY_CAPS (self->sinkpad);
461 GST_PAD_SET_PROXY_ALLOCATION (self->sinkpad);
462 gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
464 self->pt = DEFAULT_PT;
465 self->num_received = 0;
466 self->rtp_history = g_queue_new ();
471 gst_rtp_red_dec_set_property (GObject * object, guint prop_id,
472 const GValue * value, GParamSpec * pspec)
474 GstRtpRedDec *self = GST_RTP_RED_DEC (object);
478 self->pt = g_value_get_int (value);
481 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
487 gst_rtp_red_dec_get_property (GObject * object, guint prop_id,
488 GValue * value, GParamSpec * pspec)
490 GstRtpRedDec *self = GST_RTP_RED_DEC (object);
493 g_value_set_int (value, self->pt);
496 g_value_set_uint (value, self->num_received);
499 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
505 gst_rtp_red_dec_class_init (GstRtpRedDecClass * klass)
507 GObjectClass *gobject_class;
508 GstElementClass *element_class;
510 gobject_class = G_OBJECT_CLASS (klass);
511 element_class = GST_ELEMENT_CLASS (klass);
513 gst_element_class_add_pad_template (element_class,
514 gst_static_pad_template_get (&src_template));
515 gst_element_class_add_pad_template (element_class,
516 gst_static_pad_template_get (&sink_template));
518 gst_element_class_set_metadata (element_class,
519 "Redundant Audio Data (RED) Decoder",
520 "Codec/Depayloader/Network/RTP",
521 "Decode Redundant Audio Data (RED)",
522 "Hani Mustafa <hani@pexip.com>, Mikhail Fludkov <misha@pexip.com>");
524 gobject_class->set_property =
525 GST_DEBUG_FUNCPTR (gst_rtp_red_dec_set_property);
526 gobject_class->get_property =
527 GST_DEBUG_FUNCPTR (gst_rtp_red_dec_get_property);
528 gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_rtp_red_dec_dispose);
530 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_PT,
531 g_param_spec_int ("pt", "payload type",
532 "Payload type FEC packets",
533 MIN_PT, MAX_PT, DEFAULT_PT,
534 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
536 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_RECEIVED,
537 g_param_spec_uint ("received", "Received",
538 "Count of received packets",
539 0, G_MAXUINT32, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
541 GST_DEBUG_CATEGORY_INIT (gst_rtp_red_dec_debug, "rtpreddec", 0,