1 /* GStreamer plugin for forward error correction
2 * Copyright (C) 2017 Pexip
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 * Author: Mikhail Fludkov <misha@pexip.com>
22 * SECTION:element-rtpreddec
23 * @short_description: RTP Redundant Audio Data (RED) decoder
26 * Decode Redundant Audio Data (RED) as per RFC 2198.
28 * This element is mostly provided for chrome webrtc compatibility:
29 * chrome will wrap ulpfec-protected streams in RED packets, and such
30 * streams need to be unwrapped by this element before being passed on
31 * to #GstRtpUlpFecDec.
33 * The #GstRtpRedDec:pt property should be set to the expected payload
34 * types of the RED packets.
36 * When using #GstRtpBin, this element should be inserted through the
37 * #GstRtpBin::request-aux-receiver signal.
40 * <title>Example pipeline</title>
42 * gst-launch-1.0 udpsrc port=8888 caps="application/x-rtp, payload=96, clock-rate=90000" ! rtpreddec pt=122 ! rtpstorage size-time=220000000 ! rtpssrcdemux ! application/x-rtp, payload=96, clock-rate=90000, media=video, encoding-name=H264 ! rtpjitterbuffer do-lost=1 latency=200 ! rtpulpfecdec pt=122 ! rtph264depay ! avdec_h264 ! videoconvert ! autovideosink
43 * ]| This example will receive a stream with RED and ULP FEC and try to reconstruct the packets.
46 * See also: #GstRtpRedEnc, #GstWebRTCBin, #GstRtpBin
50 #include <gst/rtp/gstrtpbuffer.h>
52 #include "rtpredcommon.h"
53 #include "gstrtpreddec.h"
54 #include "rtpulpfeccommon.h"
56 #define RTP_HISTORY_MAX_SIZE (16)
64 #define RTP_HIST_ITEM_TIMESTAMP(p) ((RTPHistItem *)p)->timestamp
65 #define RTP_HIST_ITEM_SEQ(p) ((RTPHistItem *)p)->seq
67 static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
70 GST_STATIC_CAPS ("application/x-rtp"));
72 static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
75 GST_STATIC_CAPS ("application/x-rtp"));
78 #define MIN_PT UNDEF_PT
80 #define DEFAULT_PT UNDEF_PT
82 GST_DEBUG_CATEGORY_STATIC (gst_rtp_red_dec_debug);
83 #define GST_CAT_DEFAULT gst_rtp_red_dec_debug
85 G_DEFINE_TYPE (GstRtpRedDec, gst_rtp_red_dec, GST_TYPE_ELEMENT);
95 rtp_hist_item_alloc (void)
97 return g_slice_new (RTPHistItem);
101 rtp_hist_item_free (gpointer item)
103 g_slice_free (RTPHistItem, item);
107 gst_rtp_red_history_find_less_or_equal (gconstpointer item,
108 gconstpointer timestamp)
110 guint32 t = GPOINTER_TO_UINT (timestamp);
111 gint32 diff = t - RTP_HIST_ITEM_TIMESTAMP (item);
116 gst_rtp_red_history_find_less (gconstpointer item, gconstpointer timestamp)
118 guint32 t = GPOINTER_TO_UINT (timestamp);
119 gint32 diff = t - RTP_HIST_ITEM_TIMESTAMP (item);
124 gst_rtp_red_history_update (GstRtpRedDec * self, GstRTPBuffer * rtp)
127 GList *link, *sibling;
129 /* If we have not reached MAX number of elements in the history,
130 * allocate a new link and a new item,
131 * otherwise reuse the tail (the oldest data) without any reallocations
133 if (self->rtp_history->length < RTP_HISTORY_MAX_SIZE) {
134 item = rtp_hist_item_alloc ();
135 link = g_list_alloc ();
138 link = g_queue_pop_tail_link (self->rtp_history);
142 item->timestamp = gst_rtp_buffer_get_timestamp (rtp);
143 item->seq = gst_rtp_buffer_get_seq (rtp);
145 /* Looking for a place to insert new link.
146 * The queue has newest to oldest rtp timestamps, so in 99% cases
147 * it is inserted before the head of the queue */
148 sibling = g_list_find_custom (self->rtp_history->head,
149 GUINT_TO_POINTER (item->timestamp),
150 gst_rtp_red_history_find_less_or_equal);
151 g_queue_push_nth_link (self->rtp_history,
152 g_list_position (self->rtp_history->head, sibling), link);
156 rtp_red_buffer_is_valid (GstRtpRedDec * self, GstRTPBuffer * red_rtp,
157 gsize * dst_first_red_payload_offset)
159 guint8 *payload = gst_rtp_buffer_get_payload (red_rtp);
160 gsize payload_len = gst_rtp_buffer_get_payload_len (red_rtp);
161 gsize red_hdrs_offset = 0;
162 guint red_hdrs_checked = 0;
163 guint redundant_payload_len = 0;
166 gpointer red_hdr = payload + red_hdrs_offset;
168 gboolean is_redundant;
172 /* Can we address the first byte where F bit is located ? */
173 if (red_hdrs_offset + 1 > payload_len)
174 goto red_buffer_invalid;
176 is_redundant = rtp_red_block_is_redundant (red_hdr);
178 /* Is it the last block? */
180 red_hdr_len = rtp_red_block_header_get_length (TRUE);
182 /* Can we address all the other bytes in RED block header? */
183 if (red_hdrs_offset + red_hdr_len > payload_len)
184 goto red_buffer_invalid;
186 redundant_payload_len += rtp_red_block_get_payload_length (red_hdr);
187 red_hdrs_offset += red_hdr_len;
189 red_hdr_len = rtp_red_block_header_get_length (FALSE);
190 red_hdrs_offset += red_hdr_len;
195 /* Do we have enough data to create redundant packets & main packet. Keep in
196 * mind that redundant_payload_len contains the length of redundant packets only.
198 if (red_hdrs_offset + redundant_payload_len >= payload_len)
199 goto red_buffer_invalid;
201 *dst_first_red_payload_offset = red_hdrs_offset;
203 GST_LOG_OBJECT (self, "RED packet has %u blocks", red_hdrs_checked);
207 GST_WARNING_OBJECT (self, "Received invalid RED packet "
208 "ssrc=0x%08x pt=%u tstamp=%u seq=%u size=%u, "
210 gst_rtp_buffer_get_ssrc (red_rtp),
211 gst_rtp_buffer_get_payload_type (red_rtp),
212 gst_rtp_buffer_get_timestamp (red_rtp),
213 gst_rtp_buffer_get_seq (red_rtp),
214 gst_rtp_buffer_get_packet_len (red_rtp), red_hdrs_checked);
219 gst_red_history_lost_seq_num_for_timestamp (GstRtpRedDec * self,
220 guint32 timestamp, guint16 * dst_seq_num)
222 GList *older_sibling = g_list_find_custom (self->rtp_history->head,
223 GUINT_TO_POINTER (timestamp),
224 gst_rtp_red_history_find_less);
227 guint32 timestamp_diff;
228 gint seq_diff, lost_packet_idx;
230 if (NULL == older_sibling) {
231 if (self->rtp_history->length == RTP_HISTORY_MAX_SIZE)
232 GST_WARNING_OBJECT (self, "History is too short. "
233 "Oldest rtp timestamp %u, looking for %u, size %u",
234 RTP_HIST_ITEM_TIMESTAMP (self->rtp_history->tail->data),
235 timestamp, self->rtp_history->length);
239 if (NULL == older_sibling->prev) {
240 GST_WARNING_OBJECT (self, "RED block timestamp offset probably wrong. "
241 "Latest rtp timestamp %u, looking for %u, size %u",
242 RTP_HIST_ITEM_TIMESTAMP (self->rtp_history->head->data),
243 timestamp, self->rtp_history->length);
247 older = older_sibling->data;
248 newer = older_sibling->prev->data;
249 /* We know for sure @older has lower timestamp than we are looking for,
250 * if @newer has the same timestamp, there is no packet loss and we
251 * don't need to use redundant data */
252 if (newer->timestamp == timestamp)
255 seq_diff = gst_rtp_buffer_compare_seqnum (older->seq, newer->seq);
258 GST_WARNING_OBJECT (self, "RED block timestamp offset is wrong: "
259 "#%u,%u #%u,%u looking for %u",
260 older->seq, older->timestamp,
261 newer->seq, newer->timestamp, timestamp);
263 GST_WARNING_OBJECT (self, "RTP timestamps increasing while "
264 "sequence numbers decreasing: #%u,%u #%u,%u",
265 older->seq, older->timestamp, newer->seq, newer->timestamp);
269 timestamp_diff = newer->timestamp - older->timestamp;
270 for (lost_packet_idx = 1; lost_packet_idx < seq_diff; ++lost_packet_idx) {
271 guint32 lost_timestamp = older->timestamp +
272 lost_packet_idx * timestamp_diff / seq_diff;
273 if (lost_timestamp == timestamp) {
274 *dst_seq_num = older->seq + lost_packet_idx;
279 GST_WARNING_OBJECT (self, "Can't find RED block timestamp "
280 "#%u,%u #%u,%u looking for %u",
281 older->seq, older->timestamp, newer->seq, newer->timestamp, timestamp);
286 gst_rtp_red_create_packet (GstRtpRedDec * self, GstRTPBuffer * red_rtp,
287 gboolean marker, guint8 pt, guint16 seq_num, guint32 timestamp,
288 gsize red_payload_subbuffer_start, gsize red_payload_subbuffer_len)
290 guint csrc_count = gst_rtp_buffer_get_csrc_count (red_rtp);
291 GstBuffer *ret = gst_rtp_buffer_new_allocate (0, 0, csrc_count);
292 GstRTPBuffer ret_rtp = GST_RTP_BUFFER_INIT;
294 if (!gst_rtp_buffer_map (ret, GST_MAP_WRITE, &ret_rtp))
295 g_assert_not_reached ();
297 gst_rtp_buffer_set_marker (&ret_rtp, marker);
298 gst_rtp_buffer_set_payload_type (&ret_rtp, pt);
299 gst_rtp_buffer_set_seq (&ret_rtp, seq_num);
300 gst_rtp_buffer_set_timestamp (&ret_rtp, timestamp);
301 gst_rtp_buffer_set_ssrc (&ret_rtp, gst_rtp_buffer_get_ssrc (red_rtp));
302 for (i = 0; i < csrc_count; ++i)
303 gst_rtp_buffer_set_csrc (&ret_rtp, i, gst_rtp_buffer_get_csrc (red_rtp, i));
304 gst_rtp_buffer_unmap (&ret_rtp);
306 ret = gst_buffer_append (ret,
307 gst_rtp_buffer_get_payload_subbuffer (red_rtp,
308 red_payload_subbuffer_start, red_payload_subbuffer_len));
310 /* Timestamps, meta, flags from the RED packet should go to main block packet */
311 gst_buffer_copy_into (ret, red_rtp->buffer, GST_BUFFER_COPY_METADATA, 0, -1);
316 gst_rtp_red_create_from_redundant_block (GstRtpRedDec * self,
317 GstRTPBuffer * red_rtp, gsize * red_hdr_offset, gsize * red_payload_offset)
319 guint8 *payload = gst_rtp_buffer_get_payload (red_rtp);
320 guint8 *red_hdr = payload + *red_hdr_offset;
321 guint32 lost_timestamp = gst_rtp_buffer_get_timestamp (red_rtp) -
322 rtp_red_block_get_timestamp_offset (red_hdr);
324 GstBuffer *ret = NULL;
325 guint16 lost_seq = 0;
326 if (gst_red_history_lost_seq_num_for_timestamp (self, lost_timestamp,
328 GST_LOG_OBJECT (self, "Recovering from RED packet pt=%u ts=%u seq=%u"
329 " len=%u present", rtp_red_block_get_payload_type (red_hdr),
330 lost_timestamp, lost_seq, rtp_red_block_get_payload_length (red_hdr));
332 gst_rtp_red_create_packet (self, red_rtp, FALSE,
333 rtp_red_block_get_payload_type (red_hdr), lost_seq, lost_timestamp,
334 *red_payload_offset, rtp_red_block_get_payload_length (red_hdr));
335 GST_BUFFER_FLAG_SET (ret, GST_RTP_BUFFER_FLAG_REDUNDANT);
337 GST_LOG_OBJECT (self, "Ignore RED packet pt=%u ts=%u len=%u because already"
338 " present", rtp_red_block_get_payload_type (red_hdr), lost_timestamp,
339 rtp_red_block_get_payload_length (red_hdr));
342 *red_hdr_offset += rtp_red_block_header_get_length (TRUE);
343 *red_payload_offset += rtp_red_block_get_payload_length (red_hdr);
348 gst_rtp_red_create_from_main_block (GstRtpRedDec * self,
349 GstRTPBuffer * red_rtp, gsize red_hdr_offset, gsize * red_payload_offset)
351 guint8 *payload = gst_rtp_buffer_get_payload (red_rtp);
352 GstBuffer *ret = gst_rtp_red_create_packet (self, red_rtp,
353 gst_rtp_buffer_get_marker (red_rtp),
354 rtp_red_block_get_payload_type (payload + red_hdr_offset),
355 gst_rtp_buffer_get_seq (red_rtp),
356 gst_rtp_buffer_get_timestamp (red_rtp),
357 *red_payload_offset, -1);
358 *red_payload_offset = gst_rtp_buffer_get_payload_len (red_rtp);
359 GST_LOG_OBJECT (self, "Extracting main payload from RED pt=%u seq=%u ts=%u"
360 " marker=%u", rtp_red_block_get_payload_type (payload + red_hdr_offset),
361 gst_rtp_buffer_get_seq (red_rtp), gst_rtp_buffer_get_timestamp (red_rtp),
362 gst_rtp_buffer_get_marker (red_rtp));
368 gst_rtp_red_create_from_block (GstRtpRedDec * self, GstRTPBuffer * red_rtp,
369 gsize * red_hdr_offset, gsize * red_payload_offset)
371 guint8 *payload = gst_rtp_buffer_get_payload (red_rtp);
373 if (rtp_red_block_is_redundant (payload + (*red_hdr_offset)))
374 return gst_rtp_red_create_from_redundant_block (self, red_rtp,
375 red_hdr_offset, red_payload_offset);
377 return gst_rtp_red_create_from_main_block (self, red_rtp, *red_hdr_offset,
382 gst_rtp_red_process (GstRtpRedDec * self, GstRTPBuffer * red_rtp,
383 gsize first_red_payload_offset)
385 gsize red_hdr_offset = 0;
386 gsize red_payload_offset = first_red_payload_offset;
387 gsize payload_len = gst_rtp_buffer_get_payload_len (red_rtp);
388 GstFlowReturn ret = GST_FLOW_OK;
392 gst_rtp_red_create_from_block (self, red_rtp, &red_hdr_offset,
393 &red_payload_offset);
395 ret = gst_pad_push (self->srcpad, buf);
396 } while (GST_FLOW_OK == ret && red_payload_offset < payload_len);
402 gst_rtp_red_dec_chain (GstPad * pad, GstObject * parent, GstBuffer * buffer)
404 GstRtpRedDec *self = GST_RTP_RED_DEC (parent);
405 GstRTPBuffer irtp = GST_RTP_BUFFER_INIT;
406 GstFlowReturn ret = GST_FLOW_OK;
407 gsize first_red_payload_offset = 0;
409 if (self->pt == UNDEF_PT)
410 return gst_pad_push (self->srcpad, buffer);
412 if (!gst_rtp_buffer_map (buffer, GST_MAP_READ, &irtp))
413 return gst_pad_push (self->srcpad, buffer);
415 gst_rtp_red_history_update (self, &irtp);
417 if (self->pt != gst_rtp_buffer_get_payload_type (&irtp)) {
418 GST_LOG_RTP_PACKET (self, "rtp header (incoming)", &irtp);
420 gst_rtp_buffer_unmap (&irtp);
421 return gst_pad_push (self->srcpad, buffer);
424 self->num_received++;
426 if (rtp_red_buffer_is_valid (self, &irtp, &first_red_payload_offset)) {
427 GST_DEBUG_RTP_PACKET (self, "rtp header (red)", &irtp);
428 ret = gst_rtp_red_process (self, &irtp, first_red_payload_offset);
431 gst_rtp_buffer_unmap (&irtp);
432 gst_buffer_unref (buffer);
437 gst_rtp_red_dec_dispose (GObject * obj)
439 GstRtpRedDec *self = GST_RTP_RED_DEC (obj);
441 g_queue_free_full (self->rtp_history, rtp_hist_item_free);
443 G_OBJECT_CLASS (gst_rtp_red_dec_parent_class)->dispose (obj);
447 gst_rtp_red_dec_init (GstRtpRedDec * self)
449 GstPadTemplate *pad_template;
452 gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (self), "src");
453 self->srcpad = gst_pad_new_from_template (pad_template, "src");
454 gst_element_add_pad (GST_ELEMENT_CAST (self), self->srcpad);
457 gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (self), "sink");
458 self->sinkpad = gst_pad_new_from_template (pad_template, "sink");
459 gst_pad_set_chain_function (self->sinkpad,
460 GST_DEBUG_FUNCPTR (gst_rtp_red_dec_chain));
461 GST_PAD_SET_PROXY_CAPS (self->sinkpad);
462 GST_PAD_SET_PROXY_ALLOCATION (self->sinkpad);
463 gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
465 self->pt = DEFAULT_PT;
466 self->num_received = 0;
467 self->rtp_history = g_queue_new ();
472 gst_rtp_red_dec_set_property (GObject * object, guint prop_id,
473 const GValue * value, GParamSpec * pspec)
475 GstRtpRedDec *self = GST_RTP_RED_DEC (object);
479 self->pt = g_value_get_int (value);
482 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
488 gst_rtp_red_dec_get_property (GObject * object, guint prop_id,
489 GValue * value, GParamSpec * pspec)
491 GstRtpRedDec *self = GST_RTP_RED_DEC (object);
494 g_value_set_int (value, self->pt);
497 g_value_set_uint (value, self->num_received);
500 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
506 gst_rtp_red_dec_class_init (GstRtpRedDecClass * klass)
508 GObjectClass *gobject_class;
509 GstElementClass *element_class;
511 gobject_class = G_OBJECT_CLASS (klass);
512 element_class = GST_ELEMENT_CLASS (klass);
514 gst_element_class_add_pad_template (element_class,
515 gst_static_pad_template_get (&src_template));
516 gst_element_class_add_pad_template (element_class,
517 gst_static_pad_template_get (&sink_template));
519 gst_element_class_set_metadata (element_class,
520 "Redundant Audio Data (RED) Decoder",
521 "Codec/Depayloader/Network/RTP",
522 "Decode Redundant Audio Data (RED)",
523 "Hani Mustafa <hani@pexip.com>, Mikhail Fludkov <misha@pexip.com>");
525 gobject_class->set_property =
526 GST_DEBUG_FUNCPTR (gst_rtp_red_dec_set_property);
527 gobject_class->get_property =
528 GST_DEBUG_FUNCPTR (gst_rtp_red_dec_get_property);
529 gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_rtp_red_dec_dispose);
531 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_PT,
532 g_param_spec_int ("pt", "payload type",
533 "Payload type FEC packets",
534 MIN_PT, MAX_PT, DEFAULT_PT,
535 G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
537 g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_RECEIVED,
538 g_param_spec_uint ("received", "Received",
539 "Count of received packets",
540 0, G_MAXUINT32, 0, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));
542 GST_DEBUG_CATEGORY_INIT (gst_rtp_red_dec_debug, "rtpreddec", 0,