1 /* GStreamer plugin for forward error correction
2 * Copyright (C) 2017 Pexip
4 * This library is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
9 * This library is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with this library; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 * Author: Mikhail Fludkov <misha@pexip.com>
21 #include <gst/rtp/gstrtpbuffer.h>
22 #include <gst/rtp/gstrtp-enumtypes.h>
24 #include "rtpulpfeccommon.h"
25 #include "gstrtpulpfecdec.h"
27 static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink",
30 GST_STATIC_CAPS ("application/x-rtp")
33 static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src",
36 GST_STATIC_CAPS ("application/x-rtp")
49 #define DEFAULT_FEC_PT 0
51 static GParamSpec *klass_properties[N_PROPERTIES] = { NULL, };
53 GST_DEBUG_CATEGORY (gst_rtp_ulpfec_dec_debug);
54 #define GST_CAT_DEFAULT (gst_rtp_ulpfec_dec_debug)
56 G_DEFINE_TYPE (GstRtpUlpFecDec, gst_rtp_ulpfec_dec, GST_TYPE_ELEMENT);
58 #define RTP_FEC_MAP_INFO_NTH(dec, data) (&g_array_index (\
59 ((GstRtpUlpFecDec *)dec)->info_arr, \
61 GPOINTER_TO_UINT(data)))
64 _compare_fec_map_info (gconstpointer a, gconstpointer b, gpointer userdata)
67 gst_rtp_buffer_get_seq (&RTP_FEC_MAP_INFO_NTH (userdata, a)->rtp);
69 gst_rtp_buffer_get_seq (&RTP_FEC_MAP_INFO_NTH (userdata, b)->rtp);
70 return gst_rtp_buffer_compare_seqnum (bseq, aseq);
74 gst_rtp_ulpfec_dec_start (GstRtpUlpFecDec * self, GstBufferList * buflist,
75 guint8 fec_pt, guint16 lost_seq)
77 guint fec_packets = 0;
79 g_assert (NULL == self->info_media);
80 g_assert (0 == self->info_fec->len);
81 g_assert (0 == self->info_arr->len);
83 g_array_set_size (self->info_arr, gst_buffer_list_length (buflist));
86 i < gst_buffer_list_length (buflist) && !self->lost_packet_from_storage;
88 GstBuffer *buffer = gst_buffer_list_get (buflist, i);
89 RtpUlpFecMapInfo *info = RTP_FEC_MAP_INFO_NTH (self, i);
91 if (!rtp_ulpfec_map_info_map (gst_buffer_ref (buffer), info))
92 g_assert_not_reached ();
94 if (fec_pt == gst_rtp_buffer_get_payload_type (&info->rtp)) {
95 GST_DEBUG_RTP_PACKET (self, "rtp header (fec)", &info->rtp);
98 if (rtp_ulpfec_buffer_is_valid (&info->rtp)) {
99 GST_DEBUG_FEC_PACKET (self, &info->rtp);
100 g_ptr_array_add (self->info_fec, GUINT_TO_POINTER (i));
103 GST_LOG_RTP_PACKET (self, "rtp header (incoming)", &info->rtp);
105 if (lost_seq == gst_rtp_buffer_get_seq (&info->rtp)) {
106 GST_DEBUG_OBJECT (self, "Received lost packet from from the storage");
107 g_list_free (self->info_media);
108 self->info_media = NULL;
109 self->lost_packet_from_storage = TRUE;
112 g_list_insert_sorted_with_data (self->info_media,
113 GUINT_TO_POINTER (i), _compare_fec_map_info, self);
116 if (!self->lost_packet_from_storage) {
117 self->fec_packets_received += fec_packets;
118 self->fec_packets_rejected += fec_packets - self->info_fec->len;
123 gst_rtp_ulpfec_dec_stop (GstRtpUlpFecDec * self)
125 g_array_set_size (self->info_arr, 0);
126 g_ptr_array_set_size (self->info_fec, 0);
127 g_list_free (self->info_media);
128 self->info_media = NULL;
129 self->lost_packet_from_storage = FALSE;
130 self->lost_packet_returned = FALSE;
134 gst_rtp_ulpfec_dec_get_media_buffers_mask (GstRtpUlpFecDec * self,
135 guint16 fec_seq_base)
138 for (GList * it = self->info_media; it; it = it->next) {
139 RtpUlpFecMapInfo *info = RTP_FEC_MAP_INFO_NTH (self, it->data);
141 rtp_ulpfec_packet_mask_from_seqnum (gst_rtp_buffer_get_seq (&info->rtp),
148 gst_rtp_ulpfec_dec_is_recovered_pt_valid (GstRtpUlpFecDec * self, gint media_pt,
151 if (media_pt == recovered_pt)
154 for (GList * it = self->info_media; it; it = it->next) {
155 RtpUlpFecMapInfo *info = RTP_FEC_MAP_INFO_NTH (self, it->data);
156 if (gst_rtp_buffer_get_payload_type (&info->rtp) == recovered_pt)
163 gst_rtp_ulpfec_dec_recover_from_fec (GstRtpUlpFecDec * self,
164 RtpUlpFecMapInfo * info_fec, guint32 ssrc, gint media_pt, guint16 seq,
167 guint64 fec_mask = rtp_ulpfec_buffer_get_mask (&info_fec->rtp);
168 gboolean fec_mask_long = rtp_ulpfec_buffer_get_fechdr (&info_fec->rtp)->L;
169 guint16 fec_seq_base = rtp_ulpfec_buffer_get_seq_base (&info_fec->rtp);
172 g_array_set_size (self->scratch_buf, 0);
173 rtp_buffer_to_ulpfec_bitstring (&info_fec->rtp, self->scratch_buf, TRUE,
176 for (GList * it = self->info_media; it; it = it->next) {
177 RtpUlpFecMapInfo *info = RTP_FEC_MAP_INFO_NTH (self, it->data);
178 guint64 packet_mask =
179 rtp_ulpfec_packet_mask_from_seqnum (gst_rtp_buffer_get_seq (&info->rtp),
182 if (fec_mask & packet_mask) {
183 fec_mask ^= packet_mask;
184 rtp_buffer_to_ulpfec_bitstring (&info->rtp, self->scratch_buf, FALSE,
190 rtp_ulpfec_bitstring_to_media_rtp_buffer (self->scratch_buf,
191 fec_mask_long, ssrc, seq);
193 /* We are about to put recovered packet back in self->info_media to be able
194 * to reuse it later for recovery of other packets
196 gint i = self->info_arr->len;
197 RtpUlpFecMapInfo *info;
200 g_array_set_size (self->info_arr, self->info_arr->len + 1);
201 info = RTP_FEC_MAP_INFO_NTH (self, i);
203 if (!rtp_ulpfec_map_info_map (gst_buffer_ref (ret), info)) {
204 GST_WARNING_OBJECT (self, "Invalid recovered packet");
205 goto recovered_packet_invalid;
208 recovered_pt = gst_rtp_buffer_get_payload_type (&info->rtp);
209 if (!gst_rtp_ulpfec_dec_is_recovered_pt_valid (self, media_pt,
211 GST_WARNING_OBJECT (self,
212 "Recovered packet has unexpected payload type (%u)", recovered_pt);
213 goto recovered_packet_invalid;
216 GST_DEBUG_RTP_PACKET (self, "rtp header (recovered)", &info->rtp);
218 g_list_insert_sorted_with_data (self->info_media, GUINT_TO_POINTER (i),
219 _compare_fec_map_info, self);
220 *dst_pt = recovered_pt;
224 recovered_packet_invalid:
225 g_array_set_size (self->info_arr, self->info_arr->len - 1);
226 gst_buffer_unref (ret);
231 gst_rtp_ulpfec_dec_recover_from_storage (GstRtpUlpFecDec * self,
232 guint8 * dst_pt, guint16 * dst_seq)
234 RtpUlpFecMapInfo *info;
236 if (self->lost_packet_returned)
239 g_assert (g_list_length (self->info_media) == 1);
241 info = RTP_FEC_MAP_INFO_NTH (self, self->info_media->data);
242 *dst_seq = gst_rtp_buffer_get_seq (&info->rtp);
243 *dst_pt = gst_rtp_buffer_get_payload_type (&info->rtp);
244 self->lost_packet_returned = TRUE;
245 GST_DEBUG_RTP_PACKET (self, "rtp header (recovered)", &info->rtp);
246 return gst_buffer_ref (info->rtp.buffer);
250 gst_rtp_ulpfec_dec_recover (GstRtpUlpFecDec * self, guint32 ssrc, gint media_pt,
251 guint8 * dst_pt, guint16 * dst_seq)
253 guint64 media_mask = 0;
254 gint media_mask_seq_base = -1;
256 if (self->lost_packet_from_storage)
257 return gst_rtp_ulpfec_dec_recover_from_storage (self, dst_pt, dst_seq);
259 /* Looking for a FEC packet which can be used for recovery */
260 for (gsize i = 0; i < self->info_fec->len; ++i) {
261 RtpUlpFecMapInfo *info = RTP_FEC_MAP_INFO_NTH (self,
262 g_ptr_array_index (self->info_fec, i));
263 guint16 seq_base = rtp_ulpfec_buffer_get_seq_base (&info->rtp);
264 guint64 fec_mask = rtp_ulpfec_buffer_get_mask (&info->rtp);
265 guint64 missing_packets_mask;
267 if (media_mask_seq_base != (gint) seq_base) {
268 media_mask_seq_base = seq_base;
269 media_mask = gst_rtp_ulpfec_dec_get_media_buffers_mask (self, seq_base);
272 /* media_mask has 1s if packet exist.
273 * fec_mask is the mask of protected packets
274 * The statement below excludes existing packets from the protected. So
275 * we are left with 1s only for missing packets which can be recovered
276 * by this FEC packet. */
277 missing_packets_mask = fec_mask & (~media_mask);
279 /* Do we have any 1s? Checking if current FEC packet can be used for recovery */
280 if (0 != missing_packets_mask) {
281 guint trailing_zeros = __builtin_ctzll (missing_packets_mask);
283 /* Is it the only 1 in the mask? Checking if we lacking single packet in
284 * that case FEC packet can be used for recovery */
285 if (missing_packets_mask == (1ULL << trailing_zeros)) {
289 seq_base + (RTP_ULPFEC_SEQ_BASE_OFFSET_MAX (TRUE) - trailing_zeros);
291 gst_rtp_ulpfec_dec_recover_from_fec (self, info, ssrc, media_pt,
302 gst_rtp_ulpfec_dec_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
304 GstRtpUlpFecDec *self = GST_RTP_ULPFEC_DEC (parent);
306 if (G_LIKELY (GST_FLOW_OK == self->chain_return_val)) {
307 if (G_UNLIKELY (self->unset_discont_flag)) {
308 self->unset_discont_flag = FALSE;
309 buf = gst_buffer_make_writable (buf);
310 GST_BUFFER_FLAG_UNSET (buf, GST_BUFFER_FLAG_DISCONT);
312 return gst_pad_push (self->srcpad, buf);
315 gst_buffer_unref (buf);
316 return self->chain_return_val;
320 gst_rtp_ulpfec_dec_handle_packet_loss (GstRtpUlpFecDec * self, guint16 seqnum,
321 GstClockTime timestamp, GstClockTime duration)
323 gint caps_pt = self->have_caps_pt ? self->caps_pt : -1;
325 GstBufferList *buflist =
326 rtp_storage_get_packets_for_recovery (self->storage, self->fec_pt,
327 self->caps_ssrc, seqnum);
330 GstBuffer *recovered_buffer = NULL;
331 guint16 recovered_seq = 0;
332 guint8 recovered_pt = 0;
334 gst_rtp_ulpfec_dec_start (self, buflist, self->fec_pt, seqnum);
336 while (NULL != (recovered_buffer =
337 gst_rtp_ulpfec_dec_recover (self, self->caps_ssrc, caps_pt,
338 &recovered_pt, &recovered_seq))) {
339 if (seqnum == recovered_seq) {
340 recovered_buffer = gst_buffer_make_writable (recovered_buffer);
341 GST_BUFFER_PTS (recovered_buffer) = timestamp;
342 /* GST_BUFFER_DURATION (recovered_buffer) = duration;
343 * JB does not set the duration, so we will not too */
345 if (!self->lost_packet_from_storage)
346 rtp_storage_put_recovered_packet (self->storage,
347 gst_buffer_ref (recovered_buffer), recovered_pt, self->caps_ssrc,
350 GST_DEBUG_OBJECT (self,
351 "Pushing recovered packet ssrc=0x%08x seq=%u %" GST_PTR_FORMAT,
352 self->caps_ssrc, seqnum, recovered_buffer);
355 self->unset_discont_flag = TRUE;
356 self->chain_return_val = gst_pad_push (self->srcpad, recovered_buffer);
360 rtp_storage_put_recovered_packet (self->storage,
361 recovered_buffer, recovered_pt, self->caps_ssrc, recovered_seq);
364 gst_rtp_ulpfec_dec_stop (self);
365 gst_buffer_list_unref (buflist);
368 GST_DEBUG_OBJECT (self, "Packet lost ssrc=0x%08x seq=%u", self->caps_ssrc,
375 gst_rtp_ulpfec_dec_handle_sink_event (GstPad * pad, GstObject * parent,
378 GstRtpUlpFecDec *self = GST_RTP_ULPFEC_DEC (parent);
379 gboolean forward = TRUE;
381 GST_LOG_OBJECT (self, "Received event %" GST_PTR_FORMAT, event);
383 if (GST_FLOW_OK == self->chain_return_val &&
384 GST_EVENT_CUSTOM_DOWNSTREAM == GST_EVENT_TYPE (event) &&
385 gst_event_has_name (event, "GstRTPPacketLost")) {
387 GstClockTime timestamp, duration;
389 g_assert (self->have_caps_ssrc);
390 g_assert (self->storage);
392 if (!gst_structure_get (gst_event_get_structure (event),
393 "seqnum", G_TYPE_UINT, &seqnum,
394 "timestamp", G_TYPE_UINT64, ×tamp,
395 "duration", G_TYPE_UINT64, &duration, NULL))
396 g_assert_not_reached ();
399 gst_rtp_ulpfec_dec_handle_packet_loss (self, seqnum, timestamp,
402 ++self->packets_unrecovered;
404 ++self->packets_recovered;
405 GST_DEBUG_OBJECT (self, "Unrecovered / Recovered: %lu / %lu",
406 self->packets_unrecovered, self->packets_recovered);
407 } else if (GST_EVENT_CAPS == GST_EVENT_TYPE (event)) {
409 gboolean have_caps_pt = FALSE;
410 gboolean have_caps_ssrc = FALSE;
414 gst_event_parse_caps (event, &caps);
416 gst_structure_get_uint (gst_caps_get_structure (caps, 0), "ssrc",
419 gst_structure_get_int (gst_caps_get_structure (caps, 0), "payload",
422 if (self->have_caps_ssrc != have_caps_ssrc || self->caps_ssrc != caps_ssrc)
423 GST_DEBUG_OBJECT (self, "SSRC changed %u, 0x%08x -> %u, 0x%08x",
424 self->have_caps_ssrc, self->caps_ssrc, have_caps_ssrc, caps_ssrc);
425 if (self->have_caps_pt != have_caps_pt || self->caps_pt != caps_pt)
426 GST_DEBUG_OBJECT (self, "PT changed %u, %u -> %u, %u",
427 self->have_caps_pt, self->caps_pt, have_caps_pt, caps_pt);
429 self->have_caps_ssrc = have_caps_ssrc;
430 self->have_caps_pt = have_caps_pt;
431 self->caps_ssrc = caps_ssrc;
432 self->caps_pt = caps_pt;
436 return gst_pad_push_event (self->srcpad, event);
437 gst_event_unref (event);
442 gst_rtp_ulpfec_dec_init (GstRtpUlpFecDec * self)
444 self->srcpad = gst_pad_new_from_static_template (&srctemplate, "src");
445 self->sinkpad = gst_pad_new_from_static_template (&sinktemplate, "sink");
446 GST_PAD_SET_PROXY_CAPS (self->sinkpad);
447 GST_PAD_SET_PROXY_ALLOCATION (self->sinkpad);
448 gst_pad_set_chain_function (self->sinkpad,
449 GST_DEBUG_FUNCPTR (gst_rtp_ulpfec_dec_chain));
450 gst_pad_set_event_function (self->sinkpad,
451 GST_DEBUG_FUNCPTR (gst_rtp_ulpfec_dec_handle_sink_event));
453 gst_element_add_pad (GST_ELEMENT (self), self->srcpad);
454 gst_element_add_pad (GST_ELEMENT (self), self->sinkpad);
456 self->fec_pt = DEFAULT_FEC_PT;
458 self->chain_return_val = GST_FLOW_OK;
459 self->have_caps_ssrc = FALSE;
461 self->info_fec = g_ptr_array_new ();
462 self->info_arr = g_array_new (FALSE, TRUE, sizeof (RtpUlpFecMapInfo));
463 g_array_set_clear_func (self->info_arr,
464 (GDestroyNotify) rtp_ulpfec_map_info_unmap);
465 self->scratch_buf = g_array_new (FALSE, TRUE, sizeof (guint8));
469 gst_rtp_ulpfec_dec_dispose (GObject * obj)
471 GstRtpUlpFecDec *self = GST_RTP_ULPFEC_DEC (obj);
473 GST_INFO_OBJECT (self,
475 " packets_recovered=%" G_GSIZE_FORMAT
476 " packets_unrecovered=%" G_GSIZE_FORMAT,
477 self->caps_ssrc, self->caps_pt,
478 self->packets_recovered, self->packets_unrecovered);
481 g_object_unref (self->storage);
483 g_assert (NULL == self->info_media);
484 g_assert (0 == self->info_fec->len);
485 g_assert (0 == self->info_arr->len);
487 if (self->fec_packets_received) {
488 GST_INFO_OBJECT (self,
489 " fec_packets_received=%" G_GSIZE_FORMAT
490 " fec_packets_rejected=%" G_GSIZE_FORMAT
491 " packets_rejected=%" G_GSIZE_FORMAT,
492 self->fec_packets_received,
493 self->fec_packets_rejected, self->packets_rejected);
496 g_ptr_array_free (self->info_fec, TRUE);
497 g_array_free (self->info_arr, TRUE);
498 g_array_free (self->scratch_buf, TRUE);
500 G_OBJECT_CLASS (gst_rtp_ulpfec_dec_parent_class)->dispose (obj);
504 gst_rtp_ulpfec_dec_set_property (GObject * object, guint prop_id,
505 const GValue * value, GParamSpec * pspec)
507 GstRtpUlpFecDec *self = GST_RTP_ULPFEC_DEC (object);
509 if (GST_LEVEL_LOG <= gst_debug_category_get_threshold (GST_CAT_DEFAULT)) {
510 gchar *val_str = gst_value_serialize (value);
511 GST_LOG_OBJECT (object, "Setting property \"%s\" to %s", pspec->name,
518 self->fec_pt = g_value_get_uint (value);
522 g_object_unref (self->storage);
523 self->storage = g_value_get_object (value);
525 g_object_ref (self->storage);
528 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
534 gst_rtp_ulpfec_dec_get_property (GObject * object, guint prop_id,
535 GValue * value, GParamSpec * pspec)
537 GstRtpUlpFecDec *self = GST_RTP_ULPFEC_DEC (object);
541 g_value_set_uint (value, self->fec_pt);
544 g_value_set_object (value, self->storage);
547 g_value_set_uint (value, (guint) self->packets_recovered);
549 case PROP_UNRECOVERED:
550 g_value_set_uint (value, (guint) self->packets_unrecovered);
553 G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
557 if (GST_LEVEL_LOG <= gst_debug_category_get_threshold (GST_CAT_DEFAULT)) {
558 gchar *val_str = gst_value_serialize (value);
559 GST_LOG_OBJECT (object, "Returning property \"%s\" %s", pspec->name,
566 gst_rtp_ulpfec_dec_class_init (GstRtpUlpFecDecClass * klass)
568 GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
569 GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
571 GST_DEBUG_CATEGORY_INIT (gst_rtp_ulpfec_dec_debug,
572 "rtpulpfecdec", 0, "RTP FEC Decoder");
574 gst_element_class_add_pad_template (element_class,
575 gst_static_pad_template_get (&srctemplate));
576 gst_element_class_add_pad_template (element_class,
577 gst_static_pad_template_get (&sinktemplate));
579 gst_element_class_set_static_metadata (element_class,
581 "Codec/Depayloader/Network/RTP",
582 "Decodes RTP FEC (RFC5109)", "Mikhail Fludkov <misha@pexip.com>");
584 gobject_class->set_property =
585 GST_DEBUG_FUNCPTR (gst_rtp_ulpfec_dec_set_property);
586 gobject_class->get_property =
587 GST_DEBUG_FUNCPTR (gst_rtp_ulpfec_dec_get_property);
588 gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_rtp_ulpfec_dec_dispose);
590 klass_properties[PROP_PT] = g_param_spec_uint ("pt", "pt",
591 "FEC packets payload type", 0, 127,
592 DEFAULT_FEC_PT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS);
593 klass_properties[PROP_STORAGE] =
594 g_param_spec_object ("storage", "RTP storage", "RTP storage",
595 G_TYPE_OBJECT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS);
596 klass_properties[PROP_RECOVERED] =
597 g_param_spec_uint ("recovered", "recovered",
598 "The number of recovered packets", 0, G_MAXUINT, 0,
599 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS);
600 klass_properties[PROP_UNRECOVERED] =
601 g_param_spec_uint ("unrecovered", "unrecovered",
602 "The number of unrecovered packets", 0, G_MAXUINT, 0,
603 G_PARAM_READABLE | G_PARAM_STATIC_STRINGS);
605 g_object_class_install_properties (gobject_class, N_PROPERTIES,