From b7d0ddd1a40953b80f5e847884ad0fa6a1a4939c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 3 Aug 2021 12:14:49 +1000 Subject: [PATCH] webrtc: support renegotiating adding/removing RTX We need to always add the RTX/RED/ULPFEC elements as rtpbin will only call us once to request aux/fec senders/receivers. We also need to regenerate the media section of the SDP instead of blindly copying from the previous offer. Part-of: --- .../gst-plugins-bad/ext/webrtc/gstwebrtcbin.c | 692 ++++++++++++--------- .../gst-plugins-bad/ext/webrtc/transportstream.c | 29 +- .../gst-plugins-bad/ext/webrtc/transportstream.h | 3 + .../gst-plugins-bad/ext/webrtc/webrtctransceiver.c | 7 +- .../gst-plugins-bad/ext/webrtc/webrtctransceiver.h | 4 + .../tests/check/elements/webrtcbin.c | 55 ++ 6 files changed, 462 insertions(+), 328 deletions(-) diff --git a/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcbin.c b/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcbin.c index 9986657..0fd1145 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcbin.c +++ b/subprojects/gst-plugins-bad/ext/webrtc/gstwebrtcbin.c @@ -2943,10 +2943,10 @@ _copy_field (GQuark field_id, const GValue * value, GstStructure * s) /* based off https://tools.ietf.org/html/draft-ietf-rtcweb-jsep-18#section-5.2.1 */ static gboolean sdp_media_from_transceiver (GstWebRTCBin * webrtc, GstSDPMedia * media, - GstWebRTCRTPTransceiver * trans, guint media_idx, - GString * bundled_mids, guint bundle_idx, gchar * bundle_ufrag, - gchar * bundle_pwd, GArray * reserved_pts, GHashTable * all_mids, - GError ** error) + const GstSDPMedia * last_media, GstWebRTCRTPTransceiver * trans, + guint media_idx, GString * bundled_mids, guint bundle_idx, + gchar * bundle_ufrag, gchar * bundle_pwd, GArray * reserved_pts, + GHashTable * all_mids, GError ** error) { /* TODO: * rtp header extensions @@ -2965,8 +2965,7 @@ sdp_media_from_transceiver (GstWebRTCBin * webrtc, GstSDPMedia * media, GstStructure *extmap; int i; - if (trans->direction == GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_NONE - || trans->direction == GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_INACTIVE) + if (trans->direction == GST_WEBRTC_RTP_TRANSCEIVER_DIRECTION_NONE) return FALSE; g_assert (trans->mline == -1 || trans->mline == media_idx); @@ -2974,8 +2973,49 @@ sdp_media_from_transceiver (GstWebRTCBin * webrtc, GstSDPMedia * media, bundle_only = bundled_mids && bundle_idx != media_idx && webrtc->bundle_policy == GST_WEBRTC_BUNDLE_POLICY_MAX_BUNDLE; - /* mandated by JSEP */ - gst_sdp_media_add_attribute (media, "setup", "actpass"); + caps = _find_codec_preferences (webrtc, trans, media_idx, error); + caps = _add_supported_attributes_to_caps (webrtc, WEBRTC_TRANSCEIVER (trans), + caps); + + if (!caps || gst_caps_is_empty (caps) || gst_caps_is_any (caps)) { + gst_clear_caps (&caps); + + if (last_media) { + guint i, n; + + n = gst_sdp_media_formats_len (last_media); + if (n > 0) { + caps = gst_caps_new_empty (); + for (i = 0; i < n; i++) { + guint fmt = atoi (gst_sdp_media_get_format (last_media, i)); + GstCaps *tmp = gst_sdp_media_get_caps_from_media (last_media, fmt); + GstStructure *s = gst_caps_get_structure (tmp, 0); + gst_structure_set_name (s, "application/x-rtp"); + gst_caps_append_structure (caps, gst_structure_copy (s)); + gst_clear_caps (&tmp); + } + GST_DEBUG_OBJECT (webrtc, "using previously negotiated caps for " + "transceiver %" GST_PTR_FORMAT " %" GST_PTR_FORMAT, trans, caps); + } + } + + if (!caps) { + GST_WARNING_OBJECT (webrtc, "no caps available for transceiver %" + GST_PTR_FORMAT ", skipping", trans); + return FALSE; + } + } + + if (last_media) { + const char *setup = gst_sdp_media_get_attribute_val (last_media, "setup"); + if (setup) + gst_sdp_media_add_attribute (media, "setup", setup); + else + return FALSE; + } else { + /* mandated by JSEP */ + gst_sdp_media_add_attribute (media, "setup", "actpass"); + } /* FIXME: deal with ICE restarts */ if (last_offer && trans->mline != -1 && trans->mid) { @@ -3022,15 +3062,6 @@ sdp_media_from_transceiver (GstWebRTCBin * webrtc, GstSDPMedia * media, gst_sdp_media_add_attribute (media, direction, ""); g_free (direction); - caps = _find_codec_preferences (webrtc, trans, media_idx, error); - - if (!caps || gst_caps_is_empty (caps) || gst_caps_is_any (caps)) { - GST_WARNING_OBJECT (webrtc, "no caps available for transceiver, skipping"); - if (caps) - gst_caps_unref (caps); - return FALSE; - } - caps = gst_caps_make_writable (caps); /* When an extmap is defined twice for the same ID, firefox complains and @@ -3385,9 +3416,11 @@ _create_offer_task (GstWebRTCBin * webrtc, const GstStructure * options, trans = g_ptr_array_index (webrtc->priv->transceivers, j); if (trans->mid && g_strcmp0 (trans->mid, last_mid) == 0) { - GstSDPMedia *media; - const gchar *mid; WebRTCTransceiver *wtrans = WEBRTC_TRANSCEIVER (trans); + const char *mid; + GstSDPMedia media; + + memset (&media, 0, sizeof (media)); g_assert (!g_list_find (seen_transceivers, trans)); @@ -3405,30 +3438,35 @@ _create_offer_task (GstWebRTCBin * webrtc, const GstStructure * options, GST_PTR_FORMAT " with mid %s into media index %u", trans, trans->mid, media_idx); - /* FIXME: deal with format changes */ - gst_sdp_media_copy (last_media, &media); - _media_replace_direction (media, trans->direction); - - mid = gst_sdp_media_get_attribute_val (media, "mid"); - g_assert (mid); + if (webrtc->bundle_policy == GST_WEBRTC_BUNDLE_POLICY_NONE) { + reserved_pts = g_array_new (FALSE, FALSE, sizeof (guint)); + } - if (g_hash_table_contains (all_mids, mid)) { - gst_sdp_media_free (media); - g_set_error (error, GST_WEBRTC_ERROR, - GST_WEBRTC_ERROR_INTERNAL_FAILURE, - "Duplicate mid %s when creating offer", mid); - goto cancel_offer; + gst_sdp_media_init (&media); + if (!sdp_media_from_transceiver (webrtc, &media, last_media, trans, + media_idx, bundled_mids, 0, bundle_ufrag, bundle_pwd, + reserved_pts, all_mids, error)) { + gst_sdp_media_uninit (&media); + if (!*error) + g_set_error_literal (error, GST_WEBRTC_ERROR, + GST_WEBRTC_ERROR_INTERNAL_FAILURE, + "Could not reuse transceiver"); } - g_hash_table_insert (all_mids, g_strdup (mid), NULL); + if (webrtc->bundle_policy == GST_WEBRTC_BUNDLE_POLICY_NONE) { + g_array_free (reserved_pts, TRUE); + reserved_pts = NULL; + } + if (*error) + goto cancel_offer; - if (bundled_mids) - g_string_append_printf (bundled_mids, " %s", mid); + mid = gst_sdp_media_get_attribute_val (&media, "mid"); + g_assert (mid && g_strcmp0 (last_mid, mid) == 0); - gst_sdp_message_add_media (ret, media); + gst_sdp_message_add_media (ret, &media); media_idx++; - gst_sdp_media_free (media); + gst_sdp_media_uninit (&media); seen_transceivers = g_list_prepend (seen_transceivers, trans); break; } @@ -3559,7 +3597,7 @@ _create_offer_task (GstWebRTCBin * webrtc, const GstStructure * options, GST_LOG_OBJECT (webrtc, "adding transceiver %" GST_PTR_FORMAT " at media " "index %u", trans, media_idx); - if (sdp_media_from_transceiver (webrtc, &media, trans, media_idx, + if (sdp_media_from_transceiver (webrtc, &media, NULL, trans, media_idx, bundled_mids, 0, bundle_ufrag, bundle_pwd, reserved_pts, all_mids, error)) { /* as per JSEP, a=rtcp-mux-only is only added for new streams */ @@ -3728,6 +3766,7 @@ _media_add_rtx (GstSDPMedia * media, WebRTCTransceiver * trans, str = g_strdup_printf ("%u", target_ssrc); gst_structure_set (trans->local_rtx_ssrc_map, str, G_TYPE_UINT, g_random_int (), NULL); + g_free (str); } } } @@ -4379,12 +4418,10 @@ out: static GstElement * _build_fec_encoder (GstWebRTCBin * webrtc, WebRTCTransceiver * trans) { - GstElement *ret = NULL; - GstElement *prev = NULL; - guint ulpfec_pt = 0; - guint red_pt = 0; - GstPad *sinkpad = NULL; GstWebRTCRTPTransceiver *rtp_trans = GST_WEBRTC_RTP_TRANSCEIVER (trans); + guint ulpfec_pt = 0, red_pt = 0; + GstPad *sinkpad, *srcpad, *ghost; + GstElement *ret; if (trans->stream) { ulpfec_pt = @@ -4392,70 +4429,216 @@ _build_fec_encoder (GstWebRTCBin * webrtc, WebRTCTransceiver * trans) red_pt = transport_stream_get_pt (trans->stream, "RED", rtp_trans->mline); } - if (ulpfec_pt || red_pt) - ret = gst_bin_new (NULL); + if (trans->ulpfecenc || trans->redenc) { + g_critical ("webrtcbin: duplicate call to create a fec encoder or " + "red encoder!"); + return NULL; + } - if (ulpfec_pt) { - GstElement *fecenc = gst_element_factory_make ("rtpulpfecenc", NULL); - GstCaps *caps = transport_stream_get_caps_for_pt (trans->stream, ulpfec_pt); + GST_DEBUG_OBJECT (webrtc, + "Creating ULPFEC encoder for mline %u with pt %d", rtp_trans->mline, + ulpfec_pt); - GST_DEBUG_OBJECT (webrtc, - "Creating ULPFEC encoder for mline %u with pt %d", rtp_trans->mline, - ulpfec_pt); + ret = gst_bin_new (NULL); + + trans->ulpfecenc = gst_element_factory_make ("rtpulpfecenc", NULL); + gst_object_ref_sink (trans->ulpfecenc); + if (!gst_bin_add (GST_BIN (ret), trans->ulpfecenc)) + g_warn_if_reached (); + sinkpad = gst_element_get_static_pad (trans->ulpfecenc, "sink"); + + g_object_bind_property (rtp_trans, "fec-percentage", trans->ulpfecenc, + "percentage", G_BINDING_BIDIRECTIONAL); + + trans->redenc = gst_element_factory_make ("rtpredenc", NULL); + gst_object_ref_sink (trans->redenc); + + GST_DEBUG_OBJECT (webrtc, "Creating RED encoder for mline %u with pt %d", + rtp_trans->mline, red_pt); + + gst_bin_add (GST_BIN (ret), trans->redenc); + gst_element_link (trans->ulpfecenc, trans->redenc); + + ghost = gst_ghost_pad_new ("sink", sinkpad); + gst_clear_object (&sinkpad); + gst_element_add_pad (ret, ghost); + ghost = NULL; + + srcpad = gst_element_get_static_pad (trans->redenc, "src"); + ghost = gst_ghost_pad_new ("src", srcpad); + gst_clear_object (&srcpad); + gst_element_add_pad (ret, ghost); + ghost = NULL; + + return ret; +} + +static gboolean +_merge_structure (GQuark field_id, const GValue * value, gpointer user_data) +{ + GstStructure *s = user_data; - gst_bin_add (GST_BIN (ret), fecenc); - sinkpad = gst_element_get_static_pad (fecenc, "sink"); - g_object_set (fecenc, "pt", ulpfec_pt, "percentage", - trans->fec_percentage, NULL); + gst_structure_id_set_value (s, field_id, value); - g_object_bind_property (rtp_trans, "fec-percentage", fecenc, "percentage", - G_BINDING_BIDIRECTIONAL); + return TRUE; +} - if (caps && !gst_caps_is_empty (caps)) { - const GstStructure *s = gst_caps_get_structure (caps, 0); - const gchar *media = gst_structure_get_string (s, "media"); +#define GST_WEBRTC_PAYLOAD_TYPE "gst.webrtcbin.payload.type" - if (!g_strcmp0 (media, "video")) - g_object_set (fecenc, "multipacket", TRUE, NULL); +static void +try_match_transceiver_with_fec_decoder (GstWebRTCBin * webrtc, + WebRTCTransceiver * trans) +{ + GList *l; + + for (l = trans->stream->fecdecs; l; l = l->next) { + GstElement *fecdec = GST_ELEMENT (l->data); + gboolean found_transceiver = FALSE; + int original_pt; + guint i; + + original_pt = + GPOINTER_TO_INT (g_object_get_data (G_OBJECT (fecdec), + GST_WEBRTC_PAYLOAD_TYPE)); + if (original_pt <= 0) { + GST_WARNING_OBJECT (trans, "failed to match fec decoder with " + "transceiver, fec decoder %" GST_PTR_FORMAT " does not contain a " + "valid payload type", fecdec); + continue; } - prev = fecenc; + for (i = 0; i < trans->stream->ptmap->len; i++) { + PtMapItem *item = &g_array_index (trans->stream->ptmap, PtMapItem, i); + + /* FIXME: this only works for a 1-1 original_pt->fec_pt mapping */ + if (original_pt == item->pt && item->media_idx != -1 + && item->media_idx == trans->parent.mline) { + if (trans->ulpfecdec) { + GST_FIXME_OBJECT (trans, "cannot"); + gst_clear_object (&trans->ulpfecdec); + } + trans->ulpfecdec = gst_object_ref (fecdec); + found_transceiver = TRUE; + break; + } + } + + if (!found_transceiver) { + GST_WARNING_OBJECT (trans, "failed to match fec decoder with " + "transceiver"); + } } +} - if (red_pt) { - GstElement *redenc = gst_element_factory_make ("rtpredenc", NULL); +static void +_set_internal_rtpbin_element_props_from_stream (GstWebRTCBin * webrtc, + TransportStream * stream) +{ + GstStructure *merged_local_rtx_ssrc_map; + GstStructure *pt_map = gst_structure_new_empty ("application/x-rtp-pt-map"); + GValue red_pt_array = { 0, }; + gint *rtx_pt; + gsize rtx_count; + gsize i; - GST_DEBUG_OBJECT (webrtc, "Creating RED encoder for mline %u with pt %d", - rtp_trans->mline, red_pt); + gst_value_array_init (&red_pt_array, 0); - gst_bin_add (GST_BIN (ret), redenc); - if (prev) - gst_element_link (prev, redenc); - else - sinkpad = gst_element_get_static_pad (redenc, "sink"); + rtx_pt = transport_stream_get_all_pt (stream, "RTX", &rtx_count); + GST_DEBUG_OBJECT (stream, "have %" G_GSIZE_FORMAT " rtx payloads", rtx_count); - g_object_set (redenc, "pt", red_pt, "allow-no-red-blocks", TRUE, NULL); + for (i = 0; i < rtx_count; i++) { + GstCaps *rtx_caps = transport_stream_get_caps_for_pt (stream, rtx_pt[i]); + const GstStructure *s = gst_caps_get_structure (rtx_caps, 0); + const gchar *apt = gst_structure_get_string (s, "apt"); - prev = redenc; + GST_LOG_OBJECT (stream, "setting rtx mapping: %s -> %u", apt, rtx_pt[i]); + gst_structure_set (pt_map, apt, G_TYPE_UINT, rtx_pt[i], NULL); } - if (sinkpad) { - GstPad *ghost = gst_ghost_pad_new ("sink", sinkpad); - gst_object_unref (sinkpad); - gst_element_add_pad (ret, ghost); + GST_DEBUG_OBJECT (stream, "setting payload map on %" GST_PTR_FORMAT " : %" + GST_PTR_FORMAT " and %" GST_PTR_FORMAT, stream->rtxreceive, + stream->rtxsend, pt_map); + + if (stream->rtxreceive) + g_object_set (stream->rtxreceive, "payload-type-map", pt_map, NULL); + if (stream->rtxsend) + g_object_set (stream->rtxsend, "payload-type-map", pt_map, NULL); + + gst_structure_free (pt_map); + g_clear_pointer (&rtx_pt, g_free); + + merged_local_rtx_ssrc_map = + gst_structure_new_empty ("application/x-rtp-ssrc-map"); + + for (i = 0; i < webrtc->priv->transceivers->len; i++) { + GstWebRTCRTPTransceiver *rtp_trans = + g_ptr_array_index (webrtc->priv->transceivers, i); + WebRTCTransceiver *trans = WEBRTC_TRANSCEIVER (rtp_trans); + + if (trans->stream == stream) { + gint ulpfec_pt, red_pt = 0; + + ulpfec_pt = transport_stream_get_pt (stream, "ULPFEC", rtp_trans->mline); + if (ulpfec_pt <= 0) + ulpfec_pt = 0; + + red_pt = transport_stream_get_pt (stream, "RED", rtp_trans->mline); + if (red_pt <= 0) { + red_pt = -1; + } else { + GValue ptval = { 0, }; + + g_value_init (&ptval, G_TYPE_INT); + g_value_set_int (&ptval, red_pt); + gst_value_array_append_value (&red_pt_array, &ptval); + g_value_unset (&ptval); + } + + GST_DEBUG_OBJECT (webrtc, "stream %" GST_PTR_FORMAT " transceiever %" + GST_PTR_FORMAT " has FEC payload %d and RED payload %d", stream, + trans, ulpfec_pt, red_pt); + + if (trans->ulpfecenc) { + g_object_set (trans->ulpfecenc, "pt", ulpfec_pt, "multipacket", + rtp_trans->kind == GST_WEBRTC_KIND_VIDEO, "percentage", + trans->fec_percentage, NULL); + } + + try_match_transceiver_with_fec_decoder (webrtc, trans); + if (trans->ulpfecdec) { + g_object_set (trans->ulpfecdec, "pt", ulpfec_pt, NULL); + } + + if (trans->redenc) { + gboolean always_produce = TRUE; + if (red_pt == -1) { + /* passthrough settings */ + red_pt = 0; + always_produce = FALSE; + } + g_object_set (trans->redenc, "pt", red_pt, "allow-no-red-blocks", + always_produce, NULL); + } + + if (trans->local_rtx_ssrc_map) { + gst_structure_foreach (trans->local_rtx_ssrc_map, + _merge_structure, merged_local_rtx_ssrc_map); + } + } } - if (prev) { - GstPad *srcpad = gst_element_get_static_pad (prev, "src"); - GstPad *ghost = gst_ghost_pad_new ("src", srcpad); - gst_object_unref (srcpad); - gst_element_add_pad (ret, ghost); + if (stream->rtxsend) + g_object_set (stream->rtxsend, "ssrc-map", merged_local_rtx_ssrc_map, NULL); + gst_clear_structure (&merged_local_rtx_ssrc_map); + + if (stream->reddec) { + g_object_set_property (G_OBJECT (stream->reddec), "payloads", + &red_pt_array); } - return ret; + g_value_unset (&red_pt_array); } - static GstPad * _connect_input_stream (GstWebRTCBin * webrtc, GstWebRTCBinPad * pad) { @@ -4511,20 +4694,25 @@ _connect_input_stream (GstWebRTCBin * webrtc, GstWebRTCBinPad * pad) gst_element_sync_state_with_parent (clocksync); srcpad = gst_element_get_static_pad (clocksync, "src"); - sinkpad = gst_element_get_static_pad (clocksync, "sink"); - if ((fec_encoder = _build_fec_encoder (webrtc, trans))) { - GstPad *fec_sink; + fec_encoder = _build_fec_encoder (webrtc, trans); + if (!fec_encoder) { + g_warn_if_reached (); + return NULL; + } - gst_bin_add (GST_BIN (webrtc), fec_encoder); - gst_element_sync_state_with_parent (fec_encoder); + _set_internal_rtpbin_element_props_from_stream (webrtc, trans->stream); - fec_sink = gst_element_get_static_pad (fec_encoder, "sink"); - gst_pad_link (srcpad, fec_sink); - gst_object_unref (srcpad); - gst_object_unref (fec_sink); - srcpad = gst_element_get_static_pad (fec_encoder, "src"); - } + gst_bin_add (GST_BIN (webrtc), fec_encoder); + gst_element_sync_state_with_parent (fec_encoder); + + sinkpad = gst_element_get_static_pad (fec_encoder, "sink"); + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) + g_warn_if_reached (); + gst_clear_object (&srcpad); + gst_clear_object (&sinkpad); + sinkpad = gst_element_get_static_pad (clocksync, "sink"); + srcpad = gst_element_get_static_pad (fec_encoder, "src"); if (!webrtc->rtpfunnel) { rtp_templ = @@ -4539,8 +4727,6 @@ _connect_input_stream (GstWebRTCBin * webrtc, GstWebRTCBinPad * pad) gst_pad_link (srcpad, rtp_sink); gst_object_unref (rtp_sink); - gst_ghost_pad_set_target (GST_GHOST_PAD (pad), sinkpad); - pad_name = g_strdup_printf ("send_rtp_src_%u", pad->trans->mline); if (!gst_element_link_pads (GST_ELEMENT (webrtc->rtpbin), pad_name, GST_ELEMENT (trans->stream->send_bin), "rtp_sink")) @@ -4552,14 +4738,15 @@ _connect_input_stream (GstWebRTCBin * webrtc, GstWebRTCBinPad * pad) gst_element_request_pad_simple (webrtc->rtpfunnel, pad_name); gst_pad_link (srcpad, funnel_sinkpad); - gst_ghost_pad_set_target (GST_GHOST_PAD (pad), sinkpad); g_free (pad_name); gst_object_unref (funnel_sinkpad); } - gst_object_unref (srcpad); - gst_object_unref (sinkpad); + gst_ghost_pad_set_target (GST_GHOST_PAD (pad), sinkpad); + + gst_clear_object (&srcpad); + gst_clear_object (&sinkpad); gst_element_sync_state_with_parent (GST_ELEMENT (trans->stream->send_bin)); @@ -4716,41 +4903,6 @@ _filter_sdp_fields (GQuark field_id, const GValue * value, } static void -_set_rtx_ptmap_from_stream (GstWebRTCBin * webrtc, TransportStream * stream) -{ - gint *rtx_pt; - gsize rtx_count; - - rtx_pt = transport_stream_get_all_pt (stream, "RTX", &rtx_count); - GST_LOG_OBJECT (stream, "have %" G_GSIZE_FORMAT " rtx payloads", rtx_count); - if (rtx_pt) { - GstStructure *pt_map = gst_structure_new_empty ("application/x-rtp-pt-map"); - gsize i; - - for (i = 0; i < rtx_count; i++) { - GstCaps *rtx_caps = transport_stream_get_caps_for_pt (stream, rtx_pt[i]); - const GstStructure *s = gst_caps_get_structure (rtx_caps, 0); - const gchar *apt = gst_structure_get_string (s, "apt"); - - GST_LOG_OBJECT (stream, "setting rtx mapping: %s -> %u", apt, rtx_pt[i]); - gst_structure_set (pt_map, apt, G_TYPE_UINT, rtx_pt[i], NULL); - } - - GST_DEBUG_OBJECT (stream, "setting payload map on %" GST_PTR_FORMAT " : %" - GST_PTR_FORMAT " and %" GST_PTR_FORMAT, stream->rtxreceive, - stream->rtxsend, pt_map); - - if (stream->rtxreceive) - g_object_set (stream->rtxreceive, "payload-type-map", pt_map, NULL); - if (stream->rtxsend) - g_object_set (stream->rtxsend, "payload-type-map", pt_map, NULL); - - gst_structure_free (pt_map); - g_free (rtx_pt); - } -} - -static void _update_transport_ptmap_from_media (GstWebRTCBin * webrtc, TransportStream * stream, const GstSDPMessage * sdp, guint media_idx) { @@ -5026,7 +5178,7 @@ _update_transceiver_from_sdp_media (GstWebRTCBin * webrtc, if (!bundled || bundle_idx == media_idx) { if (stream->rtxsend || stream->rtxreceive) { - _set_rtx_ptmap_from_stream (webrtc, stream); + _set_internal_rtpbin_element_props_from_stream (webrtc, stream); } g_object_set (stream, "dtls-client", @@ -6467,82 +6619,56 @@ unknown_session: } } -static gboolean -_merge_structure (GQuark field_id, const GValue * value, gpointer user_data) -{ - GstStructure *s = user_data; - - gst_structure_id_set_value (s, field_id, value); - - return TRUE; -} - static GstElement * on_rtpbin_request_aux_sender (GstElement * rtpbin, guint session_id, GstWebRTCBin * webrtc) { TransportStream *stream; - gboolean have_rtx = FALSE; - GstElement *ret = NULL; + GstElement *ret, *rtx; + GstPad *pad; + char *name; stream = _find_transport_for_session (webrtc, session_id); + if (!stream) { + /* a rtp session without a stream is a webrtcbin bug */ + g_warn_if_reached (); + return NULL; + } - if (stream) - have_rtx = transport_stream_get_pt (stream, "RTX", -1) != 0; - - GST_LOG_OBJECT (webrtc, "requesting aux sender for stream %" GST_PTR_FORMAT, - stream); - - if (have_rtx) { - GstElement *rtx; - GstPad *pad; - gchar *name; - GstStructure *merged_local_rtx_ssrc_map = - gst_structure_new_empty ("application/x-rtp-ssrc-map"); - guint i; - - if (stream->rtxsend) { - GST_WARNING_OBJECT (webrtc, "rtprtxsend already created! rtpbin bug?!"); - goto out; - } - - GST_INFO ("creating AUX sender"); - ret = gst_bin_new (NULL); - rtx = gst_element_factory_make ("rtprtxsend", NULL); - g_object_set (rtx, "max-size-packets", 500, NULL); - _set_rtx_ptmap_from_stream (webrtc, stream); - - for (i = 0; i < webrtc->priv->transceivers->len; i++) { - WebRTCTransceiver *trans = - WEBRTC_TRANSCEIVER (g_ptr_array_index (webrtc->priv->transceivers, - i)); + if (stream->rtxsend) { + GST_WARNING_OBJECT (webrtc, "rtprtxsend already created! rtpbin bug?!"); + g_warn_if_reached (); + return NULL; + } - if (trans->stream == stream && trans->local_rtx_ssrc_map) - gst_structure_foreach (trans->local_rtx_ssrc_map, - _merge_structure, merged_local_rtx_ssrc_map); - } + GST_DEBUG_OBJECT (webrtc, "requesting aux sender for session %u " + "stream %" GST_PTR_FORMAT, session_id, stream); - g_object_set (rtx, "ssrc-map", merged_local_rtx_ssrc_map, NULL); - gst_structure_free (merged_local_rtx_ssrc_map); + ret = gst_bin_new (NULL); + rtx = gst_element_factory_make ("rtprtxsend", NULL); + /* XXX: allow control from outside? */ + g_object_set (rtx, "max-size-packets", 500, NULL); - gst_bin_add (GST_BIN (ret), rtx); + if (!gst_bin_add (GST_BIN (ret), rtx)) + g_warn_if_reached (); - pad = gst_element_get_static_pad (rtx, "src"); - name = g_strdup_printf ("src_%u", session_id); - gst_element_add_pad (ret, gst_ghost_pad_new (name, pad)); - g_free (name); - gst_object_unref (pad); + stream->rtxsend = gst_object_ref (rtx); + _set_internal_rtpbin_element_props_from_stream (webrtc, stream); - pad = gst_element_get_static_pad (rtx, "sink"); - name = g_strdup_printf ("sink_%u", session_id); - gst_element_add_pad (ret, gst_ghost_pad_new (name, pad)); - g_free (name); - gst_object_unref (pad); + name = g_strdup_printf ("src_%u", session_id); + pad = gst_element_get_static_pad (rtx, "src"); + if (!gst_element_add_pad (ret, gst_ghost_pad_new (name, pad))) + g_warn_if_reached (); + gst_clear_object (&pad); + g_clear_pointer (&name, g_free); - stream->rtxsend = gst_object_ref (rtx); - } + name = g_strdup_printf ("sink_%u", session_id); + pad = gst_element_get_static_pad (rtx, "sink"); + if (!gst_element_add_pad (ret, gst_ghost_pad_new (name, pad))) + g_warn_if_reached (); + gst_clear_object (&pad); + g_clear_pointer (&name, g_free); -out: return ret; } @@ -6550,108 +6676,68 @@ static GstElement * on_rtpbin_request_aux_receiver (GstElement * rtpbin, guint session_id, GstWebRTCBin * webrtc) { - GstElement *ret = NULL; - GstElement *prev = NULL; - GstPad *sinkpad = NULL; TransportStream *stream; - gint rtx_pt = 0; - GValue red_pt_array = { 0, }; - gboolean have_red_pt = FALSE; - - g_value_init (&red_pt_array, GST_TYPE_ARRAY); + GstPad *pad, *ghost; + GstElement *ret; + char *name; stream = _find_transport_for_session (webrtc, session_id); - - if (stream) { - guint i = 0; - - for (i = 0; i < stream->ptmap->len; i++) { - PtMapItem *item = &g_array_index (stream->ptmap, PtMapItem, i); - - if (!gst_caps_is_empty (item->caps)) { - GstStructure *s = gst_caps_get_structure (item->caps, 0); - - if (!g_strcmp0 (gst_structure_get_string (s, "encoding-name"), "RED")) { - GValue ptval = { 0, }; - - g_value_init (&ptval, G_TYPE_INT); - g_value_set_int (&ptval, item->pt); - gst_value_array_append_value (&red_pt_array, &ptval); - g_value_unset (&ptval); - - have_red_pt = TRUE; - } - } - } - - rtx_pt = transport_stream_get_pt (stream, "RTX", -1); + if (!stream) { + /* no transport stream before the session has been created is a webrtcbin + * programming error! */ + g_warn_if_reached (); + return NULL; } - GST_LOG_OBJECT (webrtc, "requesting aux receiver for stream %" GST_PTR_FORMAT, - stream); - - if (have_red_pt || rtx_pt) - ret = gst_bin_new (NULL); - - if (rtx_pt) { - if (stream->rtxreceive) { - GST_WARNING_OBJECT (webrtc, - "rtprtxreceive already created! rtpbin bug?!"); - goto error; - } - - stream->rtxreceive = gst_element_factory_make ("rtprtxreceive", NULL); - _set_rtx_ptmap_from_stream (webrtc, stream); - - gst_bin_add (GST_BIN (ret), stream->rtxreceive); - - sinkpad = gst_element_get_static_pad (stream->rtxreceive, "sink"); + if (stream->rtxreceive) { + GST_WARNING_OBJECT (webrtc, "rtprtxreceive already created! rtpbin bug?!"); + g_warn_if_reached (); + return NULL; + } - prev = gst_object_ref (stream->rtxreceive); + if (stream->reddec) { + GST_WARNING_OBJECT (webrtc, "rtpreddec already created! rtpbin bug?!"); + g_warn_if_reached (); + return NULL; } - if (have_red_pt) { - GstElement *rtpreddec = gst_element_factory_make ("rtpreddec", NULL); + GST_DEBUG_OBJECT (webrtc, "requesting aux receiver for session %u " + "stream %" GST_PTR_FORMAT, session_id, stream); - GST_DEBUG_OBJECT (webrtc, "Creating RED decoder in session %u", session_id); + ret = gst_bin_new (NULL); - gst_bin_add (GST_BIN (ret), rtpreddec); + stream->rtxreceive = gst_element_factory_make ("rtprtxreceive", NULL); + gst_object_ref (stream->rtxreceive); + if (!gst_bin_add (GST_BIN (ret), stream->rtxreceive)) + g_warn_if_reached (); - g_object_set_property (G_OBJECT (rtpreddec), "payloads", &red_pt_array); + stream->reddec = gst_element_factory_make ("rtpreddec", NULL); + gst_object_ref (stream->reddec); + if (!gst_bin_add (GST_BIN (ret), stream->reddec)) + g_warn_if_reached (); - if (prev) - gst_element_link (prev, rtpreddec); - else - sinkpad = gst_element_get_static_pad (rtpreddec, "sink"); + _set_internal_rtpbin_element_props_from_stream (webrtc, stream); - prev = rtpreddec; - } + if (!gst_element_link (stream->rtxreceive, stream->reddec)) + g_warn_if_reached (); - if (sinkpad) { - gchar *name = g_strdup_printf ("sink_%u", session_id); - GstPad *ghost = gst_ghost_pad_new (name, sinkpad); - g_free (name); - gst_object_unref (sinkpad); - gst_element_add_pad (ret, ghost); - } + name = g_strdup_printf ("sink_%u", session_id); + pad = gst_element_get_static_pad (stream->rtxreceive, "sink"); + ghost = gst_ghost_pad_new (name, pad); + g_clear_pointer (&name, g_free); + gst_clear_object (&pad); + if (!gst_element_add_pad (ret, ghost)) + g_warn_if_reached (); - if (prev) { - gchar *name = g_strdup_printf ("src_%u", session_id); - GstPad *srcpad = gst_element_get_static_pad (prev, "src"); - GstPad *ghost = gst_ghost_pad_new (name, srcpad); - g_free (name); - gst_object_unref (srcpad); - gst_element_add_pad (ret, ghost); - } + name = g_strdup_printf ("src_%u", session_id); + pad = gst_element_get_static_pad (stream->reddec, "src"); + ghost = gst_ghost_pad_new (name, pad); + g_clear_pointer (&name, g_free); + gst_clear_object (&pad); + if (!gst_element_add_pad (ret, ghost)) + g_warn_if_reached (); -out: - g_value_unset (&red_pt_array); return ret; - -error: - if (ret) - gst_object_unref (ret); - goto out; } static GstElement * @@ -6660,10 +6746,14 @@ on_rtpbin_request_fec_decoder_full (GstElement * rtpbin, guint session_id, { TransportStream *stream; GstElement *ret = NULL; - gint fec_pt = 0; GObject *internal_storage; stream = _find_transport_for_session (webrtc, session_id); + if (!stream) { + /* a rtp session without a stream is a webrtcbin bug */ + g_warn_if_reached (); + return NULL; + } /* TODO: for now, we only support ulpfec, but once we support * more algorithms, if the remote may use more than one algorithm, @@ -6671,33 +6761,25 @@ on_rtpbin_request_fec_decoder_full (GstElement * rtpbin, guint session_id, * * + Return a bin here, with the relevant FEC decoders plugged in * and their payload type set to 0 - * + Enable the decoders by setting the payload type only when - * we detect it (by connecting to ptdemux:new-payload-type for - * example) */ - if (stream) { - guint i; + GST_DEBUG_OBJECT (webrtc, "Creating ULPFEC decoder for pt %d in session %u " + "stream %" GST_PTR_FORMAT, pt, session_id, stream); - for (i = 0; i < stream->ptmap->len; i++) { - PtMapItem *item = &g_array_index (stream->ptmap, PtMapItem, i); + ret = gst_element_factory_make ("rtpulpfecdec", NULL); - if (item->pt == pt) { - fec_pt = transport_stream_get_pt (stream, "ULPFEC", item->media_idx); - break; - } - } - } + g_signal_emit_by_name (webrtc->rtpbin, "get-internal-storage", session_id, + &internal_storage); - if (fec_pt) { - GST_DEBUG_OBJECT (webrtc, "Creating ULPFEC decoder for pt %d in session %u", - fec_pt, session_id); - ret = gst_element_factory_make ("rtpulpfecdec", NULL); - g_signal_emit_by_name (webrtc->rtpbin, "get-internal-storage", session_id, - &internal_storage); + g_object_set (ret, "storage", internal_storage, NULL); + g_clear_object (&internal_storage); - g_object_set (ret, "pt", fec_pt, "storage", internal_storage, NULL); - g_object_unref (internal_storage); - } + g_object_set_data (G_OBJECT (ret), GST_WEBRTC_PAYLOAD_TYPE, + GINT_TO_POINTER (pt)); + + PC_LOCK (webrtc); + stream->fecdecs = g_list_prepend (stream->fecdecs, gst_object_ref (ret)); + _set_internal_rtpbin_element_props_from_stream (webrtc, stream); + PC_UNLOCK (webrtc); return ret; } diff --git a/subprojects/gst-plugins-bad/ext/webrtc/transportstream.c b/subprojects/gst-plugins-bad/ext/webrtc/transportstream.c index a556669..de40971 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/transportstream.c +++ b/subprojects/gst-plugins-bad/ext/webrtc/transportstream.c @@ -59,7 +59,7 @@ transport_stream_get_pt (TransportStream * stream, const gchar * encoding_name, guint media_idx) { guint i; - gint ret = 0; + gint ret = -1; for (i = 0; i < stream->ptmap->len; i++) { PtMapItem *item = &g_array_index (stream->ptmap, PtMapItem, i); @@ -165,25 +165,14 @@ transport_stream_dispose (GObject * object) { TransportStream *stream = TRANSPORT_STREAM (object); - if (stream->send_bin) - gst_object_unref (stream->send_bin); - stream->send_bin = NULL; - - if (stream->receive_bin) - gst_object_unref (stream->receive_bin); - stream->receive_bin = NULL; - - if (stream->transport) - gst_object_unref (stream->transport); - stream->transport = NULL; - - if (stream->rtxsend) - gst_object_unref (stream->rtxsend); - stream->rtxsend = NULL; - - if (stream->rtxreceive) - gst_object_unref (stream->rtxreceive); - stream->rtxreceive = NULL; + gst_clear_object (&stream->send_bin); + gst_clear_object (&stream->receive_bin); + gst_clear_object (&stream->transport); + gst_clear_object (&stream->rtxsend); + gst_clear_object (&stream->rtxreceive); + gst_clear_object (&stream->reddec); + g_list_free_full (stream->fecdecs, (GDestroyNotify) gst_object_unref); + stream->fecdecs = NULL; GST_OBJECT_PARENT (object) = NULL; diff --git a/subprojects/gst-plugins-bad/ext/webrtc/transportstream.h b/subprojects/gst-plugins-bad/ext/webrtc/transportstream.h index 24afe27..39794b6 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/transportstream.h +++ b/subprojects/gst-plugins-bad/ext/webrtc/transportstream.h @@ -67,6 +67,9 @@ struct _TransportStream GstElement *rtxsend; GstElement *rtxreceive; + + GstElement *reddec; + GList *fecdecs; }; struct _TransportStreamClass diff --git a/subprojects/gst-plugins-bad/ext/webrtc/webrtctransceiver.c b/subprojects/gst-plugins-bad/ext/webrtc/webrtctransceiver.c index 963dfc6..ed4f906 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/webrtctransceiver.c +++ b/subprojects/gst-plugins-bad/ext/webrtc/webrtctransceiver.c @@ -148,9 +148,10 @@ webrtc_transceiver_finalize (GObject * object) { WebRTCTransceiver *trans = WEBRTC_TRANSCEIVER (object); - if (trans->stream) - gst_object_unref (trans->stream); - trans->stream = NULL; + gst_clear_object (&trans->stream); + gst_clear_object (&trans->ulpfecdec); + gst_clear_object (&trans->ulpfecenc); + gst_clear_object (&trans->redenc); if (trans->local_rtx_ssrc_map) gst_structure_free (trans->local_rtx_ssrc_map); diff --git a/subprojects/gst-plugins-bad/ext/webrtc/webrtctransceiver.h b/subprojects/gst-plugins-bad/ext/webrtc/webrtctransceiver.h index 2e2abaf..a3c80c5 100644 --- a/subprojects/gst-plugins-bad/ext/webrtc/webrtctransceiver.h +++ b/subprojects/gst-plugins-bad/ext/webrtc/webrtctransceiver.h @@ -51,6 +51,10 @@ struct _WebRTCTransceiver GstCaps *last_configured_caps; gboolean mline_locked; + + GstElement *ulpfecdec; + GstElement *ulpfecenc; + GstElement *redenc; }; struct _WebRTCTransceiverClass diff --git a/subprojects/gst-plugins-bad/tests/check/elements/webrtcbin.c b/subprojects/gst-plugins-bad/tests/check/elements/webrtcbin.c index 9ed6937..8508bd1 100644 --- a/subprojects/gst-plugins-bad/tests/check/elements/webrtcbin.c +++ b/subprojects/gst-plugins-bad/tests/check/elements/webrtcbin.c @@ -4371,6 +4371,60 @@ GST_START_TEST (test_codec_preferences_in_on_new_transceiver) GST_END_TEST; +GST_START_TEST (test_renego_rtx) +{ + struct test_webrtc *t = create_audio_video_test (); + VAL_SDP_INIT (no_duplicate_payloads, on_sdp_media_no_duplicate_payloads, + NULL, NULL); + guint media_format_count[] = { 1, 1 }; + VAL_SDP_INIT (media_formats, on_sdp_media_count_formats, + media_format_count, &no_duplicate_payloads); + VAL_SDP_INIT (count_media, _count_num_sdp_media, GUINT_TO_POINTER (2), + &media_formats); + VAL_SDP_INIT (payloads, on_sdp_media_payload_types, + GUINT_TO_POINTER (1), &count_media); + const gchar *expected_offer_direction[] = { "sendrecv", "sendrecv", }; + VAL_SDP_INIT (offer_direction, on_sdp_media_direction, + expected_offer_direction, &payloads); + const gchar *expected_answer_direction[] = { "recvonly", "recvonly", }; + VAL_SDP_INIT (answer_direction, on_sdp_media_direction, + expected_answer_direction, &payloads); + const gchar *expected_offer_setup[] = { "actpass", "actpass", }; + VAL_SDP_INIT (offer, on_sdp_media_setup, expected_offer_setup, + &offer_direction); + const gchar *expected_answer_setup[] = { "active", "active", }; + VAL_SDP_INIT (answer, on_sdp_media_setup, expected_answer_setup, + &answer_direction); + GstWebRTCRTPTransceiver *trans; + + t->on_negotiation_needed = NULL; + t->on_ice_candidate = NULL; + t->on_pad_added = _pad_added_fakesink; + + test_validate_sdp (t, &offer, &answer); + + test_webrtc_reset_negotiation (t); + + g_signal_emit_by_name (t->webrtc1, "get-transceiver", 1, &trans); + g_object_set (trans, "do-nack", TRUE, "fec-type", + GST_WEBRTC_FEC_TYPE_ULP_RED, NULL); + g_clear_object (&trans); + + g_signal_emit_by_name (t->webrtc2, "get-transceiver", 1, &trans); + g_object_set (trans, "do-nack", TRUE, "fec-type", + GST_WEBRTC_FEC_TYPE_ULP_RED, NULL); + g_clear_object (&trans); + + /* adding RTX/RED/FEC increases the number of media formats */ + media_format_count[1] = 5; + + test_validate_sdp (t, &offer, &answer); + + test_webrtc_free (t); +} + +GST_END_TEST; + static Suite * webrtcbin_suite (void) { @@ -4425,6 +4479,7 @@ webrtcbin_suite (void) tcase_add_test (tc, test_codec_preferences_no_duplicate_extmaps); tcase_add_test (tc, test_codec_preferences_incompatible_extmaps); tcase_add_test (tc, test_codec_preferences_invalid_extmap); + tcase_add_test (tc, test_renego_rtx); if (sctpenc && sctpdec) { tcase_add_test (tc, test_data_channel_create); tcase_add_test (tc, test_data_channel_remote_notify); -- 2.7.4