From 3c9e4f488602794473d9d4e2b6fc67670a401424 Mon Sep 17 00:00:00 2001 From: Patricia Muscalu Date: Tue, 5 Jul 2022 16:15:19 +0200 Subject: [PATCH] rtph265: keep delta unit flag Without this patch all buffers that pass the payloader are marked as non-delta-unit buffers. Part-of: --- .../gst-plugins-good/gst/rtp/gstrtph265pay.c | 89 +++++++++--- .../gst-plugins-good/gst/rtp/gstrtph265pay.h | 3 + .../tests/check/elements/rtph265.c | 152 +++++++++++++++++++++ 3 files changed, 224 insertions(+), 20 deletions(-) diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c b/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c index 054af14..82b68cc 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c @@ -922,21 +922,24 @@ gst_rtp_h265_pay_decode_nal (GstRtpH265Pay * payloader, } static GstFlowReturn gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * - basepayload, GPtrArray * paybufs, GstClockTime dts, GstClockTime pts); + basepayload, GPtrArray * paybufs, GstClockTime dts, GstClockTime pts, + gboolean delta_unit); static GstFlowReturn gst_rtp_h265_pay_payload_nal_single (GstRTPBasePayload * basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, - gboolean marker); + gboolean marker, gboolean delta_unit); static GstFlowReturn gst_rtp_h265_pay_payload_nal_fragment (GstRTPBasePayload * basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, - gboolean marker, guint mtu, guint8 nal_type, const guint8 * nal_header, - int size); + gboolean marker, gboolean delta_unit, guint mtu, guint8 nal_type, + const guint8 * nal_header, int size); static GstFlowReturn gst_rtp_h265_pay_payload_nal_bundle (GstRTPBasePayload * basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, - gboolean marker, guint8 nal_type, const guint8 * nal_header, int size); + gboolean marker, gboolean delta_unit, guint8 nal_type, + const guint8 * nal_header, int size); static GstFlowReturn gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload, - GstRtpH265Pay * rtph265pay, GstClockTime dts, GstClockTime pts) + GstRtpH265Pay * rtph265pay, GstClockTime dts, GstClockTime pts, + gboolean delta_unit) { GstFlowReturn ret = GST_FLOW_OK; gboolean sent_all_vps_sps_pps = TRUE; @@ -967,7 +970,7 @@ gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload, g_ptr_array_add (bufs, gst_buffer_ref (pps_buf)); } - ret = gst_rtp_h265_pay_payload_nal (basepayload, bufs, dts, pts); + ret = gst_rtp_h265_pay_payload_nal (basepayload, bufs, dts, pts, FALSE); if (ret != GST_FLOW_OK) { /* not critical but warn */ GST_WARNING_OBJECT (basepayload, "failed pushing VPS/SPS/PPS"); @@ -993,7 +996,8 @@ gst_rtp_h265_pay_reset_bundle (GstRtpH265Pay * rtph265pay) static GstFlowReturn gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, - GPtrArray * paybufs, GstClockTime dts, GstClockTime pts) + GPtrArray * paybufs, GstClockTime dts, GstClockTime pts, + gboolean delta_unit) { GstRtpH265Pay *rtph265pay; guint mtu; @@ -1103,7 +1107,8 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, sent_ps = TRUE; GST_DEBUG_OBJECT (rtph265pay, "sending VPS/SPS/PPS before current frame"); ret = - gst_rtp_h265_pay_send_vps_sps_pps (basepayload, rtph265pay, dts, pts); + gst_rtp_h265_pay_send_vps_sps_pps (basepayload, rtph265pay, dts, pts, + delta_unit); if (ret != GST_FLOW_OK) { gst_buffer_unref (paybuf); continue; @@ -1112,10 +1117,10 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, if (rtph265pay->aggregate_mode != GST_RTP_H265_AGGREGATE_NONE) ret = gst_rtp_h265_pay_payload_nal_bundle (basepayload, paybuf, dts, pts, - marker, nal_type, nal_header, size); + marker, delta_unit, nal_type, nal_header, size); else ret = gst_rtp_h265_pay_payload_nal_fragment (basepayload, paybuf, dts, - pts, marker, mtu, nal_type, nal_header, size); + pts, marker, delta_unit, mtu, nal_type, nal_header, size); } g_ptr_array_free (paybufs, TRUE); @@ -1125,7 +1130,8 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, static GstFlowReturn gst_rtp_h265_pay_payload_nal_single (GstRTPBasePayload * basepayload, - GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean marker) + GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean marker, + gboolean delta_unit) { GstBufferList *outlist; GstBuffer *outbuf; @@ -1142,6 +1148,9 @@ gst_rtp_h265_pay_payload_nal_single (GstRTPBasePayload * basepayload, gst_rtp_buffer_set_marker (&rtp, marker); GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_MARKER); + if (delta_unit) + GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + /* timestamp the outbuffer */ GST_BUFFER_PTS (outbuf) = pts; GST_BUFFER_DTS (outbuf) = dts; @@ -1164,6 +1173,7 @@ gst_rtp_h265_pay_payload_nal_single (GstRTPBasePayload * basepayload, static GstFlowReturn gst_rtp_h265_pay_payload_nal_fragment (GstRTPBasePayload * basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean marker, + gboolean delta_unit, guint mtu, guint8 nal_type, const guint8 * nal_header, int size) { GstRtpH265Pay *rtph265pay = (GstRtpH265Pay *) basepayload; @@ -1179,7 +1189,7 @@ gst_rtp_h265_pay_payload_nal_fragment (GstRTPBasePayload * basepayload, "NAL Unit fit in one packet datasize=%d mtu=%d", size, mtu); /* will fit in one packet */ return gst_rtp_h265_pay_payload_nal_single (basepayload, paybuf, dts, pts, - marker); + marker, delta_unit); } GST_DEBUG_OBJECT (basepayload, @@ -1238,6 +1248,12 @@ gst_rtp_h265_pay_payload_nal_fragment (GstRTPBasePayload * basepayload, gst_rtp_copy_video_meta (rtph265pay, outbuf, paybuf); gst_buffer_copy_into (outbuf, paybuf, GST_BUFFER_COPY_MEMORY, pos, fragment_size); + if (!delta_unit) + /* only the first packet sent should not have the flag */ + delta_unit = TRUE; + else + GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + /* add the buffer to the buffer list */ gst_buffer_list_add (outlist, outbuf); } @@ -1256,6 +1272,7 @@ gst_rtp_h265_pay_send_bundle (GstRtpH265Pay * rtph265pay, gboolean marker) guint length, bundle_size; GstBuffer *first, *outbuf; GstClockTime dts, pts; + gboolean delta_unit; bundle_size = rtph265pay->bundle_size; @@ -1271,6 +1288,7 @@ gst_rtp_h265_pay_send_bundle (GstRtpH265Pay * rtph265pay, gboolean marker) first = gst_buffer_list_get (bundle, 0); dts = GST_BUFFER_DTS (first); pts = GST_BUFFER_PTS (first); + delta_unit = GST_BUFFER_FLAG_IS_SET (first, GST_BUFFER_FLAG_DELTA_UNIT); if (length == 1) { /* Push unaggregated NALU */ @@ -1330,13 +1348,14 @@ gst_rtp_h265_pay_send_bundle (GstRtpH265Pay * rtph265pay, gboolean marker) gst_rtp_h265_pay_reset_bundle (rtph265pay); return gst_rtp_h265_pay_payload_nal_single (basepayload, outbuf, dts, pts, - marker); + marker, delta_unit); } static gboolean gst_rtp_h265_pay_payload_nal_bundle (GstRTPBasePayload * basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, - gboolean marker, guint8 nal_type, const guint8 * nal_header, int size) + gboolean marker, gboolean delta_unit, guint8 nal_type, + const guint8 * nal_header, int size) { GstRtpH265Pay *rtph265pay; GstFlowReturn ret; @@ -1386,7 +1405,7 @@ gst_rtp_h265_pay_payload_nal_bundle (GstRTPBasePayload * basepayload, goto out; return gst_rtp_h265_pay_payload_nal_fragment (basepayload, paybuf, dts, pts, - marker, mtu, nal_type, nal_header, size); + marker, delta_unit, mtu, nal_type, nal_header, size); } bundle_size = rtph265pay->bundle_size + pay_size; @@ -1418,6 +1437,11 @@ gst_rtp_h265_pay_payload_nal_bundle (GstRTPBasePayload * basepayload, GST_BUFFER_PTS (paybuf) = pts; GST_BUFFER_DTS (paybuf) = dts; + if (delta_unit) + GST_BUFFER_FLAG_SET (paybuf, GST_BUFFER_FLAG_DELTA_UNIT); + else + GST_BUFFER_FLAG_UNSET (paybuf, GST_BUFFER_FLAG_DELTA_UNIT); + gst_buffer_list_add (bundle, gst_buffer_ref (paybuf)); rtph265pay->bundle_size += pay_size; ret = GST_FLOW_OK; @@ -1452,6 +1476,7 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, gboolean hevc; GstBuffer *paybuf = NULL; gsize skip; + gboolean delayed_not_delta_unit = FALSE; gboolean marker = FALSE; gboolean discont = FALSE; gboolean draining = (buffer == NULL); @@ -1469,8 +1494,14 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, return GST_FLOW_OK; } else { if (buffer) { - if (gst_adapter_available (rtph265pay->adapter) == 0) - discont = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT); + if (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)) { + if (gst_adapter_available (rtph265pay->adapter) == 0) + rtph265pay->delta_unit = FALSE; + else + delayed_not_delta_unit = TRUE; + } + + discont = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT); marker = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_MARKER); gst_adapter_push (rtph265pay->adapter, buffer); buffer = NULL; @@ -1506,6 +1537,8 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, pts = GST_BUFFER_PTS (buffer); dts = GST_BUFFER_DTS (buffer); + rtph265pay->delta_unit = GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT); marker = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_MARKER); GST_DEBUG_OBJECT (basepayload, "got %" G_GSIZE_FORMAT " bytes", remaining_buffer_size); @@ -1559,7 +1592,13 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, offset += nal_len; remaining_buffer_size -= nal_len; } - ret = gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts); + ret = + gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts, + rtph265pay->delta_unit); + + if (!rtph265pay->delta_unit) + /* only the first outgoing packet doesn't have the DELTA_UNIT flag */ + rtph265pay->delta_unit = TRUE; gst_buffer_memory_unmap (&memory); gst_buffer_unref (buffer); @@ -1674,12 +1713,22 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, discont = FALSE; } + if (delayed_not_delta_unit) { + rtph265pay->delta_unit = FALSE; + delayed_not_delta_unit = FALSE; + } else { + /* only the first outgoing packet doesn't have the DELTA_UNIT flag */ + rtph265pay->delta_unit = TRUE; + } + /* move to next NAL packet */ /* Skips the trailing zeros */ gst_adapter_flush (rtph265pay->adapter, nal_len - size); } /* put the data in one or more RTP packets */ - ret = gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts); + ret = + gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts, + rtph265pay->delta_unit); g_array_set_size (nal_queue, 0); } diff --git a/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.h b/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.h index af80c72..f2829d0 100644 --- a/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.h +++ b/subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.h @@ -73,6 +73,9 @@ struct _GstRtpH265Pay gboolean send_vps_sps_pps; GstClockTime last_vps_sps_pps; + /* TRUE if the next NALU processed should have the DELTA_UNIT flag */ + gboolean delta_unit; + /* aggregate buffers with AP */ GstBufferList *bundle; guint bundle_size; diff --git a/subprojects/gst-plugins-good/tests/check/elements/rtph265.c b/subprojects/gst-plugins-good/tests/check/elements/rtph265.c index e7735ce..c35d05c 100644 --- a/subprojects/gst-plugins-good/tests/check/elements/rtph265.c +++ b/subprojects/gst-plugins-good/tests/check/elements/rtph265.c @@ -1128,6 +1128,156 @@ GST_START_TEST (test_rtph265pay_aggregate_verify_nalu_hdr) } GST_END_TEST; + +static guint8 h265_hvc1_idr_data[] = { + 0x00, 0x00, 0x00, 0x1a, 0x28, 0x01, 0xaf, 0x05, 0x38, 0x4a, 0x03, 0x06, 0x7c, + 0x7a, 0xb1, 0x8b, 0xff, 0xfe, 0xfd, 0xb7, 0xff, 0xff, 0xd1, 0xff, 0x40, 0x06, + 0xd8, 0xd3, 0xb2, 0xf8 +}; + +static guint8 h265_hvc1_non_idr_data[] = { + 0x00, 0x00, 0x00, 0x0d, 0x02, 0x01, 0xd0, 0x09, 0x7e, 0x10, 0xc2, 0x02, 0xbc, + 0x38, 0x6d, 0xcf, 0x80 +}; + +GST_START_TEST (test_rtph265pay_delta_unit_flag) +{ + GstHarness *h = gst_harness_new_parse ("rtph265pay timestamp-offset=123" + " name=p"); + GstFlowReturn ret; + GstBuffer *buffer; + + gst_harness_set_src_caps_str (h, + "video/x-h265,alignment=au,stream-format=hvc1," + "codec_data=(buffer)0104080000009e28000000003ff000fcfff8f800000f032000" + "01001740010c01ffff0408000003009e2800000300003fba0240210001002f4201010" + "408000003009e2800000300003f90041020b2dd492657ff80008000b5060606040000" + "03000400000300782022000100074401c172b02240"); + + /* key frame */ + buffer = wrap_static_buffer_with_pts (h265_hvc1_idr_data, + sizeof (h265_hvc1_idr_data), 0); + ret = gst_harness_push (h, buffer); + fail_unless_equals_int (ret, GST_FLOW_OK); + + /* delta unit frame */ + buffer = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, + h265_hvc1_non_idr_data, sizeof (h265_hvc1_non_idr_data), 0, + sizeof (h265_hvc1_non_idr_data), NULL, NULL); + GST_BUFFER_PTS (buffer) = 0; + GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT); + ret = gst_harness_push (h, buffer); + fail_unless_equals_int (ret, GST_FLOW_OK); + + buffer = gst_harness_pull (h); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)); + gst_buffer_unref (buffer); + buffer = gst_harness_pull (h); + fail_unless (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)); + gst_buffer_unref (buffer); + + gst_harness_teardown (h); +} + +GST_END_TEST; + +GST_START_TEST (test_rtph265pay_delta_unit_flag_config_interval) +{ + GstHarness *h = gst_harness_new_parse ("rtph265pay timestamp-offset=123" + " name=p config-interval=-1"); + GstFlowReturn ret; + GstBuffer *buffer; + guint num_buffers; + guint8 *payload = NULL; + guint8 nal_type; + GstRTPBuffer rtp = GST_RTP_BUFFER_INIT; + + gst_harness_set_src_caps_str (h, + "video/x-h265,alignment=au,stream-format=hvc1," + "codec_data=(buffer)0104080000009e28000000003ff000fcfff8f800000f032000" + "01001740010c01ffff0408000003009e2800000300003fba0240210001002f4201010" + "408000003009e2800000300003f90041020b2dd492657ff80008000b5060606040000" + "03000400000300782022000100074401c172b02240"); + + /* key frame */ + buffer = wrap_static_buffer_with_pts (h265_hvc1_idr_data, + sizeof (h265_hvc1_idr_data), 0); + ret = gst_harness_push (h, buffer); + fail_unless_equals_int (ret, GST_FLOW_OK); + + /* delta unit */ + buffer = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY, + h265_hvc1_non_idr_data, sizeof (h265_hvc1_non_idr_data), 0, + sizeof (h265_hvc1_non_idr_data), NULL, NULL); + GST_BUFFER_PTS (buffer) = 0; + GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT); + ret = gst_harness_push (h, buffer); + fail_unless_equals_int (ret, GST_FLOW_OK); + + /* another key frame */ + buffer = wrap_static_buffer_with_pts (h265_hvc1_idr_data, + sizeof (h265_hvc1_idr_data), 0); + ret = gst_harness_push (h, buffer); + fail_unless_equals_int (ret, GST_FLOW_OK); + + /* VSP SPS PPS I P VSP SPS PPS I */ + num_buffers = gst_harness_buffers_in_queue (h); + fail_unless_equals_int (num_buffers, 9); + + for (guint i = 0; i < num_buffers; i++) { + buffer = gst_harness_pull (h); + fail_unless (gst_rtp_buffer_map (buffer, GST_MAP_READ, &rtp)); + payload = gst_rtp_buffer_get_payload (&rtp); + nal_type = (GST_READ_UINT8 (payload) >> 1) & 0x3f; + GST_INFO ("nal_type=%d,delta_unit=%d", nal_type, + GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)); + if (i == 0) { + fail_unless_equals_int (nal_type, GST_H265_NAL_VPS); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT)); + } else if (i == 1) { + fail_unless_equals_int (nal_type, GST_H265_NAL_SPS); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT)); + } else if (i == 2) { + fail_unless_equals_int (nal_type, GST_H265_NAL_PPS); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT)); + } else if (i == 3) { + fail_unless_equals_int (nal_type, GST_H265_NAL_SLICE_IDR_N_LP); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT)); + } else if (i == 4) { + fail_unless_equals_int (nal_type, GST_H265_NAL_SLICE_TRAIL_R); + fail_unless (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)); + } else if (i == 5) { + fail_unless_equals_int (nal_type, GST_H265_NAL_VPS); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT)); + } else if (i == 6) { + fail_unless_equals_int (nal_type, GST_H265_NAL_SPS); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT)); + } else if (i == 7) { + fail_unless_equals_int (nal_type, GST_H265_NAL_PPS); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT)); + } else if (i == 8) { + fail_unless_equals_int (nal_type, GST_H265_NAL_SLICE_IDR_N_LP); + fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, + GST_BUFFER_FLAG_DELTA_UNIT)); + } + + gst_rtp_buffer_unmap (&rtp); + gst_buffer_unref (buffer); + } + + gst_harness_teardown (h); +} + +GST_END_TEST; + + static Suite * rtph265_suite (void) { @@ -1154,6 +1304,8 @@ rtph265_suite (void) tcase_add_test (tc_chain, test_rtph265pay_aggregate_with_discont); tcase_add_test (tc_chain, test_rtph265pay_aggregate_until_vcl); tcase_add_test (tc_chain, test_rtph265pay_aggregate_verify_nalu_hdr); + tcase_add_test (tc_chain, test_rtph265pay_delta_unit_flag); + tcase_add_test (tc_chain, test_rtph265pay_delta_unit_flag_config_interval); return s; } -- 2.7.4