avauddec: Add clipping meta support for gapless playback

Co-authored-by: Sebastian Dröge <sebastian@centricular.com>
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1028>

diff --git a/subprojects/gst-libav/ext/libav/gstavauddec.c b/subprojects/gst-libav/ext/libav/gstavauddec.c
index b03a724..4bdb197 100644 (file)
--- a/subprojects/gst-libav/ext/libav/gstavauddec.c
+++ b/subprojects/gst-libav/ext/libav/gstavauddec.c
@@ -30,6 +30,7 @@
 #include <libavutil/channel_layout.h>
 
 #include <gst/gst.h>
+#include <gst/base/gstbytewriter.h>
 
 #include "gstav.h"
 #include "gstavcodecmap.h"
@@ -463,7 +464,8 @@ gst_avpacket_init (AVPacket * packet, guint8 * data, guint size)
  */
 static gboolean
 gst_ffmpegauddec_audio_frame (GstFFMpegAudDec * ffmpegdec,
-    AVCodec * in_plugin, GstBuffer ** outbuf, GstFlowReturn * ret)
+    AVCodec * in_plugin, GstBuffer ** outbuf, GstFlowReturn * ret,
+    gboolean * need_more_data)
 {
   gboolean got_frame = FALSE;
   gint res;
@@ -533,6 +535,7 @@ gst_ffmpegauddec_audio_frame (GstFFMpegAudDec * ffmpegdec,
       GST_BUFFER_FLAG_SET (*outbuf, GST_BUFFER_FLAG_CORRUPTED);
   } else if (res == AVERROR (EAGAIN)) {
     *outbuf = NULL;
+    *need_more_data = TRUE;
   } else if (res == AVERROR_EOF) {
     *ret = GST_FLOW_EOS;
     GST_DEBUG_OBJECT (ffmpegdec, "Context was entirely flushed");
@@ -552,7 +555,8 @@ beach:
  * Returns: whether a frame was decoded
  */
 static gboolean
-gst_ffmpegauddec_frame (GstFFMpegAudDec * ffmpegdec, GstFlowReturn * ret)
+gst_ffmpegauddec_frame (GstFFMpegAudDec * ffmpegdec, GstFlowReturn * ret,
+    gboolean * need_more_data)
 {
   GstFFMpegAudDecClass *oclass;
   GstBuffer *outbuf = NULL;
@@ -567,7 +571,8 @@ gst_ffmpegauddec_frame (GstFFMpegAudDec * ffmpegdec, GstFlowReturn * ret)
   oclass = (GstFFMpegAudDecClass *) (G_OBJECT_GET_CLASS (ffmpegdec));
 
   got_frame =
-      gst_ffmpegauddec_audio_frame (ffmpegdec, oclass->in_plugin, &outbuf, ret);
+      gst_ffmpegauddec_audio_frame (ffmpegdec, oclass->in_plugin, &outbuf, ret,
+      need_more_data);
 
   if (outbuf) {
     GST_LOG_OBJECT (ffmpegdec, "Decoded data, buffer %" GST_PTR_FORMAT, outbuf);
@@ -594,16 +599,17 @@ gst_ffmpegauddec_drain (GstFFMpegAudDec * ffmpegdec, gboolean force)
 {
   GstFlowReturn ret = GST_FLOW_OK;
   gboolean got_any_frames = FALSE;
+  gboolean need_more_data = FALSE;
   gboolean got_frame;
 
   if (avcodec_send_packet (ffmpegdec->context, NULL))
     goto send_packet_failed;
 
   do {
-    got_frame = gst_ffmpegauddec_frame (ffmpegdec, &ret);
+    got_frame = gst_ffmpegauddec_frame (ffmpegdec, &ret, &need_more_data);
     if (got_frame)
       got_any_frames = TRUE;
-  } while (got_frame);
+  } while (got_frame && !need_more_data);
   avcodec_flush_buffers (ffmpegdec->context);
 
   /* FFMpeg will return AVERROR_EOF if it's internal was fully drained
@@ -653,6 +659,10 @@ gst_ffmpegauddec_handle_frame (GstAudioDecoder * decoder, GstBuffer * inbuf)
   GstFlowReturn ret = GST_FLOW_OK;
   gboolean is_header;
   AVPacket packet;
+  GstAudioClippingMeta *clipping_meta = NULL;
+  guint32 num_clipped_samples = 0;
+  gboolean fully_clipped = FALSE;
+  gboolean need_more_data = FALSE;
 
   ffmpegdec = (GstFFMpegAudDec *) decoder;
 
@@ -684,6 +694,18 @@ gst_ffmpegauddec_handle_frame (GstAudioDecoder * decoder, GstBuffer * inbuf)
     inbuf = gst_buffer_make_writable (inbuf);
   }
 
+  /* mpegaudioparse is setting buffer flags for the Xing/LAME header. This
+   * should not be passed to the decoder as it results in unnecessary silence
+   * samples to be output */
+  if (oclass->in_plugin->id == AV_CODEC_ID_MP3 &&
+      GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_DECODE_ONLY) &&
+      GST_BUFFER_FLAG_IS_SET (inbuf, GST_BUFFER_FLAG_DROPPABLE)) {
+    gst_buffer_unref (inbuf);
+    return gst_audio_decoder_finish_frame (decoder, NULL, 1);
+  }
+
+  clipping_meta = gst_buffer_get_audio_clipping_meta (inbuf);
+
   gst_buffer_map (inbuf, &map, GST_MAP_READ);
 
   data = map.data;
@@ -711,13 +733,37 @@ gst_ffmpegauddec_handle_frame (GstAudioDecoder * decoder, GstBuffer * inbuf)
   if (!packet.size)
     goto unmap;
 
+  if (clipping_meta != NULL) {
+    if (clipping_meta->format == GST_FORMAT_DEFAULT) {
+      uint8_t *p = av_packet_new_side_data (&packet, AV_PKT_DATA_SKIP_SAMPLES,
+          10);
+      if (p != NULL) {
+        GstByteWriter writer;
+        guint32 start = clipping_meta->start;
+        guint32 end = clipping_meta->end;
+
+        num_clipped_samples = start + end;
+
+        gst_byte_writer_init_with_data (&writer, p, 10, FALSE);
+        gst_byte_writer_put_uint32_le (&writer, start);
+        gst_byte_writer_put_uint32_le (&writer, end);
+        GST_LOG_OBJECT (ffmpegdec, "buffer has clipping metadata; added skip "
+            "side data to avpacket with start %u and end %u", start, end);
+      }
+    } else {
+      GST_WARNING_OBJECT (ffmpegdec,
+          "buffer has clipping metadata in unsupported format %s",
+          gst_format_get_name (clipping_meta->format));
+    }
+  }
+
   if (avcodec_send_packet (ffmpegdec->context, &packet) < 0) {
     goto send_packet_failed;
   }
 
   do {
     /* decode a frame of audio now */
-    got_frame = gst_ffmpegauddec_frame (ffmpegdec, &ret);
+    got_frame = gst_ffmpegauddec_frame (ffmpegdec, &ret, &need_more_data);
 
     if (got_frame)
       got_any_frames = TRUE;
@@ -728,9 +774,18 @@ gst_ffmpegauddec_handle_frame (GstAudioDecoder * decoder, GstBuffer * inbuf)
       /* bad flow return, make sure we discard all data and exit */
       break;
     }
-  } while (got_frame);
-
-  if (is_header || got_any_frames) {
+  } while (got_frame && !need_more_data);
+
+  /* The frame was fully clipped if we have samples to be clipped and
+   * it's either more than the known fixed frame size, or the decoder returned
+   * that it needs more data (EAGAIN) and we didn't decode any frames at all.
+   */
+  fully_clipped = (clipping_meta != NULL && num_clipped_samples > 0)
+      && ((ffmpegdec->context->frame_size != 0
+          && num_clipped_samples >= ffmpegdec->context->frame_size)
+      || (need_more_data && !got_any_frames));
+
+  if (is_header || got_any_frames || fully_clipped) {
     /* Even if previous return wasn't GST_FLOW_OK, we need to call
      * _finish_frame() since baseclass is expecting that _finish_frame()
      * is followed by _finish_subframe()