avviddec: Temporarily unlock stream lock while flushing buffers

This can call into the decoder again from other threads and try to take
the stream lock from there, which would cause a deadlock.

Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/2558

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4625>

diff --git a/subprojects/gst-libav/ext/libav/gstavviddec.c b/subprojects/gst-libav/ext/libav/gstavviddec.c
index ff38742..1c10bb2 100644 (file)
--- a/subprojects/gst-libav/ext/libav/gstavviddec.c
+++ b/subprojects/gst-libav/ext/libav/gstavviddec.c
@@ -2052,7 +2052,10 @@ gst_ffmpegviddec_drain (GstVideoDecoder * decoder)
   do {
     got_frame = gst_ffmpegviddec_frame (ffmpegdec, NULL, &ret);
   } while (got_frame && ret == GST_FLOW_OK);
+
+  GST_VIDEO_DECODER_STREAM_UNLOCK (ffmpegdec);
   avcodec_flush_buffers (ffmpegdec->context);
+  GST_VIDEO_DECODER_STREAM_LOCK (ffmpegdec);
 
   /* FFMpeg will return AVERROR_EOF if it's internal was fully drained
    * then we are translating it to GST_FLOW_EOS. However, because this behavior
@@ -2265,7 +2268,9 @@ gst_ffmpegviddec_flush (GstVideoDecoder * decoder)
 
   if (ffmpegdec->opened) {
     GST_LOG_OBJECT (decoder, "flushing buffers");
+    GST_VIDEO_DECODER_STREAM_UNLOCK (ffmpegdec);
     avcodec_flush_buffers (ffmpegdec->context);
+    GST_VIDEO_DECODER_STREAM_LOCK (ffmpegdec);
   }
 
   return TRUE;