h264: reset num_reorder_frames if it is invalid

An invalid VUI is not considered a fatal error, so the SPS containing it
may still be used. Leaving an invalid value of num_reorder_frames there
can result in writing over the bounds of H264Context.delayed_pic.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC:libav-stable@libav.org

diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c
index 9eb252d..033bb79 100644 (file)
--- a/libavcodec/h264_ps.c
+++ b/libavcodec/h264_ps.c
@@ -224,7 +224,9 @@ static inline int decode_vui_parameters(H264Context *h, SPS *sps)
         if (sps->num_reorder_frames > 16U
             /* max_dec_frame_buffering || max_dec_frame_buffering > 16 */) {
             av_log(h->avctx, AV_LOG_ERROR,
-                   "illegal num_reorder_frames %d\n", sps->num_reorder_frames);
+                   "Clipping illegal num_reorder_frames %d\n",
+                   sps->num_reorder_frames);
+            sps->num_reorder_frames = 16;
             return AVERROR_INVALIDDATA;
         }
     }