smacker: Avoid integer overflow when allocating packets

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
Signed-off-by: Martin Storsjö <martin@martin.st>

diff --git a/libavformat/smacker.c b/libavformat/smacker.c
index a15bded..f8a0769 100644 (file)
--- a/libavformat/smacker.c
+++ b/libavformat/smacker.c
@@ -327,7 +327,7 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
             }
             flags >>= 1;
         }
-        if (frame_size < 0)
+        if (frame_size < 0 || frame_size >= INT_MAX/2)
             return AVERROR_INVALIDDATA;
         if (av_new_packet(pkt, frame_size + 769))
             return AVERROR(ENOMEM);