sierravmd: Do sanity checking of frame sizes

Limit the size to INT_MAX/2 (for simplicity) to be sure that
size + BYTES_PER_FRAME_RECORD won't overflow.

Also factorize other existing error return paths.

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
Signed-off-by: Martin Storsjö <martin@martin.st>
(cherry picked from commit 0ef1660a6365ce60ead8858936b6f3f8ea862826)
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>
(cherry picked from commit 153deed18bed43d16b272e8681b2a9b988d2682a)

diff --git a/libavformat/sierravmd.c b/libavformat/sierravmd.c
index 81ff46fea0e953fad7fc518d9d22f394bf5c06ef..3b8da4a769bc1ae396e33bcccb6e62bc4544c814 100644 (file)
--- a/libavformat/sierravmd.c
+++ b/libavformat/sierravmd.c
@@ -88,7 +88,7 @@ static int vmd_read_header(AVFormatContext *s,
     unsigned char *raw_frame_table;
     int raw_frame_table_size;
     int64_t current_offset;
-    int i, j;
+    int i, j, ret;
     unsigned int total_frames;
     int64_t current_audio_pts = 0;
     unsigned char chunk[BYTES_PER_FRAME_RECORD];
@@ -169,15 +169,13 @@ static int vmd_read_header(AVFormatContext *s,
     raw_frame_table = av_malloc(raw_frame_table_size);
     vmd->frame_table = av_malloc((vmd->frame_count * vmd->frames_per_block + sound_buffers) * sizeof(vmd_frame));
     if (!raw_frame_table || !vmd->frame_table) {
-        av_free(raw_frame_table);
-        av_free(vmd->frame_table);
-        return AVERROR(ENOMEM);
+        ret = AVERROR(ENOMEM);
+        goto error;
     }
     if (avio_read(pb, raw_frame_table, raw_frame_table_size) !=
         raw_frame_table_size) {
-        av_free(raw_frame_table);
-        av_free(vmd->frame_table);
-        return AVERROR(EIO);
+        ret = AVERROR(EIO);
+        goto error;
     }
 
     total_frames = 0;
@@ -193,6 +191,11 @@ static int vmd_read_header(AVFormatContext *s,
             avio_read(pb, chunk, BYTES_PER_FRAME_RECORD);
             type = chunk[0];
             size = AV_RL32(&chunk[2]);
+            if (size > INT_MAX / 2) {
+                av_log(s, AV_LOG_ERROR, "Invalid frame size\n");
+                ret = AVERROR_INVALIDDATA;
+                goto error;
+            }
             if(!size && type != 1)
                 continue;
             switch(type) {
@@ -229,6 +232,11 @@ static int vmd_read_header(AVFormatContext *s,
     vmd->frame_count = total_frames;
 
     return 0;
+
+error:
+    av_free(raw_frame_table);
+    av_free(vmd->frame_table);
+    return ret;
 }
 
 static int vmd_read_packet(AVFormatContext *s,