From 474a50c64837d05c68e6aa3d24ae096c53f2757d Mon Sep 17 00:00:00 2001 From: Jerome Jiang Date: Wed, 6 Apr 2022 09:46:57 -0700 Subject: [PATCH] Fix int overflow in intermediate calculation This is not a complete fix to webm:1751. Bug: webm:1751 Change-Id: Ieed6c823744f5f0625d529db3746cfe4f549c8c0 --- vp8/encoder/bitstream.c | 6 ++---- vp8/encoder/boolhuff.h | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/vp8/encoder/bitstream.c b/vp8/encoder/bitstream.c index 87825fa..0e97af5 100644 --- a/vp8/encoder/bitstream.c +++ b/vp8/encoder/bitstream.c @@ -172,9 +172,8 @@ void vp8_pack_tokens(vp8_writer *w, const TOKENEXTRA *p, int xcount) { validate_buffer(w->buffer + w->pos, 1, w->buffer_end, w->error); w->buffer[w->pos++] = (lowvalue >> (24 - offset)) & 0xff; - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } @@ -223,9 +222,8 @@ void vp8_pack_tokens(vp8_writer *w, const TOKENEXTRA *p, int xcount) { validate_buffer(w->buffer + w->pos, 1, w->buffer_end, w->error); w->buffer[w->pos++] = (lowvalue >> (24 - offset)) & 0xff; - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } diff --git a/vp8/encoder/boolhuff.h b/vp8/encoder/boolhuff.h index 8cc61bd..a8c536b 100644 --- a/vp8/encoder/boolhuff.h +++ b/vp8/encoder/boolhuff.h @@ -94,9 +94,8 @@ static void vp8_encode_bool(BOOL_CODER *bc, int bit, int probability) { validate_buffer(bc->buffer + bc->pos, 1, bc->buffer_end, bc->error); bc->buffer[bc->pos++] = (lowvalue >> (24 - offset) & 0xff); - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } -- 2.7.4