From 5f345a9246b71374cbedeb28b0e0b0101701732a Mon Sep 17 00:00:00 2001 From: "Jorge E. Moreira" Date: Wed, 30 Jun 2021 11:33:51 -0700 Subject: [PATCH] Avoid overflow in calc_iframe_target_size The changed product was observed to attempt to multiply 1800 by 2500000, which overflows unsigned 32 bits. Converting to unsigned 64 bits first and testing whether the final result fits in 32 bits solves the problem. BUG=b:179686142 Change-Id: I5d27317bf14b0311b739144c451d8e172db01945 --- vp8/encoder/ratectrl.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vp8/encoder/ratectrl.c b/vp8/encoder/ratectrl.c index ba124c3..d2b8dff 100644 --- a/vp8/encoder/ratectrl.c +++ b/vp8/encoder/ratectrl.c @@ -349,8 +349,12 @@ static void calc_iframe_target_size(VP8_COMP *cpi) { } if (cpi->oxcf.rc_max_intra_bitrate_pct) { - unsigned int max_rate = - cpi->per_frame_bandwidth * cpi->oxcf.rc_max_intra_bitrate_pct / 100; + unsigned int max_rate; + // This product may overflow unsigned int + uint64_t product = cpi->per_frame_bandwidth; + product *= cpi->oxcf.rc_max_intra_bitrate_pct; + product /= 100; + max_rate = (unsigned int)VPXMIN(INT_MAX, product); if (target > max_rate) target = max_rate; } -- 2.7.4