libs: encoder: h264_fei: fix potential overflow before widen

Found by static analysis. encoder->mb_width * encoder->mb_height
is evaluated using 32-bit arithmetic before widen.  Thus, cast
at least one of these to guint64 to avoid overflow.

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c
index ed392a0..ea5c495 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264_fei.c
@@ -2600,7 +2600,7 @@ ensure_bitrate (GstVaapiEncoderH264Fei * encoder)
         if (!encoder->use_dct8x8)
           bits_per_mb += (bits_per_mb * 10) / 100;
 
-        factor = encoder->mb_width * encoder->mb_height * bits_per_mb;
+        factor = (guint64) encoder->mb_width * encoder->mb_height * bits_per_mb;
         base_encoder->bitrate =
             gst_util_uint64_scale (factor, GST_VAAPI_ENCODER_FPS_N (encoder),
             GST_VAAPI_ENCODER_FPS_D (encoder)) / 1000;