libs: encoder: h265: fix possible integer overflow

Coverity scan bug:

Unintentional integer overflow. The expression's value may not be what
the programmer intended, because the expression is evaluated using a
narrow (i.e. few bits) integer type.

Cast operator to guint64 before computation to avoid narrowing.

merge with 3c5a6add

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
index 61ad175a8619c5747137d4bc2f95c634e2fe430e..1a14f6c3de16109b4621f1383cbdef6789dfe0bc 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
@@ -2479,7 +2479,7 @@ ensure_bitrate (GstVaapiEncoderH264 * encoder)
         if (!encoder->use_dct8x8)
           bits_per_mb += (bits_per_mb * 10) / 100;
 
-        factor = encoder->mb_width * encoder->mb_height * bits_per_mb;
+        factor = (guint64) encoder->mb_width * encoder->mb_height * bits_per_mb;
         base_encoder->bitrate =
             gst_util_uint64_scale (factor, GST_VAAPI_ENCODER_FPS_N (encoder),
             GST_VAAPI_ENCODER_FPS_D (encoder)) / 1000;

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h265.c b/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
index 7bcb54f3f6d4d7cdc64ee26cc04764ce97bd8760..b2bf494d23621f023b9d46997e871bef7a7cfd4b 100644 (file)
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
@@ -1902,7 +1902,9 @@ ensure_bitrate (GstVaapiEncoderH265 * encoder)
         /* FIXME: Provide better estimation */
         /* Using a 1/6 compression ratio */
         /* 12 bits per pixel for YUV420 */
-        guint64 factor = encoder->luma_width * encoder->luma_height * 12 / 6;
+        guint64 factor;
+
+        factor = (guint64) encoder->luma_width * encoder->luma_height * 12 / 6;
         base_encoder->bitrate =
             gst_util_uint64_scale (factor, GST_VAAPI_ENCODER_FPS_N (encoder),
             GST_VAAPI_ENCODER_FPS_D (encoder)) / 1000;