basic/hexdecoct: be more careful in overflow check

CID #139583: plen + 1 is evaluated as int, and could in principle overflow.
So cast to ssize_t and add an additional check that our overflow calculation
doesn't overflow itself.

diff --git a/src/basic/hexdecoct.c b/src/basic/hexdecoct.c
index c0f9640..c5987ee 100644 (file)
--- a/src/basic/hexdecoct.c
+++ b/src/basic/hexdecoct.c
@@ -601,10 +601,11 @@ static int base64_append_width(
         lines = DIV_ROUND_UP(len, width);
 
         slen = strlen_ptr(sep);
-        if (lines > (SSIZE_MAX - plen - 1 - slen) / (indent + width + 1))
+        if (plen >= SSIZE_MAX - 1 - slen ||
+            lines > (SSIZE_MAX - plen - 1 - slen) / (indent + width + 1))
                 return -ENOMEM;
 
-        t = realloc(*prefix, plen + 1 + slen + (indent + width + 1) * lines);
+        t = realloc(*prefix, (ssize_t) plen + 1 + slen + (indent + width + 1) * lines);
         if (!t)
                 return -ENOMEM;