[pixman-image] Avoid integer overflow when allocating bits.

Check for potential overflows at every step of the calculation of the
buffer size required for the pixels.

(Fixes https://bugs.freedesktop.org/show_bug.cgi?id=11627)

diff --git a/pixman/pixman-image.c b/pixman/pixman-image.c
index fa32208..986f8ab 100644 (file)
--- a/pixman/pixman-image.c
+++ b/pixman/pixman-image.c
@@ -285,9 +285,32 @@ create_bits (pixman_format_code_t format,
     int stride;
     int buf_size;
     int bpp;
-    
+
+    /* what follows is a long-winded way, avoiding any possibility of integer
+     * overflows, of saying:
+     * stride = ((width * bpp + FB_MASK) >> FB_SHIFT) * sizeof (uint32_t);
+     */
+
     bpp = PIXMAN_FORMAT_BPP (format);
-    stride = ((width * bpp + FB_MASK) >> FB_SHIFT) * sizeof (uint32_t);
+    if (pixman_multiply_overflows_int (width, bpp))
+       return NULL;
+
+    stride = width * bpp;
+    if (pixman_addition_overflows_int (stride, FB_MASK))
+       return NULL;
+
+    stride += FB_MASK;
+    stride >>= FB_SHIFT;
+
+#if FB_SHIFT < 2
+    if (pixman_multiply_overflows_int (stride, sizeof (uint32_t)))
+       return NULL;
+#endif
+    stride *= sizeof (uint32_t);
+
+    if (pixman_multiply_overflows_int (height, stride))
+       return NULL;
+
     buf_size = height * stride;
 
     if (rowstride_bytes)

diff --git a/pixman/pixman-private.h b/pixman/pixman-private.h
index 6487bfd..0c5942f 100644 (file)
--- a/pixman/pixman-private.h
+++ b/pixman/pixman-private.h
@@ -69,6 +69,8 @@
 /* Memory allocation helpers */
 void *pixman_malloc_ab (unsigned int n, unsigned int b);
 void *pixman_malloc_abc (unsigned int a, unsigned int b, unsigned int c);
+pixman_bool_t pixman_multiply_overflows_int (unsigned int a, unsigned int b);
+pixman_bool_t pixman_addition_overflows_int (unsigned int a, unsigned int b);
 
 #if DEBUG
 

diff --git a/pixman/pixman-utils.c b/pixman/pixman-utils.c
index fc93608..1d1dec9 100644 (file)
--- a/pixman/pixman-utils.c
+++ b/pixman/pixman-utils.c
@@ -371,6 +371,20 @@ pixman_line_fixed_edge_init (pixman_edge_t *e,
                    bot->y + y_off_fixed);
 }
 
+pixman_bool_t
+pixman_multiply_overflows_int (unsigned int a,
+                              unsigned int b)
+{
+    return a >= INT32_MAX / b;
+}
+
+pixman_bool_t
+pixman_addition_overflows_int (unsigned int a,
+                              unsigned int b)
+{
+    return a > INT32_MAX - b;
+}
+
 void *
 pixman_malloc_ab(unsigned int a,
                 unsigned int b)