allocator: Avoid integer overflow when allocating sysmem

Thanks to Antonio Morales for finding and reporting the issue.

Fixes GHSL-2024-166
Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3851

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8044>

diff --git a/subprojects/gstreamer/gst/gstallocator.c b/subprojects/gstreamer/gst/gstallocator.c
index 996f5dc946c7371bf599d67b03cf37d4d16772ce..198cfe9523075b735232a5fc3864610fb266caf0 100644 (file)
--- a/subprojects/gstreamer/gst/gstallocator.c
+++ b/subprojects/gstreamer/gst/gstallocator.c
@@ -427,8 +427,20 @@ _sysmem_new_block (GstMemoryFlags flags,
   /* ensure configured alignment */
   align |= gst_memory_alignment;
   /* allocate more to compensate for alignment */
+  if (align > G_MAXSIZE || maxsize > G_MAXSIZE - align) {
+    GST_CAT_WARNING (GST_CAT_MEMORY,
+        "Allocating %" G_GSIZE_FORMAT " bytes with alignment %" G_GSIZE_FORMAT
+        "x overflows", maxsize, align);
+    return NULL;
+  }
   maxsize += align;
   /* alloc header and data in one block */
+  if (maxsize > G_MAXSIZE - sizeof (GstMemorySystem)) {
+    GST_CAT_WARNING (GST_CAT_MEMORY,
+        "Allocating %" G_GSIZE_FORMAT " bytes with alignment %" G_GSIZE_FORMAT
+        "x overflows", maxsize, align);
+    return NULL;
+  }
   slice_size = sizeof (GstMemorySystem) + maxsize;
 
   mem = g_malloc (slice_size);
@@ -478,6 +490,8 @@ _sysmem_copy (GstMemorySystem * mem, gssize offset, gsize size)
     size = mem->mem.size > offset ? mem->mem.size - offset : 0;
 
   copy = _sysmem_new_block (0, size, mem->mem.align, 0, size);
+  if (!copy)
+    return NULL;
   GST_CAT_DEBUG (GST_CAT_PERFORMANCE,
       "memcpy %" G_GSIZE_FORMAT " memory %p -> %p", size, mem, copy);
   memcpy (copy->data, mem->data + mem->mem.offset + offset, size);