smpte: Fix integer overflow with possible heap corruption in GstMask creation.

Check that width*height*sizeof(guint32) doesn't overflow when
allocated user_data for mask, potential for heap overwrite when
inverting.

Fixes https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1231

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2603>

diff --git a/subprojects/gst-plugins-good/gst/smpte/gstmask.c b/subprojects/gst-plugins-good/gst/smpte/gstmask.c
index 92b591936c71b78874e48f983b7dca4505f78312..9b00061d508320fe1867e4b2732fc5e32dfc406b 100644 (file)
--- a/subprojects/gst-plugins-good/gst/smpte/gstmask.c
+++ b/subprojects/gst-plugins-good/gst/smpte/gstmask.c
@@ -85,6 +85,13 @@ gst_mask_factory_new (gint type, gboolean invert, gint bpp, gint width,
     mask->height = height;
     mask->destroy_func = definition->destroy_func;
     mask->user_data = definition->user_data;
+
+    if (((guint64) width * (guint64) height * sizeof (guint32)) > G_MAXUINT) {
+      GST_WARNING ("width x height overflows");
+      g_free (mask);
+      return NULL;
+    }
+
     mask->data = g_malloc (width * height * sizeof (guint32));
 
     definition->draw_func (mask);