compositor: Ensure all arguments to CLAMP are signed int

If any of the arguments to CLAMP are unsigned integers, the comparison causes
an automatic conversion of the signed int to unsigned, which causes -1 to become
UINT_MAX and get clamped to the high value of the CLAMP instead of 0.

See 716 at http://c0x.coding-guidelines.com/6.3.1.8.html

Also add a test for this.

https://bugzilla.gnome.org/show_bug.cgi?id=754576

diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c
index 64d05ac..d015d85 100644 (file)
--- a/gst/compositor/compositor.c
+++ b/gst/compositor/compositor.c
@@ -351,8 +351,8 @@ is_rectangle_contained (GstVideoRectangle rect1, GstVideoRectangle rect2)
 }
 
 static GstVideoRectangle
-clamp_rectangle (gint x, gint y, gint w, gint h, guint outer_width,
-    guint outer_height)
+clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width,
+    gint outer_height)
 {
   gint x2 = x + w;
   gint y2 = y + h;

diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c
index dc1a58f..ccd68a1 100644 (file)
--- a/tests/check/elements/compositor.c
+++ b/tests/check/elements/compositor.c
@@ -1326,6 +1326,17 @@ GST_START_TEST (test_obscured_skipped)
   out_width = out_height = 0;
   buffer_mapped = FALSE;
 
+  xpos1 = -1;
+  xpos0 = ypos0 = width0 = height0 = width1 = height1 = 10;
+  out_width = out_height = 20;
+  GST_INFO ("testing bug 754576");
+  _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1,
+      width1, height1, alpha1, out_width, out_height);
+  fail_unless (buffer_mapped == TRUE);
+  xpos0 = xpos1 = ypos1 = width0 = height0 = width1 = height1 = 0;
+  out_width = out_height = 0;
+  buffer_mapped = FALSE;
+
   xpos0 = ypos0 = 10000;
   out_width = 320;
   out_height = 240;