Fix overflows during trap rasterization. [Bug 16560].

Avoid overflows when rasterizing traps that fall entirely in the space
between the final sample row and the end of the coordinate system, or
in the space between the beginning of the coordinate system and the
first sample row. Such traps don't contain any sample points, so the
top and bottom of the edges can safely be moved to the beginning/end.

diff --git a/pixman/pixman-utils.c b/pixman/pixman-utils.c
index a1b7492..cdf0220 100644 (file)
--- a/pixman/pixman-utils.c
+++ b/pixman/pixman-utils.c
@@ -201,8 +201,13 @@ pixman_sample_ceil_y (pixman_fixed_t y, int n)
     f = ((f + Y_FRAC_FIRST(n)) / STEP_Y_SMALL(n)) * STEP_Y_SMALL(n) + Y_FRAC_FIRST(n);
     if (f > Y_FRAC_LAST(n))
     {
-       f = Y_FRAC_FIRST(n);
-       i += pixman_fixed_1;
+       if (pixman_fixed_to_int(i) == 0x7fff)
+       {
+           f = 0xffff; /* saturate */
+       } else {
+           f = Y_FRAC_FIRST(n);
+           i += pixman_fixed_1;
+       }
     }
     return (i | f);
 }
@@ -222,8 +227,13 @@ pixman_sample_floor_y (pixman_fixed_t y, int n)
     f = _div(f - Y_FRAC_FIRST(n), STEP_Y_SMALL(n)) * STEP_Y_SMALL(n) + Y_FRAC_FIRST(n);
     if (f < Y_FRAC_FIRST(n))
     {
-       f = Y_FRAC_LAST(n);
-       i -= pixman_fixed_1;
+       if (pixman_fixed_to_int(i) == 0x8000)
+       {
+           f = 0; /* saturate */
+       } else {
+           f = Y_FRAC_LAST(n);
+           i -= pixman_fixed_1;
+       }
     }
     return (i | f);
 }