From e483af47db769fcba559dda72699bc80d154b575 Mon Sep 17 00:00:00 2001 From: Adam Jackson Date: Fri, 15 May 2009 06:26:48 -0400 Subject: [PATCH] Fix overflows during trap rasterization. [Bug 16560]. Avoid overflows when rasterizing traps that fall entirely in the space between the final sample row and the end of the coordinate system, or in the space between the beginning of the coordinate system and the first sample row. Such traps don't contain any sample points, so the top and bottom of the edges can safely be moved to the beginning/end. --- pixman/pixman-utils.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pixman/pixman-utils.c b/pixman/pixman-utils.c index a1b7492..cdf0220 100644 --- a/pixman/pixman-utils.c +++ b/pixman/pixman-utils.c @@ -201,8 +201,13 @@ pixman_sample_ceil_y (pixman_fixed_t y, int n) f = ((f + Y_FRAC_FIRST(n)) / STEP_Y_SMALL(n)) * STEP_Y_SMALL(n) + Y_FRAC_FIRST(n); if (f > Y_FRAC_LAST(n)) { - f = Y_FRAC_FIRST(n); - i += pixman_fixed_1; + if (pixman_fixed_to_int(i) == 0x7fff) + { + f = 0xffff; /* saturate */ + } else { + f = Y_FRAC_FIRST(n); + i += pixman_fixed_1; + } } return (i | f); } @@ -222,8 +227,13 @@ pixman_sample_floor_y (pixman_fixed_t y, int n) f = _div(f - Y_FRAC_FIRST(n), STEP_Y_SMALL(n)) * STEP_Y_SMALL(n) + Y_FRAC_FIRST(n); if (f < Y_FRAC_FIRST(n)) { - f = Y_FRAC_LAST(n); - i -= pixman_fixed_1; + if (pixman_fixed_to_int(i) == 0x8000) + { + f = 0; /* saturate */ + } else { + f = Y_FRAC_LAST(n); + i -= pixman_fixed_1; + } } return (i | f); } -- 2.7.4