From e8b1c6d6f55f5be3bef25084fdd8b6127517e137 Mon Sep 17 00:00:00 2001 From: Adam Jackson Date: Fri, 20 May 2011 18:21:15 -0400 Subject: [PATCH] mesa: Fix return type of _mesa_get_format_bytes() (#37351) Despite that negative values aren't sensible here, making this unsigned is dangerous. Consider get_pointer_generic, which computes a value of the form: void *base + (int x * int stride + int y) * unsigned bpp The usual arithmetic conversions will coerce the (x*stride + y) subexpression to unsigned. Since stride can be negative, this is disastrous. Fixes at least the following piglit tests on Ironlake: fbo/fbo-blit-d24s8 spec/ARB_depth_texture/fbo-clear-formats spec/EXT_packed_depth_stencil/fbo-clear-formats NOTE: This is a candidate for the 7.10 branch. Reviewed-by: Chad Versace Signed-off-by: Adam Jackson --- src/mesa/main/formats.c | 4 +++- src/mesa/main/formats.h | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mesa/main/formats.c b/src/mesa/main/formats.c index 60e8ae3..e88ba43 100644 --- a/src/mesa/main/formats.c +++ b/src/mesa/main/formats.c @@ -1118,8 +1118,10 @@ _mesa_get_format_name(gl_format format) * Return bytes needed to store a block of pixels in the given format. * Normally, a block is 1x1 (a single pixel). But for compressed formats * a block may be 4x4 or 8x4, etc. + * + * Note: not GLuint, so as not to coerce math to unsigned. cf. fdo #37351 */ -GLuint +GLint _mesa_get_format_bytes(gl_format format) { const struct gl_format_info *info = _mesa_get_format_info(format); diff --git a/src/mesa/main/formats.h b/src/mesa/main/formats.h index df9ed70..0640bbc 100644 --- a/src/mesa/main/formats.h +++ b/src/mesa/main/formats.h @@ -216,7 +216,7 @@ typedef enum extern const char * _mesa_get_format_name(gl_format format); -extern GLuint +extern GLint _mesa_get_format_bytes(gl_format format); extern GLint -- 2.7.4