From: Aaro Koskinen <aaro.koskinen@iki.fi>
Date: Tue, 11 Sep 2012 21:44:37 +0000 (+0300)
Subject: staging: xgifb: validate the mode against video memory size
X-Git-Tag: upstream/snapshot3+hdmi~6593^2~287
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=a09f347c6cc0b2821557d1346c4733cc78a79ffa;p=platform%2Fadaptation%2Frenesas_rcar%2Frenesas_kernel.git

staging: xgifb: validate the mode against video memory size

It's possible to select video mode that exceeds the available video
memory. This is potentially dangerous, fix by adding a check.

The patch fixes system hangs seen occasionally when playing random videos
with mplayer.

Signed-off-by: Aaro Koskinen <aaro.koskinen@iki.fi>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---

diff --git a/drivers/staging/xgifb/XGI_main_26.c b/drivers/staging/xgifb/XGI_main_26.c
index 7fc3049..ba6c347 100644
--- a/drivers/staging/xgifb/XGI_main_26.c
+++ b/drivers/staging/xgifb/XGI_main_26.c
@@ -329,6 +329,7 @@ static int XGIfb_validate_mode(struct xgifb_video_info *xgifb_info, int myindex)
 {
 	u16 xres, yres;
 	struct xgi_hw_device_info *hw_info = &xgifb_info->hw_info;
+	unsigned long required_mem;
 
 	if (xgifb_info->chip == XG21) {
 		if (xgifb_info->display2 == XGIFB_DISP_LCD) {
@@ -345,13 +346,13 @@ static int XGIfb_validate_mode(struct xgifb_video_info *xgifb_info, int myindex)
 			}
 
 		}
-		return myindex;
+		goto check_memory;
 
 	}
 
 	/* FIXME: for now, all is valid on XG27 */
 	if (xgifb_info->chip == XG27)
-		return myindex;
+		goto check_memory;
 
 	if (!(XGIbios_mode[myindex].chipset & MD_XGI315))
 		return -1;
@@ -539,6 +540,12 @@ static int XGIfb_validate_mode(struct xgifb_video_info *xgifb_info, int myindex)
 	case XGIFB_DISP_NONE:
 		break;
 	}
+
+check_memory:
+	required_mem = XGIbios_mode[myindex].xres * XGIbios_mode[myindex].yres *
+		       XGIbios_mode[myindex].bpp / 8;
+	if (required_mem > xgifb_info->video_size)
+		return -1;
 	return myindex;
 
 }