From: akallabeth <akallabeth@posteo.net>
Date: Fri, 15 May 2020 14:30:02 +0000 (+0200)
Subject: Fixed undefined behaviour
X-Git-Tag: 2.1.1^2~33
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=6dfd915004b63e91c417f569af9f8516f7f942fc;p=platform%2Fupstream%2Ffreerdp.git

Fixed undefined behaviour

(cherry picked from commit c15929d821ce99aa8c1069f790f4addbe9439593)
---

diff --git a/libfreerdp/primitives/prim_colors.c b/libfreerdp/primitives/prim_colors.c
index a9d36e3..54c7ecb 100644
--- a/libfreerdp/primitives/prim_colors.c
+++ b/libfreerdp/primitives/prim_colors.c
@@ -50,13 +50,13 @@ static pstatus_t general_yCbCrToRGB_16s8u_P3AC4R_BGRX(const INT16* const pSrc[3]
 		{
 			INT16 R, G, B;
 			const INT32 divisor = 16;
-			const INT32 Y = ((*pY++) + 4096) << divisor;
+			const INT32 Y = (INT32)((UINT32)((*pY++) + 4096) << divisor);
 			const INT32 Cb = (*pCb++);
 			const INT32 Cr = (*pCr++);
-			const INT32 CrR = Cr * (INT32)(1.402525f * (1 << divisor));
-			const INT32 CrG = Cr * (INT32)(0.714401f * (1 << divisor));
-			const INT32 CbG = Cb * (INT32)(0.343730f * (1 << divisor));
-			const INT32 CbB = Cb * (INT32)(1.769905f * (1 << divisor));
+			const INT64 CrR = Cr * (INT64)(1.402525f * (1 << divisor)) * 1LL;
+			const INT64 CrG = Cr * (INT64)(0.714401f * (1 << divisor)) * 1LL;
+			const INT64 CbG = Cb * (INT64)(0.343730f * (1 << divisor)) * 1LL;
+			const INT64 CbB = Cb * (INT64)(1.769905f * (1 << divisor)) * 1LL;
 			R = ((INT16)((CrR + Y) >> divisor) >> 5);
 			G = ((INT16)((Y - CbG - CrG) >> divisor) >> 5);
 			B = ((INT16)((CbB + Y) >> divisor) >> 5);
@@ -90,18 +90,18 @@ static pstatus_t general_yCbCrToRGB_16s8u_P3AC4R_general(const INT16* const pSrc
 	{
 		for (x = 0; x < roi->width; x++)
 		{
-			INT16 R, G, B;
+			INT64 R, G, B;
 			const INT32 divisor = 16;
-			const INT32 Y = ((*pY++) + 4096) << divisor;
+			const INT32 Y = (INT32)((UINT32)((*pY++) + 4096) << divisor);
 			const INT32 Cb = (*pCb++);
 			const INT32 Cr = (*pCr++);
-			const INT32 CrR = Cr * (INT32)(1.402525f * (1 << divisor));
-			const INT32 CrG = Cr * (INT32)(0.714401f * (1 << divisor));
-			const INT32 CbG = Cb * (INT32)(0.343730f * (1 << divisor));
-			const INT32 CbB = Cb * (INT32)(1.769905f * (1 << divisor));
-			R = ((INT16)((CrR + Y) >> divisor) >> 5);
-			G = ((INT16)((Y - CbG - CrG) >> divisor) >> 5);
-			B = ((INT16)((CbB + Y) >> divisor) >> 5);
+			const INT64 CrR = Cr * (INT64)(1.402525f * (1 << divisor)) * 1LL;
+			const INT64 CrG = Cr * (INT64)(0.714401f * (1 << divisor)) * 1LL;
+			const INT64 CbG = Cb * (INT64)(0.343730f * (1 << divisor)) * 1LL;
+			const INT64 CbB = Cb * (INT64)(1.769905f * (1 << divisor)) * 1LL;
+			R = (INT64)((CrR + Y) >> (divisor + 5));
+			G = (INT64)((Y - CbG - CrG) >> (divisor + 5));
+			B = (INT64)((CbB + Y) >> (divisor + 5));
 			pRGB = (*writePixel)(pRGB, formatSize, DstFormat, CLIP(R), CLIP(G), CLIP(B), 0xFF);
 		}
 
@@ -170,7 +170,7 @@ static pstatus_t general_yCbCrToRGB_16s16s_P3P3(const INT16* const pSrc[3], INT3
 			INT32 y = (INT32)(*yptr++);
 			INT32 cb = (INT32)(*cbptr++);
 			INT32 cr = (INT32)(*crptr++);
-			INT32 r, g, b;
+			INT64 r, g, b;
 			/*
 			 * This is the slow floating point version kept here for reference.
 			 * y = y + 4096; // 128<<5=4096 so that we can scale the sum by>>5
@@ -191,10 +191,10 @@ static pstatus_t general_yCbCrToRGB_16s16s_P3P3(const INT16* const pSrc[3], INT3
 			 * G: 0.344 << 16 = 22544, 0.714 << 16 = 46792
 			 * B: 1.770 << 16 = 115998
 			 */
-			y = (y + 4096) << 16;
-			r = y + cr * 91947;
-			g = y - cb * 22544 - cr * 46792;
-			b = y + cb * 115998;
+			y = (INT32)((UINT32)(y + 4096) << 16);
+			r = y + cr * 91947LL;
+			g = y - cb * 22544LL - cr * 46792LL;
+			b = y + cb * 115998LL;
 			*rptr++ = CLIP(r >> 21);
 			*gptr++ = CLIP(g >> 21);
 			*bptr++ = CLIP(b >> 21);