From 6dfd915004b63e91c417f569af9f8516f7f942fc Mon Sep 17 00:00:00 2001 From: akallabeth Date: Fri, 15 May 2020 16:30:02 +0200 Subject: [PATCH] Fixed undefined behaviour (cherry picked from commit c15929d821ce99aa8c1069f790f4addbe9439593) --- libfreerdp/primitives/prim_colors.c | 38 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/libfreerdp/primitives/prim_colors.c b/libfreerdp/primitives/prim_colors.c index a9d36e3..54c7ecb 100644 --- a/libfreerdp/primitives/prim_colors.c +++ b/libfreerdp/primitives/prim_colors.c @@ -50,13 +50,13 @@ static pstatus_t general_yCbCrToRGB_16s8u_P3AC4R_BGRX(const INT16* const pSrc[3] { INT16 R, G, B; const INT32 divisor = 16; - const INT32 Y = ((*pY++) + 4096) << divisor; + const INT32 Y = (INT32)((UINT32)((*pY++) + 4096) << divisor); const INT32 Cb = (*pCb++); const INT32 Cr = (*pCr++); - const INT32 CrR = Cr * (INT32)(1.402525f * (1 << divisor)); - const INT32 CrG = Cr * (INT32)(0.714401f * (1 << divisor)); - const INT32 CbG = Cb * (INT32)(0.343730f * (1 << divisor)); - const INT32 CbB = Cb * (INT32)(1.769905f * (1 << divisor)); + const INT64 CrR = Cr * (INT64)(1.402525f * (1 << divisor)) * 1LL; + const INT64 CrG = Cr * (INT64)(0.714401f * (1 << divisor)) * 1LL; + const INT64 CbG = Cb * (INT64)(0.343730f * (1 << divisor)) * 1LL; + const INT64 CbB = Cb * (INT64)(1.769905f * (1 << divisor)) * 1LL; R = ((INT16)((CrR + Y) >> divisor) >> 5); G = ((INT16)((Y - CbG - CrG) >> divisor) >> 5); B = ((INT16)((CbB + Y) >> divisor) >> 5); @@ -90,18 +90,18 @@ static pstatus_t general_yCbCrToRGB_16s8u_P3AC4R_general(const INT16* const pSrc { for (x = 0; x < roi->width; x++) { - INT16 R, G, B; + INT64 R, G, B; const INT32 divisor = 16; - const INT32 Y = ((*pY++) + 4096) << divisor; + const INT32 Y = (INT32)((UINT32)((*pY++) + 4096) << divisor); const INT32 Cb = (*pCb++); const INT32 Cr = (*pCr++); - const INT32 CrR = Cr * (INT32)(1.402525f * (1 << divisor)); - const INT32 CrG = Cr * (INT32)(0.714401f * (1 << divisor)); - const INT32 CbG = Cb * (INT32)(0.343730f * (1 << divisor)); - const INT32 CbB = Cb * (INT32)(1.769905f * (1 << divisor)); - R = ((INT16)((CrR + Y) >> divisor) >> 5); - G = ((INT16)((Y - CbG - CrG) >> divisor) >> 5); - B = ((INT16)((CbB + Y) >> divisor) >> 5); + const INT64 CrR = Cr * (INT64)(1.402525f * (1 << divisor)) * 1LL; + const INT64 CrG = Cr * (INT64)(0.714401f * (1 << divisor)) * 1LL; + const INT64 CbG = Cb * (INT64)(0.343730f * (1 << divisor)) * 1LL; + const INT64 CbB = Cb * (INT64)(1.769905f * (1 << divisor)) * 1LL; + R = (INT64)((CrR + Y) >> (divisor + 5)); + G = (INT64)((Y - CbG - CrG) >> (divisor + 5)); + B = (INT64)((CbB + Y) >> (divisor + 5)); pRGB = (*writePixel)(pRGB, formatSize, DstFormat, CLIP(R), CLIP(G), CLIP(B), 0xFF); } @@ -170,7 +170,7 @@ static pstatus_t general_yCbCrToRGB_16s16s_P3P3(const INT16* const pSrc[3], INT3 INT32 y = (INT32)(*yptr++); INT32 cb = (INT32)(*cbptr++); INT32 cr = (INT32)(*crptr++); - INT32 r, g, b; + INT64 r, g, b; /* * This is the slow floating point version kept here for reference. * y = y + 4096; // 128<<5=4096 so that we can scale the sum by>>5 @@ -191,10 +191,10 @@ static pstatus_t general_yCbCrToRGB_16s16s_P3P3(const INT16* const pSrc[3], INT3 * G: 0.344 << 16 = 22544, 0.714 << 16 = 46792 * B: 1.770 << 16 = 115998 */ - y = (y + 4096) << 16; - r = y + cr * 91947; - g = y - cb * 22544 - cr * 46792; - b = y + cb * 115998; + y = (INT32)((UINT32)(y + 4096) << 16); + r = y + cr * 91947LL; + g = y - cb * 22544LL - cr * 46792LL; + b = y + cb * 115998LL; *rptr++ = CLIP(r >> 21); *gptr++ = CLIP(g >> 21); *bptr++ = CLIP(b >> 21); -- 2.7.4