Fixed undefined behaviour in primitives shift

(cherry picked from commit 436bd3dbb103c7c49e51ffb1cafad6f25ab05d66)

diff --git a/libfreerdp/primitives/prim_shift.c b/libfreerdp/primitives/prim_shift.c
index 3fabfe2..cab9481 100644 (file)
--- a/libfreerdp/primitives/prim_shift.c
+++ b/libfreerdp/primitives/prim_shift.c
@@ -26,9 +26,11 @@ static INLINE pstatus_t general_lShiftC_16s(const INT16* pSrc, UINT32 val, INT16
 {
        if (val == 0)
                return PRIMITIVES_SUCCESS;
+       if (val >= 16)
+               return -1;
 
        while (len--)
-               *pDst++ = *pSrc++ << val;
+               *pDst++ = (INT16)((UINT16)*pSrc++ << val);
 
        return PRIMITIVES_SUCCESS;
 }
@@ -38,6 +40,8 @@ static INLINE pstatus_t general_rShiftC_16s(const INT16* pSrc, UINT32 val, INT16
 {
        if (val == 0)
                return PRIMITIVES_SUCCESS;
+       if (val >= 16)
+               return -1;
 
        while (len--)
                *pDst++ = *pSrc++ >> val;
@@ -51,9 +55,11 @@ static INLINE pstatus_t general_lShiftC_16u(const UINT16* pSrc, UINT32 val, UINT
 {
        if (val == 0)
                return PRIMITIVES_SUCCESS;
+       if (val >= 16)
+               return -1;
 
        while (len--)
-               *pDst++ = *pSrc++ << val;
+               *pDst++ = (INT16)((UINT16)*pSrc++ << val);
 
        return PRIMITIVES_SUCCESS;
 }
@@ -64,6 +70,8 @@ static INLINE pstatus_t general_rShiftC_16u(const UINT16* pSrc, UINT32 val, UINT
 {
        if (val == 0)
                return PRIMITIVES_SUCCESS;
+       if (val >= 16)
+               return -1;
 
        while (len--)
                *pDst++ = *pSrc++ >> val;

diff --git a/libfreerdp/primitives/prim_shift_opt.c b/libfreerdp/primitives/prim_shift_opt.c
index 481363a..2515dfb 100644 (file)
--- a/libfreerdp/primitives/prim_shift_opt.c
+++ b/libfreerdp/primitives/prim_shift_opt.c
@@ -39,13 +39,13 @@ static primitives_t* generic = NULL;
 #if !defined(WITH_IPP) || defined(ALL_PRIMITIVES_VERSIONS)
 /* ------------------------------------------------------------------------- */
 SSE3_SCD_ROUTINE(sse2_lShiftC_16s, INT16, generic->lShiftC_16s, _mm_slli_epi16,
-                 *dptr++ = *sptr++ << val)
+                 *dptr++ = (INT16)((UINT16)*sptr++ << val))
 /* ------------------------------------------------------------------------- */
 SSE3_SCD_ROUTINE(sse2_rShiftC_16s, INT16, generic->rShiftC_16s, _mm_srai_epi16,
                  *dptr++ = *sptr++ >> val)
 /* ------------------------------------------------------------------------- */
 SSE3_SCD_ROUTINE(sse2_lShiftC_16u, UINT16, generic->lShiftC_16u, _mm_slli_epi16,
-                 *dptr++ = *sptr++ << val)
+                 *dptr++ = (INT16)((UINT16)*sptr++ << val))
 /* ------------------------------------------------------------------------- */
 SSE3_SCD_ROUTINE(sse2_rShiftC_16u, UINT16, generic->rShiftC_16u, _mm_srli_epi16,
                  *dptr++ = *sptr++ >> val)

diff --git a/libfreerdp/primitives/prim_templates.h b/libfreerdp/primitives/prim_templates.h
index 4787265..46996e2 100644 (file)
--- a/libfreerdp/primitives/prim_templates.h
+++ b/libfreerdp/primitives/prim_templates.h
@@ -51,6 +51,10 @@
                const _type_* sptr = pSrc;                                                    \
                _type_* dptr = pDst;                                                          \
                size_t count;                                                                 \
+               if (val == 0)                                                                 \
+                       return PRIMITIVES_SUCCESS;                                                \
+               if (val >= 16)                                                                \
+                       return -1;                                                                \
                if (len < 16) /* pointless if too small */                                    \
                {                                                                             \
                        return _fallback_(pSrc, val, pDst, len);                                  \

diff --git a/libfreerdp/primitives/test/TestPrimitivesShift.c b/libfreerdp/primitives/test/TestPrimitivesShift.c
index 448c2d1..487e188 100644 (file)
--- a/libfreerdp/primitives/test/TestPrimitivesShift.c
+++ b/libfreerdp/primitives/test/TestPrimitivesShift.c
@@ -29,7 +29,18 @@ static BOOL test_lShift_16s_func(void)
        UINT32 val;
        winpr_RAND((BYTE*)&val, sizeof(val));
        winpr_RAND((BYTE*)src, sizeof(src));
-       val = (val % (FUNC_TEST_SIZE - 1)) + 1;
+       val = val % 16;
+       /* Negative tests */
+       status = generic->lShiftC_16s(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
+       status = optimized->lShiftC_16s(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
        /* Aligned */
        status = generic->lShiftC_16s(src + 1, val, d1 + 1, FUNC_TEST_SIZE);
 
@@ -63,7 +74,19 @@ static BOOL test_lShift_16u_func(void)
        UINT32 val;
        winpr_RAND((BYTE*)&val, sizeof(val));
        winpr_RAND((BYTE*)src, sizeof(src));
-       val = (val % (FUNC_TEST_SIZE - 1)) + 1;
+       val = val % 16;
+
+       /* Negative tests */
+       status = generic->lShiftC_16u(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
+       status = optimized->lShiftC_16u(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
        /* Aligned */
        status = generic->lShiftC_16u(src + 1, val, d1 + 1, FUNC_TEST_SIZE);
 
@@ -97,7 +120,19 @@ static BOOL test_rShift_16s_func(void)
        UINT32 val;
        winpr_RAND((BYTE*)&val, sizeof(val));
        winpr_RAND((BYTE*)src, sizeof(src));
-       val = (val % (FUNC_TEST_SIZE - 1)) + 1;
+       val = val % 16;
+
+       /* Negative Tests */
+       status = generic->rShiftC_16s(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
+       status = optimized->rShiftC_16s(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
        /* Aligned */
        status = generic->rShiftC_16s(src + 1, val, d1 + 1, FUNC_TEST_SIZE);
 
@@ -131,7 +166,18 @@ static BOOL test_rShift_16u_func(void)
        UINT32 val;
        winpr_RAND((BYTE*)&val, sizeof(val));
        winpr_RAND((BYTE*)src, sizeof(src));
-       val = (val % (FUNC_TEST_SIZE - 1)) + 1;
+       val = val % 16;
+       /* Negative tests */
+       status = generic->rShiftC_16u(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
+       status = optimized->rShiftC_16u(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
        /* Aligned */
        status = generic->rShiftC_16u(src + 1, val, d1 + 1, FUNC_TEST_SIZE);
 
@@ -166,7 +212,19 @@ static BOOL test_ShiftWrapper_16s_func(void)
        INT32 val;
        winpr_RAND((BYTE*)&tmp, sizeof(tmp));
        winpr_RAND((BYTE*)src, sizeof(src));
-       val = (tmp % (FUNC_TEST_SIZE - 1)) + 1;
+       val = tmp % 16;
+
+       /* Negative tests */
+       status = generic->shiftC_16s(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
+       status = optimized->shiftC_16s(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
        /* Aligned */
        status = generic->shiftC_16s(src + 1, val, d1 + 1, FUNC_TEST_SIZE);
 
@@ -221,7 +279,18 @@ static BOOL test_ShiftWrapper_16u_func(void)
        INT32 val;
        winpr_RAND((BYTE*)&tmp, sizeof(tmp));
        winpr_RAND((BYTE*)src, sizeof(src));
-       val = (tmp % (FUNC_TEST_SIZE - 1)) + 1;
+       val = tmp % 16;
+
+       /* Negative */
+       status = generic->shiftC_16u(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
+       status = optimized->shiftC_16u(src + 1, 16, d1 + 1, FUNC_TEST_SIZE);
+
+       if (status == PRIMITIVES_SUCCESS)
+               return FALSE;
+
        /* Aligned */
        status = generic->shiftC_16u(src + 1, val, d1 + 1, FUNC_TEST_SIZE);
 
@@ -275,6 +344,7 @@ static BOOL test_lShift_16s_speed(void)
        winpr_RAND((BYTE*)src, sizeof(src));
        winpr_RAND((BYTE*)&val, sizeof(val));
 
+       val = val % 16;
        if (!speed_test("lShift_16s", "aligned", g_Iterations, (speed_test_fkt)generic->lShiftC_16s,
                        (speed_test_fkt)optimized->lShiftC_16s, src, val, dst, MAX_TEST_SIZE))
                return FALSE;
@@ -294,6 +364,7 @@ static BOOL test_lShift_16u_speed(void)
        winpr_RAND((BYTE*)&val, sizeof(val));
        winpr_RAND((BYTE*)src, sizeof(src));
 
+       val = val % 16;
        if (!speed_test("lShift_16u", "aligned", g_Iterations, (speed_test_fkt)generic->lShiftC_16u,
                        (speed_test_fkt)optimized->lShiftC_16u, src, val, dst, MAX_TEST_SIZE))
                return FALSE;
@@ -313,6 +384,7 @@ static BOOL test_rShift_16s_speed(void)
        winpr_RAND((BYTE*)src, sizeof(src));
        winpr_RAND((BYTE*)&val, sizeof(val));
 
+       val = val % 16;
        if (!speed_test("rShift_16s", "aligned", g_Iterations, (speed_test_fkt)generic->rShiftC_16s,
                        (speed_test_fkt)optimized->rShiftC_16s, src, val, dst, MAX_TEST_SIZE))
                return FALSE;
@@ -332,6 +404,7 @@ static BOOL test_rShift_16u_speed(void)
        winpr_RAND((BYTE*)&val, sizeof(val));
        winpr_RAND((BYTE*)src, sizeof(src));
 
+       val = val % 16;
        if (!speed_test("rShift_16u", "aligned", g_Iterations, (speed_test_fkt)generic->rShiftC_16u,
                        (speed_test_fkt)optimized->rShiftC_16u, src, val, dst, MAX_TEST_SIZE))
                return FALSE;