Fixed sign-compare warnings

diff --git a/libfreerdp/codec/color.c b/libfreerdp/codec/color.c
index 477c0ef..37a2283 100644 (file)
--- a/libfreerdp/codec/color.c
+++ b/libfreerdp/codec/color.c
@@ -415,6 +415,9 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat,
        UINT32 dstVOffset = 0;
        INT32 dstVMultiplier = 1;
 
+       if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX))
+               return FALSE;
+
        if (!pDstData || !pSrcData)
                return FALSE;
 
@@ -441,7 +444,7 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat,
                        /* Copy down */
                        if (nYDst < nYSrc)
                        {
-                               for (y = 0; y < nHeight; y++)
+                               for (y = 0; y < (INT32)nHeight; y++)
                                {
                                        const BYTE* srcLine = &pSrcData[(y + nYSrc) *
                                                                                    nSrcStep * srcVMultiplier +
@@ -471,7 +474,7 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat,
                        /* Copy left */
                        else if (nXSrc > nXDst)
                        {
-                               for (y = 0; y < nHeight; y++)
+                               for (y = 0; y < (INT32)nHeight; y++)
                                {
                                        const BYTE* srcLine = &pSrcData[(y + nYSrc) *
                                                                                    nSrcStep * srcVMultiplier +
@@ -486,7 +489,7 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat,
                        /* Copy right */
                        else if (nXSrc < nXDst)
                        {
-                               for (y = nHeight - 1; y >= 0; y--)
+                               for (y = (INT32)nHeight - 1; y >= 0; y--)
                                {
                                        const BYTE* srcLine = &pSrcData[(y + nYSrc) *
                                                                                    nSrcStep * srcVMultiplier +
@@ -505,7 +508,7 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat,
                }
                else
                {
-                       for (y = 0; y < nHeight; y++)
+                       for (y = 0; y < (INT32)nHeight; y++)
                        {
                                const BYTE* srcLine = &pSrcData[(y + nYSrc) *
                                                                            nSrcStep * srcVMultiplier +
@@ -520,9 +523,9 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat,
        }
        else
        {
-               UINT32 x, y;
+               INT32 x, y;
 
-               for (y = 0; y < nHeight; y++)
+               for (y = 0; y < (INT32)nHeight; y++)
                {
                        const BYTE* srcLine = &pSrcData[(y + nYSrc) *
                                                                    nSrcStep * srcVMultiplier +
@@ -530,7 +533,7 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat,
                        BYTE* dstLine = &pDstData[(y + nYDst) *
                                                              nDstStep * dstVMultiplier + dstVOffset];
 
-                       for (x = 0; x < nWidth; x++)
+                       for (x = 0; x < (INT32)nWidth; x++)
                        {
                                UINT32 dstColor;
                                UINT32 color = ReadColor(&srcLine[(x + nXSrc) * srcByte],

diff --git a/libfreerdp/codec/ncrush.c b/libfreerdp/codec/ncrush.c
index 9e79d71..1186c1a 100644 (file)
--- a/libfreerdp/codec/ncrush.c
+++ b/libfreerdp/codec/ncrush.c
@@ -2276,7 +2276,7 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, BYTE* pSrcData, UINT32 SrcSize, BYTE
        UINT32 DstSize;
        BOOL PacketAtFront;
        BOOL PacketFlushed;
-       int MatchLength;
+       UINT32 MatchLength;
        UINT32 IndexLEC;
        UINT32 IndexLOM;
        UINT32 IndexCO;
@@ -2362,11 +2362,14 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, BYTE* pSrcData, UINT32 SrcSize, BYTE
 
                if (ncrush->MatchTable[HistoryOffset])
                {
+                       int rc;
+
                        MatchOffset = 0;
-                       MatchLength = ncrush_find_best_match(ncrush, HistoryOffset, &MatchOffset);
+                       rc = ncrush_find_best_match(ncrush, HistoryOffset, &MatchOffset);
 
-                       if (MatchLength == -1)
+                       if (rc < 0)
                                return -1005;
+                       MatchLength = (UINT32)rc;
                }
 
                if (MatchLength)
@@ -2375,7 +2378,7 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, BYTE* pSrcData, UINT32 SrcSize, BYTE
                if ((MatchLength == 2) && (CopyOffset >= 64))
                        MatchLength = 0;
 
-               if (!MatchLength)
+               if (MatchLength == 0)
                {
                        /* Literal */
                        Literal = *SrcPtr++;
@@ -2606,15 +2609,16 @@ int ncrush_compress(NCRUSH_CONTEXT* ncrush, BYTE* pSrcData, UINT32 SrcSize, BYTE
 
 static int ncrush_generate_tables(NCRUSH_CONTEXT* context)
 {
-       int i, j, k, l;
+       UINT32 k, i;
+       int j, l;
        k = 0;
 
        for (i = 0; i < 28; i++)
        {
                for (j = 0; j < 1 << LOMBitsLUT[i]; j++)
                {
-                       l = k++ + 2;
-                       context->HuffTableLOM[l] = i;
+                       l = (k++) + 2;
+                       context->HuffTableLOM[l] = (int)i;
                }
        }
 

diff --git a/libfreerdp/codec/planar.c b/libfreerdp/codec/planar.c
index c8bd99c..cb59960 100644 (file)
--- a/libfreerdp/codec/planar.c
+++ b/libfreerdp/codec/planar.c
@@ -93,7 +93,7 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* pSrcData, UINT32 Src
         UINT32 nWidth, UINT32 nHeight,
         UINT32 nChannel, BOOL vFlip)
 {
-       UINT32 x, y;
+       INT32 x, y;
        UINT32 pixel;
        UINT32 cRawBytes;
        UINT32 nRunLength;
@@ -103,28 +103,32 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* pSrcData, UINT32 Src
        BYTE* currentScanline;
        BYTE* previousScanline;
        const BYTE* srcp = pSrcData;
+
+       if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX) || (nDstStep > INT32_MAX))
+               return -1;
+
        previousScanline = NULL;
 
        if (vFlip)
        {
-               beg = nHeight - 1;
+               beg = (INT32)nHeight - 1;
                end = -1;
                inc = -1;
        }
        else
        {
                beg = 0;
-               end = nHeight;
+               end = (INT32)nHeight;
                inc = 1;
        }
 
        for (y = beg; y != end; y += inc)
        {
-               BYTE* dstp = &pDstData[((nYDst + y) * nDstStep) + (nXDst * 4) + nChannel];
+               BYTE* dstp = &pDstData[((nYDst + y) * (INT32)nDstStep) + (nXDst * 4) + nChannel];
                pixel = 0;
                currentScanline = dstp;
 
-               for (x = 0; x < nWidth;)
+               for (x = 0; x < (INT32)nWidth;)
                {
                        controlByte = *srcp;
                        srcp++;
@@ -636,16 +640,19 @@ static INLINE BOOL freerdp_split_color_planes(const BYTE* data, UINT32 format,
         UINT32 scanline, BYTE* planes[4])
 {
        INT32 i, j, k;
+       if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
+               return FALSE;
+
        k = 0;
 
        if (scanline == 0)
                scanline = width * GetBytesPerPixel(format);
 
-       for (i = height - 1; i >= 0; i--)
+       for (i = (INT32)height - 1; i >= 0; i--)
        {
-               const BYTE* pixel = &data[scanline * i];
+               const BYTE* pixel = &data[(INT32)scanline * i];
 
-               for (j = 0; j < width; j++)
+               for (j = 0; j < (INT32)width; j++)
                {
                        const UINT32 color = ReadColor(pixel, format);
                        pixel += GetBytesPerPixel(format);

diff --git a/libfreerdp/codec/progressive.c b/libfreerdp/codec/progressive.c
index d3dc5c3..f62ea3f 100644 (file)
--- a/libfreerdp/codec/progressive.c
+++ b/libfreerdp/codec/progressive.c
@@ -969,7 +969,7 @@ static INLINE int progressive_rfx_upgrade_block(RFX_PROGRESSIVE_UPGRADE_STATE* s
         INT16* buffer, INT16* sign, UINT32 length,
         UINT32 shift, UINT32 bitPos, UINT32 numBits)
 {
-       int index;
+       UINT32 index;
        INT16 input;
        wBitStream* raw;
 

diff --git a/libfreerdp/codec/rfx.c b/libfreerdp/codec/rfx.c
index d98d58f..ef5ff7b 100644 (file)
--- a/libfreerdp/codec/rfx.c
+++ b/libfreerdp/codec/rfx.c
@@ -1409,8 +1409,11 @@ BOOL setupWorkers(RFX_CONTEXT* context, int nbTiles)
 
 RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* context, const RFX_RECT* rects,
                                 int numRects,
-                                BYTE* data, int width, int height, int scanline)
+                                BYTE* data, int w, int h, int s)
 {
+       const UINT32 width = (UINT32)w;
+       const UINT32 height = (UINT32)h;
+       const UINT32 scanline = (UINT32)s;
        UINT32 i, maxNbTiles, maxTilesX, maxTilesY;
        UINT32 xIdx, yIdx, regionNbRects;
        UINT32 gridRelX, gridRelY, ax, ay, bytesPerPixel;
@@ -1427,9 +1430,9 @@ RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* context, const RFX_RECT* rects,
        assert(data);
        assert(rects);
        assert(numRects > 0);
-       assert(width > 0);
-       assert(height > 0);
-       assert(scanline > 0);
+       assert(w > 0);
+       assert(h > 0);
+       assert(s > 0);
 
        if (!(message = (RFX_MESSAGE*)calloc(1, sizeof(RFX_MESSAGE))))
                return NULL;
@@ -1492,10 +1495,10 @@ RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* context, const RFX_RECT* rects,
        for (i = 0, rfxRect = message->rects; i < regionNbRects;
             i++, regionRect++, rfxRect++)
        {
-               int startTileX = regionRect->left / 64;
-               int endTileX = (regionRect->right - 1) / 64;
-               int startTileY = regionRect->top / 64;
-               int endTileY = (regionRect->bottom - 1) / 64;
+               UINT32 startTileX = regionRect->left / 64;
+               UINT32 endTileX = (regionRect->right - 1) / 64;
+               UINT32 startTileY = regionRect->top / 64;
+               UINT32 endTileY = (regionRect->bottom - 1) / 64;
                rfxRect->x = regionRect->left;
                rfxRect->y = regionRect->top;
                rfxRect->width = (regionRect->right - regionRect->left);
@@ -1504,7 +1507,7 @@ RFX_MESSAGE* rfx_encode_message(RFX_CONTEXT* context, const RFX_RECT* rects,
                for (yIdx = startTileY, gridRelY = startTileY * 64; yIdx <= endTileY;
                     yIdx++, gridRelY += 64)
                {
-                       int tileHeight = 64;
+                       UINT32 tileHeight = 64;
 
                        if ((yIdx == endTileY) && (gridRelY + 64 > height))
                                tileHeight = height - gridRelY;

diff --git a/libfreerdp/codec/rfx_rlgr.c b/libfreerdp/codec/rfx_rlgr.c
index 14630b9..d117275 100644 (file)
--- a/libfreerdp/codec/rfx_rlgr.c
+++ b/libfreerdp/codec/rfx_rlgr.c
@@ -111,10 +111,12 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* pSrcData, UINT32 SrcSize, INT16*
        int cnt;
        int size;
        int nbits;
-       int offset;
+       size_t offset;
        INT16 mag;
-       int k, kp;
-       int kr, krp;
+       UINT32 k;
+       INT32 kp;
+       UINT32 kr;
+       INT32 krp;
        UINT16 code;
        UINT32 sign;
        UINT32 nIdx;
@@ -305,7 +307,7 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* pSrcData, UINT32 SrcSize, INT16*
 
                        /* write to output stream */
 
-                       offset = (int) (pOutput - pDstData);
+                       offset = (pOutput - pDstData);
                        size = run;
 
                        if ((offset + size) > DstSize)
@@ -505,7 +507,7 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* pSrcData, UINT32 SrcSize, INT16*
                }
        }
 
-       offset = (int) (pOutput - pDstData);
+       offset = (pOutput - pDstData);
 
        if (offset < DstSize)
        {
@@ -514,7 +516,7 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* pSrcData, UINT32 SrcSize, INT16*
                pOutput += size;
        }
 
-       offset = (int) (pOutput - pDstData);
+       offset = (pOutput - pDstData);
 
        if (offset != DstSize)
                return -1;

diff --git a/libfreerdp/codec/yuv.c b/libfreerdp/codec/yuv.c
index 357781c..59a7e91 100644 (file)
--- a/libfreerdp/codec/yuv.c
+++ b/libfreerdp/codec/yuv.c
@@ -111,7 +111,7 @@ BOOL yuv_context_decode(YUV_CONTEXT* context, const BYTE* pYUVData[3], UINT32 iS
        UINT32 y, nobjects, i;
        PTP_WORK *work_objects = NULL;
        YUV_PROCESS_WORK_PARAM *params;
-       int waitCount = 0;
+       UINT32 waitCount = 0;
        BOOL ret = TRUE;
 
        if (!context->useThreads)

diff --git a/libfreerdp/codec/zgfx.c b/libfreerdp/codec/zgfx.c
index 551fffc..5ce552d 100644 (file)
--- a/libfreerdp/codec/zgfx.c
+++ b/libfreerdp/codec/zgfx.c
@@ -175,16 +175,16 @@ static void zgfx_history_buffer_ring_read(ZGFX_CONTEXT* zgfx, int offset, BYTE*
 {
        UINT32 front;
        UINT32 index;
-       UINT32 bytes;
+       INT32 bytes;
        UINT32 valid;
-       UINT32 bytesLeft;
+       INT32 bytesLeft;
        BYTE* dptr = dst;
        BYTE* origDst = dst;
 
-       if (count <= 0)
+       if ((count <= 0) || (count > INT32_MAX))
                return;
 
-       bytesLeft = count;
+       bytesLeft = (INT32)count;
        index = (zgfx->HistoryIndex + zgfx->HistoryBufferSize - offset) % zgfx->HistoryBufferSize;
        bytes = MIN(bytesLeft, offset);
 
@@ -225,8 +225,8 @@ static BOOL zgfx_decompress_segment(ZGFX_CONTEXT* zgfx, wStream* stream, size_t
        BYTE flags;
        UINT32 extra = 0;
        int opIndex;
-       int haveBits;
-       int inPrefix;
+       UINT32 haveBits;
+       UINT32 inPrefix;
        UINT32 count;
        UINT32 distance;
        BYTE* pbSegment;