libfreerdp-codec: simply planar RLE code

diff --git a/libfreerdp/codec/planar.c b/libfreerdp/codec/planar.c
index 2fc1a91..2fe044d 100644 (file)
--- a/libfreerdp/codec/planar.c
+++ b/libfreerdp/codec/planar.c
@@ -77,7 +77,6 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
 {
        int i, j;
        BYTE* dstp;
-       BYTE* segp;
        BYTE symbol;
        int cSegments;
        int nRunLength;
@@ -86,7 +85,6 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
        int outPlaneSize;
        int outSegmentSize;
        int nControlBytes;
-       int nSequenceLength;
 
        cSegments = 0;
        outPlaneSize = width * height;
@@ -99,240 +97,223 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
        for (i = 0; i < height; i++)
        {
                cRawBytes = 0;
-               nRunLength = 0;
-               nSequenceLength = 0;
+               nRunLength = -1;
                rawValues = &inPlane[i * width];
 
                for (j = 0; j <= width; j++)
                {
-                       if ((!nSequenceLength) && (j != width))
+                       if ((nRunLength < 0) && (j != width))
                        {
                                symbol = inPlane[(i * width) + j];
-                               nSequenceLength = 1;
+                               nRunLength = 0;
+                               continue;
+                       }
+
+                       if ((j != width) && (inPlane[(i * width) + j] == symbol))
+                       {
+                               nRunLength++;
                        }
                        else
                        {
-                               if ((j != width) && (inPlane[(i * width) + j] == symbol))
-                               {
-                                       nSequenceLength++;
-                               }
-                               else
+                               if (nRunLength >= 3)
                                {
-                                       if (nSequenceLength > 3)
-                                       {
-                                               cRawBytes += 1;
-                                               nRunLength = nSequenceLength - 1;
+                                       cRawBytes += 1;
 
 #if 0
-                                               printf("RAW[");
+                                       printf("RAW[");
 
-                                               for (k = 0; k < cRawBytes; k++)
-                                               {
-                                                       printf("0x%02X%s", rawValues[k],
-                                                                       ((k + 1) == cRawBytes) ? "" : ", ");
-                                               }
+                                       for (k = 0; k < cRawBytes; k++)
+                                       {
+                                               printf("0x%02X%s", rawValues[k],
+                                                               ((k + 1) == cRawBytes) ? "" : ", ");
+                                       }
 
-                                               printf("] RUN[%d]\n", nRunLength);
+                                       printf("] RUN[%d]\n", nRunLength);
 #endif
 
-                                               segp = dstp;
-                                               outSegmentSize = cRawBytes + 1;
+                                       while (cRawBytes > 15)
+                                       {
+                                               printf("handling cRawBytes > 15\n");
 
-                                               if (cRawBytes > 15)
+                                               nControlBytes = 1;
+                                               outSegmentSize = 15 + nControlBytes;
+
+                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
                                                {
-                                                       //printf("cRawBytes > 15 unhandled\n");
+                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
                                                        return NULL;
                                                }
 
-                                               if (nRunLength > 47)
-                                               {
-                                                       printf("nRunLength > 47 unhandled\n");
-                                                       //return NULL;
+                                               *dstp = PLANAR_CONTROL_BYTE(0, 15);
+                                               dstp++;
 
-                                                       nControlBytes = (((nRunLength - 15) + 46) / 47) + 1;
+                                               CopyMemory(dstp, rawValues, 15);
+                                               cRawBytes -= 15;
+                                               rawValues += 15;
+                                               dstp += 15;
 
-                                                       outSegmentSize = cRawBytes + nControlBytes;
+                                               return NULL;
+                                       }
 
-                                                       if (((segp - outPlane) + outSegmentSize) > outPlaneSize)
-                                                       {
-                                                               printf("overflow: %d > %d\n",
-                                                                               ((dstp - outPlane) + outSegmentSize),
-                                                                               outPlaneSize);
-                                                               return NULL;
-                                                       }
+                                       if (nRunLength > 47)
+                                       {
+                                               nControlBytes = (((nRunLength - 15) + 46) / 47) + 1;
 
-                                                       printf(">47 nRunLength: %d nControlBytes: %d\n", nRunLength, nControlBytes);
+                                               outSegmentSize = cRawBytes + nControlBytes;
 
-                                                       *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
-                                                       printf("ControlByte nRunLength: %d cRawBytes: %d\n", 15, cRawBytes);
-                                                       nRunLength -= 15;
-                                                       nControlBytes--;
-                                                       dstp++;
+                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
+                                               {
+                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                                       return NULL;
+                                               }
 
-                                                       CopyMemory(dstp, rawValues, cRawBytes);
-                                                       dstp += cRawBytes;
+                                               *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
+                                               nRunLength -= 15;
+                                               nControlBytes--;
+                                               dstp++;
 
-                                                       while (nControlBytes--)
-                                                       {
-                                                               printf("...nRunLength: %d\n", nRunLength);
-
-                                                               if (nRunLength > 47)
-                                                               {
-                                                                       *dstp = PLANAR_CONTROL_BYTE(2, (47 - 32));
-                                                                       printf("ControlByte nRunLength: %d cRawBytes: %d\n", 47, 0);
-                                                                       nRunLength -= 47;
-                                                                       dstp++;
-                                                               }
-                                                               else if (nRunLength > 31)
-                                                               {
-                                                                       *dstp = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
-                                                                       printf("ControlByte nRunLength: %d cRawBytes: %d\n", nRunLength, 0);
-                                                                       nRunLength = 0;
-                                                                       dstp++;
-                                                               }
-                                                               else if (nRunLength > 15)
-                                                               {
-                                                                       *dstp = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
-                                                                       printf("ControlByte nRunLength: %d cRawBytes: %d\n", nRunLength, 0);
-                                                                       nRunLength = 0;
-                                                                       dstp++;
-                                                               }
-                                                               else
-                                                               {
-                                                                       *dstp = PLANAR_CONTROL_BYTE(0, nRunLength);
-                                                                       printf("ControlByte nRunLength: %d cRawBytes: %d\n", nRunLength, 0);
-                                                                       nRunLength = 0;
-                                                                       dstp++;
-                                                               }
-                                                       }
+                                               CopyMemory(dstp, rawValues, cRawBytes);
+                                               dstp += cRawBytes;
 
-                                                       return NULL;
-                                               }
-                                               else if (nRunLength > 31)
+                                               while (nControlBytes--)
                                                {
-                                                       nControlBytes = 2;
-                                                       outSegmentSize = cRawBytes + nControlBytes;
-
-                                                       if (((segp - outPlane) + outSegmentSize) > outPlaneSize)
+                                                       if (nRunLength > 47)
                                                        {
-                                                               printf("overflow: %d > %d\n",
-                                                                               ((dstp - outPlane) + outSegmentSize),
-                                                                               outPlaneSize);
-                                                               return NULL;
+                                                               *dstp = PLANAR_CONTROL_BYTE(2, (47 - 32));
+                                                               nRunLength -= 47;
+                                                               dstp++;
                                                        }
+                                                       else if (nRunLength > 31)
+                                                       {
+                                                               *dstp = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
+                                                               nRunLength = 0;
+                                                               dstp++;
+                                                       }
+                                                       else if (nRunLength > 15)
+                                                       {
+                                                               *dstp = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
+                                                               nRunLength = 0;
+                                                               dstp++;
+                                                       }
+                                                       else
+                                                       {
+                                                               *dstp = PLANAR_CONTROL_BYTE(0, nRunLength);
+                                                               nRunLength = 0;
+                                                               dstp++;
+                                                       }
+                                               }
 
-                                                       *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
-                                                       dstp++;
-
-                                                       CopyMemory(dstp, rawValues, cRawBytes);
-                                                       dstp += cRawBytes;
+                                               return NULL;
+                                       }
+                                       else if (nRunLength > 31)
+                                       {
+                                               nControlBytes = 2;
+                                               outSegmentSize = cRawBytes + nControlBytes;
 
-                                                       nRunLength -= 32;
-                                                       *dstp = PLANAR_CONTROL_BYTE(2, nRunLength);
-                                                       dstp++;
-                                               }
-                                               else if (nRunLength > 15)
+                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
                                                {
-                                                       nControlBytes = 2;
-                                                       outSegmentSize = cRawBytes + nControlBytes;
+                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                                       return NULL;
+                                               }
 
-                                                       if (((segp - outPlane) + outSegmentSize) > outPlaneSize)
-                                                       {
-                                                               printf("overflow: %d > %d\n",
-                                                                               ((dstp - outPlane) + outSegmentSize),
-                                                                               outPlaneSize);
-                                                               return NULL;
-                                                       }
+                                               *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
+                                               dstp++;
 
-                                                       *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
-                                                       dstp++;
+                                               CopyMemory(dstp, rawValues, cRawBytes);
+                                               dstp += cRawBytes;
 
-                                                       CopyMemory(dstp, rawValues, cRawBytes);
-                                                       dstp += cRawBytes;
+                                               nRunLength -= 32;
+                                               *dstp = PLANAR_CONTROL_BYTE(2, nRunLength);
+                                               dstp++;
+                                       }
+                                       else if (nRunLength > 15)
+                                       {
+                                               nControlBytes = 2;
+                                               outSegmentSize = cRawBytes + nControlBytes;
 
-                                                       nRunLength -= 16;
-                                                       *dstp = PLANAR_CONTROL_BYTE(1, nRunLength);
-                                                       dstp++;
-                                               }
-                                               else
+                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
                                                {
-                                                       nControlBytes = 1;
-                                                       outSegmentSize = cRawBytes + nControlBytes;
+                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                                       return NULL;
+                                               }
 
-                                                       if (((segp - outPlane) + outSegmentSize) > outPlaneSize)
-                                                       {
-                                                               printf("overflow: %d > %d\n",
-                                                                               ((dstp - outPlane) + cRawBytes + 1),
-                                                                               outPlaneSize);
-                                                               return NULL;
-                                                       }
+                                               *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
+                                               dstp++;
+
+                                               CopyMemory(dstp, rawValues, cRawBytes);
+                                               dstp += cRawBytes;
 
-                                                       *dstp = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
-                                                       dstp++;
+                                               nRunLength -= 16;
+                                               *dstp = PLANAR_CONTROL_BYTE(1, nRunLength);
+                                               dstp++;
+                                       }
+                                       else
+                                       {
+                                               nControlBytes = 1;
+                                               outSegmentSize = cRawBytes + nControlBytes;
 
-                                                       CopyMemory(dstp, rawValues, cRawBytes);
-                                                       dstp += cRawBytes;
+                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
+                                               {
+                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                                       return NULL;
                                                }
 
-                                               //printf("Segment %d\n", ++cSegments);
-                                               //winpr_HexDump(segp, dstp - segp);
+                                               *dstp = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
+                                               dstp++;
 
-                                               rawValues = &inPlane[(i * width) + j];
-                                               cRawBytes = 0;
+                                               CopyMemory(dstp, rawValues, cRawBytes);
+                                               dstp += cRawBytes;
                                        }
-                                       else
+
+                                       rawValues = &inPlane[(i * width) + j];
+                                       cRawBytes = 0;
+                               }
+                               else
+                               {
+                                       cRawBytes += (nRunLength + 1);
+
+                                       if (j == width)
                                        {
-                                               cRawBytes += nSequenceLength;
+                                               nRunLength = 0;
 
-                                               if (j == width)
+                                               if (cRawBytes > 15)
                                                {
-                                                       nRunLength = 0;
-
-                                                       if (cRawBytes > 15)
-                                                       {
-                                                               //printf("cRawBytes > 15 unhandled\n");
-                                                               return NULL;
-                                                       }
+                                                       printf("cRawBytes > 15 unhandled\n");
+                                                       return NULL;
+                                               }
 
 #if 0
-                                                       printf("RAW[");
+                                               printf("RAW[");
 
-                                                       for (k = 0; k < cRawBytes; k++)
-                                                       {
-                                                               printf("0x%02X%s", rawValues[k],
-                                                                               ((k + 1) == cRawBytes) ? "" : ", ");
-                                                       }
+                                               for (k = 0; k < cRawBytes; k++)
+                                               {
+                                                       printf("0x%02X%s", rawValues[k],
+                                                                       ((k + 1) == cRawBytes) ? "" : ", ");
+                                               }
 
-                                                       printf("] RUN[%d]\n", nRunLength);
+                                               printf("] RUN[%d]\n", nRunLength);
 #endif
 
-                                                       segp = dstp;
-                                                       outSegmentSize = cRawBytes + 1;
-
-                                                       if (((segp - outPlane) + outSegmentSize) > outPlaneSize)
-                                                       {
-                                                               printf("overflow: %d > %d\n",
-                                                                               ((dstp - outPlane) + cRawBytes + 1),
-                                                                               outPlaneSize);
-                                                               return NULL;
-                                                       }
+                                               outSegmentSize = cRawBytes + 1;
 
-                                                       *dstp = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
-                                                       dstp++;
+                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
+                                               {
+                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                                       return NULL;
+                                               }
 
-                                                       CopyMemory(dstp, rawValues, cRawBytes);
-                                                       dstp += cRawBytes;
+                                               *dstp = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
+                                               dstp++;
 
-                                                       //printf("Segment %d\n", ++cSegments);
-                                                       //winpr_HexDump(segp, dstp - segp);
-                                               }
+                                               CopyMemory(dstp, rawValues, cRawBytes);
+                                               dstp += cRawBytes;
                                        }
+                               }
 
-                                       if (j != width)
-                                       {
-                                               symbol = inPlane[(i * width) + j];
-                                               nSequenceLength = 1;
-                                       }
+                               if (j != width)
+                               {
+                                       symbol = inPlane[(i * width) + j];
+                                       nRunLength = 0;
                                }
                        }
                }
@@ -467,8 +448,6 @@ BYTE* freerdp_bitmap_compress_planar(BYTE* data, UINT32 format, int width, int h
 
                printf("R: [%d/%d] G: [%d/%d] B: [%d/%d]\n",
                                dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
-
-               printf("RDP6 planar compression size: %d width: %d height: %d\n", size, width, height);
        }
 
        if (!dstData)