libfreerdp-codec: detect planar overflow per set of planes, not individual planes
authorMarc-André Moreau <marcandre.moreau@gmail.com>
Wed, 27 Nov 2013 21:23:20 +0000 (16:23 -0500)
committerMarc-André Moreau <marcandre.moreau@gmail.com>
Wed, 27 Nov 2013 21:23:20 +0000 (16:23 -0500)
libfreerdp/codec/planar.c

index 2fe044d..a1f7915 100644 (file)
@@ -87,10 +87,16 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
        int nControlBytes;
 
        cSegments = 0;
-       outPlaneSize = width * height;
 
        if (!outPlane)
+       {
+               outPlaneSize = width * height * 2;
                outPlane = malloc(outPlaneSize);
+       }
+       else
+       {
+               outPlaneSize = *dstSize;
+       }
 
        dstp = outPlane;
 
@@ -112,212 +118,213 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
                        if ((j != width) && (inPlane[(i * width) + j] == symbol))
                        {
                                nRunLength++;
+                               continue;
                        }
-                       else
+
+                       if (nRunLength >= 3)
                        {
-                               if (nRunLength >= 3)
-                               {
-                                       cRawBytes += 1;
+                               cRawBytes += 1;
 
 #if 0
-                                       printf("RAW[");
+                               printf("RAW[");
 
-                                       for (k = 0; k < cRawBytes; k++)
-                                       {
-                                               printf("0x%02X%s", rawValues[k],
-                                                               ((k + 1) == cRawBytes) ? "" : ", ");
-                                       }
+                               for (k = 0; k < cRawBytes; k++)
+                               {
+                                       printf("0x%02X%s", rawValues[k],
+                                                       ((k + 1) == cRawBytes) ? "" : ", ");
+                               }
 
-                                       printf("] RUN[%d]\n", nRunLength);
+                               printf("] RUN[%d]\n", nRunLength);
 #endif
 
-                                       while (cRawBytes > 15)
+                               while (cRawBytes > 15)
+                               {
+                                       printf("handling cRawBytes > 15\n");
+
+                                       nControlBytes = 1;
+                                       outSegmentSize = 15 + nControlBytes;
+
+                                       if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
                                        {
-                                               printf("handling cRawBytes > 15\n");
+                                               printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                               return NULL;
+                                       }
 
-                                               nControlBytes = 1;
-                                               outSegmentSize = 15 + nControlBytes;
+                                       *dstp = PLANAR_CONTROL_BYTE(0, 15);
+                                       dstp++;
 
-                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
-                                               {
-                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
-                                                       return NULL;
-                                               }
+                                       CopyMemory(dstp, rawValues, 15);
+                                       cRawBytes -= 15;
+                                       rawValues += 15;
+                                       dstp += 15;
+                               }
 
-                                               *dstp = PLANAR_CONTROL_BYTE(0, 15);
-                                               dstp++;
+                               if (nRunLength > 47)
+                               {
+                                       nControlBytes = (((nRunLength - 15) + 46) / 47) + 1;
 
-                                               CopyMemory(dstp, rawValues, 15);
-                                               cRawBytes -= 15;
-                                               rawValues += 15;
-                                               dstp += 15;
+                                       outSegmentSize = cRawBytes + nControlBytes;
 
+                                       if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
+                                       {
+                                               printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
                                                return NULL;
                                        }
 
-                                       if (nRunLength > 47)
-                                       {
-                                               nControlBytes = (((nRunLength - 15) + 46) / 47) + 1;
+                                       *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
+                                       nRunLength -= 15;
+                                       nControlBytes--;
+                                       dstp++;
 
-                                               outSegmentSize = cRawBytes + nControlBytes;
+                                       CopyMemory(dstp, rawValues, cRawBytes);
+                                       dstp += cRawBytes;
 
-                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
+                                       while (nControlBytes--)
+                                       {
+                                               if (nRunLength > 47)
                                                {
-                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
-                                                       return NULL;
+                                                       *dstp = PLANAR_CONTROL_BYTE(2, (47 - 32));
+                                                       nRunLength -= 47;
+                                                       dstp++;
                                                }
-
-                                               *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
-                                               nRunLength -= 15;
-                                               nControlBytes--;
-                                               dstp++;
-
-                                               CopyMemory(dstp, rawValues, cRawBytes);
-                                               dstp += cRawBytes;
-
-                                               while (nControlBytes--)
+                                               else if (nRunLength > 31)
                                                {
-                                                       if (nRunLength > 47)
-                                                       {
-                                                               *dstp = PLANAR_CONTROL_BYTE(2, (47 - 32));
-                                                               nRunLength -= 47;
-                                                               dstp++;
-                                                       }
-                                                       else if (nRunLength > 31)
-                                                       {
-                                                               *dstp = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
-                                                               nRunLength = 0;
-                                                               dstp++;
-                                                       }
-                                                       else if (nRunLength > 15)
-                                                       {
-                                                               *dstp = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
-                                                               nRunLength = 0;
-                                                               dstp++;
-                                                       }
-                                                       else
-                                                       {
-                                                               *dstp = PLANAR_CONTROL_BYTE(0, nRunLength);
-                                                               nRunLength = 0;
-                                                               dstp++;
-                                                       }
+                                                       *dstp = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
+                                                       nRunLength = 0;
+                                                       dstp++;
                                                }
-
-                                               return NULL;
-                                       }
-                                       else if (nRunLength > 31)
-                                       {
-                                               nControlBytes = 2;
-                                               outSegmentSize = cRawBytes + nControlBytes;
-
-                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
+                                               else if (nRunLength > 15)
                                                {
-                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
-                                                       return NULL;
+                                                       *dstp = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
+                                                       nRunLength = 0;
+                                                       dstp++;
                                                }
+                                               else
+                                               {
+                                                       *dstp = PLANAR_CONTROL_BYTE(0, nRunLength);
+                                                       nRunLength = 0;
+                                                       dstp++;
+                                               }
+                                       }
 
-                                               *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
-                                               dstp++;
-
-                                               CopyMemory(dstp, rawValues, cRawBytes);
-                                               dstp += cRawBytes;
+                                       return NULL;
+                               }
+                               else if (nRunLength > 31)
+                               {
+                                       nControlBytes = 2;
+                                       outSegmentSize = cRawBytes + nControlBytes;
 
-                                               nRunLength -= 32;
-                                               *dstp = PLANAR_CONTROL_BYTE(2, nRunLength);
-                                               dstp++;
-                                       }
-                                       else if (nRunLength > 15)
+                                       if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
                                        {
-                                               nControlBytes = 2;
-                                               outSegmentSize = cRawBytes + nControlBytes;
+                                               printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                               return NULL;
+                                       }
 
-                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
-                                               {
-                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
-                                                       return NULL;
-                                               }
+                                       *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
+                                       dstp++;
 
-                                               *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
-                                               dstp++;
+                                       CopyMemory(dstp, rawValues, cRawBytes);
+                                       rawValues += cRawBytes;
+                                       dstp += cRawBytes;
+                                       cRawBytes = 0;
 
-                                               CopyMemory(dstp, rawValues, cRawBytes);
-                                               dstp += cRawBytes;
+                                       nRunLength -= 32;
+                                       *dstp = PLANAR_CONTROL_BYTE(2, nRunLength);
+                                       dstp++;
+                               }
+                               else if (nRunLength > 15)
+                               {
+                                       nControlBytes = 2;
+                                       outSegmentSize = cRawBytes + nControlBytes;
 
-                                               nRunLength -= 16;
-                                               *dstp = PLANAR_CONTROL_BYTE(1, nRunLength);
-                                               dstp++;
-                                       }
-                                       else
+                                       if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
                                        {
-                                               nControlBytes = 1;
-                                               outSegmentSize = cRawBytes + nControlBytes;
+                                               printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                               return NULL;
+                                       }
 
-                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
-                                               {
-                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
-                                                       return NULL;
-                                               }
+                                       *dstp = PLANAR_CONTROL_BYTE(15, cRawBytes);
+                                       dstp++;
 
-                                               *dstp = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
-                                               dstp++;
+                                       CopyMemory(dstp, rawValues, cRawBytes);
+                                       rawValues += cRawBytes;
+                                       dstp += cRawBytes;
+                                       cRawBytes = 0;
+
+                                       nRunLength -= 16;
+                                       *dstp = PLANAR_CONTROL_BYTE(1, nRunLength);
+                                       dstp++;
+                               }
+                               else
+                               {
+                                       nControlBytes = 1;
+                                       outSegmentSize = cRawBytes + nControlBytes;
 
-                                               CopyMemory(dstp, rawValues, cRawBytes);
-                                               dstp += cRawBytes;
+                                       if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
+                                       {
+                                               printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                               return NULL;
                                        }
 
-                                       rawValues = &inPlane[(i * width) + j];
+                                       *dstp = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
+                                       dstp++;
+
+                                       CopyMemory(dstp, rawValues, cRawBytes);
+                                       rawValues += cRawBytes;
+                                       dstp += cRawBytes;
                                        cRawBytes = 0;
                                }
-                               else
+                       }
+                       else
+                       {
+                               cRawBytes += (nRunLength + 1);
+
+                               if (j == width)
                                {
-                                       cRawBytes += (nRunLength + 1);
+                                       printf("end of scanline: cRawBytes: %d nRunLength: %d\n", cRawBytes, nRunLength);
 
-                                       if (j == width)
-                                       {
-                                               nRunLength = 0;
+                                       nRunLength = 0;
 
-                                               if (cRawBytes > 15)
-                                               {
-                                                       printf("cRawBytes > 15 unhandled\n");
-                                                       return NULL;
-                                               }
+                                       if (cRawBytes > 15)
+                                       {
+                                               printf("cRawBytes > 15 unhandled\n");
+                                               return NULL;
+                                       }
 
 #if 0
-                                               printf("RAW[");
+                                       printf("RAW[");
 
-                                               for (k = 0; k < cRawBytes; k++)
-                                               {
-                                                       printf("0x%02X%s", rawValues[k],
-                                                                       ((k + 1) == cRawBytes) ? "" : ", ");
-                                               }
+                                       for (k = 0; k < cRawBytes; k++)
+                                       {
+                                               printf("0x%02X%s", rawValues[k],
+                                                               ((k + 1) == cRawBytes) ? "" : ", ");
+                                       }
 
-                                               printf("] RUN[%d]\n", nRunLength);
+                                       printf("] RUN[%d]\n", nRunLength);
 #endif
 
-                                               outSegmentSize = cRawBytes + 1;
+                                       outSegmentSize = cRawBytes + 1;
 
-                                               if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
-                                               {
-                                                       printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
-                                                       return NULL;
-                                               }
+                                       if (((dstp - outPlane) + outSegmentSize) > outPlaneSize)
+                                       {
+                                               printf("overflow: %d > %d\n", ((dstp - outPlane) + outSegmentSize), outPlaneSize);
+                                               return NULL;
+                                       }
 
-                                               *dstp = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
-                                               dstp++;
+                                       *dstp = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
+                                       dstp++;
 
-                                               CopyMemory(dstp, rawValues, cRawBytes);
-                                               dstp += cRawBytes;
-                                       }
+                                       CopyMemory(dstp, rawValues, cRawBytes);
+                                       dstp += cRawBytes;
                                }
+                       }
 
-                               if (j != width)
-                               {
-                                       symbol = inPlane[(i * width) + j];
-                                       nRunLength = 0;
-                               }
+                       if (j != width)
+                       {
+                               symbol = inPlane[(i * width) + j];
+                               nRunLength = 0;
                        }
                }
-
                //printf("---\n");
        }
 
@@ -328,20 +335,39 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
        return outPlane;
 }
 
-int freerdp_bitmap_planar_compress_planes_rle(BYTE* inPlanes[5], int width, int height, BYTE* outPlanes[5], int* dstSizes)
+int freerdp_bitmap_planar_compress_planes_rle(BYTE* inPlanes[5], int width, int height, BYTE* outPlanes, int* dstSizes)
 {
-       if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes[0], &dstSizes[0]))
+       int outPlanesSize = width * height;
+
+       dstSizes[0] = outPlanesSize;
+
+       if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes, &dstSizes[0]))
                return 0;
 
-       if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes[1], &dstSizes[1]))
+       outPlanes += dstSizes[0];
+       outPlanesSize -= dstSizes[0];
+       dstSizes[1] = outPlanesSize;
+
+       if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes, &dstSizes[1]))
                return 0;
 
-       if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes[2], &dstSizes[2]))
+       outPlanes += dstSizes[1];
+       outPlanesSize -= dstSizes[1];
+       dstSizes[2] = outPlanesSize;
+
+       if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes, &dstSizes[2]))
                return 0;
 
-       if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes[3], &dstSizes[3]))
+       outPlanes += dstSizes[2];
+       outPlanesSize -= dstSizes[2];
+       dstSizes[3] = outPlanesSize;
+
+       if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes, &dstSizes[3]))
                return 0;
 
+       outPlanes += dstSizes[3];
+       outPlanesSize -= dstSizes[3];
+
        return 1;
 }
 
@@ -404,12 +430,12 @@ BYTE* freerdp_bitmap_compress_planar(BYTE* data, UINT32 format, int width, int h
        int size;
        BYTE* dstp;
        int planeSize;
-       int dstSizes[5];
+       int dstSizes[4];
        BYTE* planes[5];
        BYTE* planesBuffer;
        BYTE* deltaPlanes[5];
        BYTE* deltaPlanesBuffer;
-       BYTE* rlePlanes[5];
+       BYTE* rlePlanes[4];
        BYTE* rlePlanesBuffer;
        BYTE FormatHeader = 0;
 
@@ -431,21 +457,21 @@ BYTE* freerdp_bitmap_compress_planar(BYTE* data, UINT32 format, int width, int h
        deltaPlanes[3] = &deltaPlanesBuffer[planeSize * 3];
        deltaPlanes[4] = &deltaPlanesBuffer[planeSize * 4];
 
-       rlePlanesBuffer = malloc(planeSize * 5);
-       rlePlanes[0] = &rlePlanesBuffer[planeSize * 0];
-       rlePlanes[1] = &rlePlanesBuffer[planeSize * 1];
-       rlePlanes[2] = &rlePlanesBuffer[planeSize * 2];
-       rlePlanes[3] = &rlePlanesBuffer[planeSize * 3];
-       rlePlanes[4] = &rlePlanesBuffer[planeSize * 4];
+       rlePlanesBuffer = malloc(planeSize * 4);
 
        freerdp_split_color_planes(data, format, width, height, scanline, planes);
 
        freerdp_bitmap_planar_delta_encode_planes(planes, width, height, deltaPlanes);
 
-       if (freerdp_bitmap_planar_compress_planes_rle(deltaPlanes, width, height, rlePlanes, (int*) &dstSizes) > 0)
+       if (freerdp_bitmap_planar_compress_planes_rle(deltaPlanes, width, height, rlePlanesBuffer, (int*) &dstSizes) > 0)
        {
                FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
 
+               rlePlanes[0] = &rlePlanesBuffer[0];
+               rlePlanes[1] = &rlePlanesBuffer[dstSizes[0]];
+               rlePlanes[2] = &rlePlanesBuffer[dstSizes[0] + dstSizes[1]];
+               rlePlanes[3] = &rlePlanesBuffer[dstSizes[0] + dstSizes[1] + dstSizes[2]];
+
                printf("R: [%d/%d] G: [%d/%d] B: [%d/%d]\n",
                                dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
        }