From 79d0725a80d3dd17c1dd9cfc6c2b543c909b22e7 Mon Sep 17 00:00:00 2001
From: Ondrej Holy <oholy@redhat.com>
Date: Tue, 21 Aug 2018 09:16:47 +0200
Subject: [PATCH] codec/planar: Fix leak found by covscan

The patch changes API of functions instead of fixing unused and broken code.

leaked_storage: Returning without freeing "outPlane" leaks the storage that it points to.
---
 libfreerdp/codec/planar.c | 25 ++++++-------------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/libfreerdp/codec/planar.c b/libfreerdp/codec/planar.c
index deb2763..fa8152f 100644
--- a/libfreerdp/codec/planar.c
+++ b/libfreerdp/codec/planar.c
@@ -33,7 +33,7 @@
 
 #define TAG FREERDP_TAG("codec")
 
-static INLINE BYTE* freerdp_bitmap_planar_compress_plane_rle(
+static INLINE BOOL freerdp_bitmap_planar_compress_plane_rle(
     const BYTE* plane, UINT32 width, UINT32 height,
     BYTE* outPlane, UINT32* dstSize);
 static INLINE BYTE* freerdp_bitmap_planar_delta_encode_plane(
@@ -948,7 +948,7 @@ static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* pInBuffe
 	return nTotalBytesWritten;
 }
 
-BYTE* freerdp_bitmap_planar_compress_plane_rle(const BYTE* inPlane,
+BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* inPlane,
         UINT32 width, UINT32 height,
         BYTE* outPlane, UINT32* dstSize)
 {
@@ -960,25 +960,12 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(const BYTE* inPlane,
 	UINT32 nTotalBytesWritten;
 
 	if (!outPlane)
-	{
-		outBufferSize = width * height;
-
-		if (outBufferSize == 0)
-			return NULL;
-
-		outPlane = malloc(outBufferSize);
-
-		if (!outPlane)
-			return NULL;
-	}
-	else
-	{
-		outBufferSize = *dstSize;
-	}
+		return FALSE;
 
 	index = 0;
 	pInput = inPlane;
 	pOutput = outPlane;
+	outBufferSize = *dstSize;
 	nTotalBytesWritten = 0;
 
 	while (outBufferSize)
@@ -987,7 +974,7 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(const BYTE* inPlane,
 		                    pInput, width, pOutput, outBufferSize);
 
 		if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
-			return NULL;
+			return FALSE;
 
 		outBufferSize -= nBytesWritten;
 		nTotalBytesWritten += nBytesWritten;
@@ -1000,7 +987,7 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(const BYTE* inPlane,
 	}
 
 	*dstSize = nTotalBytesWritten;
-	return outPlane;
+	return TRUE;
 }
 
 static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(
-- 
2.7.4