From: akallabeth <akallabeth@posteo.net>
Date: Mon, 15 Jun 2020 06:57:21 +0000 (+0200)
Subject: Fixed possible integer overflow in crypto_rsa_common
X-Git-Tag: 2.1.2^2~2
X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=cb2adc84c0a69997c78709f20e3959b986559a7e;p=platform%2Fupstream%2Ffreerdp.git

Fixed possible integer overflow in crypto_rsa_common

Thanks @anticomputer for pointing this out

(cherry picked from commit 02c5ec66e5b47fe4cf2fc96e2cd387a18e3f2bb6)
---

diff --git a/libfreerdp/crypto/crypto.c b/libfreerdp/crypto/crypto.c
index 8414683..29c642e 100644
--- a/libfreerdp/crypto/crypto.c
+++ b/libfreerdp/crypto/crypto.c
@@ -105,11 +105,18 @@ static int crypto_rsa_common(const BYTE* input, int length, UINT32 key_length, c
 	BIGNUM* exp = NULL;
 	BIGNUM* x = NULL;
 	BIGNUM* y = NULL;
-	size_t bufferSize = 2 * key_length + exponent_size;
+	size_t bufferSize;
 
 	if (!input || (length < 0) || (exponent_size < 0) || !modulus || !exponent || !output)
 		return -1;
 
+	if (exponent_size > SIZE_MAX / 2)
+		return -1;
+
+	if (key_length >= SIZE_MAX / 2 - exponent_size)
+		return -1;
+
+	bufferSize = 2ULL * key_length + exponent_size;
 	if (length > bufferSize)
 		bufferSize = length;