Fixed possible integer overflow in crypto_rsa_common

Thanks @anticomputer for pointing this out

(cherry picked from commit 02c5ec66e5b47fe4cf2fc96e2cd387a18e3f2bb6)

diff --git a/libfreerdp/crypto/crypto.c b/libfreerdp/crypto/crypto.c
index 8414683..29c642e 100644 (file)
--- a/libfreerdp/crypto/crypto.c
+++ b/libfreerdp/crypto/crypto.c
@@ -105,11 +105,18 @@ static int crypto_rsa_common(const BYTE* input, int length, UINT32 key_length, c
        BIGNUM* exp = NULL;
        BIGNUM* x = NULL;
        BIGNUM* y = NULL;
-       size_t bufferSize = 2 * key_length + exponent_size;
+       size_t bufferSize;
 
        if (!input || (length < 0) || (exponent_size < 0) || !modulus || !exponent || !output)
                return -1;
 
+       if (exponent_size > SIZE_MAX / 2)
+               return -1;
+
+       if (key_length >= SIZE_MAX / 2 - exponent_size)
+               return -1;
+
+       bufferSize = 2ULL * key_length + exponent_size;
        if (length > bufferSize)
                bufferSize = length;