From cb2adc84c0a69997c78709f20e3959b986559a7e Mon Sep 17 00:00:00 2001 From: akallabeth Date: Mon, 15 Jun 2020 08:57:21 +0200 Subject: [PATCH] Fixed possible integer overflow in crypto_rsa_common Thanks @anticomputer for pointing this out (cherry picked from commit 02c5ec66e5b47fe4cf2fc96e2cd387a18e3f2bb6) --- libfreerdp/crypto/crypto.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libfreerdp/crypto/crypto.c b/libfreerdp/crypto/crypto.c index 8414683..29c642e 100644 --- a/libfreerdp/crypto/crypto.c +++ b/libfreerdp/crypto/crypto.c @@ -105,11 +105,18 @@ static int crypto_rsa_common(const BYTE* input, int length, UINT32 key_length, c BIGNUM* exp = NULL; BIGNUM* x = NULL; BIGNUM* y = NULL; - size_t bufferSize = 2 * key_length + exponent_size; + size_t bufferSize; if (!input || (length < 0) || (exponent_size < 0) || !modulus || !exponent || !output) return -1; + if (exponent_size > SIZE_MAX / 2) + return -1; + + if (key_length >= SIZE_MAX / 2 - exponent_size) + return -1; + + bufferSize = 2ULL * key_length + exponent_size; if (length > bufferSize) bufferSize = length; -- 2.7.4