Fix segfault found by fuzzer.

Unsigned int(input_len) is casted to int(flen), this can lead to using negative value,
unfortunately openssl doesn't check it.

According to openssl documentation, input_len is limited by RSA key size,
let's validate it in yaca to avoid segfault.

Change-Id: I8e821b94794f1b5d7231df16c591fe88c12c84e2

diff --git a/src/rsa.c b/src/rsa.c
index cbd951b..054db73 100644 (file)
--- a/src/rsa.c
+++ b/src/rsa.c
@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2016-2020 Samsung Electronics Co., Ltd All Rights Reserved
+ *  Copyright (c) 2016-2021 Samsung Electronics Co., Ltd All Rights Reserved
  *
  *  Contact: Krzysztof Jackiewicz <k.jackiewicz@samsung.com>
  *
@@ -94,6 +94,9 @@ static int encrypt_decrypt(yaca_padding_e padding,
 
        max_len = ret;
 
+       if (input_len > max_len)
+               return YACA_ERROR_INVALID_PARAMETER;
+
        ret = yaca_zalloc(max_len, (void**)&loutput);
        if (ret != YACA_ERROR_NONE)
                return ret;

diff --git a/tests/test_rsa.cpp b/tests/test_rsa.cpp
index 0f9e095..105c77c 100644 (file)
--- a/tests/test_rsa.cpp
+++ b/tests/test_rsa.cpp
@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2020 Samsung Electronics Co., Ltd All Rights Reserved
+ *  Copyright (c) 2020-2021 Samsung Electronics Co., Ltd All Rights Reserved
  *
  *  Contact: Lukasz Pawelczyk <l.pawelczyk@samsung.com>
  *
@@ -24,6 +24,7 @@
 
 #include <boost/test/unit_test.hpp>
 #include <vector>
+#include <limits>
 
 #include <yaca_crypto.h>
 #include <yaca_rsa.h>
@@ -452,6 +453,11 @@ BOOST_FIXTURE_TEST_CASE(T404__negative__public_encrypt, InitDebugFixture)
        BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
 
        ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
+                                                                 INPUT_DATA, UINT_MAX,
+                                                                 &encrypted, &encrypted_len);
+       BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
+
+       ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
                                                                  INPUT_DATA, input_len,
                                                                  NULL, &encrypted_len);
        BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
@@ -466,6 +472,11 @@ BOOST_FIXTURE_TEST_CASE(T404__negative__public_encrypt, InitDebugFixture)
                                                                  &encrypted_pkcs1, &encrypted_pkcs1_len);
        BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
 
+       ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1, key_pub,
+                                                                 INPUT_DATA, UINT_MAX,
+                                                                 &encrypted_pkcs1, &encrypted_pkcs1_len);
+       BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
+
        ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1_OAEP, key_pub,
                                                                  INPUT_DATA, input_len_pkcs1_oaep + 1,
                                                                  &encrypted_pkcs1_oaep, &encrypted_pkcs1_oaep_len);