Fix segfault found by fuzzer.

[platform/core/security/yaca.git] / src / rsa.c

diff --git a/src/rsa.c b/src/rsa.c
index 8d07f11..054db73 100644 (file)
--- a/src/rsa.c
+++ b/src/rsa.c
@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2016 Samsung Electronics Co., Ltd All Rights Reserved
+ *  Copyright (c) 2016-2021 Samsung Electronics Co., Ltd All Rights Reserved
  *
  *  Contact: Krzysztof Jackiewicz <k.jackiewicz@samsung.com>
  *
@@ -34,6 +34,11 @@
 
 #include "internal.h"
 
+#ifdef OPENSSL_MOCKUP_TESTS
+#include "../tests/openssl_mock_redefine.h"
+#endif
+
+
 int rsa_padding2openssl(yaca_padding_e padding)
 {
        switch (padding) {
@@ -50,6 +55,7 @@ int rsa_padding2openssl(yaca_padding_e padding)
        case YACA_PADDING_PKCS1_SSLV23:
                return RSA_SSLV23_PADDING;
        default:
+               assert(false);
                return -1;
        }
 }
@@ -57,12 +63,12 @@ int rsa_padding2openssl(yaca_padding_e padding)
 typedef int (*encrypt_decrypt_fn)(int, const unsigned char*, unsigned char*, RSA*, int);
 
 static int encrypt_decrypt(yaca_padding_e padding,
-                           const yaca_key_h key,
-                           const char *input,
-                           size_t input_len,
-                           char **output,
-                           size_t *output_len,
-                           encrypt_decrypt_fn fn)
+                                                  const yaca_key_h key,
+                                                  const char *input,
+                                                  size_t input_len,
+                                                  char **output,
+                                                  size_t *output_len,
+                                                  encrypt_decrypt_fn fn)
 {
        int ret;
        size_t max_len;
@@ -71,11 +77,10 @@ static int encrypt_decrypt(yaca_padding_e padding,
        int lpadding;
 
        if ((input == NULL && input_len > 0) || (input != NULL && input_len == 0) ||
-           output == NULL || output_len == NULL)
+               output == NULL || output_len == NULL)
                return YACA_ERROR_INVALID_PARAMETER;
 
        lpadding = rsa_padding2openssl(padding);
-       assert(lpadding != -1);
 
        lasym_key = key_get_evp(key);
        assert(lasym_key != NULL);
@@ -89,15 +94,18 @@ static int encrypt_decrypt(yaca_padding_e padding,
 
        max_len = ret;
 
+       if (input_len > max_len)
+               return YACA_ERROR_INVALID_PARAMETER;
+
        ret = yaca_zalloc(max_len, (void**)&loutput);
        if (ret != YACA_ERROR_NONE)
                return ret;
 
        ret = fn(input_len,
-                (const unsigned char*)input,
-                (unsigned char*)loutput,
-                lasym_key->evp->pkey.rsa,
-                lpadding);
+                        (const unsigned char*)input,
+                        (unsigned char*)loutput,
+                        EVP_PKEY_get0_RSA(lasym_key->evp),
+                        lpadding);
 
        if (ret < 0) {
                ret = ERROR_HANDLE();
@@ -121,16 +129,16 @@ exit:
 
 
 API int yaca_rsa_public_encrypt(yaca_padding_e padding,
-                                const yaca_key_h pub_key,
-                                const char *plaintext,
-                                size_t plaintext_len,
-                                char **ciphertext,
-                                size_t *ciphertext_len)
+                                                               const yaca_key_h pub_key,
+                                                               const char *plaintext,
+                                                               size_t plaintext_len,
+                                                               char **ciphertext,
+                                                               size_t *ciphertext_len)
 {
        if (pub_key == YACA_KEY_NULL || pub_key->type != YACA_KEY_TYPE_RSA_PUB)
                return YACA_ERROR_INVALID_PARAMETER;
 
-       switch(padding) {
+       switch (padding) {
        case YACA_PADDING_NONE:
        case YACA_PADDING_PKCS1:
        case YACA_PADDING_PKCS1_OAEP:
@@ -141,25 +149,25 @@ API int yaca_rsa_public_encrypt(yaca_padding_e padding,
        }
 
        return encrypt_decrypt(padding,
-                              pub_key,
-                              plaintext,
-                              plaintext_len,
-                              ciphertext,
-                              ciphertext_len,
-                              RSA_public_encrypt);
+                                                  pub_key,
+                                                  plaintext,
+                                                  plaintext_len,
+                                                  ciphertext,
+                                                  ciphertext_len,
+                                                  RSA_public_encrypt);
 }
 
 API int yaca_rsa_private_decrypt(yaca_padding_e padding,
-                                 const yaca_key_h prv_key,
-                                 const char *ciphertext,
-                                 size_t ciphertext_len,
-                                 char **plaintext,
-                                 size_t *plaintext_len)
+                                                                const yaca_key_h prv_key,
+                                                                const char *ciphertext,
+                                                                size_t ciphertext_len,
+                                                                char **plaintext,
+                                                                size_t *plaintext_len)
 {
        if (prv_key == YACA_KEY_NULL || prv_key->type != YACA_KEY_TYPE_RSA_PRIV)
                return YACA_ERROR_INVALID_PARAMETER;
 
-       switch(padding) {
+       switch (padding) {
        case YACA_PADDING_NONE:
        case YACA_PADDING_PKCS1:
        case YACA_PADDING_PKCS1_OAEP:
@@ -170,25 +178,25 @@ API int yaca_rsa_private_decrypt(yaca_padding_e padding,
        }
 
        return encrypt_decrypt(padding,
-                              prv_key,
-                              ciphertext,
-                              ciphertext_len,
-                              plaintext,
-                              plaintext_len,
-                              RSA_private_decrypt);
+                                                  prv_key,
+                                                  ciphertext,
+                                                  ciphertext_len,
+                                                  plaintext,
+                                                  plaintext_len,
+                                                  RSA_private_decrypt);
 }
 
 API int yaca_rsa_private_encrypt(yaca_padding_e padding,
-                                 const yaca_key_h prv_key,
-                                 const char *plaintext,
-                                 size_t plaintext_len,
-                                 char **ciphertext,
-                                 size_t *ciphertext_len)
+                                                                const yaca_key_h prv_key,
+                                                                const char *plaintext,
+                                                                size_t plaintext_len,
+                                                                char **ciphertext,
+                                                                size_t *ciphertext_len)
 {
        if (prv_key == YACA_KEY_NULL || prv_key->type != YACA_KEY_TYPE_RSA_PRIV)
                return YACA_ERROR_INVALID_PARAMETER;
 
-       switch(padding) {
+       switch (padding) {
        case YACA_PADDING_NONE:
        case YACA_PADDING_PKCS1:
                break;
@@ -197,25 +205,25 @@ API int yaca_rsa_private_encrypt(yaca_padding_e padding,
        }
 
        return encrypt_decrypt(padding,
-                              prv_key,
-                              plaintext,
-                              plaintext_len,
-                              ciphertext,
-                              ciphertext_len,
-                              RSA_private_encrypt);
+                                                  prv_key,
+                                                  plaintext,
+                                                  plaintext_len,
+                                                  ciphertext,
+                                                  ciphertext_len,
+                                                  RSA_private_encrypt);
 }
 
 API int yaca_rsa_public_decrypt(yaca_padding_e padding,
-                                const yaca_key_h pub_key,
-                                const char *ciphertext,
-                                size_t ciphertext_len,
-                                char **plaintext,
-                                size_t *plaintext_len)
+                                                               const yaca_key_h pub_key,
+                                                               const char *ciphertext,
+                                                               size_t ciphertext_len,
+                                                               char **plaintext,
+                                                               size_t *plaintext_len)
 {
        if (pub_key == YACA_KEY_NULL || pub_key->type != YACA_KEY_TYPE_RSA_PUB)
                return YACA_ERROR_INVALID_PARAMETER;
 
-       switch(padding) {
+       switch (padding) {
        case YACA_PADDING_NONE:
        case YACA_PADDING_PKCS1:
                break;
@@ -224,10 +232,10 @@ API int yaca_rsa_public_decrypt(yaca_padding_e padding,
        }
 
        return encrypt_decrypt(padding,
-                              pub_key,
-                              ciphertext,
-                              ciphertext_len,
-                              plaintext,
-                              plaintext_len,
-                              RSA_public_decrypt);
+                                                  pub_key,
+                                                  ciphertext,
+                                                  ciphertext_len,
+                                                  plaintext,
+                                                  plaintext_len,
+                                                  RSA_public_decrypt);
 }