Fix segfault found by fuzzer.
[platform/core/security/yaca.git] / src / rsa.c
1 /*
2  *  Copyright (c) 2016-2021 Samsung Electronics Co., Ltd All Rights Reserved
3  *
4  *  Contact: Krzysztof Jackiewicz <k.jackiewicz@samsung.com>
5  *
6  *  Licensed under the Apache License, Version 2.0 (the "License");
7  *  you may not use this file except in compliance with the License.
8  *  You may obtain a copy of the License at
9  *
10  *      http://www.apache.org/licenses/LICENSE-2.0
11  *
12  *  Unless required by applicable law or agreed to in writing, software
13  *  distributed under the License is distributed on an "AS IS" BASIS,
14  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  *  See the License for the specific language governing permissions and
16  *  limitations under the License
17  */
18
19 /**
20  * @file   rsa.c
21  * @brief  Advanced API for low-level RSA operations
22  */
23
24 #include <assert.h>
25
26 #include <openssl/evp.h>
27 #include <openssl/rsa.h>
28
29 #include <yaca_rsa.h>
30 #include <yaca_error.h>
31 #include <yaca_types.h>
32 #include <yaca_crypto.h>
33 #include <yaca_key.h>
34
35 #include "internal.h"
36
37 #ifdef OPENSSL_MOCKUP_TESTS
38 #include "../tests/openssl_mock_redefine.h"
39 #endif
40
41
42 int rsa_padding2openssl(yaca_padding_e padding)
43 {
44         switch (padding) {
45         case YACA_PADDING_NONE:
46                 return RSA_NO_PADDING;
47         case YACA_PADDING_X931:
48                 return RSA_X931_PADDING;
49         case YACA_PADDING_PKCS1:
50                 return RSA_PKCS1_PADDING;
51         case YACA_PADDING_PKCS1_PSS:
52                 return RSA_PKCS1_PSS_PADDING;
53         case YACA_PADDING_PKCS1_OAEP:
54                 return RSA_PKCS1_OAEP_PADDING;
55         case YACA_PADDING_PKCS1_SSLV23:
56                 return RSA_SSLV23_PADDING;
57         default:
58                 assert(false);
59                 return -1;
60         }
61 }
62
63 typedef int (*encrypt_decrypt_fn)(int, const unsigned char*, unsigned char*, RSA*, int);
64
65 static int encrypt_decrypt(yaca_padding_e padding,
66                                                    const yaca_key_h key,
67                                                    const char *input,
68                                                    size_t input_len,
69                                                    char **output,
70                                                    size_t *output_len,
71                                                    encrypt_decrypt_fn fn)
72 {
73         int ret;
74         size_t max_len;
75         char *loutput = NULL;
76         struct yaca_key_evp_s *lasym_key;
77         int lpadding;
78
79         if ((input == NULL && input_len > 0) || (input != NULL && input_len == 0) ||
80                 output == NULL || output_len == NULL)
81                 return YACA_ERROR_INVALID_PARAMETER;
82
83         lpadding = rsa_padding2openssl(padding);
84
85         lasym_key = key_get_evp(key);
86         assert(lasym_key != NULL);
87
88         ret = EVP_PKEY_size(lasym_key->evp);
89         if (ret <= 0) {
90                 ret = YACA_ERROR_INTERNAL;
91                 ERROR_DUMP(ret);
92                 return ret;
93         }
94
95         max_len = ret;
96
97         if (input_len > max_len)
98                 return YACA_ERROR_INVALID_PARAMETER;
99
100         ret = yaca_zalloc(max_len, (void**)&loutput);
101         if (ret != YACA_ERROR_NONE)
102                 return ret;
103
104         ret = fn(input_len,
105                          (const unsigned char*)input,
106                          (unsigned char*)loutput,
107                          EVP_PKEY_get0_RSA(lasym_key->evp),
108                          lpadding);
109
110         if (ret < 0) {
111                 ret = ERROR_HANDLE();
112                 goto exit;
113         }
114
115         if (ret == 0) {
116                 yaca_free(loutput);
117                 loutput = NULL;
118         }
119
120         *output_len = ret;
121         *output = loutput;
122         loutput = NULL;
123         ret = YACA_ERROR_NONE;
124
125 exit:
126         yaca_free(loutput);
127         return ret;
128 }
129
130
131 API int yaca_rsa_public_encrypt(yaca_padding_e padding,
132                                                                 const yaca_key_h pub_key,
133                                                                 const char *plaintext,
134                                                                 size_t plaintext_len,
135                                                                 char **ciphertext,
136                                                                 size_t *ciphertext_len)
137 {
138         if (pub_key == YACA_KEY_NULL || pub_key->type != YACA_KEY_TYPE_RSA_PUB)
139                 return YACA_ERROR_INVALID_PARAMETER;
140
141         switch (padding) {
142         case YACA_PADDING_NONE:
143         case YACA_PADDING_PKCS1:
144         case YACA_PADDING_PKCS1_OAEP:
145         case YACA_PADDING_PKCS1_SSLV23:
146                 break;
147         default:
148                 return YACA_ERROR_INVALID_PARAMETER;
149         }
150
151         return encrypt_decrypt(padding,
152                                                    pub_key,
153                                                    plaintext,
154                                                    plaintext_len,
155                                                    ciphertext,
156                                                    ciphertext_len,
157                                                    RSA_public_encrypt);
158 }
159
160 API int yaca_rsa_private_decrypt(yaca_padding_e padding,
161                                                                  const yaca_key_h prv_key,
162                                                                  const char *ciphertext,
163                                                                  size_t ciphertext_len,
164                                                                  char **plaintext,
165                                                                  size_t *plaintext_len)
166 {
167         if (prv_key == YACA_KEY_NULL || prv_key->type != YACA_KEY_TYPE_RSA_PRIV)
168                 return YACA_ERROR_INVALID_PARAMETER;
169
170         switch (padding) {
171         case YACA_PADDING_NONE:
172         case YACA_PADDING_PKCS1:
173         case YACA_PADDING_PKCS1_OAEP:
174         case YACA_PADDING_PKCS1_SSLV23:
175                 break;
176         default:
177                 return YACA_ERROR_INVALID_PARAMETER;
178         }
179
180         return encrypt_decrypt(padding,
181                                                    prv_key,
182                                                    ciphertext,
183                                                    ciphertext_len,
184                                                    plaintext,
185                                                    plaintext_len,
186                                                    RSA_private_decrypt);
187 }
188
189 API int yaca_rsa_private_encrypt(yaca_padding_e padding,
190                                                                  const yaca_key_h prv_key,
191                                                                  const char *plaintext,
192                                                                  size_t plaintext_len,
193                                                                  char **ciphertext,
194                                                                  size_t *ciphertext_len)
195 {
196         if (prv_key == YACA_KEY_NULL || prv_key->type != YACA_KEY_TYPE_RSA_PRIV)
197                 return YACA_ERROR_INVALID_PARAMETER;
198
199         switch (padding) {
200         case YACA_PADDING_NONE:
201         case YACA_PADDING_PKCS1:
202                 break;
203         default:
204                 return YACA_ERROR_INVALID_PARAMETER;
205         }
206
207         return encrypt_decrypt(padding,
208                                                    prv_key,
209                                                    plaintext,
210                                                    plaintext_len,
211                                                    ciphertext,
212                                                    ciphertext_len,
213                                                    RSA_private_encrypt);
214 }
215
216 API int yaca_rsa_public_decrypt(yaca_padding_e padding,
217                                                                 const yaca_key_h pub_key,
218                                                                 const char *ciphertext,
219                                                                 size_t ciphertext_len,
220                                                                 char **plaintext,
221                                                                 size_t *plaintext_len)
222 {
223         if (pub_key == YACA_KEY_NULL || pub_key->type != YACA_KEY_TYPE_RSA_PUB)
224                 return YACA_ERROR_INVALID_PARAMETER;
225
226         switch (padding) {
227         case YACA_PADDING_NONE:
228         case YACA_PADDING_PKCS1:
229                 break;
230         default:
231                 return YACA_ERROR_INVALID_PARAMETER;
232         }
233
234         return encrypt_decrypt(padding,
235                                                    pub_key,
236                                                    ciphertext,
237                                                    ciphertext_len,
238                                                    plaintext,
239                                                    plaintext_len,
240                                                    RSA_public_decrypt);
241 }