Fix segfault found by fuzzer.
[platform/core/security/yaca.git] / tests / test_rsa.cpp
1 /*
2  *  Copyright (c) 2020-2021 Samsung Electronics Co., Ltd All Rights Reserved
3  *
4  *  Contact: Lukasz Pawelczyk <l.pawelczyk@samsung.com>
5  *
6  *  Licensed under the Apache License, Version 2.0 (the "License");
7  *  you may not use this file except in compliance with the License.
8  *  You may obtain a copy of the License at
9  *
10  *      http://www.apache.org/licenses/LICENSE-2.0
11  *
12  *  Unless required by applicable law or agreed to in writing, software
13  *  distributed under the License is distributed on an "AS IS" BASIS,
14  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  *  See the License for the specific language governing permissions and
16  *  limitations under the License
17  */
18
19 /**
20  * @file    test_rsa.c
21  * @author  Lukasz Pawelczyk <l.pawelczyk@samsung.com>
22  * @brief   RSA API unit tests.
23  */
24
25 #include <boost/test/unit_test.hpp>
26 #include <vector>
27 #include <limits>
28
29 #include <yaca_crypto.h>
30 #include <yaca_rsa.h>
31 #include <yaca_key.h>
32 #include <yaca_error.h>
33
34 #include "common.h"
35
36
37 BOOST_AUTO_TEST_SUITE(TESTS_RSA)
38
39 BOOST_FIXTURE_TEST_CASE(T401__positive__private_encrypt, InitDebugFixture)
40 {
41         struct rsa_args {
42                 yaca_padding_e pad;
43                 yaca_key_bit_length_e bit_len;
44                 size_t shorter;
45         };
46
47         const std::vector<struct rsa_args> rargs = {
48                 {YACA_PADDING_NONE,         YACA_KEY_LENGTH_512BIT,   0},
49                 {YACA_PADDING_NONE,         YACA_KEY_LENGTH_2048BIT,  0},
50                 {YACA_PADDING_PKCS1,        YACA_KEY_LENGTH_512BIT,  11},
51                 {YACA_PADDING_PKCS1,        YACA_KEY_LENGTH_4096BIT, 11},
52         };
53
54         for (const auto &ra: rargs) {
55                 int ret;
56                 yaca_key_h rsa_prv = YACA_KEY_NULL, rsa_pub = YACA_KEY_NULL;
57                 size_t input_len = ra.bit_len / 8 - ra.shorter;
58                 char *encrypted = NULL, *decrypted = NULL;
59                 size_t encrypted_len, decrypted_len;
60
61                 generate_asymmetric_keys(YACA_KEY_TYPE_RSA_PRIV, ra.bit_len, &rsa_prv, &rsa_pub);
62
63                 ret = yaca_rsa_private_encrypt(ra.pad, rsa_prv, INPUT_DATA, input_len,
64                                                                            &encrypted, &encrypted_len);
65                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
66
67                 ret = yaca_rsa_public_decrypt(ra.pad, rsa_pub, encrypted, encrypted_len,
68                                                                           &decrypted, &decrypted_len);
69                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
70
71                 BOOST_REQUIRE(decrypted_len == input_len);
72                 ret = yaca_memcmp(decrypted, INPUT_DATA, input_len);
73                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
74
75                 yaca_key_destroy(rsa_prv);
76                 yaca_key_destroy(rsa_pub);
77                 yaca_free(encrypted);
78                 yaca_free(decrypted);
79         }
80
81         /* Empty input, must use padding */
82         for (const auto &ra: rargs) {
83                 if (ra.pad == YACA_PADDING_NONE)
84                         continue;
85
86                 int ret;
87                 yaca_key_h rsa_prv = YACA_KEY_NULL, rsa_pub = YACA_KEY_NULL;
88                 char *encrypted = NULL, *decrypted = NULL;
89                 size_t encrypted_len, decrypted_len;
90
91                 generate_asymmetric_keys(YACA_KEY_TYPE_RSA_PRIV, ra.bit_len, &rsa_prv, &rsa_pub);
92
93                 ret = yaca_rsa_private_encrypt(ra.pad, rsa_prv, NULL, 0,
94                                                                            &encrypted, &encrypted_len);
95                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
96
97                 ret = yaca_rsa_public_decrypt(ra.pad, rsa_pub, encrypted, encrypted_len,
98                                                                           &decrypted, &decrypted_len);
99                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
100
101                 BOOST_REQUIRE(decrypted == NULL);
102                 BOOST_REQUIRE(decrypted_len == 0);
103
104                 yaca_key_destroy(rsa_prv);
105                 yaca_key_destroy(rsa_pub);
106                 yaca_free(encrypted);
107                 yaca_free(decrypted);
108         }
109 }
110
111 BOOST_FIXTURE_TEST_CASE(T402__negative__private_encrypt, InitDebugFixture)
112 {
113         int ret;
114         size_t bit_len = YACA_KEY_LENGTH_1024BIT;
115         yaca_key_h key_prv = YACA_KEY_NULL, key_pub = YACA_KEY_NULL;
116         yaca_key_h key_prv2 = YACA_KEY_NULL, key_pub2 = YACA_KEY_NULL;
117         yaca_key_h key_dsa = YACA_KEY_NULL, key_ec = YACA_KEY_NULL;
118         size_t input_len = bit_len / 8;
119         size_t input_len_pkcs1 = bit_len / 8 - 11;
120         char *encrypted = NULL, *encrypted_pkcs1 = NULL, *decrypted = NULL;
121         size_t encrypted_len, encrypted_pkcs1_len, decrypted_len;
122
123         generate_asymmetric_keys(YACA_KEY_TYPE_RSA_PRIV, bit_len, &key_prv, &key_pub);
124         generate_asymmetric_keys(YACA_KEY_TYPE_RSA_PRIV, bit_len, &key_prv2, &key_pub2);
125         generate_asymmetric_keys(YACA_KEY_TYPE_DSA_PRIV, bit_len, &key_dsa);
126         generate_asymmetric_keys(YACA_KEY_TYPE_EC_PRIV, YACA_KEY_LENGTH_EC_PRIME256V1, &key_ec);
127
128         ret = yaca_rsa_private_encrypt(YACA_INVALID_PADDING, key_prv,
129                                                                    INPUT_DATA, input_len,
130                                                                    &encrypted, &encrypted_len);
131         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
132
133         ret = yaca_rsa_private_encrypt(YACA_PADDING_X931, key_prv,
134                                                                    INPUT_DATA, input_len,
135                                                                    &encrypted, &encrypted_len);
136         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
137
138         ret = yaca_rsa_private_encrypt(YACA_PADDING_PKCS1_OAEP, key_prv,
139                                                                    INPUT_DATA, input_len,
140                                                                    &encrypted, &encrypted_len);
141         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
142
143         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, YACA_KEY_NULL,
144                                                                    INPUT_DATA, input_len,
145                                                                    &encrypted, &encrypted_len);
146         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
147
148         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_pub,
149                                                                    INPUT_DATA, input_len,
150                                                                    &encrypted, &encrypted_len);
151         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
152
153         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_ec,
154                                                                    INPUT_DATA, input_len,
155                                                                    &encrypted, &encrypted_len);
156         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
157
158         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_dsa,
159                                                                    INPUT_DATA, input_len,
160                                                                    &encrypted, &encrypted_len);
161         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
162
163         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_prv,
164                                                                    NULL, input_len,
165                                                                    &encrypted, &encrypted_len);
166         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
167
168         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_prv,
169                                                                    INPUT_DATA, 0,
170                                                                    &encrypted, &encrypted_len);
171         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
172
173         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_prv,
174                                                                    INPUT_DATA, input_len + 1,
175                                                                    &encrypted, &encrypted_len);
176         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
177
178         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_prv,
179                                                                    INPUT_DATA, input_len - 1,
180                                                                    &encrypted, &encrypted_len);
181         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
182
183         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_prv,
184                                                                    INPUT_DATA, input_len - 8,
185                                                                    &encrypted, &encrypted_len);
186         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
187
188         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_prv,
189                                                                    INPUT_DATA, input_len,
190                                                                    NULL, &encrypted_len);
191         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
192
193         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_prv,
194                                                                    INPUT_DATA, input_len,
195                                                                    &encrypted, NULL);
196         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
197
198         ret = yaca_rsa_private_encrypt(YACA_PADDING_PKCS1, key_prv,
199                                                                    INPUT_DATA, input_len_pkcs1 + 1,
200                                                                    &encrypted_pkcs1, &encrypted_pkcs1_len);
201         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
202
203         ret = yaca_rsa_private_encrypt(YACA_PADDING_NONE, key_prv,
204                                                                    INPUT_DATA, input_len,
205                                                                    &encrypted, &encrypted_len);
206         BOOST_REQUIRE(ret == YACA_ERROR_NONE);
207         ret = yaca_rsa_private_encrypt(YACA_PADDING_PKCS1, key_prv,
208                                                                    INPUT_DATA, input_len_pkcs1,
209                                                                    &encrypted_pkcs1, &encrypted_pkcs1_len);
210         BOOST_REQUIRE(ret == YACA_ERROR_NONE);
211
212         ret = yaca_rsa_public_decrypt(YACA_INVALID_PADDING, key_pub,
213                                                                   encrypted, encrypted_len,
214                                                                   &decrypted, &decrypted_len);
215         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
216
217         ret = yaca_rsa_public_decrypt(YACA_PADDING_PKCS1_SSLV23, key_pub,
218                                                                   encrypted, encrypted_len,
219                                                                   &decrypted, &decrypted_len);
220         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
221
222         ret = yaca_rsa_public_decrypt(YACA_PADDING_PKCS1, key_pub,
223                                                                   encrypted, encrypted_len,
224                                                                   &decrypted, &decrypted_len);
225         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
226
227         ret = yaca_rsa_public_decrypt(YACA_PADDING_NONE, YACA_KEY_NULL,
228                                                                   encrypted, encrypted_len,
229                                                                   &decrypted, &decrypted_len);
230         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
231
232         ret = yaca_rsa_public_decrypt(YACA_PADDING_NONE, key_prv,
233                                                                   encrypted, encrypted_len,
234                                                                   &decrypted, &decrypted_len);
235         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
236
237         ret = yaca_rsa_public_decrypt(YACA_PADDING_NONE, key_ec,
238                                                                   encrypted, encrypted_len,
239                                                                   &decrypted, &decrypted_len);
240         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
241
242         ret = yaca_rsa_public_decrypt(YACA_PADDING_NONE, key_dsa,
243                                                                   encrypted, encrypted_len,
244                                                                   &decrypted, &decrypted_len);
245         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
246
247         ret = yaca_rsa_public_decrypt(YACA_PADDING_NONE, key_pub,
248                                                                   NULL, encrypted_len,
249                                                                   &decrypted, &decrypted_len);
250         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
251
252         ret = yaca_rsa_public_decrypt(YACA_PADDING_NONE, key_pub,
253                                                                   encrypted, 0,
254                                                                   &decrypted, &decrypted_len);
255         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
256
257         ret = yaca_rsa_public_decrypt(YACA_PADDING_NONE, key_pub,
258                                                                   encrypted, encrypted_len,
259                                                                   NULL, &decrypted_len);
260         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
261
262         ret = yaca_rsa_public_decrypt(YACA_PADDING_NONE, key_pub,
263                                                                   encrypted, encrypted_len,
264                                                                   &decrypted, NULL);
265         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
266
267         ret = yaca_rsa_public_decrypt(YACA_PADDING_PKCS1_PSS, key_pub,
268                                                                   encrypted_pkcs1, encrypted_pkcs1_len,
269                                                                   &decrypted, &decrypted_len);
270         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
271
272         ret = yaca_rsa_public_decrypt(YACA_PADDING_X931, key_pub,
273                                                                   encrypted_pkcs1, encrypted_pkcs1_len,
274                                                                   &decrypted, &decrypted_len);
275         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
276
277         ret = yaca_rsa_public_decrypt(YACA_PADDING_PKCS1, key_pub2,
278                                                                   encrypted_pkcs1, encrypted_pkcs1_len,
279                                                                   &decrypted, &decrypted_len);
280         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
281
282         ret = yaca_rsa_public_decrypt(YACA_PADDING_PKCS1, key_pub,
283                                                                   encrypted_pkcs1, encrypted_pkcs1_len - 1,
284                                                                   &decrypted, &decrypted_len);
285         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
286
287         ret = yaca_rsa_public_decrypt(YACA_PADDING_PKCS1, key_pub,
288                                                                   encrypted_pkcs1 + 1, encrypted_pkcs1_len - 1,
289                                                                   &decrypted, &decrypted_len);
290         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
291
292         yaca_key_destroy(key_prv);
293         yaca_key_destroy(key_pub);
294         yaca_key_destroy(key_prv2);
295         yaca_key_destroy(key_pub2);
296         yaca_key_destroy(key_dsa);
297         yaca_key_destroy(key_ec);
298         yaca_free(encrypted);
299         yaca_free(encrypted_pkcs1);
300 }
301
302 BOOST_FIXTURE_TEST_CASE(T403__positive__public_encrypt, InitDebugFixture)
303 {
304         struct rsa_args {
305                 yaca_padding_e pad;
306                 yaca_key_bit_length_e bit_len;
307                 size_t shorter;
308         };
309
310         const std::vector<struct rsa_args> rargs = {
311                 {YACA_PADDING_NONE,         YACA_KEY_LENGTH_512BIT,   0},
312                 {YACA_PADDING_PKCS1,        YACA_KEY_LENGTH_1024BIT, 11},
313                 {YACA_PADDING_PKCS1_OAEP,   YACA_KEY_LENGTH_2048BIT, 42},
314         };
315
316         for (const auto &ra: rargs) {
317                 int ret;
318                 yaca_key_h rsa_prv = YACA_KEY_NULL, rsa_pub = YACA_KEY_NULL;
319                 size_t input_len = ra.bit_len / 8 - ra.shorter;
320                 char *encrypted = NULL, *decrypted = NULL;
321                 size_t encrypted_len, decrypted_len;
322
323                 generate_asymmetric_keys(YACA_KEY_TYPE_RSA_PRIV, ra.bit_len, &rsa_prv, &rsa_pub);
324
325                 ret = yaca_rsa_public_encrypt(ra.pad, rsa_pub, INPUT_DATA, input_len,
326                                                                           &encrypted, &encrypted_len);
327                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
328
329                 ret = yaca_rsa_private_decrypt(ra.pad, rsa_prv, encrypted, encrypted_len,
330                                                                            &decrypted, &decrypted_len);
331                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
332
333                 BOOST_REQUIRE(decrypted_len == input_len);
334                 ret = yaca_memcmp(decrypted, INPUT_DATA, input_len);
335                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
336
337                 yaca_key_destroy(rsa_prv);
338                 yaca_key_destroy(rsa_pub);
339                 yaca_free(encrypted);
340                 yaca_free(decrypted);
341         }
342
343         /* Empty input, must use padding */
344         for (const auto &ra: rargs) {
345                 if (ra.pad == YACA_PADDING_NONE)
346                         continue;
347
348                 int ret;
349                 yaca_key_h rsa_prv = YACA_KEY_NULL, rsa_pub = YACA_KEY_NULL;
350                 char *encrypted = NULL, *decrypted = NULL;
351                 size_t encrypted_len, decrypted_len;
352
353                 generate_asymmetric_keys(YACA_KEY_TYPE_RSA_PRIV, ra.bit_len, &rsa_prv, &rsa_pub);
354
355                 ret = yaca_rsa_public_encrypt(ra.pad, rsa_pub, NULL, 0,
356                                                                           &encrypted, &encrypted_len);
357                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
358
359                 ret = yaca_rsa_private_decrypt(ra.pad, rsa_prv, encrypted, encrypted_len,
360                                                                            &decrypted, &decrypted_len);
361                 BOOST_REQUIRE(ret == YACA_ERROR_NONE);
362
363                 BOOST_REQUIRE(decrypted_len == 0);
364                 BOOST_REQUIRE(decrypted == NULL);
365
366                 yaca_key_destroy(rsa_prv);
367                 yaca_key_destroy(rsa_pub);
368                 yaca_free(encrypted);
369                 yaca_free(decrypted);
370         }
371 }
372
373 BOOST_FIXTURE_TEST_CASE(T404__negative__public_encrypt, InitDebugFixture)
374 {
375         int ret;
376         size_t bit_len = YACA_KEY_LENGTH_1024BIT;
377         yaca_key_h key_prv = YACA_KEY_NULL, key_pub = YACA_KEY_NULL;
378         yaca_key_h key_prv2 = YACA_KEY_NULL, key_pub2 = YACA_KEY_NULL;
379         yaca_key_h key_dsa = YACA_KEY_NULL, key_ec = YACA_KEY_NULL;
380         size_t input_len = bit_len / 8;
381         size_t input_len_pkcs1 = bit_len / 8 - 11;
382         size_t input_len_pkcs1_oaep = bit_len / 8 - 42;
383         size_t input_len_pkcs1_sslv23 = bit_len / 8 - 11;
384         char *encrypted = NULL, *encrypted_pkcs1 = NULL;
385         char *encrypted_pkcs1_oaep = NULL, *encrypted_pkcs1_sslv23 = NULL;
386         char *decrypted = NULL;
387         size_t encrypted_len, encrypted_pkcs1_len, encrypted_pkcs1_oaep_len;
388         size_t encrypted_pkcs1_sslv23_len, decrypted_len;
389
390         generate_asymmetric_keys(YACA_KEY_TYPE_RSA_PRIV, bit_len, &key_prv, &key_pub);
391         generate_asymmetric_keys(YACA_KEY_TYPE_RSA_PRIV, bit_len, &key_prv2, &key_pub2);
392         generate_asymmetric_keys(YACA_KEY_TYPE_DSA_PRIV, bit_len, &key_dsa);
393         generate_asymmetric_keys(YACA_KEY_TYPE_EC_PRIV, YACA_KEY_LENGTH_EC_PRIME256V1, &key_ec);
394
395         ret = yaca_rsa_public_encrypt(YACA_INVALID_PADDING, key_pub,
396                                                                   INPUT_DATA, input_len,
397                                                                   &encrypted, &encrypted_len);
398         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
399
400         ret = yaca_rsa_public_encrypt(YACA_PADDING_X931, key_pub,
401                                                                   INPUT_DATA, input_len,
402                                                                   &encrypted, &encrypted_len);
403         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
404
405         ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1_PSS, key_pub,
406                                                                   INPUT_DATA, input_len,
407                                                                   &encrypted, &encrypted_len);
408         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
409
410         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, YACA_KEY_NULL,
411                                                                   INPUT_DATA, input_len,
412                                                                   &encrypted, &encrypted_len);
413         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
414
415         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_prv,
416                                                                   INPUT_DATA, input_len,
417                                                                   &encrypted, &encrypted_len);
418         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
419
420         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_ec,
421                                                                   INPUT_DATA, input_len,
422                                                                   &encrypted, &encrypted_len);
423         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
424
425         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_dsa,
426                                                                   INPUT_DATA, input_len,
427                                                                   &encrypted, &encrypted_len);
428         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
429
430         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
431                                                                   NULL, input_len,
432                                                                   &encrypted, &encrypted_len);
433         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
434
435         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
436                                                                   INPUT_DATA, 0,
437                                                                   &encrypted, &encrypted_len);
438         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
439
440         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
441                                                                   INPUT_DATA, input_len + 1,
442                                                                   &encrypted, &encrypted_len);
443         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
444
445         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
446                                                                   INPUT_DATA, input_len - 1,
447                                                                   &encrypted, &encrypted_len);
448         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
449
450         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
451                                                                   INPUT_DATA, input_len - 8,
452                                                                   &encrypted, &encrypted_len);
453         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
454
455         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
456                                                                   INPUT_DATA, UINT_MAX,
457                                                                   &encrypted, &encrypted_len);
458         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
459
460         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
461                                                                   INPUT_DATA, input_len,
462                                                                   NULL, &encrypted_len);
463         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
464
465         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
466                                                                   INPUT_DATA, input_len,
467                                                                   &encrypted, NULL);
468         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
469
470         ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1, key_pub,
471                                                                   INPUT_DATA, input_len_pkcs1 + 1,
472                                                                   &encrypted_pkcs1, &encrypted_pkcs1_len);
473         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
474
475         ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1, key_pub,
476                                                                   INPUT_DATA, UINT_MAX,
477                                                                   &encrypted_pkcs1, &encrypted_pkcs1_len);
478         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
479
480         ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1_OAEP, key_pub,
481                                                                   INPUT_DATA, input_len_pkcs1_oaep + 1,
482                                                                   &encrypted_pkcs1_oaep, &encrypted_pkcs1_oaep_len);
483         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
484
485         ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1_SSLV23, key_pub,
486                                                                   INPUT_DATA, input_len_pkcs1_sslv23 + 1,
487                                                                   &encrypted_pkcs1_sslv23, &encrypted_pkcs1_sslv23_len);
488         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
489
490         ret = yaca_rsa_public_encrypt(YACA_PADDING_NONE, key_pub,
491                                                                   INPUT_DATA, input_len,
492                                                                   &encrypted, &encrypted_len);
493         BOOST_REQUIRE(ret == YACA_ERROR_NONE);
494         ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1, key_pub,
495                                                                   INPUT_DATA, input_len_pkcs1,
496                                                                   &encrypted_pkcs1, &encrypted_pkcs1_len);
497         BOOST_REQUIRE(ret == YACA_ERROR_NONE);
498         ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1_OAEP, key_pub,
499                                                                   INPUT_DATA, input_len_pkcs1_oaep,
500                                                                   &encrypted_pkcs1_oaep, &encrypted_pkcs1_oaep_len);
501         BOOST_REQUIRE(ret == YACA_ERROR_NONE);
502         ret = yaca_rsa_public_encrypt(YACA_PADDING_PKCS1_SSLV23, key_pub,
503                                                                   INPUT_DATA, input_len_pkcs1_sslv23,
504                                                                   &encrypted_pkcs1_sslv23, &encrypted_pkcs1_sslv23_len);
505         BOOST_REQUIRE(ret == YACA_ERROR_NONE);
506
507         ret = yaca_rsa_private_decrypt(YACA_INVALID_PADDING, key_prv,
508                                                                    encrypted, encrypted_len,
509                                                                    &decrypted, &decrypted_len);
510         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
511
512         ret = yaca_rsa_private_decrypt(YACA_PADDING_X931, key_prv,
513                                                                    encrypted, encrypted_len,
514                                                                    &decrypted, &decrypted_len);
515         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
516
517         ret = yaca_rsa_private_decrypt(YACA_PADDING_PKCS1, key_prv,
518                                                                    encrypted, encrypted_len,
519                                                                    &decrypted, &decrypted_len);
520         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
521
522         ret = yaca_rsa_private_decrypt(YACA_PADDING_NONE, YACA_KEY_NULL,
523                                                                    encrypted, encrypted_len,
524                                                                    &decrypted, &decrypted_len);
525         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
526
527         ret = yaca_rsa_private_decrypt(YACA_PADDING_NONE, key_pub,
528                                                                    encrypted, encrypted_len,
529                                                                    &decrypted, &decrypted_len);
530         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
531
532         ret = yaca_rsa_private_decrypt(YACA_PADDING_NONE, key_ec,
533                                                                    encrypted, encrypted_len,
534                                                                    &decrypted, &decrypted_len);
535         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
536
537         ret = yaca_rsa_private_decrypt(YACA_PADDING_NONE, key_dsa,
538                                                                    encrypted, encrypted_len,
539                                                                    &decrypted, &decrypted_len);
540         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
541
542         ret = yaca_rsa_private_decrypt(YACA_PADDING_NONE, key_prv,
543                                                                    NULL, encrypted_len,
544                                                                    &decrypted, &decrypted_len);
545         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
546
547         ret = yaca_rsa_private_decrypt(YACA_PADDING_NONE, key_prv,
548                                                                    encrypted, 0,
549                                                                    &decrypted, &decrypted_len);
550         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
551
552         ret = yaca_rsa_private_decrypt(YACA_PADDING_NONE, key_prv,
553                                                                    encrypted, encrypted_len,
554                                                                    NULL, &decrypted_len);
555         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
556
557         ret = yaca_rsa_private_decrypt(YACA_PADDING_NONE, key_prv,
558                                                                    encrypted, encrypted_len,
559                                                                    &decrypted, NULL);
560         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
561
562         ret = yaca_rsa_private_decrypt(YACA_PADDING_PKCS1_PSS, key_prv,
563                                                                    encrypted_pkcs1, encrypted_pkcs1_len,
564                                                                    &decrypted, &decrypted_len);
565         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
566
567         ret = yaca_rsa_private_decrypt(YACA_PADDING_PKCS1, key_prv,
568                                                                    encrypted_pkcs1, encrypted_pkcs1_len - 1,
569                                                                    &decrypted, &decrypted_len);
570         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
571
572         ret = yaca_rsa_private_decrypt(YACA_PADDING_PKCS1, key_prv,
573                                                                    encrypted_pkcs1_oaep, encrypted_pkcs1_oaep_len,
574                                                                    &decrypted, &decrypted_len);
575         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
576
577         ret = yaca_rsa_private_decrypt(YACA_PADDING_PKCS1_SSLV23, key_prv,
578                                                                    encrypted_pkcs1_oaep, encrypted_pkcs1_oaep_len,
579                                                                    &decrypted, &decrypted_len);
580         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
581
582         ret = yaca_rsa_private_decrypt(YACA_PADDING_PKCS1_OAEP, key_prv,
583                                                                    encrypted_pkcs1_oaep, encrypted_pkcs1_oaep_len - 1,
584                                                                    &decrypted, &decrypted_len);
585         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
586
587         ret = yaca_rsa_private_decrypt(YACA_PADDING_PKCS1_OAEP, key_prv,
588                                                                    encrypted_pkcs1_sslv23, encrypted_pkcs1_sslv23_len,
589                                                                    &decrypted, &decrypted_len);
590         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
591
592         ret = yaca_rsa_private_decrypt(YACA_PADDING_PKCS1_SSLV23, key_prv,
593                                                                    encrypted_pkcs1_sslv23, encrypted_pkcs1_sslv23_len - 1,
594                                                                    &decrypted, &decrypted_len);
595         BOOST_REQUIRE(ret == YACA_ERROR_INVALID_PARAMETER);
596
597         yaca_key_destroy(key_prv);
598         yaca_key_destroy(key_pub);
599         yaca_key_destroy(key_prv2);
600         yaca_key_destroy(key_pub2);
601         yaca_key_destroy(key_dsa);
602         yaca_key_destroy(key_ec);
603         yaca_free(encrypted);
604         yaca_free(encrypted_pkcs1);
605         yaca_free(encrypted_pkcs1_oaep);
606         yaca_free(encrypted_pkcs1_sslv23);
607 }
608
609 BOOST_AUTO_TEST_SUITE_END()