Fix CVE-2017-6891 in minitasn1 code
[platform/upstream/gnutls.git] / lib / gnutls_hash_int.c
1 /*
2  * Copyright (C) 2000-2012 Free Software Foundation, Inc.
3  *
4  * Author: Nikos Mavrogiannopoulos
5  *
6  * This file is part of GnuTLS.
7  *
8  * The GnuTLS is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public License
10  * as published by the Free Software Foundation; either version 2.1 of
11  * the License, or (at your option) any later version.
12  *
13  * This library is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public License
19  * along with this program.  If not, see <http://www.gnu.org/licenses/>
20  *
21  */
22
23 /* This file handles all the internal functions that cope with hashes
24  * and HMACs.
25  */
26
27 #include <gnutls_int.h>
28 #include <gnutls_hash_int.h>
29 #include <gnutls_errors.h>
30 #include <algorithms.h>
31 #include <fips.h>
32
33 int _gnutls_hash_init(digest_hd_st * dig, const mac_entry_st * e)
34 {
35         int result;
36         const gnutls_crypto_digest_st *cc = NULL;
37
38         FAIL_IF_LIB_ERROR;
39
40         if (unlikely(e == NULL || e->id == GNUTLS_MAC_NULL))
41                 return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
42
43         dig->e = e;
44
45         /* check if a digest has been registered 
46          */
47         cc = _gnutls_get_crypto_digest((gnutls_digest_algorithm_t)e->id);
48         if (cc != NULL && cc->init) {
49                 if (cc->init((gnutls_digest_algorithm_t)e->id, &dig->handle) < 0) {
50                         gnutls_assert();
51                         return GNUTLS_E_HASH_FAILED;
52                 }
53
54                 dig->hash = cc->hash;
55                 dig->output = cc->output;
56                 dig->deinit = cc->deinit;
57
58                 return 0;
59         }
60
61         result = _gnutls_digest_ops.init((gnutls_digest_algorithm_t)e->id, &dig->handle);
62         if (result < 0) {
63                 gnutls_assert();
64                 return result;
65         }
66
67         dig->hash = _gnutls_digest_ops.hash;
68         dig->output = _gnutls_digest_ops.output;
69         dig->deinit = _gnutls_digest_ops.deinit;
70
71         return 0;
72 }
73
74 /* Returns true(non-zero) or false(0) if the 
75  * provided hash exists
76  */
77 int _gnutls_digest_exists(gnutls_digest_algorithm_t algo)
78 {
79         const gnutls_crypto_digest_st *cc = NULL;
80
81         cc = _gnutls_get_crypto_digest(algo);
82         if (cc != NULL)
83                 return 1;
84
85         return _gnutls_digest_ops.exists(algo);
86 }
87
88 void _gnutls_hash_deinit(digest_hd_st * handle, void *digest)
89 {
90         if (handle->handle == NULL) {
91                 return;
92         }
93
94         if (digest != NULL)
95                 _gnutls_hash_output(handle, digest);
96
97         handle->deinit(handle->handle);
98         handle->handle = NULL;
99 }
100
101 int
102 _gnutls_hash_fast(gnutls_digest_algorithm_t algorithm,
103                   const void *text, size_t textlen, void *digest)
104 {
105         int ret;
106         const gnutls_crypto_digest_st *cc = NULL;
107         
108         FAIL_IF_LIB_ERROR;
109
110         /* check if a digest has been registered 
111          */
112         cc = _gnutls_get_crypto_digest(algorithm);
113         if (cc != NULL) {
114                 if (cc->fast(algorithm, text, textlen, digest) < 0) {
115                         gnutls_assert();
116                         return GNUTLS_E_HASH_FAILED;
117                 }
118
119                 return 0;
120         }
121
122         ret = _gnutls_digest_ops.fast(algorithm, text, textlen, digest);
123         if (ret < 0) {
124                 gnutls_assert();
125                 return ret;
126         }
127
128         return 0;
129 }
130
131
132 /* HMAC interface */
133
134 int
135 _gnutls_mac_fast(gnutls_mac_algorithm_t algorithm, const void *key,
136                  int keylen, const void *text, size_t textlen,
137                  void *digest)
138 {
139         int ret;
140         const gnutls_crypto_mac_st *cc = NULL;
141
142         FAIL_IF_LIB_ERROR;
143
144         /* check if a digest has been registered 
145          */
146         cc = _gnutls_get_crypto_mac(algorithm);
147         if (cc != NULL) {
148                 if (cc->
149                     fast(algorithm, NULL, 0, key, keylen, text, textlen,
150                          digest) < 0) {
151                         gnutls_assert();
152                         return GNUTLS_E_HASH_FAILED;
153                 }
154
155                 return 0;
156         }
157
158         ret =
159             _gnutls_mac_ops.fast(algorithm, NULL, 0, key, keylen, text,
160                                  textlen, digest);
161         if (ret < 0) {
162                 gnutls_assert();
163                 return ret;
164         }
165
166         return 0;
167
168 }
169
170 /* Returns true(non-zero) or false(0) if the 
171  * provided hash exists
172  */
173 int _gnutls_mac_exists(gnutls_mac_algorithm_t algo)
174 {
175         const gnutls_crypto_mac_st *cc = NULL;
176
177         cc = _gnutls_get_crypto_mac(algo);
178         if (cc != NULL)
179                 return 1;
180
181         return _gnutls_mac_ops.exists(algo);
182 }
183
184 int
185 _gnutls_mac_init(mac_hd_st * mac, const mac_entry_st * e,
186                  const void *key, int keylen)
187 {
188         int result;
189         const gnutls_crypto_mac_st *cc = NULL;
190
191         FAIL_IF_LIB_ERROR;
192
193         if (unlikely(e == NULL || e->id == GNUTLS_MAC_NULL))
194                 return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);
195
196         mac->e = e;
197         mac->mac_len = _gnutls_mac_get_algo_len(e);
198
199         /* check if a digest has been registered 
200          */
201         cc = _gnutls_get_crypto_mac(e->id);
202         if (cc != NULL && cc->init != NULL) {
203                 if (cc->init(e->id, &mac->handle) < 0) {
204                         gnutls_assert();
205                         return GNUTLS_E_HASH_FAILED;
206                 }
207
208                 if (cc->setkey(mac->handle, key, keylen) < 0) {
209                         gnutls_assert();
210                         cc->deinit(mac->handle);
211                         return GNUTLS_E_HASH_FAILED;
212                 }
213
214                 mac->hash = cc->hash;
215                 mac->setnonce = cc->setnonce;
216                 mac->output = cc->output;
217                 mac->deinit = cc->deinit;
218
219                 return 0;
220         }
221
222         result = _gnutls_mac_ops.init(e->id, &mac->handle);
223         if (result < 0) {
224                 gnutls_assert();
225                 return result;
226         }
227
228         mac->hash = _gnutls_mac_ops.hash;
229         mac->setnonce = _gnutls_mac_ops.setnonce;
230         mac->output = _gnutls_mac_ops.output;
231         mac->deinit = _gnutls_mac_ops.deinit;
232
233         if (_gnutls_mac_ops.setkey(mac->handle, key, keylen) < 0) {
234                 gnutls_assert();
235                 mac->deinit(mac->handle);
236                 return GNUTLS_E_HASH_FAILED;
237         }
238
239         return 0;
240 }
241
242 void _gnutls_mac_deinit(mac_hd_st * handle, void *digest)
243 {
244         if (handle->handle == NULL) {
245                 return;
246         }
247
248         if (digest)
249                 _gnutls_mac_output(handle, digest);
250
251         handle->deinit(handle->handle);
252         handle->handle = NULL;
253 }
254
255 inline static int get_padsize(gnutls_mac_algorithm_t algorithm)
256 {
257         switch (algorithm) {
258         case GNUTLS_MAC_MD5:
259                 return 48;
260         case GNUTLS_MAC_SHA1:
261                 return 40;
262         default:
263                 return 0;
264         }
265 }
266
267 /* Special functions for SSL3 MAC
268  */
269
270 int
271 _gnutls_mac_init_ssl3(digest_hd_st * ret, const mac_entry_st * e,
272                       void *key, int keylen)
273 {
274         uint8_t ipad[48];
275         int padsize, result;
276
277         FAIL_IF_LIB_ERROR;
278
279         padsize = get_padsize(e->id);
280         if (padsize == 0) {
281                 gnutls_assert();
282                 return GNUTLS_E_HASH_FAILED;
283         }
284
285         memset(ipad, 0x36, padsize);
286
287         result = _gnutls_hash_init(ret, e);
288         if (result < 0) {
289                 gnutls_assert();
290                 return result;
291         }
292
293         ret->key = key;
294         ret->keysize = keylen;
295
296         if (keylen > 0)
297                 _gnutls_hash(ret, key, keylen);
298         _gnutls_hash(ret, ipad, padsize);
299
300         return 0;
301 }
302
303 int _gnutls_mac_output_ssl3(digest_hd_st * handle, void *digest)
304 {
305         uint8_t ret[MAX_HASH_SIZE];
306         digest_hd_st td;
307         uint8_t opad[48];
308         int padsize;
309         int block, rc;
310
311         padsize = get_padsize(handle->e->id);
312         if (padsize == 0) {
313                 gnutls_assert();
314                 return GNUTLS_E_INTERNAL_ERROR;
315         }
316
317         memset(opad, 0x5C, padsize);
318
319         rc = _gnutls_hash_init(&td, handle->e);
320         if (rc < 0) {
321                 gnutls_assert();
322                 return rc;
323         }
324
325         if (handle->keysize > 0)
326                 _gnutls_hash(&td, handle->key, handle->keysize);
327
328         _gnutls_hash(&td, opad, padsize);
329         block = _gnutls_mac_get_algo_len(handle->e);
330         _gnutls_hash_output(handle, ret);       /* get the previous hash */
331         _gnutls_hash(&td, ret, block);
332
333         _gnutls_hash_deinit(&td, digest);
334
335         /* reset handle */
336         memset(opad, 0x36, padsize);
337
338         if (handle->keysize > 0)
339                 _gnutls_hash(handle, handle->key, handle->keysize);
340         _gnutls_hash(handle, opad, padsize);
341
342         return 0;
343 }
344
345 int _gnutls_mac_deinit_ssl3(digest_hd_st * handle, void *digest)
346 {
347         int ret = 0;
348
349         if (digest != NULL)
350                 ret = _gnutls_mac_output_ssl3(handle, digest);
351         _gnutls_hash_deinit(handle, NULL);
352
353         return ret;
354 }
355
356 int
357 _gnutls_mac_deinit_ssl3_handshake(digest_hd_st * handle,
358                                   void *digest, uint8_t * key,
359                                   uint32_t key_size)
360 {
361         uint8_t ret[MAX_HASH_SIZE];
362         digest_hd_st td;
363         uint8_t opad[48];
364         uint8_t ipad[48];
365         int padsize;
366         int block, rc;
367
368         padsize = get_padsize(handle->e->id);
369         if (padsize == 0) {
370                 gnutls_assert();
371                 rc = GNUTLS_E_INTERNAL_ERROR;
372                 goto cleanup;
373         }
374
375         memset(opad, 0x5C, padsize);
376         memset(ipad, 0x36, padsize);
377
378         rc = _gnutls_hash_init(&td, handle->e);
379         if (rc < 0) {
380                 gnutls_assert();
381                 goto cleanup;
382         }
383
384         if (key_size > 0)
385                 _gnutls_hash(&td, key, key_size);
386
387         _gnutls_hash(&td, opad, padsize);
388         block = _gnutls_mac_get_algo_len(handle->e);
389
390         if (key_size > 0)
391                 _gnutls_hash(handle, key, key_size);
392         _gnutls_hash(handle, ipad, padsize);
393         _gnutls_hash_deinit(handle, ret);       /* get the previous hash */
394
395         _gnutls_hash(&td, ret, block);
396
397         _gnutls_hash_deinit(&td, digest);
398
399         return 0;
400
401       cleanup:
402         _gnutls_hash_deinit(handle, NULL);
403         return rc;
404 }
405
406 static int
407 ssl3_sha(int i, uint8_t * secret, int secret_len,
408          uint8_t * rnd, int rnd_len, void *digest)
409 {
410         int j, ret;
411         uint8_t text1[26];
412
413         digest_hd_st td;
414
415         for (j = 0; j < i + 1; j++) {
416                 text1[j] = 65 + i;      /* A==65 */
417         }
418
419         ret = _gnutls_hash_init(&td, mac_to_entry(GNUTLS_MAC_SHA1));
420         if (ret < 0) {
421                 gnutls_assert();
422                 return ret;
423         }
424
425         _gnutls_hash(&td, text1, i + 1);
426         _gnutls_hash(&td, secret, secret_len);
427         _gnutls_hash(&td, rnd, rnd_len);
428
429         _gnutls_hash_deinit(&td, digest);
430         return 0;
431 }
432
433 #define SHA1_DIGEST_OUTPUT 20
434 #define MD5_DIGEST_OUTPUT 16
435
436 static int
437 ssl3_md5(int i, uint8_t * secret, int secret_len,
438          uint8_t * rnd, int rnd_len, void *digest)
439 {
440         uint8_t tmp[MAX_HASH_SIZE];
441         digest_hd_st td;
442         int ret;
443
444         ret = _gnutls_hash_init(&td, mac_to_entry(GNUTLS_MAC_MD5));
445         if (ret < 0) {
446                 gnutls_assert();
447                 return ret;
448         }
449
450         _gnutls_hash(&td, secret, secret_len);
451
452         ret = ssl3_sha(i, secret, secret_len, rnd, rnd_len, tmp);
453         if (ret < 0) {
454                 gnutls_assert();
455                 _gnutls_hash_deinit(&td, digest);
456                 return ret;
457         }
458
459         _gnutls_hash(&td, tmp, SHA1_DIGEST_OUTPUT);
460
461         _gnutls_hash_deinit(&td, digest);
462         return 0;
463 }
464
465 int
466 _gnutls_ssl3_hash_md5(const void *first, int first_len,
467                       const void *second, int second_len,
468                       int ret_len, uint8_t * ret)
469 {
470         uint8_t digest[MAX_HASH_SIZE];
471         digest_hd_st td;
472         int block = MD5_DIGEST_OUTPUT;
473         int rc;
474
475         rc = _gnutls_hash_init(&td, mac_to_entry(GNUTLS_MAC_MD5));
476         if (rc < 0) {
477                 gnutls_assert();
478                 return rc;
479         }
480
481         _gnutls_hash(&td, first, first_len);
482         _gnutls_hash(&td, second, second_len);
483
484         _gnutls_hash_deinit(&td, digest);
485
486         if (ret_len > block) {
487                 gnutls_assert();
488                 return GNUTLS_E_INTERNAL_ERROR;
489         }
490
491         memcpy(ret, digest, ret_len);
492
493         return 0;
494
495 }
496
497 int
498 _gnutls_ssl3_generate_random(void *secret, int secret_len,
499                              void *rnd, int rnd_len,
500                              int ret_bytes, uint8_t * ret)
501 {
502         int i = 0, copy, output_bytes;
503         uint8_t digest[MAX_HASH_SIZE];
504         int block = MD5_DIGEST_OUTPUT;
505         int result, times;
506
507         output_bytes = 0;
508         do {
509                 output_bytes += block;
510         }
511         while (output_bytes < ret_bytes);
512
513         times = output_bytes / block;
514
515         for (i = 0; i < times; i++) {
516
517                 result =
518                     ssl3_md5(i, secret, secret_len, rnd, rnd_len, digest);
519                 if (result < 0) {
520                         gnutls_assert();
521                         return result;
522                 }
523
524                 if ((1 + i) * block < ret_bytes) {
525                         copy = block;
526                 } else {
527                         copy = ret_bytes - (i) * block;
528                 }
529
530                 memcpy(&ret[i * block], digest, copy);
531         }
532
533         return 0;
534 }