Better error handling in encrypt_ctx_init() and encrypt_ctx_setup_iv()

Some calls to encrypt_ctx_init() and encrypt_ctx_setup_iv() asserts
their return code that it cannot be EINVAL. But the OpenSSL functions
there (namely EVP_CIPHER_CTX_set_key_length() and EVP_CIPHER_CTX_ctrl)
can fail because of other reasons. Handle this properly.

Side effect of this change is that while setting wrong IVLEN for CCM
it's impossible to distinguish error codes, as OpenSSL does not set
them in most cases. Handle this internally.

Change-Id: Ib82871c8f4bf348c9ff4b90467886edcc19f6f9e

diff --git a/src/debug.c b/src/debug.c
index 1e6035d0cbc5e5b0aaa0ec5de5a12c936136e91c..fbb4dc2fd454dc3c93fd56b615be4c1ebb2a29c6 100644 (file)
--- a/src/debug.c
+++ b/src/debug.c
@@ -149,6 +149,8 @@ int error_handle(const char *file, int line, const char *function)
        case ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_PKEY_DERIVE_SET_PEER, EVP_R_DIFFERENT_PARAMETERS):
        case ERR_PACK(ERR_LIB_EC, EC_F_PKEY_EC_CTRL, EC_R_INVALID_DIGEST_TYPE):
        case ERR_PACK(ERR_LIB_DSA, DSA_F_PKEY_DSA_CTRL, DSA_R_INVALID_DIGEST_TYPE):
+       case ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_CIPHER_CTX_SET_KEY_LENGTH, EVP_R_INVALID_KEY_LENGTH):
+       case ERR_PACK(ERR_LIB_EVP, EVP_F_EVP_CIPHER_CTX_CTRL, EVP_R_CTRL_OPERATION_NOT_IMPLEMENTED):
                ret = YACA_ERROR_INVALID_PARAMETER;
                break;
        case ERR_PACK(ERR_LIB_ASN1, ASN1_F_ASN1_GET_OBJECT, ASN1_R_TOO_LONG):

diff --git a/src/encrypt.c b/src/encrypt.c
index 459eca82a6bb3c6f737be1a04ab65139184b9a4e..51ca1a95b02a78788f8233db55787b3ac63fef28 100644 (file)
--- a/src/encrypt.c
+++ b/src/encrypt.c
@@ -431,11 +431,8 @@ static int encrypt_ctx_init(struct yaca_encrypt_context_s *c,
 
        /* Handling of algorithms with variable key length */
        ret = EVP_CIPHER_CTX_set_key_length(c->cipher_ctx, key_bit_len / 8);
-       if (ret != 1) {
-               ret = YACA_ERROR_INVALID_PARAMETER;
-               ERROR_CLEAR();
-               return ret;
-       }
+       if (ret != 1)
+               return ERROR_HANDLE();
 
        return YACA_ERROR_NONE;
 }
@@ -473,20 +470,28 @@ static int encrypt_ctx_setup_iv(struct yaca_encrypt_context_s *c,
 
                /* IV length doesn't match cipher (GCM & CCM supports variable IV length) */
                if (default_iv_bit_len != iv->bit_len) {
+                       size_t iv_len = iv->bit_len / 8;
                        int mode = EVP_CIPHER_CTX_mode(c->cipher_ctx);
 
                        if (mode == EVP_CIPH_GCM_MODE) {
                                ret = EVP_CIPHER_CTX_ctrl(c->cipher_ctx, EVP_CTRL_GCM_SET_IVLEN,
-                                                                                 iv->bit_len / 8, NULL);
+                                                                                 iv_len, NULL);
                        } else if (mode == EVP_CIPH_CCM_MODE) {
+                               /* OpenSSL does not return a specific error code when
+                                * wrong IVLEN is passed. It just returns 0. So there
+                                * is no way to distinguish this error from ENOMEM for
+                                * example. Handle this in our code then.
+                                */
+                               if (iv_len < 7 || iv_len > 13)
+                                       return YACA_ERROR_INVALID_PARAMETER;
                                ret = EVP_CIPHER_CTX_ctrl(c->cipher_ctx, EVP_CTRL_CCM_SET_IVLEN,
-                                                                                 iv->bit_len / 8, NULL);
+                                                                                 iv_len, NULL);
                        } else {
                                return YACA_ERROR_INVALID_PARAMETER;
                        }
 
                        if (ret != 1)
-                               return YACA_ERROR_INVALID_PARAMETER;
+                               return ERROR_HANDLE();
                }
        }
 
@@ -626,6 +631,8 @@ static int encrypt_ctx_restore(struct yaca_encrypt_context_s *c)
 
        ret = encrypt_ctx_init(c, c->backup_ctx->cipher, key->bit_len);
        assert(ret != YACA_ERROR_INVALID_PARAMETER);
+       if (ret != YACA_ERROR_NONE)
+               return ret;
 
        if (c->backup_ctx->padding == YACA_PADDING_NONE &&
                EVP_CIPHER_CTX_set_padding(c->cipher_ctx, 0) != 1) {