Fix for bug in OpenSSL's 3DES CFB1 implementation

OpenSSL 3DES CFB1 implementation assumes that the size of input data is in
bits. It is indeed possible to use bits instead of bytes by setting an
EVP_CIPHER_CTX flag EVP_CIPH_FLAG_LENGTH_BITS. However, this flag is not being
checked in the implementation. The fix has been already applied to OpenSSL but
not yet released.

This commit causes EVP_CIPH_FLAG_LENGTH_BITS flag to be set always when 3DES
CFB1 is used. It also performes conversion between bit and byte units if
needed. It's a temporary fix and should also work with fixed OpenSSL. Anyway,
as soon as the fix in OpenSSL is released this commit should be reverted.

Change-Id: I97807f0afeecace86adb974e08e6f00fa66f22de

diff --git a/src/encrypt.c b/src/encrypt.c
index ac53d66..d1b0698 100644 (file)
--- a/src/encrypt.c
+++ b/src/encrypt.c
@@ -269,6 +269,11 @@ static int encrypt_ctx_setup(struct yaca_encrypt_context_s *c,
        if (ret != YACA_ERROR_NONE)
                return ret;
 
+       /* Fix for OpenSSL error in 3DES CFB1 */
+       int nid = EVP_CIPHER_CTX_nid(c->cipher_ctx);
+       if (nid == NID_des_ede3_cfb1)
+               EVP_CIPHER_CTX_set_flags(c->cipher_ctx, EVP_CIPH_FLAG_LENGTH_BITS);
+
        if (liv != NULL)
                iv_data = (unsigned char*)liv->d;
 
@@ -746,6 +751,13 @@ int encrypt_update(yaca_context_h ctx,
                if (input == NULL || output == NULL)
                        return YACA_ERROR_INVALID_PARAMETER;
 
+       /* Fix for OpenSSL error in 3DES CFB1 */
+       if ((c->cipher_ctx->flags & EVP_CIPH_FLAG_LENGTH_BITS) != 0) {
+               if (input_len > INT_MAX / 8)
+                       return YACA_ERROR_INVALID_PARAMETER;
+               input_len *= 8;
+       }
+
        ret = EVP_CipherUpdate(c->cipher_ctx, output, &loutput_len, input, input_len);
        if (ret != 1 || loutput_len < 0) {
                ret = YACA_ERROR_INTERNAL;
@@ -754,6 +766,11 @@ int encrypt_update(yaca_context_h ctx,
        }
 
        *output_len = loutput_len;
+
+       /* Fix for OpenSSL error in 3DES CFB1 */
+       if ((c->cipher_ctx->flags & EVP_CIPH_FLAG_LENGTH_BITS) != 0)
+               *output_len /= 8;
+
        return YACA_ERROR_NONE;
 }
 
@@ -776,6 +793,11 @@ int encrypt_finalize(yaca_context_h ctx,
        }
 
        *output_len = loutput_len;
+
+       /* Fix for OpenSSL error in 3DES CFB1 */
+       if ((c->cipher_ctx->flags & EVP_CIPH_FLAG_LENGTH_BITS) != 0)
+               *output_len /= 8;
+
        return YACA_ERROR_NONE;
 }