Fix OpenSSL 1.1 issues

- 3DES CFB1 implementation is fixed but function that checks buffer overlapping
  ignores the input data length unit resulting in an error. It's impossible to
  create a code that works with both version so #if was used.
- OpenSSL 1.1's EVP_CIPHER_CTX_flags() gets flags from EVP_CIPHER instead of
  EVP_CIPHER_CTX. Call replaced with EVP_CIPHER_CTX_test_flags().

Change-Id: Ic43fd9e02b982bc767259ad3a818314a6e319eb7

diff --git a/src/encrypt.c b/src/encrypt.c
index 6ce25cc54a3aca47c86e7c12a74f95c4e8b8a229..7e9fd586abc87a73a0f3d96832130f0ba41d9d97 100644 (file)
--- a/src/encrypt.c
+++ b/src/encrypt.c
@@ -464,10 +464,12 @@ static int encrypt_ctx_setup(struct yaca_encrypt_context_s *c,
        if (ret != YACA_ERROR_NONE)
                return ret;
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
        /* Fix for OpenSSL error in 3DES CFB1 */
        int nid = EVP_CIPHER_CTX_nid(c->cipher_ctx);
        if (nid == NID_des_ede3_cfb1)
                EVP_CIPHER_CTX_set_flags(c->cipher_ctx, EVP_CIPH_FLAG_LENGTH_BITS);
+#endif
 
        if (liv != NULL)
                iv_data = (unsigned char*)liv->d;
@@ -1024,7 +1026,7 @@ int encrypt_update(yaca_context_h ctx,
        }
 
        /* Fix for OpenSSL error in 3DES CFB1 */
-       if ((EVP_CIPHER_CTX_flags(c->cipher_ctx) & EVP_CIPH_FLAG_LENGTH_BITS) != 0) {
+       if (EVP_CIPHER_CTX_test_flags(c->cipher_ctx, EVP_CIPH_FLAG_LENGTH_BITS) != 0) {
                if (input_len > INT_MAX / 8)
                        return YACA_ERROR_INVALID_PARAMETER;
                input_len *= 8;
@@ -1042,7 +1044,7 @@ int encrypt_update(yaca_context_h ctx,
        c->state = target_state;
 
        /* Fix for OpenSSL error in 3DES CFB1 */
-       if ((EVP_CIPHER_CTX_flags(c->cipher_ctx) & EVP_CIPH_FLAG_LENGTH_BITS) != 0)
+       if (EVP_CIPHER_CTX_test_flags(c->cipher_ctx, EVP_CIPH_FLAG_LENGTH_BITS) != 0)
                *output_len /= 8;
 
        return YACA_ERROR_NONE;
@@ -1071,7 +1073,7 @@ int encrypt_finalize(yaca_context_h ctx,
        *output_len = loutput_len;
 
        /* Fix for OpenSSL error in 3DES CFB1 */
-       if ((EVP_CIPHER_CTX_flags(c->cipher_ctx) & EVP_CIPH_FLAG_LENGTH_BITS) != 0)
+       if (EVP_CIPHER_CTX_test_flags(c->cipher_ctx, EVP_CIPH_FLAG_LENGTH_BITS) != 0)
                *output_len /= 8;
 
        c->state = STATE_FINALIZED;