Distinguish different cases with the same OpenSSL error code

When importing a key with a wrong password and decrypting data with
wrong key/bcm or simply broken data OpenSSL can return exactly the
same error code (ERR_LIB_EVP, EVP_F_EVP_DECRYPTFINAL_EX,
EVP_R_BAD_DECRYPT). As we need to distinguish INVALID_PARAM and
INVALID_PASS in import_key, but decryption cannot return INVALID_PASS
handle this manually in the decryption.

Change-Id: Iba2b5fccfb1660c20b76a345bc799a0b145d700c

diff --git a/src/encrypt.c b/src/encrypt.c
index 9c6a1a2..27632f8 100644 (file)
--- a/src/encrypt.c
+++ b/src/encrypt.c
@@ -1144,14 +1144,23 @@ int encrypt_finalize(yaca_context_h ctx,
        if (mode != EVP_CIPH_WRAP_MODE && mode != EVP_CIPH_CCM_MODE) {
                ret = EVP_CipherFinal(c->cipher_ctx, output, &loutput_len);
                if (ret != 1 || loutput_len < 0) {
-                       if (mode == EVP_CIPH_GCM_MODE && (op_type == OP_DECRYPT || op_type == OP_OPEN))
-                       /* A non positive return value from EVP_CipherFinal should be considered as
-                        * a failure to authenticate ciphertext and/or AAD.
-                        * It does not necessarily indicate a more serious error.
-                        */
+                       if (mode == EVP_CIPH_GCM_MODE && (op_type == OP_DECRYPT || op_type == OP_OPEN)) {
+                               /* A non positive return value from EVP_CipherFinal should be
+                                * considered as a failure to authenticate ciphertext and/or
+                                * AAD. It does not necessarily indicate a more serious error.
+                                */
                                return YACA_ERROR_INVALID_PARAMETER;
-                       else
-                               return ERROR_HANDLE();
+                       } else {
+                               /* The same error code is used if trying to import a key with a
+                                * wrong password and in case of a decrypt error due to wrong
+                                * BCM or a key. Finalize cannot return INVALID_PASS so handle
+                                * this here.
+                                */
+                               ret = ERROR_HANDLE();
+                               if (ret == YACA_ERROR_INVALID_PASSWORD)
+                                       ret = YACA_ERROR_INVALID_PARAMETER;
+                               return ret;
+                       }
                }
        }