openssl: guard against OOM on context creation

EVP_MD_CTX_create will allocate memory for the context and returns
NULL in case the allocation fails. Make sure to catch any allocation
failures and exit early if so.

In passing, also move to EVP_DigestInit rather than EVP_DigestInit_ex
as the latter is intended for ENGINE selection which we don't do.

Closes #6224

Backported: https://github.com/curl/curl/pull/6224

Change-Id: Ibcd3a0782405d3db6aa08d65892af15c3ea8431b

diff --git a/lib/vtls/openssl.c b/lib/vtls/openssl.c
index f961c1f..6045679 100644 (file)
--- a/lib/vtls/openssl.c
+++ b/lib/vtls/openssl.c
@@ -4357,7 +4357,9 @@ static CURLcode Curl_ossl_md5sum(unsigned char *tmp, /* input */
   (void) unused;
 
   mdctx = EVP_MD_CTX_create();
-  EVP_DigestInit_ex(mdctx, EVP_md5(), NULL);
+  if(!mdctx)
+    return CURLE_OUT_OF_MEMORY;
+  EVP_DigestInit(mdctx, EVP_md5());
   EVP_DigestUpdate(mdctx, tmp, tmplen);
   EVP_DigestFinal_ex(mdctx, md5sum, &len);
   EVP_MD_CTX_destroy(mdctx);
@@ -4375,7 +4377,9 @@ static CURLcode Curl_ossl_sha256sum(const unsigned char *tmp, /* input */
   (void) unused;
 
   mdctx = EVP_MD_CTX_create();
-  EVP_DigestInit_ex(mdctx, EVP_sha256(), NULL);
+  if(!mdctx)
+    return CURLE_OUT_OF_MEMORY;
+  EVP_DigestInit(mdctx, EVP_sha256());
   EVP_DigestUpdate(mdctx, tmp, tmplen);
   EVP_DigestFinal_ex(mdctx, sha256sum, &len);
   EVP_MD_CTX_destroy(mdctx);