Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

3 files changed, 5 insertions, 13 deletions

diff --git a/providers/implementations/exchange/dh_exch.c b/providers/implementations/exchange/dh_exch.c
index ebed25a08f..5d459e1c69 100644
--- a/providers/implementations/exchange/dh_exch.c
+++ b/providers/implementations/exchange/dh_exch.c
@@ -194,10 +194,8 @@ static int dh_X9_42_kdf_derive(void *vpdhctx, unsigned char *secret,
     }
     if (!dh_plain_derive(pdhctx, NULL, &stmplen, 0, 1))
         return 0;
-    if ((stmp = OPENSSL_secure_malloc(stmplen)) == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if ((stmp = OPENSSL_secure_malloc(stmplen)) == NULL)
         return 0;
-    }
     if (!dh_plain_derive(pdhctx, stmp, &stmplen, stmplen, 1))
         goto err;
 
diff --git a/providers/implementations/exchange/ecdh_exch.c b/providers/implementations/exchange/ecdh_exch.c
index af24616eef..53b4249778 100644
--- a/providers/implementations/exchange/ecdh_exch.c
+++ b/providers/implementations/exchange/ecdh_exch.c
@@ -126,7 +126,7 @@ int ecdh_match_params(const EC_KEY *priv, const EC_KEY *peer)
 
     ctx = BN_CTX_new_ex(ossl_ec_key_get_libctx(priv));
     if (ctx == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_BN_LIB);
         return 0;
     }
     ret = group_priv != NULL
@@ -524,10 +524,8 @@ int ecdh_X9_63_kdf_derive(void *vpecdhctx, unsigned char *secret,
     }
     if (!ecdh_plain_derive(vpecdhctx, NULL, &stmplen, 0))
         return 0;
-    if ((stmp = OPENSSL_secure_malloc(stmplen)) == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if ((stmp = OPENSSL_secure_malloc(stmplen)) == NULL)
         return 0;
-    }
     if (!ecdh_plain_derive(vpecdhctx, stmp, &stmplen, stmplen))
         goto err;
 
diff --git a/providers/implementations/exchange/ecx_exch.c b/providers/implementations/exchange/ecx_exch.c
index 7e223f28c8..374f436c89 100644
--- a/providers/implementations/exchange/ecx_exch.c
+++ b/providers/implementations/exchange/ecx_exch.c
@@ -46,10 +46,8 @@ static void *ecx_newctx(void *provctx, size_t keylen)
         return NULL;
 
     ctx = OPENSSL_zalloc(sizeof(PROV_ECX_CTX));
-    if (ctx == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (ctx == NULL)
         return NULL;
-    }
 
     ctx->keylen = keylen;
 
@@ -140,10 +138,8 @@ static void *ecx_dupctx(void *vecxctx)
         return NULL;
 
     dstctx = OPENSSL_zalloc(sizeof(*srcctx));
-    if (dstctx == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (dstctx == NULL)
         return NULL;
-    }
 
     *dstctx = *srcctx;
     if (dstctx->key != NULL && !ossl_ecx_key_up_ref(dstctx->key)) {