Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 3 insertions, 9 deletions

diff --git a/crypto/o_str.c b/crypto/o_str.c
index 3354ce0927..119d791e20 100644
--- a/crypto/o_str.c
+++ b/crypto/o_str.c
@@ -56,10 +56,8 @@ void *CRYPTO_memdup(const void *data, size_t siz, const char* file, int line)
         return NULL;
 
     ret = CRYPTO_malloc(siz, file, line);
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
     return memcpy(ret, data, siz);
 }
 
@@ -196,10 +194,8 @@ unsigned char *ossl_hexstr2buf_sep(const char *str, long *buflen,
         return NULL;
     }
     buf_n /= 2;
-    if ((buf = OPENSSL_malloc(buf_n)) == NULL) {
-        ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+    if ((buf = OPENSSL_malloc(buf_n)) == NULL)
         return NULL;
-    }
 
     if (buflen != NULL)
         *buflen = 0;
@@ -272,10 +268,8 @@ char *ossl_buf2hexstr_sep(const unsigned char *buf, long buflen, char sep)
         return OPENSSL_zalloc(1);
 
     tmp_n = (sep != CH_ZERO) ? buflen * 3 : 1 + buflen * 2;
-    if ((tmp = OPENSSL_malloc(tmp_n)) == NULL) {
-        ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+    if ((tmp = OPENSSL_malloc(tmp_n)) == NULL)
         return NULL;
-    }
 
     if (buf2hexstr_sep(tmp, tmp_n, NULL, buf, buflen, sep))
         return tmp;