Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

3 files changed, 7 insertions, 17 deletions

diff --git a/crypto/rand/prov_seed.c b/crypto/rand/prov_seed.c
index b394242f71..546c204094 100644
--- a/crypto/rand/prov_seed.c
+++ b/crypto/rand/prov_seed.c
@@ -22,7 +22,7 @@ size_t ossl_rand_get_entropy(ossl_unused const OSSL_CORE_HANDLE *handle,
 
     pool = ossl_rand_pool_new(entropy, 1, min_len, max_len);
     if (pool == NULL) {
-        ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_RAND, ERR_R_RAND_LIB);
         return 0;
     }
 
@@ -53,7 +53,7 @@ size_t ossl_rand_get_nonce(ossl_unused const OSSL_CORE_HANDLE *handle,
 
     pool = ossl_rand_pool_new(0, 0, min_len, max_len);
     if (pool == NULL) {
-        ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_RAND, ERR_R_RAND_LIB);
         return 0;
     }
 
diff --git a/crypto/rand/rand_lib.c b/crypto/rand/rand_lib.c
index 227c505117..c69fc4f2af 100644
--- a/crypto/rand/rand_lib.c
+++ b/crypto/rand/rand_lib.c
@@ -731,10 +731,8 @@ static int random_set_string(char **p, const char *s)
 
     if (s != NULL) {
         d = OPENSSL_strdup(s);
-        if (d == NULL) {
-            ERR_raise(ERR_LIB_CRYPTO, ERR_R_MALLOC_FAILURE);
+        if (d == NULL)
             return 0;
-        }
     }
     OPENSSL_free(*p);
     *p = d;
diff --git a/crypto/rand/rand_pool.c b/crypto/rand/rand_pool.c
index 55f14be60e..8dc230b540 100644
--- a/crypto/rand/rand_pool.c
+++ b/crypto/rand/rand_pool.c
@@ -25,10 +25,8 @@ RAND_POOL *ossl_rand_pool_new(int entropy_requested, int secure,
     RAND_POOL *pool = OPENSSL_zalloc(sizeof(*pool));
     size_t min_alloc_size = RAND_POOL_MIN_ALLOCATION(secure);
 
-    if (pool == NULL) {
-        ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE);
+    if (pool == NULL)
         return NULL;
-    }
 
     pool->min_len = min_len;
     pool->max_len = (max_len > RAND_POOL_MAX_LENGTH) ?
@@ -42,10 +40,8 @@ RAND_POOL *ossl_rand_pool_new(int entropy_requested, int secure,
     else
         pool->buffer = OPENSSL_zalloc(pool->alloc_len);
 
-    if (pool->buffer == NULL) {
-        ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE);
+    if (pool->buffer == NULL)
         goto err;
-    }
 
     pool->entropy_requested = entropy_requested;
     pool->secure = secure;
@@ -67,10 +63,8 @@ RAND_POOL *ossl_rand_pool_attach(const unsigned char *buffer, size_t len,
 {
     RAND_POOL *pool = OPENSSL_zalloc(sizeof(*pool));
 
-    if (pool == NULL) {
-        ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE);
+    if (pool == NULL)
         return NULL;
-    }
 
     /*
      * The const needs to be cast away, but attached buffers will not be
@@ -222,10 +216,8 @@ static int rand_pool_grow(RAND_POOL *pool, size_t len)
             p = OPENSSL_secure_zalloc(newlen);
         else
             p = OPENSSL_zalloc(newlen);
-        if (p == NULL) {
-            ERR_raise(ERR_LIB_RAND, ERR_R_MALLOC_FAILURE);
+        if (p == NULL)
             return 0;
-        }
         memcpy(p, pool->buffer, pool->len);
         if (pool->secure)
             OPENSSL_secure_clear_free(pool->buffer, pool->alloc_len);