Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

5 files changed, 11 insertions, 27 deletions

diff --git a/providers/implementations/rands/drbg.c b/providers/implementations/rands/drbg.c
index 007a181c89..11ba455233 100644
--- a/providers/implementations/rands/drbg.c
+++ b/providers/implementations/rands/drbg.c
@@ -160,10 +160,8 @@ size_t ossl_drbg_get_seed(void *vdrbg, unsigned char **pout,
 
     /* Allocate storage */
     buffer = OPENSSL_secure_malloc(bytes_needed);
-    if (buffer == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (buffer == NULL)
         return 0;
-    }
 
     /*
      * Get random data.  Include our DRBG address as
@@ -777,10 +775,8 @@ PROV_DRBG *ossl_rand_drbg_new
         return NULL;
 
     drbg = OPENSSL_zalloc(sizeof(*drbg));
-    if (drbg == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (drbg == NULL)
         return NULL;
-    }
 
     drbg->provctx = provctx;
     drbg->instantiate = instantiate;
diff --git a/providers/implementations/rands/drbg_ctr.c b/providers/implementations/rands/drbg_ctr.c
index 451113c4d1..89e0ca5573 100644
--- a/providers/implementations/rands/drbg_ctr.c
+++ b/providers/implementations/rands/drbg_ctr.c
@@ -538,7 +538,7 @@ static int drbg_ctr_init(PROV_DRBG *drbg)
     if (ctr->ctx_ctr == NULL)
         ctr->ctx_ctr = EVP_CIPHER_CTX_new();
     if (ctr->ctx_ecb == NULL || ctr->ctx_ctr == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_EVP_LIB);
         goto err;
     }
 
@@ -565,7 +565,7 @@ static int drbg_ctr_init(PROV_DRBG *drbg)
         if (ctr->ctx_df == NULL)
             ctr->ctx_df = EVP_CIPHER_CTX_new();
         if (ctr->ctx_df == NULL) {
-            ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PROV, ERR_R_EVP_LIB);
             goto err;
         }
         /* Set key schedule for df_key */
@@ -589,10 +589,8 @@ static int drbg_ctr_new(PROV_DRBG *drbg)
     PROV_DRBG_CTR *ctr;
 
     ctr = OPENSSL_secure_zalloc(sizeof(*ctr));
-    if (ctr == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (ctr == NULL)
         return 0;
-    }
 
     ctr->use_df = 1;
     drbg->data = ctr;
@@ -693,10 +691,8 @@ static int drbg_ctr_set_ctx_params(void *vctx, const OSSL_PARAM params[])
             ERR_raise(ERR_LIB_PROV, PROV_R_REQUIRE_CTR_MODE_CIPHER);
             return 0;
         }
-        if ((ecb = OPENSSL_strndup(base, p->data_size)) == NULL) {
-            ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        if ((ecb = OPENSSL_strndup(base, p->data_size)) == NULL)
             return 0;
-        }
         strcpy(ecb + p->data_size - ecb_str_len, "ECB");
         EVP_CIPHER_free(ctr->cipher_ecb);
         EVP_CIPHER_free(ctr->cipher_ctr);
diff --git a/providers/implementations/rands/drbg_hash.c b/providers/implementations/rands/drbg_hash.c
index 99853a7979..12faa993d0 100644
--- a/providers/implementations/rands/drbg_hash.c
+++ b/providers/implementations/rands/drbg_hash.c
@@ -390,10 +390,8 @@ static int drbg_hash_new(PROV_DRBG *ctx)
     PROV_DRBG_HASH *hash;
 
     hash = OPENSSL_secure_zalloc(sizeof(*hash));
-    if (hash == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (hash == NULL)
         return 0;
-    }
 
     ctx->data = hash;
     ctx->seedlen = HASH_PRNG_MAX_SEEDLEN;
diff --git a/providers/implementations/rands/drbg_hmac.c b/providers/implementations/rands/drbg_hmac.c
index e68465a78c..ffeb70f8c3 100644
--- a/providers/implementations/rands/drbg_hmac.c
+++ b/providers/implementations/rands/drbg_hmac.c
@@ -276,10 +276,8 @@ static int drbg_hmac_new(PROV_DRBG *drbg)
     PROV_DRBG_HMAC *hmac;
 
     hmac = OPENSSL_secure_zalloc(sizeof(*hmac));
-    if (hmac == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (hmac == NULL)
         return 0;
-    }
 
     drbg->data = hmac;
     /* See SP800-57 Part1 Rev4 5.6.1 Table 3 */
diff --git a/providers/implementations/rands/seed_src.c b/providers/implementations/rands/seed_src.c
index 7a4b780bb4..5e599775eb 100644
--- a/providers/implementations/rands/seed_src.c
+++ b/providers/implementations/rands/seed_src.c
@@ -53,10 +53,8 @@ static void *seed_src_new(void *provctx, void *parent,
     }
 
     s = OPENSSL_zalloc(sizeof(*s));
-    if (s == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (s == NULL)
         return NULL;
-    }
 
     s->provctx = provctx;
     s->state = EVP_RAND_STATE_UNINITIALISED;
@@ -106,7 +104,7 @@ static int seed_src_generate(void *vseed, unsigned char *out, size_t outlen,
 
     pool = ossl_rand_pool_new(strength, 1, outlen, outlen);
     if (pool == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_RAND_LIB);
         return 0;
     }
 
@@ -197,10 +195,8 @@ static size_t seed_get_seed(void *vseed, unsigned char **pout,
     }
 
     p = OPENSSL_secure_malloc(bytes_needed);
-    if (p == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (p == NULL)
         return 0;
-    }
     if (seed_src_generate(vseed, p, bytes_needed, 0, prediction_resistance,
                           adin, adin_len) != 0) {
         *pout = p;