Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

3 files changed, 5 insertions, 15 deletions

diff --git a/crypto/async/arch/async_posix.c b/crypto/async/arch/async_posix.c
index f7432c3298..f2b507c7d0 100644
--- a/crypto/async/arch/async_posix.c
+++ b/crypto/async/arch/async_posix.c
@@ -119,7 +119,6 @@ int async_fibre_makecontext(async_fibre *fibre)
             makecontext(&fibre->fibre, async_start_func, 0);
             return 1;
         }
-        ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE);
     } else {
         fibre->fibre.uc_stack.ss_sp = NULL;
     }
diff --git a/crypto/async/async.c b/crypto/async/async.c
index 076197c79b..46c87d6a5a 100644
--- a/crypto/async/async.c
+++ b/crypto/async/async.c
@@ -40,10 +40,8 @@ static async_ctx *async_ctx_new(void)
         return NULL;
 
     nctx = OPENSSL_malloc(sizeof(*nctx));
-    if (nctx == NULL) {
-        ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE);
+    if (nctx == NULL)
         goto err;
-    }
 
     async_fibre_init_dispatcher(&nctx->dispatcher);
     nctx->currjob = NULL;
@@ -82,10 +80,8 @@ static ASYNC_JOB *async_job_new(void)
     ASYNC_JOB *job = NULL;
 
     job = OPENSSL_zalloc(sizeof(*job));
-    if (job == NULL) {
-        ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE);
+    if (job == NULL)
         return NULL;
-    }
 
     job->status = ASYNC_JOB_RUNNING;
 
@@ -256,7 +252,6 @@ int ASYNC_start_job(ASYNC_JOB **job, ASYNC_WAIT_CTX *wctx, int *ret,
         if (args != NULL) {
             ctx->currjob->funcargs = OPENSSL_malloc(size);
             if (ctx->currjob->funcargs == NULL) {
-                ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE);
                 async_release_job(ctx->currjob);
                 ctx->currjob = NULL;
                 return ASYNC_ERR;
@@ -367,14 +362,12 @@ int ASYNC_init_thread(size_t max_size, size_t init_size)
         return 0;
 
     pool = OPENSSL_zalloc(sizeof(*pool));
-    if (pool == NULL) {
-        ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE);
+    if (pool == NULL)
         return 0;
-    }
 
     pool->jobs = sk_ASYNC_JOB_new_reserve(NULL, init_size);
     if (pool->jobs == NULL) {
-        ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_ASYNC, ERR_R_CRYPTO_LIB);
         OPENSSL_free(pool);
         return 0;
     }
diff --git a/crypto/async/async_wait.c b/crypto/async/async_wait.c
index df7d293021..c5d000a03b 100644
--- a/crypto/async/async_wait.c
+++ b/crypto/async/async_wait.c
@@ -47,10 +47,8 @@ int ASYNC_WAIT_CTX_set_wait_fd(ASYNC_WAIT_CTX *ctx, const void *key,
 {
     struct fd_lookup_st *fdlookup;
 
-    if ((fdlookup = OPENSSL_zalloc(sizeof(*fdlookup))) == NULL) {
-        ERR_raise(ERR_LIB_ASYNC, ERR_R_MALLOC_FAILURE);
+    if ((fdlookup = OPENSSL_zalloc(sizeof(*fdlookup))) == NULL)
         return 0;
-    }
 
     fdlookup->key = key;
     fdlookup->fd = fd;