Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 10 insertions, 26 deletions

diff --git a/crypto/ct/ct_log.c b/crypto/ct/ct_log.c
index ec6ac1dd7f..95084dc76f 100644
--- a/crypto/ct/ct_log.c
+++ b/crypto/ct/ct_log.c
@@ -62,9 +62,6 @@ static CTLOG_STORE_LOAD_CTX *ctlog_store_load_ctx_new(void)
 {
     CTLOG_STORE_LOAD_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx));
 
-    if (ctx == NULL)
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
-
     return ctx;
 }
 
@@ -104,23 +101,19 @@ CTLOG_STORE *CTLOG_STORE_new_ex(OSSL_LIB_CTX *libctx, const char *propq)
 {
     CTLOG_STORE *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
 
     ret->libctx = libctx;
     if (propq != NULL) {
         ret->propq = OPENSSL_strdup(propq);
-        if (ret->propq == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        if (ret->propq == NULL)
             goto err;
-        }
     }
 
     ret->logs = sk_CTLOG_new_null();
     if (ret->logs == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CT, ERR_R_CRYPTO_LIB);
         goto err;
     }
 
@@ -196,7 +189,7 @@ static int ctlog_store_load_log(const char *log_name, int log_name_len,
 
     tmp = OPENSSL_strndup(log_name, log_name_len);
     if (tmp == NULL)
-        goto mem_err;
+        return -1;
 
     ret = ctlog_new_from_conf(load_ctx->log_store, &ct_log, load_ctx->conf, tmp);
     OPENSSL_free(tmp);
@@ -212,14 +205,11 @@ static int ctlog_store_load_log(const char *log_name, int log_name_len,
     }
 
     if (!sk_CTLOG_push(load_ctx->log_store->logs, ct_log)) {
-        goto mem_err;
+        CTLOG_free(ct_log);
+        ERR_raise(ERR_LIB_CT, ERR_R_CRYPTO_LIB);
+        return -1;
     }
     return 1;
-
-mem_err:
-    CTLOG_free(ct_log);
-    ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
-    return -1;
 }
 
 int CTLOG_STORE_load_file(CTLOG_STORE *store, const char *file)
@@ -269,25 +259,19 @@ CTLOG *CTLOG_new_ex(EVP_PKEY *public_key, const char *name, OSSL_LIB_CTX *libctx
 {
     CTLOG *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
 
     ret->libctx = libctx;
     if (propq != NULL) {
         ret->propq = OPENSSL_strdup(propq);
-        if (ret->propq == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        if (ret->propq == NULL)
             goto err;
-        }
     }
 
     ret->name = OPENSSL_strdup(name);
-    if (ret->name == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (ret->name == NULL)
         goto err;
-    }
 
     if (ct_v1_log_id_from_pkey(ret, public_key) != 1)
         goto err;