Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

6 files changed, 21 insertions, 59 deletions

diff --git a/crypto/ct/ct_b64.c b/crypto/ct/ct_b64.c
index d3f783962a..2535442063 100644
--- a/crypto/ct/ct_b64.c
+++ b/crypto/ct/ct_b64.c
@@ -34,10 +34,8 @@ static int ct_base64_decode(const char *in, unsigned char **out)
 
     outlen = (inlen / 4) * 3;
     outbuf = OPENSSL_malloc(outlen);
-    if (outbuf == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (outbuf == NULL)
         goto err;
-    }
 
     outlen = EVP_DecodeBlock(outbuf, (unsigned char *)in, inlen);
     if (outlen < 0) {
@@ -71,7 +69,7 @@ SCT *SCT_new_from_base64(unsigned char version, const char *logid_base64,
     int declen;
 
     if (sct == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CT, ERR_R_CT_LIB);
         return NULL;
     }
 
diff --git a/crypto/ct/ct_log.c b/crypto/ct/ct_log.c
index ec6ac1dd7f..95084dc76f 100644
--- a/crypto/ct/ct_log.c
+++ b/crypto/ct/ct_log.c
@@ -62,9 +62,6 @@ static CTLOG_STORE_LOAD_CTX *ctlog_store_load_ctx_new(void)
 {
     CTLOG_STORE_LOAD_CTX *ctx = OPENSSL_zalloc(sizeof(*ctx));
 
-    if (ctx == NULL)
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
-
     return ctx;
 }
 
@@ -104,23 +101,19 @@ CTLOG_STORE *CTLOG_STORE_new_ex(OSSL_LIB_CTX *libctx, const char *propq)
 {
     CTLOG_STORE *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
 
     ret->libctx = libctx;
     if (propq != NULL) {
         ret->propq = OPENSSL_strdup(propq);
-        if (ret->propq == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        if (ret->propq == NULL)
             goto err;
-        }
     }
 
     ret->logs = sk_CTLOG_new_null();
     if (ret->logs == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CT, ERR_R_CRYPTO_LIB);
         goto err;
     }
 
@@ -196,7 +189,7 @@ static int ctlog_store_load_log(const char *log_name, int log_name_len,
 
     tmp = OPENSSL_strndup(log_name, log_name_len);
     if (tmp == NULL)
-        goto mem_err;
+        return -1;
 
     ret = ctlog_new_from_conf(load_ctx->log_store, &ct_log, load_ctx->conf, tmp);
     OPENSSL_free(tmp);
@@ -212,14 +205,11 @@ static int ctlog_store_load_log(const char *log_name, int log_name_len,
     }
 
     if (!sk_CTLOG_push(load_ctx->log_store->logs, ct_log)) {
-        goto mem_err;
+        CTLOG_free(ct_log);
+        ERR_raise(ERR_LIB_CT, ERR_R_CRYPTO_LIB);
+        return -1;
     }
     return 1;
-
-mem_err:
-    CTLOG_free(ct_log);
-    ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
-    return -1;
 }
 
 int CTLOG_STORE_load_file(CTLOG_STORE *store, const char *file)
@@ -269,25 +259,19 @@ CTLOG *CTLOG_new_ex(EVP_PKEY *public_key, const char *name, OSSL_LIB_CTX *libctx
 {
     CTLOG *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
 
     ret->libctx = libctx;
     if (propq != NULL) {
         ret->propq = OPENSSL_strdup(propq);
-        if (ret->propq == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        if (ret->propq == NULL)
             goto err;
-        }
     }
 
     ret->name = OPENSSL_strdup(name);
-    if (ret->name == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (ret->name == NULL)
         goto err;
-    }
 
     if (ct_v1_log_id_from_pkey(ret, public_key) != 1)
         goto err;
diff --git a/crypto/ct/ct_oct.c b/crypto/ct/ct_oct.c
index 72a4337479..145b277109 100644
--- a/crypto/ct/ct_oct.c
+++ b/crypto/ct/ct_oct.c
@@ -178,10 +178,8 @@ int i2o_SCT_signature(const SCT *sct, unsigned char **out)
             *out += len;
         } else {
             pstart = p = OPENSSL_malloc(len);
-            if (p == NULL) {
-                ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+            if (p == NULL)
                 goto err;
-            }
             *out = p;
         }
 
@@ -225,10 +223,8 @@ int i2o_SCT(const SCT *sct, unsigned char **out)
         *out += len;
     } else {
         pstart = p = OPENSSL_malloc(len);
-        if (p == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        if (p == NULL)
             goto err;
-        }
         *out = p;
     }
 
@@ -330,10 +326,8 @@ int i2o_SCT_LIST(const STACK_OF(SCT) *a, unsigned char **pp)
                 ERR_raise(ERR_LIB_CT, CT_R_SCT_LIST_INVALID);
                 return -1;
             }
-            if ((*pp = OPENSSL_malloc(len)) == NULL) {
-                ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+            if ((*pp = OPENSSL_malloc(len)) == NULL)
                 return -1;
-            }
             is_pp_new = 1;
         }
         p = *pp + 2;
diff --git a/crypto/ct/ct_policy.c b/crypto/ct/ct_policy.c
index ad792b740d..725be7ce2a 100644
--- a/crypto/ct/ct_policy.c
+++ b/crypto/ct/ct_policy.c
@@ -31,16 +31,13 @@ CT_POLICY_EVAL_CTX *CT_POLICY_EVAL_CTX_new_ex(OSSL_LIB_CTX *libctx,
     CT_POLICY_EVAL_CTX *ctx = OPENSSL_zalloc(sizeof(CT_POLICY_EVAL_CTX));
     OSSL_TIME now;
 
-    if (ctx == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (ctx == NULL)
         return NULL;
-    }
 
     ctx->libctx = libctx;
     if (propq != NULL) {
         ctx->propq = OPENSSL_strdup(propq);
         if (ctx->propq == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
             OPENSSL_free(ctx);
             return NULL;
         }
diff --git a/crypto/ct/ct_sct.c b/crypto/ct/ct_sct.c
index 10a67ed6d6..ec87d02309 100644
--- a/crypto/ct/ct_sct.c
+++ b/crypto/ct/ct_sct.c
@@ -23,10 +23,8 @@ SCT *SCT_new(void)
 {
     SCT *sct = OPENSSL_zalloc(sizeof(*sct));
 
-    if (sct == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (sct == NULL)
         return NULL;
-    }
 
     sct->entry_type = CT_LOG_ENTRY_TYPE_NOT_SET;
     sct->version = SCT_VERSION_NOT_SET;
@@ -105,10 +103,8 @@ int SCT_set1_log_id(SCT *sct, const unsigned char *log_id, size_t log_id_len)
 
     if (log_id != NULL && log_id_len > 0) {
         sct->log_id = OPENSSL_memdup(log_id, log_id_len);
-        if (sct->log_id == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        if (sct->log_id == NULL)
             return 0;
-        }
         sct->log_id_len = log_id_len;
     }
     return 1;
@@ -157,10 +153,8 @@ int SCT_set1_extensions(SCT *sct, const unsigned char *ext, size_t ext_len)
 
     if (ext != NULL && ext_len > 0) {
         sct->ext = OPENSSL_memdup(ext, ext_len);
-        if (sct->ext == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        if (sct->ext == NULL)
             return 0;
-        }
         sct->ext_len = ext_len;
     }
     return 1;
@@ -183,10 +177,8 @@ int SCT_set1_signature(SCT *sct, const unsigned char *sig, size_t sig_len)
 
     if (sig != NULL && sig_len > 0) {
         sct->sig = OPENSSL_memdup(sig, sig_len);
-        if (sct->sig == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+        if (sct->sig == NULL)
             return 0;
-        }
         sct->sig_len = sig_len;
     }
     return 1;
diff --git a/crypto/ct/ct_sct_ctx.c b/crypto/ct/ct_sct_ctx.c
index 8653684814..effd724a0a 100644
--- a/crypto/ct/ct_sct_ctx.c
+++ b/crypto/ct/ct_sct_ctx.c
@@ -24,16 +24,13 @@ SCT_CTX *SCT_CTX_new(OSSL_LIB_CTX *libctx, const char *propq)
 {
     SCT_CTX *sctx = OPENSSL_zalloc(sizeof(*sctx));
 
-    if (sctx == NULL) {
-        ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
+    if (sctx == NULL)
         return NULL;
-    }
 
     sctx->libctx = libctx;
     if (propq != NULL) {
         sctx->propq = OPENSSL_strdup(propq);
         if (sctx->propq == NULL) {
-            ERR_raise(ERR_LIB_CT, ERR_R_MALLOC_FAILURE);
             OPENSSL_free(sctx);
             return NULL;
         }