Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

6 files changed, 22 insertions, 23 deletions

diff --git a/crypto/engine/eng_dyn.c b/crypto/engine/eng_dyn.c
index 6d402927c5..cc3a2b0aa3 100644
--- a/crypto/engine/eng_dyn.c
+++ b/crypto/engine/eng_dyn.c
@@ -159,13 +159,11 @@ static int dynamic_set_data_ctx(ENGINE *e, dynamic_data_ctx **ctx)
     dynamic_data_ctx *c = OPENSSL_zalloc(sizeof(*c));
     int ret = 0;
 
-    if (c == NULL) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+    if (c == NULL)
         return 0;
-    }
     c->dirs = sk_OPENSSL_STRING_new_null();
     if (c->dirs == NULL) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
         goto end;
     }
     c->DYNAMIC_F1 = "v_check";
@@ -357,13 +355,11 @@ static int dynamic_ctrl(ENGINE *e, int cmd, long i, void *p, void (*f) (void))
         }
         {
             char *tmp_str = OPENSSL_strdup(p);
-            if (tmp_str == NULL) {
-                ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+            if (tmp_str == NULL)
                 return 0;
-            }
             if (!sk_OPENSSL_STRING_push(ctx->dirs, tmp_str)) {
                 OPENSSL_free(tmp_str);
-                ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
                 return 0;
             }
         }
diff --git a/crypto/engine/eng_init.c b/crypto/engine/eng_init.c
index a1b9917f44..d262b16917 100644
--- a/crypto/engine/eng_init.c
+++ b/crypto/engine/eng_init.c
@@ -86,7 +86,8 @@ int ENGINE_init(ENGINE *e)
         return 0;
     }
     if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+        /* Maybe this should be raised in do_engine_lock_init() */
+        ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
         return 0;
     }
     if (!CRYPTO_THREAD_write_lock(global_engine_lock))
diff --git a/crypto/engine/eng_lib.c b/crypto/engine/eng_lib.c
index 5285200107..40e0413a26 100644
--- a/crypto/engine/eng_lib.c
+++ b/crypto/engine/eng_lib.c
@@ -28,11 +28,13 @@ ENGINE *ENGINE_new(void)
 {
     ENGINE *ret;
 
-    if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)
-        || (ret = OPENSSL_zalloc(sizeof(*ret))) == NULL) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
-        return NULL;
+    if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) {
+        /* Maybe this should be raised in do_engine_lock_init() */
+        ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
+        return 0;
     }
+    if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL)
+        return NULL;
     ret->struct_ref = 1;
     ENGINE_REF_PRINT(ret, 0, 1);
     if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_ENGINE, ret, &ret->ex_data)) {
@@ -125,10 +127,8 @@ static ENGINE_CLEANUP_ITEM *int_cleanup_item(ENGINE_CLEANUP_CB *cb)
 {
     ENGINE_CLEANUP_ITEM *item;
 
-    if ((item = OPENSSL_malloc(sizeof(*item))) == NULL) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+    if ((item = OPENSSL_malloc(sizeof(*item))) == NULL)
         return NULL;
-    }
     item->cb = cb;
     return item;
 }
diff --git a/crypto/engine/eng_list.c b/crypto/engine/eng_list.c
index 04c73c7628..dd9ada34c8 100644
--- a/crypto/engine/eng_list.c
+++ b/crypto/engine/eng_list.c
@@ -219,7 +219,8 @@ ENGINE *ENGINE_get_first(void)
     ENGINE *ret;
 
     if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+        /* Maybe this should be raised in do_engine_lock_init() */
+        ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
         return NULL;
     }
 
@@ -239,7 +240,8 @@ ENGINE *ENGINE_get_last(void)
     ENGINE *ret;
 
     if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+        /* Maybe this should be raised in do_engine_lock_init() */
+        ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
         return NULL;
     }
 
@@ -378,7 +380,8 @@ ENGINE *ENGINE_by_id(const char *id)
     ENGINE_load_builtin_engines();
 
     if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+        /* Maybe this should be raised in do_engine_lock_init() */
+        ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
         return NULL;
     }
 
diff --git a/crypto/engine/eng_openssl.c b/crypto/engine/eng_openssl.c
index 91656e6b80..8b39e3dec7 100644
--- a/crypto/engine/eng_openssl.c
+++ b/crypto/engine/eng_openssl.c
@@ -450,10 +450,8 @@ static int ossl_hmac_init(EVP_PKEY_CTX *ctx)
 {
     OSSL_HMAC_PKEY_CTX *hctx;
 
-    if ((hctx = OPENSSL_zalloc(sizeof(*hctx))) == NULL) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+    if ((hctx = OPENSSL_zalloc(sizeof(*hctx))) == NULL)
         return 0;
-    }
     hctx->ktmp.type = V_ASN1_OCTET_STRING;
     hctx->ctx = HMAC_CTX_new();
     if (hctx->ctx == NULL) {
diff --git a/crypto/engine/tb_asnmth.c b/crypto/engine/tb_asnmth.c
index bd65ede2f2..fb3953437a 100644
--- a/crypto/engine/tb_asnmth.c
+++ b/crypto/engine/tb_asnmth.c
@@ -196,7 +196,8 @@ const EVP_PKEY_ASN1_METHOD *ENGINE_pkey_asn1_find_str(ENGINE **pe,
     fstr.len = len;
 
     if (!RUN_ONCE(&engine_lock_init, do_engine_lock_init)) {
-        ERR_raise(ERR_LIB_ENGINE, ERR_R_MALLOC_FAILURE);
+        /* Maybe this should be raised in do_engine_lock_init() */
+        ERR_raise(ERR_LIB_ENGINE, ERR_R_CRYPTO_LIB);
         return NULL;
     }