Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

4 files changed, 5 insertions, 13 deletions

diff --git a/crypto/objects/o_names.c b/crypto/objects/o_names.c
index 1efa0345fb..5a468bba3a 100644
--- a/crypto/objects/o_names.c
+++ b/crypto/objects/o_names.c
@@ -89,7 +89,6 @@ int OBJ_NAME_new_index(unsigned long (*hash_func) (const char *),
     for (i = sk_NAME_FUNCS_num(name_funcs_stack); i < names_type_num; i++) {
         name_funcs = OPENSSL_zalloc(sizeof(*name_funcs));
         if (name_funcs == NULL) {
-            ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE);
             ret = 0;
             goto out;
         }
@@ -98,7 +97,7 @@ int OBJ_NAME_new_index(unsigned long (*hash_func) (const char *),
         push = sk_NAME_FUNCS_push(name_funcs_stack, name_funcs);
 
         if (!push) {
-            ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_OBJ, ERR_R_CRYPTO_LIB);
             OPENSSL_free(name_funcs);
             ret = 0;
             goto out;
diff --git a/crypto/objects/obj_dat.c b/crypto/objects/obj_dat.c
index 852039b6da..ea5d57a77e 100644
--- a/crypto/objects/obj_dat.c
+++ b/crypto/objects/obj_dat.c
@@ -254,10 +254,8 @@ static int ossl_obj_add_object(const ASN1_OBJECT *obj, int lock)
             || (o->sn != NULL
                 && (ao[ADDED_SNAME] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL)
             || (o->ln != NULL
-                && (ao[ADDED_LNAME] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL)) {
-        ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE);
+                && (ao[ADDED_LNAME] = OPENSSL_malloc(sizeof(*ao[0]))) == NULL))
         goto err2;
-    }
 
     if (!ossl_obj_write_lock(lock)) {
         ERR_raise(ERR_LIB_OBJ, ERR_R_UNABLE_TO_GET_WRITE_LOCK);
@@ -266,7 +264,7 @@ static int ossl_obj_add_object(const ASN1_OBJECT *obj, int lock)
     if (added == NULL) {
         added = lh_ADDED_OBJ_new(added_obj_hash, added_obj_cmp);
         if (added == NULL) {
-            ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_OBJ, ERR_R_CRYPTO_LIB);
             goto err;
         }
     }
@@ -420,10 +418,8 @@ ASN1_OBJECT *OBJ_txt2obj(const char *s, int no_name)
     if (j < 0)
         return NULL;
 
-    if ((buf = OPENSSL_malloc(j)) == NULL) {
-        ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE);
+    if ((buf = OPENSSL_malloc(j)) == NULL)
         return NULL;
-    }
 
     p = buf;
     /* Write out tag+length */
diff --git a/crypto/objects/obj_lib.c b/crypto/objects/obj_lib.c
index 72c0c2c81d..6cdc1d7e74 100644
--- a/crypto/objects/obj_lib.c
+++ b/crypto/objects/obj_lib.c
@@ -50,7 +50,6 @@ ASN1_OBJECT *OBJ_dup(const ASN1_OBJECT *o)
     return r;
  err:
     ASN1_OBJECT_free(r);
-    ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE);
     return NULL;
 }
 
diff --git a/crypto/objects/obj_xref.c b/crypto/objects/obj_xref.c
index 8660de2004..2eb757cb77 100644
--- a/crypto/objects/obj_xref.c
+++ b/crypto/objects/obj_xref.c
@@ -155,10 +155,8 @@ int OBJ_add_sigid(int signid, int dig_id, int pkey_id)
     if (!obj_sig_init())
         return 0;
 
-    if ((ntr = OPENSSL_malloc(sizeof(*ntr))) == NULL) {
-        ERR_raise(ERR_LIB_OBJ, ERR_R_MALLOC_FAILURE);
+    if ((ntr = OPENSSL_malloc(sizeof(*ntr))) == NULL)
         return 0;
-    }
     ntr->sign_id = signid;
     ntr->hash_id = dig_id;
     ntr->pkey_id = pkey_id;