Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 5 insertions, 8 deletions

diff --git a/crypto/x509/v3_ia5.c b/crypto/x509/v3_ia5.c
index 6722b6c01f..7b79935872 100644
--- a/crypto/x509/v3_ia5.c
+++ b/crypto/x509/v3_ia5.c
@@ -31,10 +31,8 @@ char *i2s_ASN1_IA5STRING(X509V3_EXT_METHOD *method, ASN1_IA5STRING *ia5)
 
     if (ia5 == NULL || ia5->length <= 0)
         return NULL;
-    if ((tmp = OPENSSL_malloc(ia5->length + 1)) == NULL) {
-        ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE);
+    if ((tmp = OPENSSL_malloc(ia5->length + 1)) == NULL)
         return NULL;
-    }
     memcpy(tmp, ia5->data, ia5->length);
     tmp[ia5->length] = 0;
     return tmp;
@@ -48,8 +46,10 @@ ASN1_IA5STRING *s2i_ASN1_IA5STRING(X509V3_EXT_METHOD *method,
         ERR_raise(ERR_LIB_X509V3, X509V3_R_INVALID_NULL_ARGUMENT);
         return NULL;
     }
-    if ((ia5 = ASN1_IA5STRING_new()) == NULL)
-        goto err;
+    if ((ia5 = ASN1_IA5STRING_new()) == NULL) {
+        ERR_raise(ERR_LIB_X509V3, ERR_R_ASN1_LIB);
+        return NULL;
+    }
     if (!ASN1_STRING_set((ASN1_STRING *)ia5, str, strlen(str))) {
         ASN1_IA5STRING_free(ia5);
         return NULL;
@@ -58,7 +58,4 @@ ASN1_IA5STRING *s2i_ASN1_IA5STRING(X509V3_EXT_METHOD *method,
     ebcdic2ascii(ia5->data, ia5->data, ia5->length);
 #endif                          /* CHARSET_EBCDIC */
     return ia5;
- err:
-    ERR_raise(ERR_LIB_X509V3, ERR_R_MALLOC_FAILURE);
-    return NULL;
 }