Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 21 insertions, 21 deletions

diff --git a/ssl/ssl_sess.c b/ssl/ssl_sess.c
index 720ea4bc18..fbc3224219 100644
--- a/ssl/ssl_sess.c
+++ b/ssl/ssl_sess.c
@@ -106,10 +106,8 @@ SSL_SESSION *SSL_SESSION_new(void)
         return NULL;
 
     ss = OPENSSL_zalloc(sizeof(*ss));
-    if (ss == NULL) {
-        ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE);
+    if (ss == NULL)
         return NULL;
-    }
 
     ss->verify_result = 1;      /* avoid 0 (= X509_V_OK) just in case */
     ss->references = 1;
@@ -119,7 +117,7 @@ SSL_SESSION *SSL_SESSION_new(void)
     ssl_session_calculate_timeout(ss);
     ss->lock = CRYPTO_THREAD_lock_new();
     if (ss->lock == NULL) {
-        ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB);
         OPENSSL_free(ss);
         return NULL;
     }
@@ -177,48 +175,54 @@ SSL_SESSION *ssl_session_dup(const SSL_SESSION *src, int ticket)
     dest->references = 1;
 
     dest->lock = CRYPTO_THREAD_lock_new();
-    if (dest->lock == NULL)
+    if (dest->lock == NULL) {
+        ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB);
         goto err;
+    }
 
-    if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL_SESSION, dest, &dest->ex_data))
+    if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_SSL_SESSION, dest, &dest->ex_data)) {
+        ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB);
         goto err;
+    }
 
     if (src->peer != NULL) {
-        if (!X509_up_ref(src->peer))
+        if (!X509_up_ref(src->peer)) {
+            ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB);
             goto err;
+        }
         dest->peer = src->peer;
     }
 
     if (src->peer_chain != NULL) {
         dest->peer_chain = X509_chain_up_ref(src->peer_chain);
-        if (dest->peer_chain == NULL)
+        if (dest->peer_chain == NULL) {
+            ERR_raise(ERR_LIB_SSL, ERR_R_X509_LIB);
             goto err;
+        }
     }
 #ifndef OPENSSL_NO_PSK
     if (src->psk_identity_hint) {
         dest->psk_identity_hint = OPENSSL_strdup(src->psk_identity_hint);
-        if (dest->psk_identity_hint == NULL) {
+        if (dest->psk_identity_hint == NULL)
             goto err;
-        }
     }
     if (src->psk_identity) {
         dest->psk_identity = OPENSSL_strdup(src->psk_identity);
-        if (dest->psk_identity == NULL) {
+        if (dest->psk_identity == NULL)
             goto err;
-        }
     }
 #endif
 
     if (!CRYPTO_dup_ex_data(CRYPTO_EX_INDEX_SSL_SESSION,
                             &dest->ex_data, &src->ex_data)) {
+        ERR_raise(ERR_LIB_SSL, ERR_R_CRYPTO_LIB);
         goto err;
     }
 
     if (src->ext.hostname) {
         dest->ext.hostname = OPENSSL_strdup(src->ext.hostname);
-        if (dest->ext.hostname == NULL) {
+        if (dest->ext.hostname == NULL)
             goto err;
-        }
     }
 
     if (ticket != 0 && src->ext.tick != NULL) {
@@ -241,9 +245,8 @@ SSL_SESSION *ssl_session_dup(const SSL_SESSION *src, int ticket)
 #ifndef OPENSSL_NO_SRP
     if (src->srp_username) {
         dest->srp_username = OPENSSL_strdup(src->srp_username);
-        if (dest->srp_username == NULL) {
+        if (dest->srp_username == NULL)
             goto err;
-        }
     }
 #endif
 
@@ -256,7 +259,6 @@ SSL_SESSION *ssl_session_dup(const SSL_SESSION *src, int ticket)
 
     return dest;
  err:
-    ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE);
     SSL_SESSION_free(dest);
     return NULL;
 }
@@ -409,7 +411,7 @@ int ssl_get_new_session(SSL_CONNECTION *s, int session)
     SSL_SESSION *ss = NULL;
 
     if ((ss = SSL_SESSION_new()) == NULL) {
-        SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_MALLOC_FAILURE);
+        SSLfatal(s, SSL_AD_INTERNAL_ERROR, ERR_R_SSL_LIB);
         return 0;
     }
 
@@ -1116,10 +1118,8 @@ int SSL_set_session_ticket_ext(SSL *s, void *ext_data, int ext_len)
         sc->ext.session_ticket = NULL;
         sc->ext.session_ticket =
             OPENSSL_malloc(sizeof(TLS_SESSION_TICKET_EXT) + ext_len);
-        if (sc->ext.session_ticket == NULL) {
-            ERR_raise(ERR_LIB_SSL, ERR_R_MALLOC_FAILURE);
+        if (sc->ext.session_ticket == NULL)
             return 0;
-        }
 
         if (ext_data != NULL) {
             sc->ext.session_ticket->length = ext_len;