Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 11 insertions, 7 deletions

diff --git a/crypto/cms/cms_lib.c b/crypto/cms/cms_lib.c
index e39fde9e43..d92772d41d 100644
--- a/crypto/cms/cms_lib.c
+++ b/crypto/cms/cms_lib.c
@@ -60,7 +60,6 @@ CMS_ContentInfo *CMS_ContentInfo_new_ex(OSSL_LIB_CTX *libctx, const char *propq)
             if (ci->ctx.propq == NULL) {
                 CMS_ContentInfo_free(ci);
                 ci = NULL;
-                ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
             }
         }
     }
@@ -404,7 +403,7 @@ int CMS_set_detached(CMS_ContentInfo *cms, int detached)
         (*pos)->flags |= ASN1_STRING_FLAG_CONT;
         return 1;
     }
-    ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+    ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB);
     return 0;
 }
 
@@ -702,18 +701,23 @@ int ossl_cms_set1_ias(CMS_IssuerAndSerialNumber **pias, X509 *cert)
 {
     CMS_IssuerAndSerialNumber *ias;
     ias = M_ASN1_new_of(CMS_IssuerAndSerialNumber);
-    if (!ias)
+    if (!ias) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB);
         goto err;
-    if (!X509_NAME_set(&ias->issuer, X509_get_issuer_name(cert)))
+    }
+    if (!X509_NAME_set(&ias->issuer, X509_get_issuer_name(cert))) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_X509_LIB);
         goto err;
-    if (!ASN1_STRING_copy(ias->serialNumber, X509_get0_serialNumber(cert)))
+    }
+    if (!ASN1_STRING_copy(ias->serialNumber, X509_get0_serialNumber(cert))) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB);
         goto err;
+    }
     M_ASN1_free_of(*pias, CMS_IssuerAndSerialNumber);
     *pias = ias;
     return 1;
  err:
     M_ASN1_free_of(ias, CMS_IssuerAndSerialNumber);
-    ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
     return 0;
 }
 
@@ -728,7 +732,7 @@ int ossl_cms_set1_keyid(ASN1_OCTET_STRING **pkeyid, X509 *cert)
     }
     keyid = ASN1_STRING_dup(cert_keyid);
     if (!keyid) {
-        ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CMS, ERR_R_ASN1_LIB);
         return 0;
     }
     ASN1_OCTET_STRING_free(*pkeyid);