Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

5 files changed, 30 insertions, 40 deletions

diff --git a/crypto/pkcs7/pk7_asn1.c b/crypto/pkcs7/pk7_asn1.c
index 1cd867721e..e338b8f1af 100644
--- a/crypto/pkcs7/pk7_asn1.c
+++ b/crypto/pkcs7/pk7_asn1.c
@@ -104,7 +104,6 @@ PKCS7 *PKCS7_new_ex(OSSL_LIB_CTX *libctx, const char *propq)
             if (pkcs7->ctx.propq == NULL) {
                 PKCS7_free(pkcs7);
                 pkcs7 = NULL;
-                ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
             }
         }
     }
diff --git a/crypto/pkcs7/pk7_attr.c b/crypto/pkcs7/pk7_attr.c
index e9904c5950..68f0a5c290 100644
--- a/crypto/pkcs7/pk7_attr.c
+++ b/crypto/pkcs7/pk7_attr.c
@@ -23,7 +23,7 @@ int PKCS7_add_attrib_smimecap(PKCS7_SIGNER_INFO *si,
     ASN1_STRING *seq;
 
     if ((seq = ASN1_STRING_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
         return 0;
     }
     seq->length = ASN1_item_i2d((ASN1_VALUE *)cap, &seq->data,
@@ -53,19 +53,22 @@ int PKCS7_simple_smimecap(STACK_OF(X509_ALGOR) *sk, int nid, int arg)
     X509_ALGOR *alg;
 
     if ((alg = X509_ALGOR_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
         return 0;
     }
     ASN1_OBJECT_free(alg->algorithm);
     alg->algorithm = OBJ_nid2obj(nid);
     if (arg > 0) {
         if ((alg->parameter = ASN1_TYPE_new()) == NULL) {
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
             goto err;
         }
         if ((nbit = ASN1_INTEGER_new()) == NULL) {
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
             goto err;
         }
         if (!ASN1_INTEGER_set(nbit, arg)) {
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
             goto err;
         }
         alg->parameter->value.integer = nbit;
@@ -73,11 +76,11 @@ int PKCS7_simple_smimecap(STACK_OF(X509_ALGOR) *sk, int nid, int arg)
         nbit = NULL;
     }
     if (!sk_X509_ALGOR_push(sk, alg)) {
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_CRYPTO_LIB);
         goto err;
     }
     return 1;
 err:
-    ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
     ASN1_INTEGER_free(nbit);
     X509_ALGOR_free(alg);
     return 0;
@@ -96,7 +99,7 @@ int PKCS7_add_attrib_content_type(PKCS7_SIGNER_INFO *si, ASN1_OBJECT *coid)
 int PKCS7_add0_attrib_signing_time(PKCS7_SIGNER_INFO *si, ASN1_TIME *t)
 {
     if (t == NULL && (t = X509_gmtime_adj(NULL, 0)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_X509_LIB);
         return 0;
     }
     return PKCS7_add_signed_attribute(si, NID_pkcs9_signingTime,
diff --git a/crypto/pkcs7/pk7_doit.c b/crypto/pkcs7/pk7_doit.c
index 13a820345e..31b368bda3 100644
--- a/crypto/pkcs7/pk7_doit.c
+++ b/crypto/pkcs7/pk7_doit.c
@@ -127,11 +127,8 @@ static int pkcs7_encode_rinfo(PKCS7_RECIP_INFO *ri,
         goto err;
 
     ek = OPENSSL_malloc(eklen);
-
-    if (ek == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+    if (ek == NULL)
         goto err;
-    }
 
     if (EVP_PKEY_encrypt(pctx, ek, &eklen, key, keylen) <= 0)
         goto err;
@@ -171,11 +168,8 @@ static int pkcs7_decrypt_rinfo(unsigned char **pek, int *peklen,
         goto err;
 
     ek = OPENSSL_malloc(eklen);
-
-    if (ek == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+    if (ek == NULL)
         goto err;
-    }
 
     if (EVP_PKEY_decrypt(pctx, ek, &eklen,
                          ri->enc_key->data, ri->enc_key->length) <= 0
@@ -694,7 +688,7 @@ static int do_pkcs7_signed_attrib(PKCS7_SIGNER_INFO *si, EVP_MD_CTX *mctx)
     /* Add signing time if not already present */
     if (!PKCS7_get_signed_attribute(si, NID_pkcs9_signingTime)) {
         if (!PKCS7_add0_attrib_signing_time(si, NULL)) {
-            ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB);
             return 0;
         }
     }
@@ -705,7 +699,7 @@ static int do_pkcs7_signed_attrib(PKCS7_SIGNER_INFO *si, EVP_MD_CTX *mctx)
         return 0;
     }
     if (!PKCS7_add1_attrib_digest(si, md_data, md_len)) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB);
         return 0;
     }
 
@@ -742,7 +736,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
 
     ctx_tmp = EVP_MD_CTX_new();
     if (ctx_tmp == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_EVP_LIB);
         return 0;
     }
 
@@ -760,7 +754,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
         if (os == NULL) {
             os = ASN1_OCTET_STRING_new();
             if (os == NULL) {
-                ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
                 goto err;
             }
             p7->d.signed_and_enveloped->enc_data->enc_data = os;
@@ -772,7 +766,7 @@ int PKCS7_dataFinal(PKCS7 *p7, BIO *bio)
         if (os == NULL) {
             os = ASN1_OCTET_STRING_new();
             if (os == NULL) {
-                ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
                 goto err;
             }
             p7->d.enveloped->enc_data->enc_data = os;
@@ -911,7 +905,7 @@ int PKCS7_SIGNER_INFO_sign(PKCS7_SIGNER_INFO *si)
 
     mctx = EVP_MD_CTX_new();
     if (mctx == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_EVP_LIB);
         goto err;
     }
 
@@ -1027,7 +1021,7 @@ int PKCS7_signatureVerify(BIO *bio, PKCS7 *p7, PKCS7_SIGNER_INFO *si,
 
     mdc_tmp = EVP_MD_CTX_new();
     if (mdc_tmp == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_EVP_LIB);
         goto err;
     }
 
diff --git a/crypto/pkcs7/pk7_lib.c b/crypto/pkcs7/pk7_lib.c
index eaa46a3338..188f421d87 100644
--- a/crypto/pkcs7/pk7_lib.c
+++ b/crypto/pkcs7/pk7_lib.c
@@ -224,7 +224,7 @@ int PKCS7_add_signer(PKCS7 *p7, PKCS7_SIGNER_INFO *psi)
         if ((alg = X509_ALGOR_new()) == NULL
             || (alg->parameter = ASN1_TYPE_new()) == NULL) {
             X509_ALGOR_free(alg);
-            ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
             return 0;
         }
         /*
@@ -290,7 +290,7 @@ int PKCS7_add_crl(PKCS7 *p7, X509_CRL *crl)
     if (*sk == NULL)
         *sk = sk_X509_CRL_new_null();
     if (*sk == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_CRYPTO_LIB);
         return 0;
     }
 
@@ -483,10 +483,8 @@ int ossl_pkcs7_set1_propq(PKCS7 *p7, const char *propq)
     }
     if (propq != NULL) {
         p7->ctx.propq = OPENSSL_strdup(propq);
-        if (p7->ctx.propq == NULL) {
-            ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        if (p7->ctx.propq == NULL)
             return 0;
-        }
     }
     return 1;
 }
@@ -514,7 +512,7 @@ int PKCS7_set_digest(PKCS7 *p7, const EVP_MD *md)
 {
     if (PKCS7_type_is_digest(p7)) {
         if ((p7->d.digest->md->parameter = ASN1_TYPE_new()) == NULL) {
-            ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_ASN1_LIB);
             return 0;
         }
         p7->d.digest->md->parameter->type = V_ASN1_NULL;
diff --git a/crypto/pkcs7/pk7_smime.c b/crypto/pkcs7/pk7_smime.c
index 21a317446d..a635e4ddce 100644
--- a/crypto/pkcs7/pk7_smime.c
+++ b/crypto/pkcs7/pk7_smime.c
@@ -28,7 +28,7 @@ PKCS7 *PKCS7_sign_ex(X509 *signcert, EVP_PKEY *pkey, STACK_OF(X509) *certs,
     int i;
 
     if ((p7 = PKCS7_new_ex(libctx, propq)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB);
         return NULL;
     }
 
@@ -77,7 +77,7 @@ int PKCS7_final(PKCS7 *p7, BIO *data, int flags)
     int ret = 0;
 
     if ((p7bio = PKCS7_dataInit(p7, NULL)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB);
         return 0;
     }
 
@@ -144,7 +144,7 @@ PKCS7_SIGNER_INFO *PKCS7_sign_add_signer(PKCS7 *p7, X509 *signcert,
         /* Add SMIMECapabilities */
         if (!(flags & PKCS7_NOSMIMECAP)) {
             if ((smcap = sk_X509_ALGOR_new_null()) == NULL) {
-                ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_PKCS7, ERR_R_CRYPTO_LIB);
                 goto err;
             }
             if (!add_cipher_smcap(smcap, NID_aes_256_cbc, -1)
@@ -306,7 +306,7 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,
 
     if (flags & PKCS7_TEXT) {
         if ((tmpout = BIO_new(BIO_s_mem())) == NULL) {
-            ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_BIO_LIB);
             goto err;
         }
         BIO_set_mem_eof_return(tmpout, 0);
@@ -314,10 +314,8 @@ int PKCS7_verify(PKCS7 *p7, STACK_OF(X509) *certs, X509_STORE *store,
         tmpout = out;
 
     /* We now have to 'read' from p7bio to calculate digests etc. */
-    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL)
         goto err;
-    }
     for (;;) {
         i = BIO_read(p7bio, buf, BUFFERSIZE);
         if (i <= 0)
@@ -389,7 +387,7 @@ STACK_OF(X509) *PKCS7_get0_signers(PKCS7 *p7, STACK_OF(X509) *certs,
     }
 
     if ((signers = sk_X509_new_null()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_CRYPTO_LIB);
         return NULL;
     }
 
@@ -432,7 +430,7 @@ PKCS7 *PKCS7_encrypt_ex(STACK_OF(X509) *certs, BIO *in,
     X509 *x509;
 
     if ((p7 = PKCS7_new_ex(libctx, propq)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS7, ERR_R_PKCS7_LIB);
         return NULL;
     }
 
@@ -503,12 +501,12 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags)
         BIO *tmpbuf, *bread;
         /* Encrypt BIOs can't do BIO_gets() so add a buffer BIO */
         if ((tmpbuf = BIO_new(BIO_f_buffer())) == NULL) {
-            ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_BIO_LIB);
             BIO_free_all(tmpmem);
             return 0;
         }
         if ((bread = BIO_push(tmpbuf, tmpmem)) == NULL) {
-            ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS7, ERR_R_BIO_LIB);
             BIO_free_all(tmpbuf);
             BIO_free_all(tmpmem);
             return 0;
@@ -521,10 +519,8 @@ int PKCS7_decrypt(PKCS7 *p7, EVP_PKEY *pkey, X509 *cert, BIO *data, int flags)
         BIO_free_all(bread);
         return ret;
     }
-    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS7, ERR_R_MALLOC_FAILURE);
+    if ((buf = OPENSSL_malloc(BUFFERSIZE)) == NULL)
         goto err;
-    }
     for (;;) {
         i = BIO_read(tmpmem, buf, BUFFERSIZE);
         if (i <= 0) {