Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 21 insertions, 20 deletions

diff --git a/crypto/cms/cms_smime.c b/crypto/cms/cms_smime.c
index ea40873e6a..fe0850b93e 100644
--- a/crypto/cms/cms_smime.c
+++ b/crypto/cms/cms_smime.c
@@ -39,7 +39,7 @@ static int cms_copy_content(BIO *out, BIO *in, unsigned int flags)
     tmpout = cms_get_text_bio(out, flags);
 
     if (tmpout == NULL) {
-        ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
         goto err;
     }
 
@@ -271,7 +271,7 @@ static int cms_signerinfo_verify_cert(CMS_SignerInfo *si,
     ctx = X509_STORE_CTX_new_ex(ossl_cms_ctx_get0_libctx(cms_ctx),
                                 ossl_cms_ctx_get0_propq(cms_ctx));
     if (ctx == NULL) {
-        ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_CMS, ERR_R_X509_LIB);
         goto err;
     }
     CMS_SignerInfo_get0_algs(si, NULL, &signer, NULL, NULL);
@@ -356,10 +356,8 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs,
         if (cadesVerify) {
             /* Certificate trust chain is required to check CAdES signature */
             si_chains = OPENSSL_zalloc(scount * sizeof(si_chains[0]));
-            if (si_chains == NULL) {
-                ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+            if (si_chains == NULL)
                 goto err;
-            }
         }
         cms_certs = CMS_get1_certs(cms);
         if (!(flags & CMS_NOCRL))
@@ -406,7 +404,7 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs,
         len = BIO_get_mem_data(dcont, &ptr);
         tmpin = (len == 0) ? dcont : BIO_new_mem_buf(ptr, len);
         if (tmpin == NULL) {
-            ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_CMS, ERR_R_BIO_LIB);
             goto err2;
         }
     } else {
@@ -423,7 +421,7 @@ int CMS_verify(CMS_ContentInfo *cms, STACK_OF(X509) *certs,
          */
         tmpout = cms_get_text_bio(out, flags);
         if (tmpout == NULL) {
-            ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
             goto err;
         }
         cmsbio = CMS_dataInit(cms, tmpout);
@@ -511,12 +509,16 @@ CMS_ContentInfo *CMS_sign_ex(X509 *signcert, EVP_PKEY *pkey,
     int i;
 
     cms = CMS_ContentInfo_new_ex(libctx, propq);
-    if (cms == NULL || !CMS_SignedData_init(cms))
-        goto merr;
+    if (cms == NULL || !CMS_SignedData_init(cms)) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
+        goto err;
+    }
     if (flags & CMS_ASCIICRLF
         && !CMS_set1_eContentType(cms,
-                                  OBJ_nid2obj(NID_id_ct_asciiTextWithCRLF)))
+                                  OBJ_nid2obj(NID_id_ct_asciiTextWithCRLF))) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
         goto err;
+    }
 
     if (pkey != NULL && !CMS_add1_signer(cms, signcert, pkey, NULL, flags)) {
         ERR_raise(ERR_LIB_CMS, CMS_R_ADD_SIGNER_ERROR);
@@ -526,8 +528,10 @@ CMS_ContentInfo *CMS_sign_ex(X509 *signcert, EVP_PKEY *pkey,
     for (i = 0; i < sk_X509_num(certs); i++) {
         X509 *x = sk_X509_value(certs, i);
 
-        if (!CMS_add1_cert(cms, x))
-            goto merr;
+        if (!CMS_add1_cert(cms, x)) {
+            ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
+            goto err;
+        }
     }
 
     if (!(flags & CMS_DETACHED))
@@ -539,9 +543,6 @@ CMS_ContentInfo *CMS_sign_ex(X509 *signcert, EVP_PKEY *pkey,
     else
         goto err;
 
- merr:
-    ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
-
  err:
     CMS_ContentInfo_free(cms);
     return NULL;
@@ -637,8 +638,10 @@ CMS_ContentInfo *CMS_encrypt_ex(STACK_OF(X509) *certs, BIO *data,
     cms = (EVP_CIPHER_get_flags(cipher) & EVP_CIPH_FLAG_AEAD_CIPHER)
           ? CMS_AuthEnvelopedData_create_ex(cipher, libctx, propq)
           : CMS_EnvelopedData_create_ex(cipher, libctx, propq);
-    if (cms == NULL)
-        goto merr;
+    if (cms == NULL) {
+        ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
+        goto err;
+    }
     for (i = 0; i < sk_X509_num(certs); i++) {
         recip = sk_X509_value(certs, i);
         if (!CMS_add1_recipient_cert(cms, recip, flags)) {
@@ -654,10 +657,8 @@ CMS_ContentInfo *CMS_encrypt_ex(STACK_OF(X509) *certs, BIO *data,
         || CMS_final(cms, data, NULL, flags))
         return cms;
     else
-        goto err;
+        ERR_raise(ERR_LIB_CMS, ERR_R_CMS_LIB);
 
- merr:
-    ERR_raise(ERR_LIB_CMS, ERR_R_MALLOC_FAILURE);
  err:
     CMS_ContentInfo_free(cms);
     return NULL;