Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

5 files changed, 14 insertions, 21 deletions

diff --git a/crypto/dsa/dsa_ameth.c b/crypto/dsa/dsa_ameth.c
index 482b9e1e0a..15a5266ca4 100644
--- a/crypto/dsa/dsa_ameth.c
+++ b/crypto/dsa/dsa_ameth.c
@@ -54,7 +54,7 @@ static int dsa_pub_decode(EVP_PKEY *pkey, const X509_PUBKEY *pubkey)
 
     } else if ((ptype == V_ASN1_NULL) || (ptype == V_ASN1_UNDEF)) {
         if ((dsa = DSA_new()) == NULL) {
-            ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_DSA, ERR_R_DSA_LIB);
             goto err;
         }
     } else {
@@ -101,12 +101,12 @@ static int dsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
         && dsa->params.g != NULL) {
         str = ASN1_STRING_new();
         if (str == NULL) {
-            ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB);
             goto err;
         }
         str->length = i2d_DSAparams(dsa, &str->data);
         if (str->length <= 0) {
-            ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB);
             goto err;
         }
         ptype = V_ASN1_SEQUENCE;
@@ -116,7 +116,7 @@ static int dsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
     pubint = BN_to_ASN1_INTEGER(dsa->pub_key, NULL);
 
     if (pubint == NULL) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB);
         goto err;
     }
 
@@ -124,7 +124,7 @@ static int dsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
     ASN1_INTEGER_free(pubint);
 
     if (penclen <= 0) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB);
         goto err;
     }
 
@@ -175,13 +175,13 @@ static int dsa_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
     params = ASN1_STRING_new();
 
     if (params == NULL) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB);
         goto err;
     }
 
     params->length = i2d_DSAparams(pkey->pkey.dsa, &params->data);
     if (params->length <= 0) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSA, ERR_R_ASN1_LIB);
         goto err;
     }
     params->type = V_ASN1_SEQUENCE;
@@ -483,7 +483,7 @@ static int dsa_pkey_import_from(const OSSL_PARAM params[], void *vpctx)
     DSA *dsa = ossl_dsa_new(pctx->libctx);
 
     if (dsa == NULL) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSA, ERR_R_DSA_LIB);
         return 0;
     }
 
diff --git a/crypto/dsa/dsa_backend.c b/crypto/dsa/dsa_backend.c
index f9a71bdc9e..924ccbdc0b 100644
--- a/crypto/dsa/dsa_backend.c
+++ b/crypto/dsa/dsa_backend.c
@@ -158,11 +158,11 @@ DSA *ossl_dsa_key_from_pkcs8(const PKCS8_PRIV_KEY_INFO *p8inf,
     }
     /* Calculate public key */
     if ((dsa_pubkey = BN_new()) == NULL) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSA, ERR_R_BN_LIB);
         goto dsaerr;
     }
     if ((ctx = BN_CTX_new()) == NULL) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSA, ERR_R_BN_LIB);
         goto dsaerr;
     }
 
diff --git a/crypto/dsa/dsa_lib.c b/crypto/dsa/dsa_lib.c
index ccc7016592..333885a01a 100644
--- a/crypto/dsa/dsa_lib.c
+++ b/crypto/dsa/dsa_lib.c
@@ -134,15 +134,13 @@ static DSA *dsa_new_intern(ENGINE *engine, OSSL_LIB_CTX *libctx)
 {
     DSA *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
 
     ret->references = 1;
     ret->lock = CRYPTO_THREAD_lock_new();
     if (ret->lock == NULL) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DSA, ERR_R_CRYPTO_LIB);
         OPENSSL_free(ret);
         return NULL;
     }
diff --git a/crypto/dsa/dsa_meth.c b/crypto/dsa/dsa_meth.c
index 2f0a0bf460..f2b759a9de 100644
--- a/crypto/dsa/dsa_meth.c
+++ b/crypto/dsa/dsa_meth.c
@@ -32,7 +32,6 @@ DSA_METHOD *DSA_meth_new(const char *name, int flags)
         OPENSSL_free(dsam);
     }
 
-    ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
     return NULL;
 }
 
@@ -58,7 +57,6 @@ DSA_METHOD *DSA_meth_dup(const DSA_METHOD *dsam)
         OPENSSL_free(ret);
     }
 
-    ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
     return NULL;
 }
 
@@ -71,10 +69,8 @@ int DSA_meth_set1_name(DSA_METHOD *dsam, const char *name)
 {
     char *tmpname = OPENSSL_strdup(name);
 
-    if (tmpname == NULL) {
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+    if (tmpname == NULL)
         return 0;
-    }
 
     OPENSSL_free(dsam->name);
     dsam->name = tmpname;
diff --git a/crypto/dsa/dsa_sign.c b/crypto/dsa/dsa_sign.c
index 21b0cbd5fb..d942fa2afe 100644
--- a/crypto/dsa/dsa_sign.c
+++ b/crypto/dsa/dsa_sign.c
@@ -34,8 +34,7 @@ int DSA_sign_setup(DSA *dsa, BN_CTX *ctx_in, BIGNUM **kinvp, BIGNUM **rp)
 DSA_SIG *DSA_SIG_new(void)
 {
     DSA_SIG *sig = OPENSSL_zalloc(sizeof(*sig));
-    if (sig == NULL)
-        ERR_raise(ERR_LIB_DSA, ERR_R_MALLOC_FAILURE);
+
     return sig;
 }