Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

1 files changed, 11 insertions, 13 deletions

diff --git a/providers/implementations/encode_decode/encode_key2any.c b/providers/implementations/encode_decode/encode_key2any.c
index 246826749f..bd79228a4c 100644
--- a/providers/implementations/encode_decode/encode_key2any.c
+++ b/providers/implementations/encode_decode/encode_key2any.c
@@ -91,7 +91,7 @@ static PKCS8_PRIV_KEY_INFO *key_to_p8info(const void *key, int key_nid,
         || (derlen = k2d(key, &der)) <= 0
         || !PKCS8_pkey_set0(p8info, OBJ_nid2obj(key_nid), 0,
                             params_type, params, der, derlen)) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB);
         PKCS8_PRIV_KEY_INFO_free(p8info);
         OPENSSL_free(der);
         p8info = NULL;
@@ -154,7 +154,7 @@ static X509_PUBKEY *key_to_pubkey(const void *key, int key_nid,
         || (derlen = k2d(key, &der)) <= 0
         || !X509_PUBKEY_set0_param(xpk, OBJ_nid2obj(key_nid),
                                    params_type, params, der, derlen)) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_X509_LIB);
         X509_PUBKEY_free(xpk);
         OPENSSL_free(der);
         xpk = NULL;
@@ -380,7 +380,7 @@ static int key_to_type_specific_der_bio(BIO *out, const void *key,
     int ret;
 
     if ((derlen = k2d(key, &der)) <= 0) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_PROV_LIB);
         return 0;
     }
 
@@ -446,7 +446,7 @@ static int prepare_dh_params(const void *dh, int nid, int save,
     ASN1_STRING *params = ASN1_STRING_new();
 
     if (params == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB);
         return 0;
     }
 
@@ -456,7 +456,7 @@ static int prepare_dh_params(const void *dh, int nid, int save,
         params->length = i2d_DHparams(dh, &params->data);
 
     if (params->length <= 0) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB);
         ASN1_STRING_free(params);
         return 0;
     }
@@ -550,14 +550,14 @@ static int encode_dsa_params(const void *dsa, int nid,
     ASN1_STRING *params = ASN1_STRING_new();
 
     if (params == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB);
         return 0;
     }
 
     params->length = i2d_DSAparams(dsa, &params->data);
 
     if (params->length <= 0) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB);
         ASN1_STRING_free(params);
         return 0;
     }
@@ -645,13 +645,13 @@ static int prepare_ec_explicit_params(const void *eckey,
     ASN1_STRING *params = ASN1_STRING_new();
 
     if (params == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB);
         return 0;
     }
 
     params->length = i2d_ECParameters(eckey, &params->data);
     if (params->length <= 0) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB);
         ASN1_STRING_free(params);
         return 0;
     }
@@ -762,10 +762,8 @@ static int ecx_spki_pub_to_der(const void *vecxkey, unsigned char **pder)
     }
 
     keyblob = OPENSSL_memdup(ecxkey->pubkey, ecxkey->keylen);
-    if (keyblob == NULL) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+    if (keyblob == NULL)
         return 0;
-    }
 
     *pder = keyblob;
     return ecxkey->keylen;
@@ -788,7 +786,7 @@ static int ecx_pki_priv_to_der(const void *vecxkey, unsigned char **pder)
 
     keybloblen = i2d_ASN1_OCTET_STRING(&oct, pder);
     if (keybloblen < 0) {
-        ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PROV, ERR_R_ASN1_LIB);
         return 0;
     }