Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

9 files changed, 30 insertions, 43 deletions

diff --git a/crypto/pkcs12/p12_add.c b/crypto/pkcs12/p12_add.c
index 6fd4184af5..8a56644368 100644
--- a/crypto/pkcs12/p12_add.c
+++ b/crypto/pkcs12/p12_add.c
@@ -24,16 +24,16 @@ PKCS12_SAFEBAG *PKCS12_item_pack_safebag(void *obj, const ASN1_ITEM *it,
     PKCS12_SAFEBAG *safebag;
 
     if ((bag = PKCS12_BAGS_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         return NULL;
     }
     bag->type = OBJ_nid2obj(nid1);
     if (!ASN1_item_pack(obj, it, &bag->value.octet)) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         goto err;
     }
     if ((safebag = PKCS12_SAFEBAG_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         goto err;
     }
     safebag->value.bag = bag;
@@ -51,12 +51,12 @@ PKCS7 *PKCS12_pack_p7data(STACK_OF(PKCS12_SAFEBAG) *sk)
     PKCS7 *p7;
 
     if ((p7 = PKCS7_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         return NULL;
     }
     p7->type = OBJ_nid2obj(NID_pkcs7_data);
     if ((p7->d.data = ASN1_OCTET_STRING_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         goto err;
     }
 
@@ -94,7 +94,7 @@ PKCS7 *PKCS12_pack_p7encdata_ex(int pbe_nid, const char *pass, int passlen,
     EVP_CIPHER *pbe_ciph_fetch = NULL;
 
     if ((p7 = PKCS7_new_ex(ctx, propq)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         return NULL;
     }
     if (!PKCS7_set_type(p7, NID_pkcs7_encrypted)) {
@@ -115,7 +115,7 @@ PKCS7 *PKCS12_pack_p7encdata_ex(int pbe_nid, const char *pass, int passlen,
     }
 
     if (pbe == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         goto err;
     }
     X509_ALGOR_free(p7->d.encrypted->enc_data->algorithm);
diff --git a/crypto/pkcs12/p12_decr.c b/crypto/pkcs12/p12_decr.c
index c4c63a2701..b916db0ab1 100644
--- a/crypto/pkcs12/p12_decr.c
+++ b/crypto/pkcs12/p12_decr.c
@@ -28,7 +28,7 @@ unsigned char *PKCS12_pbe_crypt_ex(const X509_ALGOR *algor,
     int max_out_len, mac_len = 0;
 
     if (ctx == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_EVP_LIB);
         goto err;
     }
 
@@ -67,10 +67,8 @@ unsigned char *PKCS12_pbe_crypt_ex(const X509_ALGOR *algor,
         }
     }
 
-    if ((out = OPENSSL_malloc(max_out_len)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+    if ((out = OPENSSL_malloc(max_out_len)) == NULL)
         goto err;
-    }
 
     if (!EVP_CipherUpdate(ctx, out, &i, in, inlen)) {
         OPENSSL_free(out);
@@ -180,7 +178,7 @@ ASN1_OCTET_STRING *PKCS12_item_i2d_encrypt_ex(X509_ALGOR *algor,
     int inlen;
 
     if ((oct = ASN1_OCTET_STRING_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         goto err;
     }
     inlen = ASN1_item_i2d(obj, &in, it);
diff --git a/crypto/pkcs12/p12_init.c b/crypto/pkcs12/p12_init.c
index 45aa2f9154..dd469b5c5c 100644
--- a/crypto/pkcs12/p12_init.c
+++ b/crypto/pkcs12/p12_init.c
@@ -20,7 +20,7 @@ PKCS12 *PKCS12_init_ex(int mode, OSSL_LIB_CTX *ctx, const char *propq)
     PKCS12 *pkcs12;
 
     if ((pkcs12 = PKCS12_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         return NULL;
     }
     if (!ASN1_INTEGER_set(pkcs12->version, 3))
@@ -29,14 +29,14 @@ PKCS12 *PKCS12_init_ex(int mode, OSSL_LIB_CTX *ctx, const char *propq)
 
     ossl_pkcs7_set0_libctx(pkcs12->authsafes, ctx);
     if (!ossl_pkcs7_set1_propq(pkcs12->authsafes, propq)) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_PKCS7_LIB);
         goto err;
     }
 
     switch (mode) {
     case NID_pkcs7_data:
         if ((pkcs12->authsafes->d.data = ASN1_OCTET_STRING_new()) == NULL) {
-            ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
             goto err;
         }
         break;
diff --git a/crypto/pkcs12/p12_key.c b/crypto/pkcs12/p12_key.c
index 41a2d7293e..9f7012a2c9 100644
--- a/crypto/pkcs12/p12_key.c
+++ b/crypto/pkcs12/p12_key.c
@@ -29,7 +29,7 @@ int PKCS12_key_gen_asc_ex(const char *pass, int passlen, unsigned char *salt,
         unipass = NULL;
         uniplen = 0;
     } else if (!OPENSSL_asc2uni(pass, passlen, &unipass, &uniplen)) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_PKCS12_LIB);
         return 0;
     }
     ret = PKCS12_key_gen_uni_ex(unipass, uniplen, salt, saltlen, id, iter,
@@ -59,7 +59,7 @@ int PKCS12_key_gen_utf8_ex(const char *pass, int passlen, unsigned char *salt,
         unipass = NULL;
         uniplen = 0;
     } else if (!OPENSSL_utf82uni(pass, passlen, &unipass, &uniplen)) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_PKCS12_LIB);
         return 0;
     }
     ret = PKCS12_key_gen_uni_ex(unipass, uniplen, salt, saltlen, id, iter,
diff --git a/crypto/pkcs12/p12_kiss.c b/crypto/pkcs12/p12_kiss.c
index b0864c1fbf..0f7a437a28 100644
--- a/crypto/pkcs12/p12_kiss.c
+++ b/crypto/pkcs12/p12_kiss.c
@@ -76,7 +76,7 @@ int PKCS12_parse(PKCS12 *p12, const char *pass, EVP_PKEY **pkey, X509 **cert,
     /* If needed, allocate stack for other certificates */
     if ((cert != NULL || ca != NULL)
             && (ocerts = sk_X509_new_null()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_CRYPTO_LIB);
         goto err;
     }
 
diff --git a/crypto/pkcs12/p12_mutl.c b/crypto/pkcs12/p12_mutl.c
index afdb8d688b..9497a5ce61 100644
--- a/crypto/pkcs12/p12_mutl.c
+++ b/crypto/pkcs12/p12_mutl.c
@@ -241,11 +241,11 @@ int PKCS12_setup_mac(PKCS12 *p12, int iter, unsigned char *salt, int saltlen,
         return PKCS12_ERROR;
     if (iter > 1) {
         if ((p12->mac->iter = ASN1_INTEGER_new()) == NULL) {
-            ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
             return 0;
         }
         if (!ASN1_INTEGER_set(p12->mac->iter, iter)) {
-            ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+            ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
             return 0;
         }
     }
@@ -253,10 +253,8 @@ int PKCS12_setup_mac(PKCS12 *p12, int iter, unsigned char *salt, int saltlen,
         saltlen = PKCS12_SALT_LEN;
     else if (saltlen < 0)
         return 0;
-    if ((p12->mac->salt->data = OPENSSL_malloc(saltlen)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+    if ((p12->mac->salt->data = OPENSSL_malloc(saltlen)) == NULL)
         return 0;
-    }
     p12->mac->salt->length = saltlen;
     if (salt == NULL) {
         if (RAND_bytes_ex(p12->authsafes->ctx.libctx, p12->mac->salt->data,
@@ -268,7 +266,7 @@ int PKCS12_setup_mac(PKCS12 *p12, int iter, unsigned char *salt, int saltlen,
     X509_SIG_getm(p12->mac->dinfo, &macalg, NULL);
     if (!X509_ALGOR_set0(macalg, OBJ_nid2obj(EVP_MD_get_type(md_type)),
                          V_ASN1_NULL, NULL)) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         return 0;
     }
 
diff --git a/crypto/pkcs12/p12_p8e.c b/crypto/pkcs12/p12_p8e.c
index 9c27534017..1230c8c88c 100644
--- a/crypto/pkcs12/p12_p8e.c
+++ b/crypto/pkcs12/p12_p8e.c
@@ -84,7 +84,6 @@ X509_SIG *PKCS8_set0_pbe_ex(const char *pass, int passlen,
     p8 = OPENSSL_zalloc(sizeof(*p8));
 
     if (p8 == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
         ASN1_OCTET_STRING_free(enckey);
         return NULL;
     }
diff --git a/crypto/pkcs12/p12_sbag.c b/crypto/pkcs12/p12_sbag.c
index 4593d595ec..7106936c62 100644
--- a/crypto/pkcs12/p12_sbag.c
+++ b/crypto/pkcs12/p12_sbag.c
@@ -119,7 +119,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create_secret(int type, int vtype, const unsigned
     PKCS12_SAFEBAG *safebag;
 
     if ((bag = PKCS12_BAGS_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         return NULL;
     }
     bag->type = OBJ_nid2obj(type);
@@ -130,7 +130,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create_secret(int type, int vtype, const unsigned
             ASN1_OCTET_STRING *strtmp = ASN1_OCTET_STRING_new();
 
             if (strtmp == NULL) {
-                ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
                 goto err;
             }
             /* Pack data into an octet string */
@@ -142,7 +142,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create_secret(int type, int vtype, const unsigned
             bag->value.other = ASN1_TYPE_new();
             if (bag->value.other == NULL) {
                 ASN1_OCTET_STRING_free(strtmp);
-                ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+                ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
                 goto err;
             }
             ASN1_TYPE_set(bag->value.other, vtype, strtmp);
@@ -155,7 +155,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create_secret(int type, int vtype, const unsigned
     }
 
     if ((safebag = PKCS12_SAFEBAG_new()) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         goto err;
     }
     safebag->value.bag = bag;
@@ -174,7 +174,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create0_p8inf(PKCS8_PRIV_KEY_INFO *p8)
     PKCS12_SAFEBAG *bag = PKCS12_SAFEBAG_new();
 
     if (bag == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         return NULL;
     }
     bag->type = OBJ_nid2obj(NID_keyBag);
@@ -190,7 +190,7 @@ PKCS12_SAFEBAG *PKCS12_SAFEBAG_create0_pkcs8(X509_SIG *p8)
 
     /* Set up the safe bag */
     if (bag == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_PKCS12, ERR_R_ASN1_LIB);
         return NULL;
     }
     bag->type = OBJ_nid2obj(NID_pkcs8ShroudedKeyBag);
diff --git a/crypto/pkcs12/p12_utl.c b/crypto/pkcs12/p12_utl.c
index 3afc8b2f13..6046b70886 100644
--- a/crypto/pkcs12/p12_utl.c
+++ b/crypto/pkcs12/p12_utl.c
@@ -24,10 +24,8 @@ unsigned char *OPENSSL_asc2uni(const char *asc, int asclen,
     if (asclen < 0)
         return NULL;
     ulen = asclen * 2 + 2;
-    if ((unitmp = OPENSSL_malloc(ulen)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+    if ((unitmp = OPENSSL_malloc(ulen)) == NULL)
         return NULL;
-    }
     for (i = 0; i < ulen - 2; i += 2) {
         unitmp[i] = 0;
         unitmp[i + 1] = asc[i >> 1];
@@ -57,10 +55,8 @@ char *OPENSSL_uni2asc(const unsigned char *uni, int unilen)
     if (!unilen || uni[unilen - 1])
         asclen++;
     uni++;
-    if ((asctmp = OPENSSL_malloc(asclen)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+    if ((asctmp = OPENSSL_malloc(asclen)) == NULL)
         return NULL;
-    }
     for (i = 0; i < unilen; i += 2)
         asctmp[i >> 1] = uni[i];
     asctmp[asclen - 1] = 0;
@@ -119,10 +115,8 @@ unsigned char *OPENSSL_utf82uni(const char *asc, int asclen,
 
     ulen += 2;  /* for trailing UTF16 zero */
 
-    if ((ret = OPENSSL_malloc(ulen)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+    if ((ret = OPENSSL_malloc(ulen)) == NULL)
         return NULL;
-    }
     /* re-run the loop writing down UTF-16 characters in big-endian order */
     for (unitmp = ret, i = 0; i < asclen; i += j) {
         j = UTF8_getc((const unsigned char *)asc+i, asclen-i, &utf32chr);
@@ -204,10 +198,8 @@ char *OPENSSL_uni2utf8(const unsigned char *uni, int unilen)
     if (!unilen || (uni[unilen-2]||uni[unilen - 1]))
         asclen++;
 
-    if ((asctmp = OPENSSL_malloc(asclen)) == NULL) {
-        ERR_raise(ERR_LIB_PKCS12, ERR_R_MALLOC_FAILURE);
+    if ((asctmp = OPENSSL_malloc(asclen)) == NULL)
         return NULL;
-    }
 
     /* re-run the loop emitting UTF-8 string */
     for (asclen = 0, i = 0; i < unilen; ) {