Stop raising ERR_R_MALLOC_FAILURE in most places

Since OPENSSL_malloc() and friends report ERR_R_MALLOC_FAILURE, and
at least handle the file name and line number they are called from,
there's no need to report ERR_R_MALLOC_FAILURE where they are called
directly, or when SSLfatal() and RLAYERfatal() is used, the reason
`ERR_R_MALLOC_FAILURE` is changed to `ERR_R_CRYPTO_LIB`.

There were a number of places where `ERR_R_MALLOC_FAILURE` was reported
even though it was a function from a different sub-system that was
called.  Those places are changed to report ERR_R_{lib}_LIB, where
{lib} is the name of that sub-system.
Some of them are tricky to get right, as we have a lot of functions
that belong in the ASN1 sub-system, and all the `sk_` calls or from
the CRYPTO sub-system.

Some extra adaptation was necessary where there were custom OPENSSL_malloc()
wrappers, and some bugs are fixed alongside these changes.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19301)

6 files changed, 17 insertions, 25 deletions

diff --git a/crypto/dh/dh_ameth.c b/crypto/dh/dh_ameth.c
index 8430872a9a..80e1612256 100644
--- a/crypto/dh/dh_ameth.c
+++ b/crypto/dh/dh_ameth.c
@@ -121,12 +121,12 @@ static int dh_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
 
     str = ASN1_STRING_new();
     if (str == NULL) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB);
         goto err;
     }
     str->length = i2d_dhp(pkey, dh, &str->data);
     if (str->length <= 0) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB);
         goto err;
     }
     ptype = V_ASN1_SEQUENCE;
@@ -140,7 +140,7 @@ static int dh_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
     ASN1_INTEGER_free(pub_key);
 
     if (penclen <= 0) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB);
         goto err;
     }
 
@@ -184,13 +184,13 @@ static int dh_priv_encode(PKCS8_PRIV_KEY_INFO *p8, const EVP_PKEY *pkey)
     params = ASN1_STRING_new();
 
     if (params == NULL) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB);
         goto err;
     }
 
     params->length = i2d_dhp(pkey, pkey->pkey.dh, &params->data);
     if (params->length <= 0) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DH, ERR_R_ASN1_LIB);
         goto err;
     }
     params->type = V_ASN1_SEQUENCE;
@@ -514,7 +514,7 @@ static int dh_pkey_import_from_type(const OSSL_PARAM params[], void *vpctx,
     DH *dh = ossl_dh_new_ex(pctx->libctx);
 
     if (dh == NULL) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DH, ERR_R_DH_LIB);
         return 0;
     }
     DH_clear_flags(dh, DH_FLAG_TYPE_MASK);
diff --git a/crypto/dh/dh_err.c b/crypto/dh/dh_err.c
index 4152397426..f1f4fb6a37 100644
--- a/crypto/dh/dh_err.c
+++ b/crypto/dh/dh_err.c
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -42,6 +42,7 @@ static const ERR_STRING_DATA DH_str_reasons[] = {
     "invalid parameter nid"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_INVALID_PUBKEY), "invalid public key"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_INVALID_SECRET), "invalid secret"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_INVALID_SIZE), "invalid size"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_KDF_PARAMETER_ERROR), "kdf parameter error"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_KEYS_NOT_SET), "keys not set"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_MISSING_PUBKEY), "missing pubkey"},
diff --git a/crypto/dh/dh_key.c b/crypto/dh/dh_key.c
index 4e9705beef..0854395fc5 100644
--- a/crypto/dh/dh_key.c
+++ b/crypto/dh/dh_key.c
@@ -418,14 +418,15 @@ size_t ossl_dh_key2buf(const DH *dh, unsigned char **pbuf_out, size_t size,
         if (!alloc) {
             if (size >= (size_t)p_size)
                 pbuf = *pbuf_out;
+            if (pbuf == NULL)
+                ERR_raise(ERR_LIB_DH, DH_R_INVALID_SIZE);
         } else {
             pbuf = OPENSSL_malloc(p_size);
         }
 
-        if (pbuf == NULL) {
-            ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        /* Errors raised above */
+        if (pbuf == NULL)
             return 0;
-        }
         /*
          * As per Section 4.2.8.1 of RFC 8446 left pad public
          * key with zeros to the size of p
diff --git a/crypto/dh/dh_lib.c b/crypto/dh/dh_lib.c
index 29cda5d7bf..575442eceb 100644
--- a/crypto/dh/dh_lib.c
+++ b/crypto/dh/dh_lib.c
@@ -75,15 +75,13 @@ static DH *dh_new_intern(ENGINE *engine, OSSL_LIB_CTX *libctx)
 {
     DH *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if (ret == NULL) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+    if (ret == NULL)
         return NULL;
-    }
 
     ret->references = 1;
     ret->lock = CRYPTO_THREAD_lock_new();
     if (ret->lock == NULL) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        ERR_raise(ERR_LIB_DH, ERR_R_CRYPTO_LIB);
         OPENSSL_free(ret);
         return NULL;
     }
diff --git a/crypto/dh/dh_meth.c b/crypto/dh/dh_meth.c
index 5c15cd2b8c..f5652e0785 100644
--- a/crypto/dh/dh_meth.c
+++ b/crypto/dh/dh_meth.c
@@ -31,7 +31,6 @@ DH_METHOD *DH_meth_new(const char *name, int flags)
         OPENSSL_free(dhm);
     }
 
-    ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
     return NULL;
 }
 
@@ -57,7 +56,6 @@ DH_METHOD *DH_meth_dup(const DH_METHOD *dhm)
         OPENSSL_free(ret);
     }
 
-    ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
     return NULL;
 }
 
@@ -70,10 +68,8 @@ int DH_meth_set1_name(DH_METHOD *dhm, const char *name)
 {
     char *tmpname = OPENSSL_strdup(name);
 
-    if (tmpname == NULL) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+    if (tmpname == NULL)
         return 0;
-    }
 
     OPENSSL_free(dhm->name);
     dhm->name = tmpname;
diff --git a/crypto/dh/dh_pmeth.c b/crypto/dh/dh_pmeth.c
index bd7902c433..c11ada9826 100644
--- a/crypto/dh/dh_pmeth.c
+++ b/crypto/dh/dh_pmeth.c
@@ -55,10 +55,8 @@ static int pkey_dh_init(EVP_PKEY_CTX *ctx)
 {
     DH_PKEY_CTX *dctx;
 
-    if ((dctx = OPENSSL_zalloc(sizeof(*dctx))) == NULL) {
-        ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+    if ((dctx = OPENSSL_zalloc(sizeof(*dctx))) == NULL)
         return 0;
-    }
     dctx->prime_len = 2048;
     dctx->subprime_len = -1;
     dctx->generator = 2;
@@ -445,10 +443,8 @@ static int pkey_dh_derive(EVP_PKEY_CTX *ctx, unsigned char *key,
         ret = 0;
         if ((Zlen = DH_size(dh)) <= 0)
             return 0;
-        if ((Z = OPENSSL_malloc(Zlen)) == NULL) {
-            ERR_raise(ERR_LIB_DH, ERR_R_MALLOC_FAILURE);
+        if ((Z = OPENSSL_malloc(Zlen)) == NULL)
             return 0;
-        }
         if (DH_compute_key_padded(Z, dhpubbn, dh) <= 0)
             goto err;
         if (!DH_KDF_X9_42(key, *keylen, Z, Zlen, dctx->kdf_oid,