Fix malloc failure handling of X509_ALGOR_set0()

Also update and slightly extend the respective documentation and simplify some code.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16251)

1 files changed, 14 insertions, 12 deletions

diff --git a/crypto/pkcs7/pk7_lib.c b/crypto/pkcs7/pk7_lib.c
index c32a666626..61ecade42d 100644
--- a/crypto/pkcs7/pk7_lib.c
+++ b/crypto/pkcs7/pk7_lib.c
@@ -305,7 +305,7 @@ int PKCS7_add_crl(PKCS7 *p7, X509_CRL *crl)
 static int pkcs7_ecdsa_or_dsa_sign_verify_setup(PKCS7_SIGNER_INFO *si,
                                                 int verify)
 {
-    if (verify == 0) {
+    if (!verify) {
         int snid, hnid;
         X509_ALGOR *alg1, *alg2;
         EVP_PKEY *pkey = si->pkey;
@@ -318,19 +318,20 @@ static int pkcs7_ecdsa_or_dsa_sign_verify_setup(PKCS7_SIGNER_INFO *si,
             return -1;
         if (!OBJ_find_sigid_by_algs(&snid, hnid, EVP_PKEY_get_id(pkey)))
             return -1;
-        X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0);
+        return X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, NULL);
     }
     return 1;
 }
 
 static int pkcs7_rsa_sign_verify_setup(PKCS7_SIGNER_INFO *si, int verify)
 {
-    if (verify == 0) {
+    if (!verify) {
         X509_ALGOR *alg = NULL;
 
         PKCS7_SIGNER_INFO_get0_algs(si, NULL, NULL, &alg);
         if (alg != NULL)
-            X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
+            return X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
+                                   V_ASN1_NULL, NULL);
     }
     return 1;
 }
@@ -342,10 +343,10 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey,
 
     /* We now need to add another PKCS7_SIGNER_INFO entry */
     if (!ASN1_INTEGER_set(p7i->version, 1))
-        goto err;
+        return 0;
     if (!X509_NAME_set(&p7i->issuer_and_serial->issuer,
                        X509_get_issuer_name(x509)))
-        goto err;
+        return 0;
 
     /*
      * because ASN1_INTEGER_set is used to set a 'long' we will do things the
@@ -354,7 +355,7 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey,
     ASN1_INTEGER_free(p7i->issuer_and_serial->serial);
     if (!(p7i->issuer_and_serial->serial =
           ASN1_INTEGER_dup(X509_get0_serialNumber(x509))))
-        goto err;
+        return 0;
 
     /* lets keep the pkey around for a while */
     EVP_PKEY_up_ref(pkey);
@@ -362,8 +363,9 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey,
 
     /* Set the algorithms */
 
-    X509_ALGOR_set0(p7i->digest_alg, OBJ_nid2obj(EVP_MD_get_type(dgst)),
-                    V_ASN1_NULL, NULL);
+    if (!X509_ALGOR_set0(p7i->digest_alg, OBJ_nid2obj(EVP_MD_get_type(dgst)),
+                         V_ASN1_NULL, NULL))
+        return 0;
 
     if (EVP_PKEY_is_a(pkey, "EC") || EVP_PKEY_is_a(pkey, "DSA"))
         return pkcs7_ecdsa_or_dsa_sign_verify_setup(p7i, 0);
@@ -380,7 +382,6 @@ int PKCS7_SIGNER_INFO_set(PKCS7_SIGNER_INFO *p7i, X509 *x509, EVP_PKEY *pkey,
         }
     }
     ERR_raise(ERR_LIB_PKCS7, PKCS7_R_SIGNING_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);
- err:
     return 0;
 }
 
@@ -598,10 +599,11 @@ static int pkcs7_rsa_encrypt_decrypt_setup(PKCS7_RECIP_INFO *ri, int decrypt)
 {
     X509_ALGOR *alg = NULL;
 
-    if (decrypt == 0) {
+    if (!decrypt) {
         PKCS7_RECIP_INFO_get0_alg(ri, &alg);
         if (alg != NULL)
-            X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
+            return X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption),
+                                   V_ASN1_NULL, NULL);
     }
     return 1;
 }