Fix malloc failure handling of X509_ALGOR_set0()

Also update and slightly extend the respective documentation and simplify some code.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16251)

2 files changed, 20 insertions, 29 deletions

diff --git a/crypto/asn1/a_sign.c b/crypto/asn1/a_sign.c
index 302045cfcd..df251719f6 100644
--- a/crypto/asn1/a_sign.c
+++ b/crypto/asn1/a_sign.c
@@ -247,16 +247,14 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it, X509_ALGOR *algor1,
             goto err;
         }
 
-        if (pkey->ameth->pkey_flags & ASN1_PKEY_SIGPARAM_NULL)
-            paramtype = V_ASN1_NULL;
-        else
-            paramtype = V_ASN1_UNDEF;
-
-        if (algor1)
-            X509_ALGOR_set0(algor1, OBJ_nid2obj(signid), paramtype, NULL);
-        if (algor2)
-            X509_ALGOR_set0(algor2, OBJ_nid2obj(signid), paramtype, NULL);
-
+        paramtype = pkey->ameth->pkey_flags & ASN1_PKEY_SIGPARAM_NULL ?
+            V_ASN1_NULL : V_ASN1_UNDEF;
+        if (algor1 != NULL
+            && !X509_ALGOR_set0(algor1, OBJ_nid2obj(signid), paramtype, NULL))
+            goto err;
+        if (algor2 != NULL
+            && !X509_ALGOR_set0(algor2, OBJ_nid2obj(signid), paramtype, NULL))
+            goto err;
     }
 
     buf_len = ASN1_item_i2d(data, &buf_in, it);
diff --git a/crypto/asn1/x_algor.c b/crypto/asn1/x_algor.c
index f56ec92f65..e78cf7a68b 100644
--- a/crypto/asn1/x_algor.c
+++ b/crypto/asn1/x_algor.c
@@ -33,12 +33,9 @@ int X509_ALGOR_set0(X509_ALGOR *alg, ASN1_OBJECT *aobj, int ptype, void *pval)
     if (alg == NULL)
         return 0;
 
-    if (ptype != V_ASN1_UNDEF) {
-        if (alg->parameter == NULL)
-            alg->parameter = ASN1_TYPE_new();
-        if (alg->parameter == NULL)
-            return 0;
-    }
+    if (ptype != V_ASN1_UNDEF && alg->parameter == NULL
+            && (alg->parameter = ASN1_TYPE_new()) == NULL)
+        return 0;
 
     ASN1_OBJECT_free(alg->algorithm);
     alg->algorithm = aobj;
@@ -68,7 +65,7 @@ X509_ALGOR *ossl_X509_ALGOR_from_nid(int nid, int ptype, void *pval)
 
  err:
     X509_ALGOR_free(alg);
-    ASN1_OBJECT_free(algo);
+    /* ASN1_OBJECT_free(algo) is not needed due to OBJ_nid2obj() */
     return NULL;
 }
 
@@ -89,18 +86,12 @@ void X509_ALGOR_get0(const ASN1_OBJECT **paobj, int *pptype,
 }
 
 /* Set up an X509_ALGOR DigestAlgorithmIdentifier from an EVP_MD */
-
 void X509_ALGOR_set_md(X509_ALGOR *alg, const EVP_MD *md)
 {
-    int param_type;
-
-    if (md->flags & EVP_MD_FLAG_DIGALGID_ABSENT)
-        param_type = V_ASN1_UNDEF;
-    else
-        param_type = V_ASN1_NULL;
-
-    X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_MD_get_type(md)), param_type, NULL);
+    int type = md->flags & EVP_MD_FLAG_DIGALGID_ABSENT ? V_ASN1_UNDEF
+                                                       : V_ASN1_NULL;
 
+    (void)X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_MD_get_type(md)), type, NULL);
 }
 
 int X509_ALGOR_cmp(const X509_ALGOR *a, const X509_ALGOR *b)
@@ -150,13 +141,15 @@ int X509_ALGOR_copy(X509_ALGOR *dest, const X509_ALGOR *src)
 /* allocate and set algorithm ID from EVP_MD, default SHA1 */
 int ossl_x509_algor_new_from_md(X509_ALGOR **palg, const EVP_MD *md)
 {
+    X509_ALGOR *alg;
+
     /* Default is SHA1 so no need to create it - still success */
     if (md == NULL || EVP_MD_is_a(md, "SHA1"))
         return 1;
-    *palg = X509_ALGOR_new();
-    if (*palg == NULL)
+    if ((alg = X509_ALGOR_new()) == NULL)
         return 0;
-    X509_ALGOR_set_md(*palg, md);
+    X509_ALGOR_set_md(alg, md);
+    *palg = alg;
     return 1;
 }