Move CMS signing code out of the algorithms and into CMS

There is a large amount of CMS sepcific code in the algorithms. This is in
the wrong place and breaks layering. This code should be in the CMS layer.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13088)

7 files changed, 160 insertions, 45 deletions

diff --git a/crypto/cms/build.info b/crypto/cms/build.info
index 2f5b653382..c7579f78c1 100644
--- a/crypto/cms/build.info
+++ b/crypto/cms/build.info
@@ -7,6 +7,6 @@ SOURCE[../../libcrypto]= \
 IF[{- !$disabled{dh} -}]
   SOURCE[../../libcrypto]=cms_dh.c
 ENDIF
-IF[{- !$disabled{ed} -}]
-  SOURCE[../../libcrypto]=cms_ecdh.c
-ENDIF
\ No newline at end of file
+IF[{- !$disabled{ec} || !$disabled{dsa}-}]
+  SOURCE[../../libcrypto]=cms_ec.c
+ENDIF
diff --git a/crypto/cms/cms_dh.c b/crypto/cms/cms_dh.c
index aca609653c..0d9eac50b3 100644
--- a/crypto/cms/cms_dh.c
+++ b/crypto/cms/cms_dh.c
@@ -7,7 +7,9 @@
  * https://www.openssl.org/source/license.html
  */
 
+#include <assert.h>
 #include <openssl/cms.h>
+#include <openssl/dh.h>
 #include <openssl/err.h>
 #include <openssl/core_names.h>
 #include "cms_local.h"
@@ -32,9 +34,7 @@ static int dh_cms_set_peerkey(EVP_PKEY_CTX *pctx,
         goto err;
 
     pk = EVP_PKEY_CTX_get0_pkey(pctx);
-    if (pk == NULL)
-        goto err;
-    if (!EVP_PKEY_is_a(pk, "DHX"))
+    if (pk == NULL || !EVP_PKEY_is_a(pk, "DHX"))
         goto err;
 
     /* Get public key */
@@ -85,10 +85,8 @@ static int dh_cms_set_shared_info(EVP_PKEY_CTX *pctx, CMS_RecipientInfo *ri)
         goto err;
     }
 
-    if (EVP_PKEY_CTX_set_dh_kdf_type(pctx, EVP_PKEY_DH_KDF_X9_42) <= 0)
-        goto err;
-
-    if (EVP_PKEY_CTX_set_dh_kdf_md(pctx, EVP_sha1()) <= 0)
+    if (EVP_PKEY_CTX_set_dh_kdf_type(pctx, EVP_PKEY_DH_KDF_X9_42) <= 0
+            || EVP_PKEY_CTX_set_dh_kdf_md(pctx, EVP_sha1()) <= 0)
         goto err;
 
     if (alg->parameter->type != V_ASN1_SEQUENCE)
@@ -139,9 +137,7 @@ static int dh_cms_set_shared_info(EVP_PKEY_CTX *pctx, CMS_RecipientInfo *ri)
 
 static int dh_cms_decrypt(CMS_RecipientInfo *ri)
 {
-    EVP_PKEY_CTX *pctx;
-
-    pctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
+    EVP_PKEY_CTX *pctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
 
     if (pctx == NULL)
         return 0;
@@ -194,8 +190,9 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri)
     if (!CMS_RecipientInfo_kari_get0_orig_id(ri, &talg, &pubkey,
                                              NULL, NULL, NULL))
         goto err;
-    X509_ALGOR_get0(&aoid, NULL, NULL, talg);
+
     /* Is everything uninitialised? */
+    X509_ALGOR_get0(&aoid, NULL, NULL, talg);
     if (aoid == OBJ_nid2obj(NID_undef)) {
         BIGNUM *bn_pub_key = NULL;
         ASN1_INTEGER *pubk;
@@ -207,8 +204,8 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri)
         BN_free(bn_pub_key);
         if (pubk == NULL)
             goto err;
-        /* Set the key */
 
+        /* Set the key */
         penclen = i2d_ASN1_INTEGER(pubk, &penc);
         ASN1_INTEGER_free(pubk);
         if (penclen <= 0)
@@ -224,9 +221,7 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri)
 
     /* See if custom parameters set */
     kdf_type = EVP_PKEY_CTX_get_dh_kdf_type(pctx);
-    if (kdf_type <= 0)
-        goto err;
-    if (!EVP_PKEY_CTX_get_dh_kdf_md(pctx, &kdf_md))
+    if (kdf_type <= 0 || !EVP_PKEY_CTX_get_dh_kdf_md(pctx, &kdf_md))
         goto err;
 
     if (kdf_type == EVP_PKEY_DH_KDF_NONE) {
@@ -312,11 +307,14 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri)
 
 int cms_dh_envelope(CMS_RecipientInfo *ri, int decrypt)
 {
+    assert(decrypt == 0 || decrypt == 1);
+
     if (decrypt == 1)
         return dh_cms_decrypt(ri);
-    else if (decrypt == 0)
+
+    if (decrypt == 0)
         return dh_cms_encrypt(ri);
 
     CMSerr(0, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);
     return 0;
-}
\ No newline at end of file
+}
diff --git a/crypto/cms/cms_ecdh.c b/crypto/cms/cms_ec.c
index b88be91211..394c8b4dc3 100644
--- a/crypto/cms/cms_ecdh.c
+++ b/crypto/cms/cms_ec.c
@@ -7,13 +7,14 @@
  * https://www.openssl.org/source/license.html
  */
 
+#include <assert.h>
 #include <openssl/cms.h>
 #include <openssl/err.h>
 #include <openssl/decoder.h>
 #include "cms_local.h"
 #include "crypto/evp.h"
 
-
+#ifndef OPENSSL_NO_EC
 static EVP_PKEY *pkey_type2param(int ptype, const void *pval,
                                  OPENSSL_CTX *libctx, const char *propq)
 {
@@ -30,7 +31,14 @@ static EVP_PKEY *pkey_type2param(int ptype, const void *pval,
         /* TODO(3.0): Need to be able to specify here that only params will do */
         ctx = OSSL_DECODER_CTX_new_by_EVP_PKEY(&pkey, "DER", "EC", libctx,
                                                propq);
+        if (ctx == NULL)
+            goto err;
+
         membio = BIO_new_mem_buf(pm, pmlen);
+        if (membio == NULL) {
+            OSSL_DECODER_CTX_free(ctx);
+            goto err;
+        }
         OSSL_DECODER_from_bio(ctx, membio);
         BIO_free(membio);
         OSSL_DECODER_CTX_free(ctx);
@@ -38,15 +46,9 @@ static EVP_PKEY *pkey_type2param(int ptype, const void *pval,
         const ASN1_OBJECT *poid = pval;
         const char *groupname;
 
-        /*
-         * type == V_ASN1_OBJECT => the parameters are given by an asn1 OID
-         */
-
+        /* type == V_ASN1_OBJECT => the parameters are given by an asn1 OID */
         pctx = EVP_PKEY_CTX_new_from_name(libctx, "EC", propq);
-
-        if (pctx == NULL)
-            goto err;
-        if (EVP_PKEY_paramgen_init(pctx) <= 0)
+        if (pctx == NULL || EVP_PKEY_paramgen_init(pctx) <= 0)
             goto err;
         groupname = OBJ_nid2sn(OBJ_obj2nid(poid));
         if (groupname == NULL
@@ -54,10 +56,8 @@ static EVP_PKEY *pkey_type2param(int ptype, const void *pval,
             CMSerr(0, CMS_R_DECODE_ERROR);
             goto err;
         }
-        if (EVP_PKEY_paramgen(pctx, &pkey) <= 0) {
-            EVP_PKEY_free(pkey);
-            pkey = NULL;
-        }
+        if (EVP_PKEY_paramgen(pctx, &pkey) <= 0)
+            goto err;
     } else {
         CMSerr(0, CMS_R_DECODE_ERROR);
         goto err;
@@ -85,9 +85,11 @@ static int ecdh_cms_set_peerkey(EVP_PKEY_CTX *pctx,
     X509_ALGOR_get0(&aoid, &atype, &aval, alg);
     if (OBJ_obj2nid(aoid) != NID_X9_62_id_ecPublicKey)
         goto err;
+
     /* If absent parameters get group from main key */
     if (atype == V_ASN1_UNDEF || atype == V_ASN1_NULL) {
         EVP_PKEY *pk;
+
         pk = EVP_PKEY_CTX_get0_pkey(pctx);
         if (pk == NULL)
             goto err;
@@ -127,6 +129,7 @@ static int ecdh_cms_set_kdf_param(EVP_PKEY_CTX *pctx, int eckdf_nid)
 {
     int kdf_nid, kdfmd_nid, cofactor;
     const EVP_MD *kdf_md;
+
     if (eckdf_nid == NID_undef)
         return 0;
 
@@ -159,7 +162,6 @@ static int ecdh_cms_set_kdf_param(EVP_PKEY_CTX *pctx, int eckdf_nid)
 static int ecdh_cms_set_shared_info(EVP_PKEY_CTX *pctx, CMS_RecipientInfo *ri)
 {
     int rv = 0;
-
     X509_ALGOR *alg, *kekalg = NULL;
     ASN1_OCTET_STRING *ukm;
     const unsigned char *p;
@@ -233,7 +235,7 @@ static int ecdh_cms_decrypt(CMS_RecipientInfo *ri)
         if (!CMS_RecipientInfo_kari_get0_orig_id(ri, &alg, &pubkey,
                                                  NULL, NULL, NULL))
             return 0;
-        if (!alg || !pubkey)
+        if (alg == NULL || pubkey == NULL)
             return 0;
         if (!ecdh_cms_set_peerkey(pctx, alg, pubkey)) {
             CMSerr(0, CMS_R_PEER_KEY_ERROR);
@@ -350,7 +352,7 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
 
     penclen = CMS_SharedInfo_encode(&penc, wrap_alg, ukm, keylen);
 
-    if (!penclen)
+    if (penclen == 0)
         goto err;
 
     if (EVP_PKEY_CTX_set0_ecdh_kdf_ukm(pctx, penc, penclen) <= 0)
@@ -362,7 +364,7 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
      * of another AlgorithmIdentifier.
      */
     penclen = i2d_X509_ALGOR(wrap_alg, &penc);
-    if (!penc || !penclen)
+    if (penc == NULL || penclen == 0)
         goto err;
     wrap_str = ASN1_STRING_new();
     if (wrap_str == NULL)
@@ -381,11 +383,38 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
 
 int cms_ecdh_envelope(CMS_RecipientInfo *ri, int decrypt)
 {
+    assert(decrypt == 0 || decrypt == 1);
+
     if (decrypt == 1)
         return ecdh_cms_decrypt(ri);
-    else if (decrypt == 0)
+
+    if (decrypt == 0)
         return ecdh_cms_encrypt(ri);
 
     CMSerr(0, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);
     return 0;
-}
\ No newline at end of file
+}
+#endif
+
+/* ECDSA and DSA implementation is the same */
+int cms_ecdsa_dsa_sign(CMS_SignerInfo *si, int verify)
+{
+    assert(verify == 0 || verify == 1);
+
+    if (verify == 0) {
+        int snid, hnid;
+        X509_ALGOR *alg1, *alg2;
+        EVP_PKEY *pkey = si->pkey;
+
+        CMS_SignerInfo_get0_algs(si, NULL, NULL, &alg1, &alg2);
+        if (alg1 == NULL || alg1->algorithm == NULL)
+            return -1;
+        hnid = OBJ_obj2nid(alg1->algorithm);
+        if (hnid == NID_undef)
+            return -1;
+        if (!OBJ_find_sigid_by_algs(&snid, hnid, EVP_PKEY_id(pkey)))
+            return -1;
+        X509_ALGOR_set0(alg2, OBJ_nid2obj(snid), V_ASN1_UNDEF, 0);
+    }
+    return 1;
+}
diff --git a/crypto/cms/cms_env.c b/crypto/cms/cms_env.c
index 395e05ce55..84d9e5b948 100644
--- a/crypto/cms/cms_env.c
+++ b/crypto/cms/cms_env.c
@@ -116,11 +116,17 @@ int cms_env_asn1_ctrl(CMS_RecipientInfo *ri, int cmd)
     } else
         return 0;
 
+#ifndef OPENSSL_NO_DH
     if (EVP_PKEY_is_a(pkey, "DHX"))
         return cms_dh_envelope(ri, cmd);
-    else if (EVP_PKEY_is_a(pkey, "EC"))
+    else
+#endif
+#ifndef OPENSSL_NO_EC
+    if (EVP_PKEY_is_a(pkey, "EC"))
         return cms_ecdh_envelope(ri, cmd);
-    else if (EVP_PKEY_is_a(pkey, "RSA"))
+    else
+#endif
+    if (EVP_PKEY_is_a(pkey, "RSA"))
         return cms_rsa_envelope(ri, cmd);
 
     /* Something else? We'll give engines etc a chance to handle this */
diff --git a/crypto/cms/cms_local.h b/crypto/cms/cms_local.h
index e162ea13ad..0b663a9e88 100644
--- a/crypto/cms/cms_local.h
+++ b/crypto/cms/cms_local.h
@@ -474,6 +474,8 @@ int ess_check_signing_certs(CMS_SignerInfo *si, STACK_OF(X509) *chain);
 int cms_dh_envelope(CMS_RecipientInfo *ri, int decrypt);
 int cms_ecdh_envelope(CMS_RecipientInfo *ri, int decrypt);
 int cms_rsa_envelope(CMS_RecipientInfo *ri, int decrypt);
+int cms_ecdsa_dsa_sign(CMS_SignerInfo *si, int verify);
+int cms_rsa_sign(CMS_SignerInfo *si, int verify);
 
 DECLARE_ASN1_ITEM(CMS_CertificateChoices)
 DECLARE_ASN1_ITEM(CMS_DigestedData)
diff --git a/crypto/cms/cms_rsa.c b/crypto/cms/cms_rsa.c
index 82d36d9b1f..88201d7b44 100644
--- a/crypto/cms/cms_rsa.c
+++ b/crypto/cms/cms_rsa.c
@@ -7,12 +7,13 @@
  * https://www.openssl.org/source/license.html
  */
 
+#include <assert.h>
 #include <openssl/cms.h>
 #include <openssl/err.h>
 #include "crypto/asn1.h"
+#include "crypto/rsa.h"
 #include "cms_local.h"
 
-
 static RSA_OAEP_PARAMS *rsa_oaep_decode(const X509_ALGOR *alg)
 {
     RSA_OAEP_PARAMS *oaep;
@@ -118,7 +119,7 @@ static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
 
     if (CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg) <= 0)
         return 0;
-    if (pkctx) {
+    if (pkctx != NULL) {
         if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
             return 0;
     }
@@ -145,6 +146,7 @@ static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
         goto err;
     if (labellen > 0) {
         ASN1_OCTET_STRING *los;
+
         oaep->pSourceFunc = X509_ALGOR_new();
         if (oaep->pSourceFunc == NULL)
             goto err;
@@ -172,11 +174,80 @@ static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
 
 int cms_rsa_envelope(CMS_RecipientInfo *ri, int decrypt)
 {
+    assert(decrypt == 0 || decrypt == 1);
+
     if (decrypt == 1)
         return rsa_cms_decrypt(ri);
-    else if (decrypt == 0)
+
+    if (decrypt == 0)
         return rsa_cms_encrypt(ri);
 
     CMSerr(0, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);
     return 0;
-}
\ No newline at end of file
+}
+
+static int rsa_cms_sign(CMS_SignerInfo *si)
+{
+    int pad_mode = RSA_PKCS1_PADDING;
+    X509_ALGOR *alg;
+    EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
+    ASN1_STRING *os = NULL;
+
+    CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
+    if (pkctx != NULL) {
+        if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
+            return 0;
+    }
+    if (pad_mode == RSA_PKCS1_PADDING) {
+        X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
+        return 1;
+    }
+    /* We don't support it */
+    if (pad_mode != RSA_PKCS1_PSS_PADDING)
+        return 0;
+    os = ossl_rsa_ctx_to_pss_string(pkctx);
+    if (os == NULL)
+        return 0;
+    X509_ALGOR_set0(alg, OBJ_nid2obj(EVP_PKEY_RSA_PSS), V_ASN1_SEQUENCE, os);
+    return 1;
+}
+
+static int rsa_cms_verify(CMS_SignerInfo *si)
+{
+    int nid, nid2;
+    X509_ALGOR *alg;
+    EVP_PKEY_CTX *pkctx = CMS_SignerInfo_get0_pkey_ctx(si);
+    EVP_PKEY *pkey = EVP_PKEY_CTX_get0_pkey(pkctx);
+
+    CMS_SignerInfo_get0_algs(si, NULL, NULL, NULL, &alg);
+    nid = OBJ_obj2nid(alg->algorithm);
+    if (nid == EVP_PKEY_RSA_PSS)
+        return ossl_rsa_pss_to_ctx(NULL, pkctx, alg, NULL);
+    /* Only PSS allowed for PSS keys */
+    if (EVP_PKEY_is_a(pkey, "RSA-PSS")) {
+        RSAerr(RSA_F_RSA_CMS_VERIFY, RSA_R_ILLEGAL_OR_UNSUPPORTED_PADDING_MODE);
+        return 0;
+    }
+    if (nid == NID_rsaEncryption)
+        return 1;
+    /* Workaround for some implementation that use a signature OID */
+    if (OBJ_find_sigid_algs(nid, NULL, &nid2)) {
+        if (nid2 == NID_rsaEncryption)
+            return 1;
+    }
+    return 0;
+}
+
+int cms_rsa_sign(CMS_SignerInfo *si, int verify)
+{
+    assert(verify == 0 || verify == 1);
+
+    if (verify == 1)
+        return rsa_cms_verify(si);
+
+    if (verify == 0)
+        return rsa_cms_sign(si);
+
+    CMSerr(0, CMS_R_NOT_SUPPORTED_FOR_THIS_KEY_TYPE);
+    return 0;
+}
diff --git a/crypto/cms/cms_sd.c b/crypto/cms/cms_sd.c
index 1338211072..377fac5917 100644
--- a/crypto/cms/cms_sd.c
+++ b/crypto/cms/cms_sd.c
@@ -233,6 +233,15 @@ static int cms_sd_asn1_ctrl(CMS_SignerInfo *si, int cmd)
     EVP_PKEY *pkey = si->pkey;
     int i;
 
+#if !defined(OPENSSL_NO_DSA) || !defined(OPENSSL_NO_EC)
+    if (EVP_PKEY_is_a(pkey, "DSA") || EVP_PKEY_is_a(pkey, "EC"))
+        return cms_ecdsa_dsa_sign(si, cmd);
+    else
+#endif
+    if (EVP_PKEY_is_a(pkey, "RSA") || EVP_PKEY_is_a(pkey, "RSA-PSS"))
+        return cms_rsa_sign(si, cmd);
+
+    /* Something else? We'll give engines etc a chance to handle this */
     if (pkey->ameth == NULL || pkey->ameth->pkey_ctrl == NULL)
         return 1;
     i = pkey->ameth->pkey_ctrl(pkey, ASN1_PKEY_CTRL_CMS_SIGN, cmd, si);