Move CMS enveloping code out of the algorithms and into CMS

There is quite a large amount of algorithm specific CMS code sitting in
the algorithm directories. However, this seems to break layering.
Algorithms really have no business knowing anything about CMS. Really it
should be the other way around. Where there is algorithm specific CMS code
it is the CMS layer that should know how to handle different algorithms.

Therefore we move this code into the CMS layer.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13088)

1 files changed, 7 insertions, 237 deletions

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 6105b7849a..562f307bcb 100644
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -29,8 +29,6 @@
 #ifndef OPENSSL_NO_CMS
 static int rsa_cms_sign(CMS_SignerInfo *si);
 static int rsa_cms_verify(CMS_SignerInfo *si);
-static int rsa_cms_decrypt(CMS_RecipientInfo *ri);
-static int rsa_cms_encrypt(CMS_RecipientInfo *ri);
 #endif
 
 static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg);
@@ -256,14 +254,6 @@ static void int_rsa_free(EVP_PKEY *pkey)
     RSA_free(pkey->pkey.rsa);
 }
 
-static X509_ALGOR *rsa_mgf1_decode(X509_ALGOR *alg)
-{
-    if (OBJ_obj2nid(alg->algorithm) != NID_mgf1)
-        return NULL;
-    return ASN1_TYPE_unpack_sequence(ASN1_ITEM_rptr(X509_ALGOR),
-                                     alg->parameter);
-}
-
 static int rsa_pss_param_print(BIO *bp, int pss_key, RSA_PSS_PARAMS *pss,
                                int indent)
 {
@@ -315,7 +305,7 @@ static int rsa_pss_param_print(BIO *bp, int pss_key, RSA_PSS_PARAMS *pss,
             goto err;
         if (BIO_puts(bp, " with ") <= 0)
             goto err;
-        maskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
+        maskHash = x509_algor_mgf1_decode(pss->maskGenAlgorithm);
         if (maskHash != NULL) {
             if (i2a_ASN1_OBJECT(bp, maskHash->algorithm) <= 0)
                 goto err;
@@ -471,7 +461,7 @@ static RSA_PSS_PARAMS *rsa_pss_decode(const X509_ALGOR *alg)
         return NULL;
 
     if (pss->maskGenAlgorithm != NULL) {
-        pss->maskHash = rsa_mgf1_decode(pss->maskGenAlgorithm);
+        pss->maskHash = x509_algor_mgf1_decode(pss->maskGenAlgorithm);
         if (pss->maskHash == NULL) {
             RSA_PSS_PARAMS_free(pss);
             return NULL;
@@ -528,15 +518,6 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
             return rsa_cms_verify(arg2);
         break;
 
-    case ASN1_PKEY_CTRL_CMS_ENVELOPE:
-        if (pkey_is_pss(pkey))
-            return -2;
-        if (arg1 == 0)
-            return rsa_cms_encrypt(arg2);
-        else if (arg1 == 1)
-            return rsa_cms_decrypt(arg2);
-        break;
-
     case ASN1_PKEY_CTRL_CMS_RI_TYPE:
         if (pkey_is_pss(pkey))
             return -2;
@@ -570,58 +551,6 @@ static int rsa_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 
 }
 
-/* allocate and set algorithm ID from EVP_MD, default SHA1 */
-static int rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md)
-{
-    if (md == NULL || EVP_MD_type(md) == NID_sha1)
-        return 1;
-    *palg = X509_ALGOR_new();
-    if (*palg == NULL)
-        return 0;
-    X509_ALGOR_set_md(*palg, md);
-    return 1;
-}
-
-/* Allocate and set MGF1 algorithm ID from EVP_MD */
-static int rsa_md_to_mgf1(X509_ALGOR **palg, const EVP_MD *mgf1md)
-{
-    X509_ALGOR *algtmp = NULL;
-    ASN1_STRING *stmp = NULL;
-
-    *palg = NULL;
-    if (mgf1md == NULL || EVP_MD_type(mgf1md) == NID_sha1)
-        return 1;
-    /* need to embed algorithm ID inside another */
-    if (!rsa_md_to_algor(&algtmp, mgf1md))
-        goto err;
-    if (ASN1_item_pack(algtmp, ASN1_ITEM_rptr(X509_ALGOR), &stmp) == NULL)
-         goto err;
-    *palg = X509_ALGOR_new();
-    if (*palg == NULL)
-        goto err;
-    X509_ALGOR_set0(*palg, OBJ_nid2obj(NID_mgf1), V_ASN1_SEQUENCE, stmp);
-    stmp = NULL;
- err:
-    ASN1_STRING_free(stmp);
-    X509_ALGOR_free(algtmp);
-    if (*palg)
-        return 1;
-    return 0;
-}
-
-/* convert algorithm ID to EVP_MD, default SHA1 */
-static const EVP_MD *rsa_algor_to_md(X509_ALGOR *alg)
-{
-    const EVP_MD *md;
-
-    if (!alg)
-        return EVP_sha1();
-    md = EVP_get_digestbyobj(alg->algorithm);
-    if (md == NULL)
-        RSAerr(RSA_F_RSA_ALGOR_TO_MD, RSA_R_UNKNOWN_DIGEST);
-    return md;
-}
-
 /*
  * Convert EVP_PKEY_CTX in PSS mode into corresponding algorithm parameter,
  * suitable for setting an AlgorithmIdentifier.
@@ -667,13 +596,13 @@ RSA_PSS_PARAMS *rsa_pss_params_create(const EVP_MD *sigmd,
         if (!ASN1_INTEGER_set(pss->saltLength, saltlen))
             goto err;
     }
-    if (!rsa_md_to_algor(&pss->hashAlgorithm, sigmd))
+    if (!x509_algor_new_from_md(&pss->hashAlgorithm, sigmd))
         goto err;
     if (mgf1md == NULL)
         mgf1md = sigmd;
-    if (!rsa_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md))
+    if (!x509_algor_md_to_mgf1(&pss->maskGenAlgorithm, mgf1md))
         goto err;
-    if (!rsa_md_to_algor(&pss->maskHash, mgf1md))
+    if (!x509_algor_new_from_md(&pss->maskHash, mgf1md))
         goto err;
     return pss;
  err:
@@ -781,10 +710,10 @@ static int rsa_pss_get_param_unverified(const RSA_PSS_PARAMS *pss,
 
     if (pss == NULL)
         return 0;
-    *pmd = rsa_algor_to_md(pss->hashAlgorithm);
+    *pmd = x509_algor_get_md(pss->hashAlgorithm);
     if (*pmd == NULL)
         return 0;
-    *pmgf1md = rsa_algor_to_md(pss->maskHash);
+    *pmgf1md = x509_algor_get_md(pss->maskHash);
     if (*pmgf1md == NULL)
         return 0;
     if (pss->saltLength)
@@ -1013,165 +942,6 @@ static int rsa_sig_info_set(X509_SIG_INFO *siginf, const X509_ALGOR *sigalg,
     return rv;
 }
 
-#ifndef OPENSSL_NO_CMS
-static RSA_OAEP_PARAMS *rsa_oaep_decode(const X509_ALGOR *alg)
-{
-    RSA_OAEP_PARAMS *oaep;
-
-    oaep = ASN1_TYPE_unpack_sequence(ASN1_ITEM_rptr(RSA_OAEP_PARAMS),
-                                     alg->parameter);
-
-    if (oaep == NULL)
-        return NULL;
-
-    if (oaep->maskGenFunc != NULL) {
-        oaep->maskHash = rsa_mgf1_decode(oaep->maskGenFunc);
-        if (oaep->maskHash == NULL) {
-            RSA_OAEP_PARAMS_free(oaep);
-            return NULL;
-        }
-    }
-    return oaep;
-}
-
-static int rsa_cms_decrypt(CMS_RecipientInfo *ri)
-{
-    EVP_PKEY_CTX *pkctx;
-    X509_ALGOR *cmsalg;
-    int nid;
-    int rv = -1;
-    unsigned char *label = NULL;
-    int labellen = 0;
-    const EVP_MD *mgf1md = NULL, *md = NULL;
-    RSA_OAEP_PARAMS *oaep;
-
-    pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
-    if (pkctx == NULL)
-        return 0;
-    if (!CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &cmsalg))
-        return -1;
-    nid = OBJ_obj2nid(cmsalg->algorithm);
-    if (nid == NID_rsaEncryption)
-        return 1;
-    if (nid != NID_rsaesOaep) {
-        RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_ENCRYPTION_TYPE);
-        return -1;
-    }
-    /* Decode OAEP parameters */
-    oaep = rsa_oaep_decode(cmsalg);
-
-    if (oaep == NULL) {
-        RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_OAEP_PARAMETERS);
-        goto err;
-    }
-
-    mgf1md = rsa_algor_to_md(oaep->maskHash);
-    if (mgf1md == NULL)
-        goto err;
-    md = rsa_algor_to_md(oaep->hashFunc);
-    if (md == NULL)
-        goto err;
-
-    if (oaep->pSourceFunc != NULL) {
-        X509_ALGOR *plab = oaep->pSourceFunc;
-
-        if (OBJ_obj2nid(plab->algorithm) != NID_pSpecified) {
-            RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_UNSUPPORTED_LABEL_SOURCE);
-            goto err;
-        }
-        if (plab->parameter->type != V_ASN1_OCTET_STRING) {
-            RSAerr(RSA_F_RSA_CMS_DECRYPT, RSA_R_INVALID_LABEL);
-            goto err;
-        }
-
-        label = plab->parameter->value.octet_string->data;
-        /* Stop label being freed when OAEP parameters are freed */
-        plab->parameter->value.octet_string->data = NULL;
-        labellen = plab->parameter->value.octet_string->length;
-    }
-
-    if (EVP_PKEY_CTX_set_rsa_padding(pkctx, RSA_PKCS1_OAEP_PADDING) <= 0)
-        goto err;
-    if (EVP_PKEY_CTX_set_rsa_oaep_md(pkctx, md) <= 0)
-        goto err;
-    if (EVP_PKEY_CTX_set_rsa_mgf1_md(pkctx, mgf1md) <= 0)
-        goto err;
-    if (label != NULL
-            && EVP_PKEY_CTX_set0_rsa_oaep_label(pkctx, label, labellen) <= 0)
-        goto err;
-    /* Carry on */
-    rv = 1;
-
- err:
-    RSA_OAEP_PARAMS_free(oaep);
-    return rv;
-}
-
-static int rsa_cms_encrypt(CMS_RecipientInfo *ri)
-{
-    const EVP_MD *md, *mgf1md;
-    RSA_OAEP_PARAMS *oaep = NULL;
-    ASN1_STRING *os = NULL;
-    X509_ALGOR *alg;
-    EVP_PKEY_CTX *pkctx = CMS_RecipientInfo_get0_pkey_ctx(ri);
-    int pad_mode = RSA_PKCS1_PADDING, rv = 0, labellen;
-    unsigned char *label;
-
-    if (CMS_RecipientInfo_ktri_get0_algs(ri, NULL, NULL, &alg) <= 0)
-        return 0;
-    if (pkctx) {
-        if (EVP_PKEY_CTX_get_rsa_padding(pkctx, &pad_mode) <= 0)
-            return 0;
-    }
-    if (pad_mode == RSA_PKCS1_PADDING) {
-        X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaEncryption), V_ASN1_NULL, 0);
-        return 1;
-    }
-    /* Not supported */
-    if (pad_mode != RSA_PKCS1_OAEP_PADDING)
-        return 0;
-    if (EVP_PKEY_CTX_get_rsa_oaep_md(pkctx, &md) <= 0)
-        goto err;
-    if (EVP_PKEY_CTX_get_rsa_mgf1_md(pkctx, &mgf1md) <= 0)
-        goto err;
-    labellen = EVP_PKEY_CTX_get0_rsa_oaep_label(pkctx, &label);
-    if (labellen < 0)
-        goto err;
-    oaep = RSA_OAEP_PARAMS_new();
-    if (oaep == NULL)
-        goto err;
-    if (!rsa_md_to_algor(&oaep->hashFunc, md))
-        goto err;
-    if (!rsa_md_to_mgf1(&oaep->maskGenFunc, mgf1md))
-        goto err;
-    if (labellen > 0) {
-        ASN1_OCTET_STRING *los;
-        oaep->pSourceFunc = X509_ALGOR_new();
-        if (oaep->pSourceFunc == NULL)
-            goto err;
-        los = ASN1_OCTET_STRING_new();
-        if (los == NULL)
-            goto err;
-        if (!ASN1_OCTET_STRING_set(los, label, labellen)) {
-            ASN1_OCTET_STRING_free(los);
-            goto err;
-        }
-        X509_ALGOR_set0(oaep->pSourceFunc, OBJ_nid2obj(NID_pSpecified),
-                        V_ASN1_OCTET_STRING, los);
-    }
-    /* create string with pss parameter encoding. */
-    if (!ASN1_item_pack(oaep, ASN1_ITEM_rptr(RSA_OAEP_PARAMS), &os))
-         goto err;
-    X509_ALGOR_set0(alg, OBJ_nid2obj(NID_rsaesOaep), V_ASN1_SEQUENCE, os);
-    os = NULL;
-    rv = 1;
- err:
-    RSA_OAEP_PARAMS_free(oaep);
-    ASN1_STRING_free(os);
-    return rv;
-}
-#endif
-
 static int rsa_pkey_check(const EVP_PKEY *pkey)
 {
     return RSA_check_key_ex(pkey->pkey.rsa, NULL);