RSA: Add a less loaded PSS-parameter structure

RSA_PSS_PARAMS carries with it a lot of baggage in form of X509_ALGOR
and ASN1_INTEGER, which we would rather avoid in our providers.
Therefore, we create a parallell structure - RSA_PSS_PARAMS_30 - that
contains the same information, but uses numeric identities (*) and C
integers (**).  This makes it simpler to handle.

Note that neither this structure nor its contents are passed between
libcrypto and the providers.  Instead, the numeric identities are
translated to and from names, which are then passed over that
boundary.

For future considerations, we might consider dropping RSA_PSS_PARAMS
entirely.  For now, it's still reserved for EVP_PKEY_ASN1_METHOD code,
which RSA_PSS_PARAMS_30 is (almost entirely) reserved for use in our
providers.

(*) We use NIDs in this case, because we already have them and because
only algorithms that libcrypto knows about are permitted in PSS
restrictions.  We could use any number series we want, as long as we
know for sure what they represent.

(**) That's for saltlen and for trailerfield, which are never expect
to surpass the set of numbers that fit in a regular 'int'.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11710)

4 files changed, 300 insertions, 2 deletions

diff --git a/crypto/rsa/rsa_backend.c b/crypto/rsa/rsa_backend.c
index cf0bff0822..7497a8579c 100644
--- a/crypto/rsa/rsa_backend.c
+++ b/crypto/rsa/rsa_backend.c
@@ -7,10 +7,16 @@
  * https://www.openssl.org/source/license.html
  */
 
+#include <string.h>
 #include <openssl/core_names.h>
 #include <openssl/params.h>
+#include <openssl/evp.h>
+#include "internal/sizes.h"
+#include "internal/param_build_set.h"
 #include "crypto/rsa.h"
 
+#include "e_os.h"                /* strcasecmp for Windows() */
+
 /*
  * The intention with the "backend" source file is to offer backend support
  * for legacy backends (EVP_PKEY_ASN1_METHOD and EVP_PKEY_METHOD) and provider
@@ -146,3 +152,139 @@ int rsa_todata(RSA *rsa, OSSL_PARAM_BLD *bld, OSSL_PARAM params[])
     sk_BIGNUM_const_free(coeffs);
     return ret;
 }
+
+int rsa_pss_params_30_todata(const RSA_PSS_PARAMS_30 *pss, const char *propq,
+                             OSSL_PARAM_BLD *bld, OSSL_PARAM params[])
+{
+    if (!rsa_pss_params_30_is_unrestricted(pss)) {
+        int hashalg_nid = rsa_pss_params_30_hashalg(pss);
+        int maskgenalg_nid = rsa_pss_params_30_maskgenalg(pss);
+        int maskgenhashalg_nid = rsa_pss_params_30_maskgenhashalg(pss);
+        int saltlen = rsa_pss_params_30_saltlen(pss);
+        int default_hashalg_nid = rsa_pss_params_30_hashalg(NULL);
+        int default_maskgenalg_nid = rsa_pss_params_30_maskgenalg(NULL);
+        int default_maskgenhashalg_nid = rsa_pss_params_30_maskgenhashalg(NULL);
+        const char *mdname =
+            (hashalg_nid == default_hashalg_nid
+             ? NULL : rsa_oaeppss_nid2name(hashalg_nid));
+        const char *mgfname =
+            (maskgenalg_nid == default_maskgenalg_nid
+             ? NULL : rsa_oaeppss_nid2name(maskgenalg_nid));
+        const char *mgf1mdname =
+            (maskgenhashalg_nid == default_maskgenhashalg_nid
+             ? NULL : rsa_oaeppss_nid2name(maskgenhashalg_nid));
+        const char *key_md = OSSL_PKEY_PARAM_RSA_DIGEST;
+        const char *key_mgf = OSSL_PKEY_PARAM_RSA_MASKGENFUNC;
+        const char *key_mgf1_md = OSSL_PKEY_PARAM_RSA_MGF1_DIGEST;
+        const char *key_saltlen = OSSL_PKEY_PARAM_RSA_PSS_SALTLEN;
+
+        /*
+         * To ensure that the key isn't seen as unrestricted by the recipient,
+         * we make sure that at least one PSS-related parameter is passed, even
+         * if it has a default value; saltlen.
+         */
+        if ((mdname != NULL
+             && !ossl_param_build_set_utf8_string(bld, params, key_md, mdname))
+            || (mgfname != NULL
+                && !ossl_param_build_set_utf8_string(bld, params,
+                                                     key_mgf, mgfname))
+            || (mgf1mdname != NULL
+                && !ossl_param_build_set_utf8_string(bld, params,
+                                                     key_mgf1_md, mgf1mdname))
+            || (!ossl_param_build_set_int(bld, params, key_saltlen, saltlen)))
+            return 0;
+    }
+    return 1;
+}
+
+int rsa_pss_params_30_fromdata(RSA_PSS_PARAMS_30 *pss_params,
+                               const OSSL_PARAM params[], OPENSSL_CTX *libctx)
+{
+    const OSSL_PARAM *param_md, *param_mgf, *param_mgf1md,  *param_saltlen;
+    EVP_MD *md = NULL, *mgf1md = NULL;
+    int saltlen;
+    int ret = 0;
+
+    if (pss_params == NULL)
+        return 0;
+
+    param_md =
+        OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_DIGEST);
+    param_mgf =
+        OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_MASKGENFUNC);
+    param_mgf1md =
+        OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_MGF1_DIGEST);
+    param_saltlen =
+        OSSL_PARAM_locate_const(params, OSSL_PKEY_PARAM_RSA_PSS_SALTLEN);
+
+    /*
+     * If we get any of the parameters, we know we have at least some
+     * restrictions, so we start by setting default values, and let each
+     * parameter override their specific restriction data.
+     */
+    if (param_md != NULL || param_mgf != NULL || param_mgf1md != NULL
+        || param_saltlen != NULL)
+        if (!rsa_pss_params_30_set_defaults(pss_params))
+            return 0;
+
+    if (param_mgf != NULL) {
+        int default_maskgenalg_nid = rsa_pss_params_30_maskgenalg(NULL);
+        const char *mgfname = NULL;
+
+        if (param_mgf->data_type == OSSL_PARAM_UTF8_STRING)
+            mgfname = param_mgf->data;
+        else if (!OSSL_PARAM_get_utf8_ptr(param_mgf, &mgfname))
+            return 0;
+
+        /* TODO Revisit this if / when a new MGF algorithm appears */
+        if (strcasecmp(param_mgf->data,
+                       rsa_mgf_nid2name(default_maskgenalg_nid)) != 0)
+            return 0;
+    }
+
+    /*
+     * We're only interested in the NIDs that correspond to the MDs, so the
+     * exact propquery is unimportant in the EVP_MD_fetch() calls below.
+     */
+
+    if (param_md != NULL) {
+        const char *mdname = NULL;
+
+        if (param_md->data_type == OSSL_PARAM_UTF8_STRING)
+            mdname = param_md->data;
+        else if (!OSSL_PARAM_get_utf8_ptr(param_mgf, &mdname))
+            goto err;
+
+        if ((md = EVP_MD_fetch(libctx, mdname, NULL)) == NULL
+            || !rsa_pss_params_30_set_hashalg(pss_params,
+                                              rsa_oaeppss_md2nid(md)))
+            goto err;
+    }
+
+    if (param_mgf1md != NULL) {
+        const char *mgf1mdname = NULL;
+
+        if (param_mgf1md->data_type == OSSL_PARAM_UTF8_STRING)
+            mgf1mdname = param_mgf1md->data;
+        else if (!OSSL_PARAM_get_utf8_ptr(param_mgf, &mgf1mdname))
+            goto err;
+
+        if ((mgf1md = EVP_MD_fetch(libctx, mgf1mdname, NULL)) == NULL
+            || !rsa_pss_params_30_set_maskgenhashalg(pss_params,
+                                                     rsa_oaeppss_md2nid(mgf1md)))
+            goto err;
+    }
+
+    if (param_saltlen != NULL) {
+        if (!OSSL_PARAM_get_int(param_saltlen, &saltlen)
+            || !rsa_pss_params_30_set_saltlen(pss_params, saltlen))
+            goto err;
+    }
+
+    ret = 1;
+
+ err:
+    EVP_MD_free(md);
+    EVP_MD_free(mgf1md);
+    return ret;
+}
diff --git a/crypto/rsa/rsa_lib.c b/crypto/rsa/rsa_lib.c
index 81daec4b34..d1fee590a8 100644
--- a/crypto/rsa/rsa_lib.c
+++ b/crypto/rsa/rsa_lib.c
@@ -162,7 +162,6 @@ void RSA_free(RSA *r)
     BN_clear_free(r->dmp1);
     BN_clear_free(r->dmq1);
     BN_clear_free(r->iqmp);
-    /* TODO(3.0): Support PSS in FIPS_MODULE */
 #ifndef FIPS_MODULE
     RSA_PSS_PARAMS_free(r->pss);
     sk_RSA_PRIME_INFO_pop_free(r->prime_infos, rsa_multip_info_free);
@@ -637,7 +636,17 @@ const BIGNUM *RSA_get0_iqmp(const RSA *r)
 
 const RSA_PSS_PARAMS *RSA_get0_pss_params(const RSA *r)
 {
+#ifdef FIPS_MODULE
+    return NULL;
+#else
     return r->pss;
+#endif
+}
+
+/* Internal */
+RSA_PSS_PARAMS_30 *rsa_get0_pss_params_30(RSA *r)
+{
+    return &r->pss_params;
 }
 
 void RSA_clear_flags(RSA *r, int flags)
diff --git a/crypto/rsa/rsa_local.h b/crypto/rsa/rsa_local.h
index 6c4ae8611b..f94fc79cdd 100644
--- a/crypto/rsa/rsa_local.h
+++ b/crypto/rsa/rsa_local.h
@@ -12,6 +12,7 @@
 
 #include <openssl/rsa.h>
 #include "internal/refcount.h"
+#include "crypto/rsa.h"
 
 #define RSA_MAX_PRIME_NUM       5
 #define RSA_MIN_MODULUS_BITS    512
@@ -50,8 +51,18 @@ struct rsa_st {
     BIGNUM *dmp1;
     BIGNUM *dmq1;
     BIGNUM *iqmp;
-    /* If a PSS only key this contains the parameter restrictions */
+
+    /*
+     * If a PSS only key this contains the parameter restrictions.
+     * There are two structures for the same thing, used in different cases.
+     */
+    /* This is used uniquely by OpenSSL provider implementations. */
+    RSA_PSS_PARAMS_30 pss_params;
+#ifndef FIPS_MODULE
+    /* This is used uniquely by rsa_ameth.c and rsa_pmeth.c. */
     RSA_PSS_PARAMS *pss;
+#endif
+
 #ifndef FIPS_MODULE
     /* for multi-prime RSA, defined in RFC 8017 */
     STACK_OF(RSA_PRIME_INFO) *prime_infos;
diff --git a/crypto/rsa/rsa_pss.c b/crypto/rsa/rsa_pss.c
index afb558cd36..a5bcdfe1ff 100644
--- a/crypto/rsa/rsa_pss.c
+++ b/crypto/rsa/rsa_pss.c
@@ -256,6 +256,142 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
 
 }
 
+/*
+ * The defaults for PSS restrictions are defined in RFC 8017, A.2.3 RSASSA-PSS
+ * (https://tools.ietf.org/html/rfc8017#appendix-A.2.3):
+ *
+ * If the default values of the hashAlgorithm, maskGenAlgorithm, and
+ * trailerField fields of RSASSA-PSS-params are used, then the algorithm
+ * identifier will have the following value:
+ *
+ *     rSASSA-PSS-Default-Identifier    RSASSA-AlgorithmIdentifier ::= {
+ *         algorithm   id-RSASSA-PSS,
+ *         parameters  RSASSA-PSS-params : {
+ *             hashAlgorithm       sha1,
+ *             maskGenAlgorithm    mgf1SHA1,
+ *             saltLength          20,
+ *             trailerField        trailerFieldBC
+ *         }
+ *     }
+ *
+ *     RSASSA-AlgorithmIdentifier ::= AlgorithmIdentifier {
+ *         {PKCS1Algorithms}
+ *     }
+ */
+static const RSA_PSS_PARAMS_30 default_RSASSA_PSS_params = {
+    NID_sha1,                    /* default hashAlgorithm */
+    {
+        NID_mgf1,                /* default maskGenAlgorithm */
+        NID_sha1                 /* default MGF1 hash */
+    },
+    20,                          /* default saltLength */
+    1                            /* default trailerField (0xBC) */
+};
+
+int rsa_pss_params_30_set_defaults(RSA_PSS_PARAMS_30 *rsa_pss_params)
+{
+    if (rsa_pss_params == NULL)
+        return 0;
+    *rsa_pss_params = default_RSASSA_PSS_params;
+    return 1;
+}
+
+int rsa_pss_params_30_is_unrestricted(const RSA_PSS_PARAMS_30 *rsa_pss_params)
+{
+    static RSA_PSS_PARAMS_30 pss_params_cmp = { 0, };
+
+    return rsa_pss_params == NULL
+        || memcmp(rsa_pss_params, &pss_params_cmp,
+                  sizeof(*rsa_pss_params)) == 0;
+}
+
+int rsa_pss_params_30_copy(RSA_PSS_PARAMS_30 *to,
+                           const RSA_PSS_PARAMS_30 *from)
+{
+    memcpy(to, from, sizeof(*to));
+    return 1;
+}
+
+int rsa_pss_params_30_set_hashalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                  int hashalg_nid)
+{
+    if (rsa_pss_params == NULL)
+        return 0;
+    rsa_pss_params->hash_algorithm_nid = hashalg_nid;
+    return 1;
+}
+
+int rsa_pss_params_30_set_maskgenalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                     int maskgenalg_nid)
+{
+    if (rsa_pss_params == NULL)
+        return 0;
+    rsa_pss_params->mask_gen.algorithm_nid = maskgenalg_nid;
+    return 1;
+}
+
+int rsa_pss_params_30_set_maskgenhashalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                         int maskgenhashalg_nid)
+{
+    if (rsa_pss_params == NULL)
+        return 0;
+    rsa_pss_params->mask_gen.hash_algorithm_nid = maskgenhashalg_nid;
+    return 1;
+}
+
+int rsa_pss_params_30_set_saltlen(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                  int saltlen)
+{
+    if (rsa_pss_params == NULL)
+        return 0;
+    rsa_pss_params->salt_len = saltlen;
+    return 1;
+}
+
+int rsa_pss_params_30_set_trailerfield(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                       int trailerfield)
+{
+    if (rsa_pss_params == NULL)
+        return 0;
+    rsa_pss_params->trailer_field = trailerfield;
+    return 1;
+}
+
+int rsa_pss_params_30_hashalg(const RSA_PSS_PARAMS_30 *rsa_pss_params)
+{
+    if (rsa_pss_params == NULL)
+        return default_RSASSA_PSS_params.hash_algorithm_nid;
+    return rsa_pss_params->hash_algorithm_nid;
+}
+
+int rsa_pss_params_30_maskgenalg(const RSA_PSS_PARAMS_30 *rsa_pss_params)
+{
+    if (rsa_pss_params == NULL)
+        return default_RSASSA_PSS_params.mask_gen.algorithm_nid;
+    return rsa_pss_params->mask_gen.algorithm_nid;
+}
+
+int rsa_pss_params_30_maskgenhashalg(const RSA_PSS_PARAMS_30 *rsa_pss_params)
+{
+    if (rsa_pss_params == NULL)
+        return default_RSASSA_PSS_params.hash_algorithm_nid;
+    return rsa_pss_params->mask_gen.hash_algorithm_nid;
+}
+
+int rsa_pss_params_30_saltlen(const RSA_PSS_PARAMS_30 *rsa_pss_params)
+{
+    if (rsa_pss_params == NULL)
+        return default_RSASSA_PSS_params.salt_len;
+    return rsa_pss_params->salt_len;
+}
+
+int rsa_pss_params_30_trailerfield(const RSA_PSS_PARAMS_30 *rsa_pss_params)
+{
+    if (rsa_pss_params == NULL)
+        return default_RSASSA_PSS_params.trailer_field;
+    return rsa_pss_params->trailer_field;
+}
+
 #if defined(_MSC_VER)
 # pragma optimize("",on)
 #endif