RSA: Add a less loaded PSS-parameter structure

RSA_PSS_PARAMS carries with it a lot of baggage in form of X509_ALGOR
and ASN1_INTEGER, which we would rather avoid in our providers.
Therefore, we create a parallell structure - RSA_PSS_PARAMS_30 - that
contains the same information, but uses numeric identities (*) and C
integers (**).  This makes it simpler to handle.

Note that neither this structure nor its contents are passed between
libcrypto and the providers.  Instead, the numeric identities are
translated to and from names, which are then passed over that
boundary.

For future considerations, we might consider dropping RSA_PSS_PARAMS
entirely.  For now, it's still reserved for EVP_PKEY_ASN1_METHOD code,
which RSA_PSS_PARAMS_30 is (almost entirely) reserved for use in our
providers.

(*) We use NIDs in this case, because we already have them and because
only algorithms that libcrypto knows about are permitted in PSS
restrictions.  We could use any number series we want, as long as we
know for sure what they represent.

(**) That's for saltlen and for trailerfield, which are never expect
to surpass the set of numbers that fit in a regular 'int'.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11710)

1 files changed, 35 insertions, 0 deletions

diff --git a/include/crypto/rsa.h b/include/crypto/rsa.h
index fb7f80da3f..69fe4c759b 100644
--- a/include/crypto/rsa.h
+++ b/include/crypto/rsa.h
@@ -13,6 +13,37 @@
 #include <openssl/core.h>
 #include <openssl/rsa.h>
 
+typedef struct rsa_pss_params_30_st {
+    int hash_algorithm_nid;
+    struct {
+        int algorithm_nid;       /* Currently always NID_mgf1 */
+        int hash_algorithm_nid;
+    } mask_gen;
+    unsigned int salt_len;
+    unsigned int trailer_field;
+} RSA_PSS_PARAMS_30;
+
+RSA_PSS_PARAMS_30 *rsa_get0_pss_params_30(RSA *r);
+int rsa_pss_params_30_set_defaults(RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_copy(RSA_PSS_PARAMS_30 *to,
+                           const RSA_PSS_PARAMS_30 *from);
+int rsa_pss_params_30_is_unrestricted(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_set_hashalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                  int hashalg_nid);
+int rsa_pss_params_30_set_maskgenalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                     int maskgenalg_nid);
+int rsa_pss_params_30_set_maskgenhashalg(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                         int maskgenhashalg_nid);
+int rsa_pss_params_30_set_saltlen(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                  int saltlen);
+int rsa_pss_params_30_set_trailerfield(RSA_PSS_PARAMS_30 *rsa_pss_params,
+                                       int trailerfield);
+int rsa_pss_params_30_hashalg(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_maskgenalg(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_maskgenhashalg(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_saltlen(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+int rsa_pss_params_30_trailerfield(const RSA_PSS_PARAMS_30 *rsa_pss_params);
+
 const char *rsa_mgf_nid2name(int mgf);
 int rsa_oaeppss_md2nid(const EVP_MD *md);
 const char *rsa_oaeppss_nid2name(int md);
@@ -28,6 +59,10 @@ int rsa_get0_all_params(RSA *r, STACK_OF(BIGNUM_const) *primes,
 
 int rsa_todata(RSA *rsa, OSSL_PARAM_BLD *bld, OSSL_PARAM params[]);
 int rsa_fromdata(RSA *rsa, const OSSL_PARAM params[]);
+int rsa_pss_params_30_todata(const RSA_PSS_PARAMS_30 *pss, const char *propq,
+                             OSSL_PARAM_BLD *bld, OSSL_PARAM params[]);
+int rsa_pss_params_30_fromdata(RSA_PSS_PARAMS_30 *pss_params,
+                               const OSSL_PARAM params[], OPENSSL_CTX *libctx);
 
 int rsa_padding_check_PKCS1_type_2_TLS(OPENSSL_CTX *ctx, unsigned char *to,
                                        size_t tlen, const unsigned char *from,