PROV & SERIALIZER: Adapt the RSA serializers for PSS-parameters

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11710)

5 files changed, 171 insertions, 34 deletions

diff --git a/providers/implementations/serializers/build.info b/providers/implementations/serializers/build.info
index 097bdcac1c..ffafbe38e5 100644
--- a/providers/implementations/serializers/build.info
+++ b/providers/implementations/serializers/build.info
@@ -10,7 +10,10 @@ $ECX_GOAL=../../libimplementations.a
 $EC_GOAL=../../libimplementations.a
 
 SOURCE[$SERIALIZER_GOAL]=serializer_common.c
+
 SOURCE[$RSA_GOAL]=serializer_rsa.c serializer_rsa_priv.c serializer_rsa_pub.c
+DEPEND[serializer_rsa.o]=../../common/include/prov/der_rsa.h
+
 IF[{- !$disabled{"dh"} || !$disabled{"dsa"} -}]
   SOURCE[$FFC_GOAL]=serializer_ffc_params.c
 ENDIF
diff --git a/providers/implementations/serializers/serializer_local.h b/providers/implementations/serializers/serializer_local.h
index 39fb3ab1e7..f4aee6fc23 100644
--- a/providers/implementations/serializers/serializer_local.h
+++ b/providers/implementations/serializers/serializer_local.h
@@ -82,6 +82,14 @@ int ossl_prov_prepare_all_dsa_params(const void *dsa, int nid,
 int ossl_prov_dsa_pub_to_der(const void *dsa, unsigned char **pder);
 int ossl_prov_dsa_priv_to_der(const void *dsa, unsigned char **pder);
 
+/*
+ * ossl_prov_prepare_rsa_params() is designed to work with the ossl_prov_write_
+ * functions, hence 'void *rsa' rather than 'RSA *rsa'.
+ */
+int ossl_prov_prepare_rsa_params(const void *rsa, int nid,
+                                 void **pstr, int *pstrtype);
+int ossl_prov_rsa_type_to_evp(const RSA *rsa);
+
 int ossl_prov_print_labeled_bignum(BIO *out, const char *label,
                                    const BIGNUM *bn);
 int ossl_prov_print_labeled_buf(BIO *out, const char *label,
diff --git a/providers/implementations/serializers/serializer_rsa.c b/providers/implementations/serializers/serializer_rsa.c
index 7578fec7c2..564210ede2 100644
--- a/providers/implementations/serializers/serializer_rsa.c
+++ b/providers/implementations/serializers/serializer_rsa.c
@@ -13,8 +13,10 @@
  */
 #include "internal/deprecated.h"
 
+#include "internal/packet.h"
 #include "crypto/rsa.h"           /* rsa_get0_all_params() */
 #include "prov/bio.h"             /* ossl_prov_bio_printf() */
+#include "prov/der_rsa.h"         /* DER_w_RSASSA_PSS_params() */
 #include "prov/implementations.h" /* rsa_keymgmt_functions */
 #include "serializer_local.h"
 
@@ -43,6 +45,7 @@ int ossl_prov_print_rsa(BIO *out, RSA *rsa, int priv)
     STACK_OF(BIGNUM_const) *factors = sk_BIGNUM_const_new_null();
     STACK_OF(BIGNUM_const) *exps = sk_BIGNUM_const_new_null();
     STACK_OF(BIGNUM_const) *coeffs = sk_BIGNUM_const_new_null();
+    RSA_PSS_PARAMS_30 *pss_params = rsa_get0_pss_params_30(rsa);
     int ret = 0;
 
     if (rsa == NULL || factors == NULL || exps == NULL || coeffs == NULL)
@@ -109,6 +112,61 @@ int ossl_prov_print_rsa(BIO *out, RSA *rsa, int priv)
                 goto err;
         }
     }
+
+    switch (RSA_test_flags(rsa, RSA_FLAG_TYPE_MASK)) {
+    case RSA_FLAG_TYPE_RSA:
+        if (!rsa_pss_params_30_is_unrestricted(pss_params)) {
+            if (ossl_prov_bio_printf(out, "(INVALID PSS PARAMETERS)\n") <= 0)
+                goto err;
+        }
+        break;
+    case RSA_FLAG_TYPE_RSASSAPSS:
+        if (rsa_pss_params_30_is_unrestricted(pss_params)) {
+            if (ossl_prov_bio_printf(out,
+                                     "No PSS parameter restrictions\n") <= 0)
+                goto err;
+        } else {
+            int hashalg_nid = rsa_pss_params_30_hashalg(pss_params);
+            int maskgenalg_nid = rsa_pss_params_30_maskgenalg(pss_params);
+            int maskgenhashalg_nid =
+                rsa_pss_params_30_maskgenhashalg(pss_params);
+            int saltlen = rsa_pss_params_30_saltlen(pss_params);
+            int trailerfield = rsa_pss_params_30_trailerfield(pss_params);
+
+            if (ossl_prov_bio_printf(out, "PSS parameter restrictions:\n") <= 0)
+                goto err;
+            if (ossl_prov_bio_printf(out, "  Hash Algorithm: %s%s\n",
+                                     rsa_oaeppss_nid2name(hashalg_nid),
+                                     (hashalg_nid == NID_sha1
+                                      ? " (default)" : "")) <= 0)
+                goto err;
+            if (ossl_prov_bio_printf(out, "  Mask Algorithm: %s with %s%s\n",
+                                     rsa_mgf_nid2name(maskgenalg_nid),
+                                     rsa_oaeppss_nid2name(maskgenhashalg_nid),
+                                     (maskgenalg_nid == NID_mgf1
+                                      && maskgenhashalg_nid == NID_sha1
+                                      ? " (default)" : "")) <= 0)
+                goto err;
+            if (ossl_prov_bio_printf(out, "  Minimum Salt Length: %d%s\n",
+                                     saltlen,
+                                     (saltlen == 20 ? " (default)" : "")) <= 0)
+                goto err;
+            /*
+             * TODO(3.0) Should we show the ASN.1 trailerField value, or
+             * the actual trailerfield byte (i.e. 0xBC for 1)?
+             * crypto/rsa/rsa_ameth.c isn't very clear on that, as it
+             * does display 0xBC when the default applies, but the ASN.1
+             * trailerField value otherwise...
+             */
+            if (ossl_prov_bio_printf(out, "  Trailer Field: 0x%x%s\n",
+                                     trailerfield,
+                                     (trailerfield == 1 ? " (default)" : ""))
+                <= 0)
+                goto err;
+        }
+        break;
+    }
+
     ret = 1;
  err:
     sk_BIGNUM_const_free(factors);
@@ -116,3 +174,90 @@ int ossl_prov_print_rsa(BIO *out, RSA *rsa, int priv)
     sk_BIGNUM_const_free(coeffs);
     return ret;
 }
+
+/*
+ * Helper functions to prepare RSA-PSS params for serialization.  We would
+ * have simply written the whole AlgorithmIdentifier, but existing libcrypto
+ * functionality doesn't allow that.
+ */
+
+int ossl_prov_prepare_rsa_params(const void *rsa, int nid,
+                                 void **pstr, int *pstrtype)
+{
+    const RSA_PSS_PARAMS_30 *pss = rsa_get0_pss_params_30((RSA *)rsa);
+
+    *pstr = NULL;
+
+    switch (RSA_test_flags(rsa, RSA_FLAG_TYPE_MASK)) {
+    case RSA_FLAG_TYPE_RSA:
+        /* If plain RSA, the parameters shall be NULL */
+        *pstrtype = V_ASN1_NULL;
+        return 1;
+    case RSA_FLAG_TYPE_RSASSAPSS:
+        if (rsa_pss_params_30_is_unrestricted(pss)) {
+            *pstrtype = V_ASN1_UNDEF;
+        } else {
+            ASN1_STRING *astr = NULL;
+            WPACKET pkt;
+            unsigned char *str = NULL;
+            size_t str_sz = 0;
+            int i;
+
+            for (i = 0; i < 2; i++) {
+                switch (i) {
+                case 0:
+                    if (!WPACKET_init_null_der(&pkt))
+                        goto err;
+                    break;
+                case 1:
+                    if ((str = OPENSSL_malloc(str_sz)) == NULL
+                        || !WPACKET_init_der(&pkt, str, str_sz)) {
+                        goto err;
+                    }
+                    break;
+                }
+                if (!DER_w_RSASSA_PSS_params(&pkt, -1, pss)
+                    || !WPACKET_finish(&pkt))
+                    goto err;
+                WPACKET_get_total_written(&pkt, &str_sz);
+                WPACKET_cleanup(&pkt);
+
+                /*
+                 * If no PSS parameters are going to be written, there's no
+                 * point going for another iteration.
+                 * This saves us from getting |str| allocated just to have it
+                 * immediately de-allocated.
+                 */
+                if (str_sz == 0)
+                    break;
+            }
+
+            if ((astr = ASN1_STRING_new()) == NULL)
+                goto err;
+            *pstrtype = V_ASN1_SEQUENCE;
+            ASN1_STRING_set0(astr, str, (int)str_sz);
+            *pstr = astr;
+
+            return 1;
+         err:
+            OPENSSL_free(str);
+            return 0;
+        }
+    }
+
+    /* Currently unsupported RSA key type */
+    return 0;
+}
+
+int ossl_prov_rsa_type_to_evp(const RSA *rsa)
+{
+    switch (RSA_test_flags(rsa, RSA_FLAG_TYPE_MASK)) {
+    case RSA_FLAG_TYPE_RSA:
+        return EVP_PKEY_RSA;
+    case RSA_FLAG_TYPE_RSASSAPSS:
+        return EVP_PKEY_RSA_PSS;
+    }
+
+    /* Currently unsupported RSA key type */
+    return EVP_PKEY_NONE;
+}
diff --git a/providers/implementations/serializers/serializer_rsa_priv.c b/providers/implementations/serializers/serializer_rsa_priv.c
index 8196473a04..8c68f5de34 100644
--- a/providers/implementations/serializers/serializer_rsa_priv.c
+++ b/providers/implementations/serializers/serializer_rsa_priv.c
@@ -21,6 +21,7 @@
 #include <openssl/types.h>
 #include <openssl/params.h>
 #include <openssl/safestack.h>
+#include "crypto/rsa.h"
 #include "prov/bio.h"
 #include "prov/implementations.h"
 #include "prov/providercommonerr.h"
@@ -49,34 +50,6 @@ struct rsa_priv_ctx_st {
     struct pkcs8_encrypt_ctx_st sc;
 };
 
-/* Helper functions to prepare RSA-PSS params for serialization */
-
-static int prepare_rsa_params(const void *rsa, int nid,
-                              void **pstr, int *pstrtype)
-{
-    const RSA_PSS_PARAMS *pss = RSA_get0_pss_params(rsa);
-    *pstr = NULL;
-
-    /* If RSA it's just NULL type */
-    if (nid != EVP_PKEY_RSA_PSS) {
-        *pstrtype = V_ASN1_NULL;
-        return 1;
-    }
-    /* If no PSS parameters we omit parameters entirely */
-    if (pss == NULL) {
-        *pstrtype = V_ASN1_UNDEF;
-        return 1;
-    }
-    /* Encode PSS parameters */
-    if (ASN1_item_pack((void *)pss, ASN1_ITEM_rptr(RSA_PSS_PARAMS),
-                       (ASN1_STRING **)pstr)
-        == NULL)
-        return 0;
-
-    *pstrtype = V_ASN1_SEQUENCE;
-    return 1;
-}
-
 /* Private key : context */
 static void *rsa_priv_newctx(void *provctx)
 {
@@ -176,8 +149,9 @@ static int rsa_priv_der(void *vctx, void *rsa, BIO *out,
     ctx->sc.cb = cb;
     ctx->sc.cbarg = cbarg;
 
-    ret = ossl_prov_write_priv_der_from_obj(out, rsa, EVP_PKEY_RSA,
-                                            prepare_rsa_params,
+    ret = ossl_prov_write_priv_der_from_obj(out, rsa,
+                                            ossl_prov_rsa_type_to_evp(rsa),
+                                            ossl_prov_prepare_rsa_params,
                                             (i2d_of_void *)i2d_RSAPrivateKey,
                                             &ctx->sc);
 
@@ -215,8 +189,9 @@ static int rsa_pem_priv(void *vctx, void *rsa, BIO *out,
     ctx->sc.cb = cb;
     ctx->sc.cbarg = cbarg;
 
-    ret = ossl_prov_write_priv_pem_from_obj(out, rsa, EVP_PKEY_RSA,
-                                            prepare_rsa_params,
+    ret = ossl_prov_write_priv_pem_from_obj(out, rsa,
+                                            ossl_prov_rsa_type_to_evp(rsa),
+                                            ossl_prov_prepare_rsa_params,
                                             (i2d_of_void *)i2d_RSAPrivateKey,
                                             &ctx->sc);
 
diff --git a/providers/implementations/serializers/serializer_rsa_pub.c b/providers/implementations/serializers/serializer_rsa_pub.c
index bcae074480..28df00877e 100644
--- a/providers/implementations/serializers/serializer_rsa_pub.c
+++ b/providers/implementations/serializers/serializer_rsa_pub.c
@@ -72,7 +72,10 @@ static int rsa_pub_der_data(void *ctx, const OSSL_PARAM params[], BIO *out,
 static int rsa_pub_der(void *ctx, void *rsa, BIO *out,
                        OSSL_PASSPHRASE_CALLBACK *cb, void *cbarg)
 {
-    return i2d_RSA_PUBKEY_bio(out, rsa);
+    return ossl_prov_write_pub_der_from_obj(out, rsa,
+                                            ossl_prov_rsa_type_to_evp(rsa),
+                                            ossl_prov_prepare_rsa_params,
+                                            (i2d_of_void *)i2d_RSAPublicKey);
 }
 
 /* Public key : PEM */
@@ -100,7 +103,10 @@ static int rsa_pub_pem_data(void *ctx, const OSSL_PARAM params[], BIO *out,
 static int rsa_pub_pem(void *ctx, void *rsa, BIO *out,
                        OSSL_PASSPHRASE_CALLBACK *cb, void *cbarg)
 {
-    return PEM_write_bio_RSA_PUBKEY(out, rsa);
+    return ossl_prov_write_pub_pem_from_obj(out, rsa,
+                                            ossl_prov_rsa_type_to_evp(rsa),
+                                            ossl_prov_prepare_rsa_params,
+                                            (i2d_of_void *)i2d_RSAPublicKey);
 }
 
 static int rsa_pub_print_data(void *ctx, const OSSL_PARAM params[], BIO *out,