EVP: fix keygen for EVP_PKEY_RSA_PSS

This commit attempts to fix the an issue when generating a key of type
EVP_PKEY_RSA_PSS. Currently, EVP_PKEY_CTX_set_rsa_keygen_bits will
return -1 if the key id is not of type EVP_PKEY_RSA. This commit adds
EVP_PKEY_RSA_PSS to also be accepted.

The macro EVP_PKEY_CTX_set_rsa_pss_keygen_md si converted into a
function and it is now called in legacy_ctrl_to_param.

Fixes #12384

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/13099)

1 files changed, 66 insertions, 1 deletions

diff --git a/crypto/rsa/rsa_lib.c b/crypto/rsa/rsa_lib.c
index da0fd4a6eb..6ca4f3a541 100644
--- a/crypto/rsa/rsa_lib.c
+++ b/crypto/rsa/rsa_lib.c
@@ -902,6 +902,70 @@ int EVP_PKEY_CTX_get_rsa_padding(EVP_PKEY_CTX *ctx, int *pad_mode)
 
 }
 
+int EVP_PKEY_CTX_set_rsa_pss_keygen_md(EVP_PKEY_CTX *ctx, const EVP_MD *md)
+{
+    const char *name;
+
+    if (ctx == NULL || md == NULL) {
+        ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED);
+        /* Uses the same return values as EVP_PKEY_CTX_ctrl */
+        return -2;
+    }
+
+    /* If key type not RSA return error */
+    if (ctx->pmeth != NULL
+        && ctx->pmeth->pkey_id != EVP_PKEY_RSA
+        && ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
+        return -1;
+
+    /* TODO(3.0): Remove this eventually when no more legacy */
+    if (ctx->op.keymgmt.genctx == NULL)
+        return EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN,
+                                 EVP_PKEY_CTRL_MD, 0, (void *)md);
+
+    name = EVP_MD_name(md);
+
+    return EVP_PKEY_CTX_set_rsa_pss_keygen_md_name(ctx, name, NULL);
+}
+
+int EVP_PKEY_CTX_set_rsa_pss_keygen_md_name(EVP_PKEY_CTX *ctx,
+                                            const char *mdname,
+                                            const char *mdprops)
+{
+    OSSL_PARAM rsa_params[3], *p = rsa_params;
+
+    if (ctx == NULL || mdname == NULL) {
+        ERR_raise(ERR_LIB_EVP, EVP_R_COMMAND_NOT_SUPPORTED);
+        /* Uses the same return values as EVP_PKEY_CTX_ctrl */
+        return -2;
+    }
+
+    /* If key type not RSA return error */
+    if (ctx->pmeth != NULL
+        && ctx->pmeth->pkey_id != EVP_PKEY_RSA
+        && ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
+        return -1;
+
+
+    *p++ = OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_RSA_DIGEST,
+                                            /*
+                                             * Cast away the const. This is read
+                                             * only so should be safe
+                                             */
+                                            (char *)mdname, 0);
+    if (mdprops != NULL) {
+        *p++ = OSSL_PARAM_construct_utf8_string(
+                    OSSL_PKEY_PARAM_RSA_DIGEST_PROPS,
+                    /*
+                     * Cast away the const. This is read only so should be safe
+                     */
+                    (char *)mdprops, 0);
+    }
+    *p++ = OSSL_PARAM_construct_end();
+
+    return EVP_PKEY_CTX_set_params(ctx, rsa_params);
+}
+
 int EVP_PKEY_CTX_set_rsa_oaep_md(EVP_PKEY_CTX *ctx, const EVP_MD *md)
 {
     const char *name;
@@ -1332,7 +1396,8 @@ int EVP_PKEY_CTX_set_rsa_keygen_bits(EVP_PKEY_CTX *ctx, int bits)
     }
 
     /* If key type not RSA return error */
-    if (ctx->pmeth != NULL && ctx->pmeth->pkey_id != EVP_PKEY_RSA)
+    if (ctx->pmeth != NULL && ctx->pmeth->pkey_id != EVP_PKEY_RSA &&
+        ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
         return -1;
 
     /* TODO(3.0): Remove this eventually when no more legacy */