set MGF1 digest correctly

Fixes #19290

update rsa_set_ctx_params() so that the digest function used in the
MGF1 construction is set correctly.  Add a test for this to
evp_extra_test.c based on the code scaro-axway provided in #19290.

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/19342)

(cherry picked from commit e5a7536eaeaacd18d1aea59edeb295fb4eb2dfca)

2 files changed, 61 insertions, 3 deletions

diff --git a/providers/implementations/asymciphers/rsa_enc.c b/providers/implementations/asymciphers/rsa_enc.c
index ce5ddff651..113c4cbfab 100644
--- a/providers/implementations/asymciphers/rsa_enc.c
+++ b/providers/implementations/asymciphers/rsa_enc.c
@@ -425,7 +425,7 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
     const OSSL_PARAM *p;
     char mdname[OSSL_MAX_NAME_SIZE];
     char mdprops[OSSL_MAX_PROPQUERY_SIZE] = { '\0' };
-    char *str = mdname;
+    char *str = NULL;
 
     if (prsactx == NULL)
         return 0;
@@ -434,13 +434,14 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
 
     p = OSSL_PARAM_locate_const(params, OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST);
     if (p != NULL) {
+        str = mdname;
         if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdname)))
             return 0;
 
-        str = mdprops;
         p = OSSL_PARAM_locate_const(params,
                                     OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST_PROPS);
         if (p != NULL) {
+            str = mdprops;
             if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdprops)))
                 return 0;
         }
@@ -496,13 +497,14 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
 
     p = OSSL_PARAM_locate_const(params, OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST);
     if (p != NULL) {
+        str = mdname;
         if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdname)))
             return 0;
 
-        str = mdprops;
         p = OSSL_PARAM_locate_const(params,
                                     OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST_PROPS);
         if (p != NULL) {
+            str = mdprops;
             if (!OSSL_PARAM_get_utf8_string(p, &str, sizeof(mdprops)))
                 return 0;
         } else {
diff --git a/test/evp_extra_test.c b/test/evp_extra_test.c
index 1a9fa8f9c4..ae41c462c5 100644
--- a/test/evp_extra_test.c
+++ b/test/evp_extra_test.c
@@ -2752,6 +2752,61 @@ static int test_RSA_get_set_params(void)
     return ret;
 }
 
+static int test_RSA_OAEP_set_get_params(void)
+{
+    int ret = 0;
+    EVP_PKEY *key = NULL;
+    EVP_PKEY_CTX *key_ctx = NULL;
+
+    if (nullprov != NULL)
+        return TEST_skip("Test does not support a non-default library context");
+
+    if (!TEST_ptr(key = load_example_rsa_key())
+        || !TEST_ptr(key_ctx = EVP_PKEY_CTX_new_from_pkey(0, key, 0)))
+        goto err;
+
+    {
+        int padding = RSA_PKCS1_OAEP_PADDING;
+        OSSL_PARAM params[4];
+
+        params[0] = OSSL_PARAM_construct_int(OSSL_SIGNATURE_PARAM_PAD_MODE, &padding);
+        params[1] = OSSL_PARAM_construct_utf8_string(OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST,
+                                                     OSSL_DIGEST_NAME_SHA2_256, 0);
+        params[2] = OSSL_PARAM_construct_utf8_string(OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST,
+                                                     OSSL_DIGEST_NAME_SHA1, 0);
+        params[3] = OSSL_PARAM_construct_end();
+
+        if (!TEST_int_gt(EVP_PKEY_encrypt_init_ex(key_ctx, params),0))
+            goto err;
+    }
+    {
+        OSSL_PARAM params[3];
+        char oaepmd[30] = { '\0' };
+        char mgf1md[30] = { '\0' };
+
+        params[0] = OSSL_PARAM_construct_utf8_string(OSSL_ASYM_CIPHER_PARAM_OAEP_DIGEST,
+                                                     oaepmd, sizeof(oaepmd));
+        params[1] = OSSL_PARAM_construct_utf8_string(OSSL_ASYM_CIPHER_PARAM_MGF1_DIGEST,
+                                                     mgf1md, sizeof(mgf1md));
+        params[2] = OSSL_PARAM_construct_end();
+
+        if (!TEST_true(EVP_PKEY_CTX_get_params(key_ctx, params)))
+            goto err;
+
+        if (!TEST_str_eq(oaepmd, OSSL_DIGEST_NAME_SHA2_256)
+            || !TEST_str_eq(mgf1md, OSSL_DIGEST_NAME_SHA1))
+            goto err;
+    }
+
+    ret = 1;
+
+ err:
+    EVP_PKEY_free(key);
+    EVP_PKEY_CTX_free(key_ctx);
+
+    return ret;
+}
+
 #if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305)
 static int test_decrypt_null_chunks(void)
 {
@@ -4666,6 +4721,7 @@ int setup_tests(void)
     ADD_TEST(test_DSA_priv_pub);
 #endif
     ADD_TEST(test_RSA_get_set_params);
+    ADD_TEST(test_RSA_OAEP_set_get_params);
 #if !defined(OPENSSL_NO_CHACHA) && !defined(OPENSSL_NO_POLY1305)
     ADD_TEST(test_decrypt_null_chunks);
 #endif