RSA: properly generate algorithm identifier for RSA-PSS signatures

Fixes #13969

- properly handle the mandatory RSA-PSS key parameters
- improve parameter checking when setting the parameters
- compute the algorithm id at the time it is requested so it
  reflects the actual parameters set
- when generating keys do not override previously set parameters
  with defaults
- tests added to the test_req recipe that should cover the PSS signature
  handling

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13988)

2 files changed, 80 insertions, 3 deletions

diff --git a/test/recipes/25-test_req.t b/test/recipes/25-test_req.t
index 3f0d9f59e7..ab6c6e681b 100644
--- a/test/recipes/25-test_req.t
+++ b/test/recipes/25-test_req.t
@@ -93,7 +93,7 @@ subtest "generating certificate requests with RSA" => sub {
 };
 
 subtest "generating certificate requests with RSA-PSS" => sub {
-    plan tests => 4;
+    plan tests => 12;
 
     SKIP: {
         skip "RSA is not supported by this OpenSSL build", 2
@@ -104,7 +104,6 @@ subtest "generating certificate requests with RSA-PSS" => sub {
                     "-new", "-out", "testreq-rsapss.pem", "-utf8",
                     "-key", srctop_file("test", "testrsapss.pem")])),
            "Generating request");
-
         ok(run(app(["openssl", "req",
                     "-config", srctop_file("test", "test.cnf"),
                     "-verify", "-in", "testreq-rsapss.pem", "-noout"])),
@@ -117,11 +116,60 @@ subtest "generating certificate requests with RSA-PSS" => sub {
                     "-sigopt", "rsa_pss_saltlen:-1",
                     "-key", srctop_file("test", "testrsapss.pem")])),
            "Generating request");
-
         ok(run(app(["openssl", "req",
                     "-config", srctop_file("test", "test.cnf"),
                     "-verify", "-in", "testreq-rsapss2.pem", "-noout"])),
            "Verifying signature on request");
+
+        ok(run(app(["openssl", "req",
+                    "-config", srctop_file("test", "test.cnf"),
+                    "-new", "-out", "testreq-rsapssmand.pem", "-utf8",
+                    "-sigopt", "rsa_padding_mode:pss",
+                    "-key", srctop_file("test", "testrsapssmandatory.pem")])),
+           "Generating request");
+        ok(run(app(["openssl", "req",
+                    "-config", srctop_file("test", "test.cnf"),
+                    "-verify", "-in", "testreq-rsapssmand.pem", "-noout"])),
+           "Verifying signature on request");
+
+        ok(run(app(["openssl", "req",
+                    "-config", srctop_file("test", "test.cnf"),
+                    "-new", "-out", "testreq-rsapssmand2.pem", "-utf8",
+                    "-sigopt", "rsa_pss_saltlen:100",
+                    "-key", srctop_file("test", "testrsapssmandatory.pem")])),
+           "Generating request");
+        ok(run(app(["openssl", "req",
+                    "-config", srctop_file("test", "test.cnf"),
+                    "-verify", "-in", "testreq-rsapssmand2.pem", "-noout"])),
+           "Verifying signature on request");
+
+        ok(!run(app(["openssl", "req",
+                     "-config", srctop_file("test", "test.cnf"),
+                     "-new", "-out", "testreq-rsapss3.pem", "-utf8",
+                     "-sigopt", "rsa_padding_mode:pkcs1",
+                     "-key", srctop_file("test", "testrsapss.pem")])),
+           "Generating request with expected failure");
+
+        ok(!run(app(["openssl", "req",
+                     "-config", srctop_file("test", "test.cnf"),
+                     "-new", "-out", "testreq-rsapss3.pem", "-utf8",
+                     "-sigopt", "rsa_pss_saltlen:-4",
+                     "-key", srctop_file("test", "testrsapss.pem")])),
+           "Generating request with expected failure");
+
+        ok(!run(app(["openssl", "req",
+                     "-config", srctop_file("test", "test.cnf"),
+                     "-new", "-out", "testreq-rsapssmand3.pem", "-utf8",
+                     "-sigopt", "rsa_pss_saltlen:10",
+                     "-key", srctop_file("test", "testrsapssmandatory.pem")])),
+           "Generating request with expected failure");
+
+        ok(!run(app(["openssl", "req",
+                     "-config", srctop_file("test", "test.cnf"),
+                     "-new", "-out", "testreq-rsapssmand3.pem", "-utf8",
+                     "-sha256",
+                     "-key", srctop_file("test", "testrsapssmandatory.pem")])),
+           "Generating request with expected failure");
     }
 };
 
diff --git a/test/testrsapssmandatory.pem b/test/testrsapssmandatory.pem
new file mode 100644
index 0000000000..d01ae82c88
--- /dev/null
+++ b/test/testrsapssmandatory.pem
@@ -0,0 +1,29 @@
+-----BEGIN PRIVATE KEY-----
+MIIE7gIBADA9BgkqhkiG9w0BAQowMKANMAsGCWCGSAFlAwQCA6EaMBgGCSqGSIb3
+DQEBCDALBglghkgBZQMEAgOiAwIBQASCBKgwggSkAgEAAoIBAQDdiLMYj8fgrXKB
+dEC704hcfmeJebCyaZbYHBE/1YthJOptbhisBbNk4onKMITO6hkYOoH12rNxqwY5
+d9J1Ray6SJETVHxYCKftJ1LlrUJGqpyRCAAff1LYjjGRyqcMzVItWffy2iCgKGud
+uUqs9Og3wsVxUeXfTSGnLo1UevVc1qTKZJuDRWD2EItuwnFt7GA89IgGx8/liLsg
+cdlnm81gGdDmNKxNGi3VeOaJqFWnP9CpL8iXybG7F32U9mgEdE+EYt8GhQfNLzjL
+j17xfLl5K0SMqL8q+phas6Md0OmTl3Xg8Tupdoo/okAoYGXrv/sHDiV1YBSkXD4i
+dbV42aUfAgMBAAECggEAEyEJrfZEYR85Avqh2FYksS/tCs7qNg2uC80opCVxWpsQ
+bxCRqtD3M5/oHABih2dpcVEkBbGzyv3klLPHBX9VseQwOsYR0pw0u+KoYtK6JVX4
+HQHe2Nlqsu5cU2V3VUCpducM5Ph21r2GxWDJlPO01ZPI7scOnWCQpln7tC7F3xU0
+jNQ0SnFZ6SO4FrrBxOMjnIFiNMexxZt0fU7khy/dGck9aN4DtmQENcQkGdXj5xRv
+lInh92mQ16yMCbEU8cslWaAwqRF/k/5QxoIwTXr8PqaWshH9TIAht0rvTilWpHPg
+zpW6Pog/wGzVat3NeU3vBDYIUayHc6n3gbfJZDNxmQKBgQD41lAkxNsA89mYY7S9
+5NkDJ1N1hKNwg+iEyCZJkjxUk+SymdO7U/iD27Hgn/XyXm4RC5aHYpXJSnuiOk7R
+Z1Az1jjqLzPxsP72sWLORzGq82smYrK+iV2rhozWNlfVyazDkBcRRz2bLSESzgvO
+JWD3K3pjvj8U9ZSUhz+zXo4sUwKBgQDj6TBTKGDb8Au8sUOC916GrIrUEq5SkMDT
+A4CiD4fmvbdNs90AhD/mmqBw/dP3TbCPNmP8tGMUT0BDev6BoRKYOt+1XGYXt2de
+P38teVU/ZUcAO2RGdMNSdWT5o9BCWQZ18qSoOR/QanckOnkhKCgU/wqSdIvBBRMQ
+5e4qdI0qhQKBgB2MJTwYfADi88WaoU2jLPmo48oik926bBPISHOX/73zScbDaVbn
+I61UmwyXMfczq1Iu1BMDa9HZHFEpJ07KO8XL/DoinMJoR/43Fgp0fbtU6DZIpfzm
+Bs9lTLfrAAcMyYz3QSX2FaSleTXobZJu8dKnwQKzBn6QorH4VWIRKkStAoGBAIYL
+M1nlaLpSf4S2OT/A376Ton9CkXaMHmy9JZ2rRsHmGPZBcB0Kq06k6PIrx8wuzEYe
+tkX9jjx2tBQ8NY3mPzp7ffF766vNOaWL8O+86e+EUHMJe1uY9vv7gaz1tNog5BTg
+5gjuuBBrXbFYFr/yj0hyDDTBCSU4J9OLeD1OGWzFAoGBAMGc9h8oLyA3rQEjIuVA
+CuzgvZxOFPbtODFPcL4EQgAKLiKS+oZK0jONfCHaQB1AhIq8/nT/4suw7tWqYoKp
+KGH/+8tKNodKZfZLjVp0k8gsehyMDz1002/RLMJyFRIJWa1BqEJs7v7XgWW3RcmC
+PWznhdpNx3BYDSao5Ibl7I5E
+-----END PRIVATE KEY-----