Add TLS 1.3 draft-23 PSS signature algorithms

We now have a split in the signature algorithms codepoint space for
whether the certificate's key is for rsaEncryption or a PSS-specific
key, which should let us get rid of some special-casing that we
previously needed to try to coax rsaEncryption keys into performing PSS.
(This will be done in a subsequent commit.)

Send the new PSS-with-PSS-specific key first in our list, so that
we prefer the new technology to the old one.

We need to update the expected certificate type in one test,
since the "RSA-PSS+SHA256" form now corresponds to a public key
of type rsaEncryption, so we should expect the server certificate
type to be just "RSA".  If we want to get a server certificate
type of "RSA-PSS", we need to use a new signature algorithm
that cannot be represented as signature+hash, so add a test for that
as well.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/5068)

3 files changed, 122 insertions, 74 deletions

diff --git a/test/recipes/70-test_sslsigalgs.t b/test/recipes/70-test_sslsigalgs.t
index 255a8c388b..7d4a0516be 100644
--- a/test/recipes/70-test_sslsigalgs.t
+++ b/test/recipes/70-test_sslsigalgs.t
@@ -247,7 +247,7 @@ sub sigalgs_filter
                     #No PSS sig algs - just send rsa_pkcs1_sha256
                     $sigalg = pack "C4", 0x00, 0x02, 0x04, 0x01;
                 } else {
-                    #PSS sig algs only - just send rsa_pss_sha256
+                    #PSS sig algs only - just send rsa_pss_rsae_sha256
                     $sigalg = pack "C4", 0x00, 0x02, 0x08, 0x04;
                 }
                 $message->set_extension(TLSProxy::Message::EXT_SIG_ALGS, $sigalg);
diff --git a/test/ssl-tests/20-cert-select.conf b/test/ssl-tests/20-cert-select.conf
index 69a80033fb..47ff667bb6 100644
--- a/test/ssl-tests/20-cert-select.conf
+++ b/test/ssl-tests/20-cert-select.conf
@@ -1,6 +1,6 @@
 # Generated with generate_ssl_tests.pl
 
-num_tests = 22
+num_tests = 23
 
 test-0 = 0-ECDSA CipherString Selection
 test-1 = 1-Ed25519 CipherString and Signature Algorithm Selection
@@ -16,14 +16,15 @@ test-10 = 10-ECDSA Signature Algorithm Selection compressed point
 test-11 = 11-ECDSA Signature Algorithm Selection, no ECDSA certificate
 test-12 = 12-RSA Signature Algorithm Selection
 test-13 = 13-RSA-PSS Signature Algorithm Selection
-test-14 = 14-RSA-PSS Certificate Signature Algorithm Selection
-test-15 = 15-Only RSA-PSS Certificate
-test-16 = 16-RSA-PSS Certificate, no PSS signature algorithms
-test-17 = 17-Suite B P-256 Hash Algorithm Selection
-test-18 = 18-Suite B P-384 Hash Algorithm Selection
-test-19 = 19-TLS 1.2 Ed25519 Client Auth
-test-20 = 20-Only RSA-PSS Certificate, TLS v1.1
-test-21 = 21-TLS 1.2 DSA Certificate Test
+test-14 = 14-RSA-PSS Certificate Legacy Signature Algorithm Selection
+test-15 = 15-RSA-PSS Certificate Unified Signature Algorithm Selection
+test-16 = 16-Only RSA-PSS Certificate
+test-17 = 17-RSA-PSS Certificate, no PSS signature algorithms
+test-18 = 18-Suite B P-256 Hash Algorithm Selection
+test-19 = 19-Suite B P-384 Hash Algorithm Selection
+test-20 = 20-TLS 1.2 Ed25519 Client Auth
+test-21 = 21-Only RSA-PSS Certificate, TLS v1.1
+test-22 = 22-TLS 1.2 DSA Certificate Test
 # ===========================================================
 
 [0-ECDSA CipherString Selection]
@@ -463,14 +464,14 @@ ExpectedServerSignType = RSA-PSS
 
 # ===========================================================
 
-[14-RSA-PSS Certificate Signature Algorithm Selection]
-ssl_conf = 14-RSA-PSS Certificate Signature Algorithm Selection-ssl
+[14-RSA-PSS Certificate Legacy Signature Algorithm Selection]
+ssl_conf = 14-RSA-PSS Certificate Legacy Signature Algorithm Selection-ssl
 
-[14-RSA-PSS Certificate Signature Algorithm Selection-ssl]
-server = 14-RSA-PSS Certificate Signature Algorithm Selection-server
-client = 14-RSA-PSS Certificate Signature Algorithm Selection-client
+[14-RSA-PSS Certificate Legacy Signature Algorithm Selection-ssl]
+server = 14-RSA-PSS Certificate Legacy Signature Algorithm Selection-server
+client = 14-RSA-PSS Certificate Legacy Signature Algorithm Selection-client
 
-[14-RSA-PSS Certificate Signature Algorithm Selection-server]
+[14-RSA-PSS Certificate Legacy Signature Algorithm Selection-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = DEFAULT
 ECDSA.Certificate = ${ENV::TEST_CERTS_DIR}/server-ecdsa-cert.pem
@@ -482,7 +483,7 @@ PSS.Certificate = ${ENV::TEST_CERTS_DIR}/server-pss-cert.pem
 PSS.PrivateKey = ${ENV::TEST_CERTS_DIR}/server-pss-key.pem
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[14-RSA-PSS Certificate Signature Algorithm Selection-client]
+[14-RSA-PSS Certificate Legacy Signature Algorithm Selection-client]
 CipherString = DEFAULT
 SignatureAlgorithms = RSA-PSS+SHA256
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
@@ -490,6 +491,40 @@ VerifyMode = Peer
 
 [test-14]
 ExpectedResult = Success
+ExpectedServerCertType = RSA
+ExpectedServerSignHash = SHA256
+ExpectedServerSignType = RSA-PSS
+
+
+# ===========================================================
+
+[15-RSA-PSS Certificate Unified Signature Algorithm Selection]
+ssl_conf = 15-RSA-PSS Certificate Unified Signature Algorithm Selection-ssl
+
+[15-RSA-PSS Certificate Unified Signature Algorithm Selection-ssl]
+server = 15-RSA-PSS Certificate Unified Signature Algorithm Selection-server
+client = 15-RSA-PSS Certificate Unified Signature Algorithm Selection-client
+
+[15-RSA-PSS Certificate Unified Signature Algorithm Selection-server]
+Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
+CipherString = DEFAULT
+ECDSA.Certificate = ${ENV::TEST_CERTS_DIR}/server-ecdsa-cert.pem
+ECDSA.PrivateKey = ${ENV::TEST_CERTS_DIR}/server-ecdsa-key.pem
+EdDSA.Certificate = ${ENV::TEST_CERTS_DIR}/server-ed25519-cert.pem
+EdDSA.PrivateKey = ${ENV::TEST_CERTS_DIR}/server-ed25519-key.pem
+MaxProtocol = TLSv1.2
+PSS.Certificate = ${ENV::TEST_CERTS_DIR}/server-pss-cert.pem
+PSS.PrivateKey = ${ENV::TEST_CERTS_DIR}/server-pss-key.pem
+PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
+
+[15-RSA-PSS Certificate Unified Signature Algorithm Selection-client]
+CipherString = DEFAULT
+SignatureAlgorithms = rsa_pss_pss_sha256
+VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
+VerifyMode = Peer
+
+[test-15]
+ExpectedResult = Success
 ExpectedServerCertType = RSA-PSS
 ExpectedServerSignHash = SHA256
 ExpectedServerSignType = RSA-PSS
@@ -497,24 +532,24 @@ ExpectedServerSignType = RSA-PSS
 
 # ===========================================================
 
-[15-Only RSA-PSS Certificate]
-ssl_conf = 15-Only RSA-PSS Certificate-ssl
+[16-Only RSA-PSS Certificate]
+ssl_conf = 16-Only RSA-PSS Certificate-ssl
 
-[15-Only RSA-PSS Certificate-ssl]
-server = 15-Only RSA-PSS Certificate-server
-client = 15-Only RSA-PSS Certificate-client
+[16-Only RSA-PSS Certificate-ssl]
+server = 16-Only RSA-PSS Certificate-server
+client = 16-Only RSA-PSS Certificate-client
 
-[15-Only RSA-PSS Certificate-server]
+[16-Only RSA-PSS Certificate-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/server-pss-cert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/server-pss-key.pem
 
-[15-Only RSA-PSS Certificate-client]
+[16-Only RSA-PSS Certificate-client]
 CipherString = DEFAULT
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
 VerifyMode = Peer
 
-[test-15]
+[test-16]
 ExpectedResult = Success
 ExpectedServerCertType = RSA-PSS
 ExpectedServerSignHash = SHA256
@@ -523,38 +558,38 @@ ExpectedServerSignType = RSA-PSS
 
 # ===========================================================
 
-[16-RSA-PSS Certificate, no PSS signature algorithms]
-ssl_conf = 16-RSA-PSS Certificate, no PSS signature algorithms-ssl
+[17-RSA-PSS Certificate, no PSS signature algorithms]
+ssl_conf = 17-RSA-PSS Certificate, no PSS signature algorithms-ssl
 
-[16-RSA-PSS Certificate, no PSS signature algorithms-ssl]
-server = 16-RSA-PSS Certificate, no PSS signature algorithms-server
-client = 16-RSA-PSS Certificate, no PSS signature algorithms-client
+[17-RSA-PSS Certificate, no PSS signature algorithms-ssl]
+server = 17-RSA-PSS Certificate, no PSS signature algorithms-server
+client = 17-RSA-PSS Certificate, no PSS signature algorithms-client
 
-[16-RSA-PSS Certificate, no PSS signature algorithms-server]
+[17-RSA-PSS Certificate, no PSS signature algorithms-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/server-pss-cert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/server-pss-key.pem
 
-[16-RSA-PSS Certificate, no PSS signature algorithms-client]
+[17-RSA-PSS Certificate, no PSS signature algorithms-client]
 CipherString = DEFAULT
 SignatureAlgorithms = RSA+SHA256
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
 VerifyMode = Peer
 
-[test-16]
+[test-17]
 ExpectedResult = ServerFail
 
 
 # ===========================================================
 
-[17-Suite B P-256 Hash Algorithm Selection]
-ssl_conf = 17-Suite B P-256 Hash Algorithm Selection-ssl
+[18-Suite B P-256 Hash Algorithm Selection]
+ssl_conf = 18-Suite B P-256 Hash Algorithm Selection-ssl
 
-[17-Suite B P-256 Hash Algorithm Selection-ssl]
-server = 17-Suite B P-256 Hash Algorithm Selection-server
-client = 17-Suite B P-256 Hash Algorithm Selection-client
+[18-Suite B P-256 Hash Algorithm Selection-ssl]
+server = 18-Suite B P-256 Hash Algorithm Selection-server
+client = 18-Suite B P-256 Hash Algorithm Selection-client
 
-[17-Suite B P-256 Hash Algorithm Selection-server]
+[18-Suite B P-256 Hash Algorithm Selection-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = SUITEB128
 ECDSA.Certificate = ${ENV::TEST_CERTS_DIR}/p256-server-cert.pem
@@ -562,13 +597,13 @@ ECDSA.PrivateKey = ${ENV::TEST_CERTS_DIR}/p256-server-key.pem
 MaxProtocol = TLSv1.2
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[17-Suite B P-256 Hash Algorithm Selection-client]
+[18-Suite B P-256 Hash Algorithm Selection-client]
 CipherString = DEFAULT
 SignatureAlgorithms = ECDSA+SHA384:ECDSA+SHA256
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/p384-root.pem
 VerifyMode = Peer
 
-[test-17]
+[test-18]
 ExpectedResult = Success
 ExpectedServerCertType = P-256
 ExpectedServerSignHash = SHA256
@@ -577,14 +612,14 @@ ExpectedServerSignType = EC
 
 # ===========================================================
 
-[18-Suite B P-384 Hash Algorithm Selection]
-ssl_conf = 18-Suite B P-384 Hash Algorithm Selection-ssl
+[19-Suite B P-384 Hash Algorithm Selection]
+ssl_conf = 19-Suite B P-384 Hash Algorithm Selection-ssl
 
-[18-Suite B P-384 Hash Algorithm Selection-ssl]
-server = 18-Suite B P-384 Hash Algorithm Selection-server
-client = 18-Suite B P-384 Hash Algorithm Selection-client
+[19-Suite B P-384 Hash Algorithm Selection-ssl]
+server = 19-Suite B P-384 Hash Algorithm Selection-server
+client = 19-Suite B P-384 Hash Algorithm Selection-client
 
-[18-Suite B P-384 Hash Algorithm Selection-server]
+[19-Suite B P-384 Hash Algorithm Selection-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = SUITEB128
 ECDSA.Certificate = ${ENV::TEST_CERTS_DIR}/p384-server-cert.pem
@@ -592,13 +627,13 @@ ECDSA.PrivateKey = ${ENV::TEST_CERTS_DIR}/p384-server-key.pem
 MaxProtocol = TLSv1.2
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[18-Suite B P-384 Hash Algorithm Selection-client]
+[19-Suite B P-384 Hash Algorithm Selection-client]
 CipherString = DEFAULT
 SignatureAlgorithms = ECDSA+SHA256:ECDSA+SHA384
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/p384-root.pem
 VerifyMode = Peer
 
-[test-18]
+[test-19]
 ExpectedResult = Success
 ExpectedServerCertType = P-384
 ExpectedServerSignHash = SHA384
@@ -607,21 +642,21 @@ ExpectedServerSignType = EC
 
 # ===========================================================
 
-[19-TLS 1.2 Ed25519 Client Auth]
-ssl_conf = 19-TLS 1.2 Ed25519 Client Auth-ssl
+[20-TLS 1.2 Ed25519 Client Auth]
+ssl_conf = 20-TLS 1.2 Ed25519 Client Auth-ssl
 
-[19-TLS 1.2 Ed25519 Client Auth-ssl]
-server = 19-TLS 1.2 Ed25519 Client Auth-server
-client = 19-TLS 1.2 Ed25519 Client Auth-client
+[20-TLS 1.2 Ed25519 Client Auth-ssl]
+server = 20-TLS 1.2 Ed25519 Client Auth-server
+client = 20-TLS 1.2 Ed25519 Client Auth-client
 
-[19-TLS 1.2 Ed25519 Client Auth-server]
+[20-TLS 1.2 Ed25519 Client Auth-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/root-cert.pem
 VerifyMode = Require
 
-[19-TLS 1.2 Ed25519 Client Auth-client]
+[20-TLS 1.2 Ed25519 Client Auth-client]
 CipherString = DEFAULT
 EdDSA.Certificate = ${ENV::TEST_CERTS_DIR}/client-ed25519-cert.pem
 EdDSA.PrivateKey = ${ENV::TEST_CERTS_DIR}/client-ed25519-key.pem
@@ -630,7 +665,7 @@ MinProtocol = TLSv1.2
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
 VerifyMode = Peer
 
-[test-19]
+[test-20]
 ExpectedClientCertType = Ed25519
 ExpectedClientSignType = Ed25519
 ExpectedResult = Success
@@ -638,38 +673,38 @@ ExpectedResult = Success
 
 # ===========================================================
 
-[20-Only RSA-PSS Certificate, TLS v1.1]
-ssl_conf = 20-Only RSA-PSS Certificate, TLS v1.1-ssl
+[21-Only RSA-PSS Certificate, TLS v1.1]
+ssl_conf = 21-Only RSA-PSS Certificate, TLS v1.1-ssl
 
-[20-Only RSA-PSS Certificate, TLS v1.1-ssl]
-server = 20-Only RSA-PSS Certificate, TLS v1.1-server
-client = 20-Only RSA-PSS Certificate, TLS v1.1-client
+[21-Only RSA-PSS Certificate, TLS v1.1-ssl]
+server = 21-Only RSA-PSS Certificate, TLS v1.1-server
+client = 21-Only RSA-PSS Certificate, TLS v1.1-client
 
-[20-Only RSA-PSS Certificate, TLS v1.1-server]
+[21-Only RSA-PSS Certificate, TLS v1.1-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/server-pss-cert.pem
 CipherString = DEFAULT
 PrivateKey = ${ENV::TEST_CERTS_DIR}/server-pss-key.pem
 
-[20-Only RSA-PSS Certificate, TLS v1.1-client]
+[21-Only RSA-PSS Certificate, TLS v1.1-client]
 CipherString = DEFAULT
 MaxProtocol = TLSv1.1
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
 VerifyMode = Peer
 
-[test-20]
+[test-21]
 ExpectedResult = ServerFail
 
 
 # ===========================================================
 
-[21-TLS 1.2 DSA Certificate Test]
-ssl_conf = 21-TLS 1.2 DSA Certificate Test-ssl
+[22-TLS 1.2 DSA Certificate Test]
+ssl_conf = 22-TLS 1.2 DSA Certificate Test-ssl
 
-[21-TLS 1.2 DSA Certificate Test-ssl]
-server = 21-TLS 1.2 DSA Certificate Test-server
-client = 21-TLS 1.2 DSA Certificate Test-client
+[22-TLS 1.2 DSA Certificate Test-ssl]
+server = 22-TLS 1.2 DSA Certificate Test-server
+client = 22-TLS 1.2 DSA Certificate Test-client
 
-[21-TLS 1.2 DSA Certificate Test-server]
+[22-TLS 1.2 DSA Certificate Test-server]
 Certificate = ${ENV::TEST_CERTS_DIR}/servercert.pem
 CipherString = ALL
 DHParameters = ${ENV::TEST_CERTS_DIR}/dhp2048.pem
@@ -679,13 +714,13 @@ MaxProtocol = TLSv1.2
 MinProtocol = TLSv1.2
 PrivateKey = ${ENV::TEST_CERTS_DIR}/serverkey.pem
 
-[21-TLS 1.2 DSA Certificate Test-client]
+[22-TLS 1.2 DSA Certificate Test-client]
 CipherString = ALL
 SignatureAlgorithms = DSA+SHA256:DSA+SHA1
 VerifyCAFile = ${ENV::TEST_CERTS_DIR}/rootcert.pem
 VerifyMode = Peer
 
-[test-21]
+[test-22]
 ExpectedResult = Success
 
 
diff --git a/test/ssl-tests/20-cert-select.conf.in b/test/ssl-tests/20-cert-select.conf.in
index 1b874b4880..ff77f6bd8c 100644
--- a/test/ssl-tests/20-cert-select.conf.in
+++ b/test/ssl-tests/20-cert-select.conf.in
@@ -232,12 +232,25 @@ our @tests = (
         },
     },
     {
-        name => "RSA-PSS Certificate Signature Algorithm Selection",
+        name => "RSA-PSS Certificate Legacy Signature Algorithm Selection",
         server => $server_pss,
         client => {
             "SignatureAlgorithms" => "RSA-PSS+SHA256",
         },
         test   => {
+            "ExpectedServerCertType" => "RSA",
+            "ExpectedServerSignHash" => "SHA256",
+            "ExpectedServerSignType" => "RSA-PSS",
+            "ExpectedResult" => "Success"
+        },
+    },
+    {
+        name => "RSA-PSS Certificate Unified Signature Algorithm Selection",
+        server => $server_pss,
+        client => {
+            "SignatureAlgorithms" => "rsa_pss_pss_sha256",
+        },
+        test   => {
             "ExpectedServerCertType" => "RSA-PSS",
             "ExpectedServerSignHash" => "SHA256",
             "ExpectedServerSignType" => "RSA-PSS",