Replace SSL_PKEY_RSA_ENC, SSL_PKEY_RSA_SIGN

The original intent of SSL_PKEY_RSA_SIGN and SSL_PKEY_RSA_ENC was to
support two different keys for RSA signing and decrypt. However this
was never implemented and we only ever set one key and the other was
always NULL. Replace with single SSL_PKEY_RSA type.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2587)

7 files changed, 33 insertions, 53 deletions

diff --git a/ssl/ssl_cert.c b/ssl/ssl_cert.c
index bbb6932210..a75fb6564a 100644
--- a/ssl/ssl_cert.c
+++ b/ssl/ssl_cert.c
@@ -63,7 +63,7 @@ CERT *ssl_cert_new(void)
         return NULL;
     }
 
-    ret->key = &(ret->pkeys[SSL_PKEY_RSA_ENC]);
+    ret->key = &(ret->pkeys[SSL_PKEY_RSA]);
     ret->references = 1;
     ret->sec_cb = ssl_security_default_callback;
     ret->sec_level = OPENSSL_TLS_SECURITY_LEVEL;
diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c
index 3149c39f5f..d28b53df92 100644
--- a/ssl/ssl_ciph.c
+++ b/ssl/ssl_ciph.c
@@ -1909,7 +1909,7 @@ int ssl_cipher_get_cert_index(const SSL_CIPHER *c)
     else if (alg_a & SSL_aDSS)
         return SSL_PKEY_DSA_SIGN;
     else if (alg_a & SSL_aRSA)
-        return SSL_PKEY_RSA_ENC;
+        return SSL_PKEY_RSA;
     else if (alg_a & SSL_aGOST12)
         return SSL_PKEY_GOST_EC;
     else if (alg_a & SSL_aGOST01)
diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index 3b18c3ebfd..11c0a80d2d 100644
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -2740,8 +2740,8 @@ void ssl_set_masks(SSL *s)
     dh_tmp = 0;
 #endif
 
-    rsa_enc = pvalid[SSL_PKEY_RSA_ENC] & CERT_PKEY_VALID;
-    rsa_sign = pvalid[SSL_PKEY_RSA_SIGN] & CERT_PKEY_SIGN;
+    rsa_enc = pvalid[SSL_PKEY_RSA] & CERT_PKEY_VALID;
+    rsa_sign = pvalid[SSL_PKEY_RSA] & CERT_PKEY_SIGN;
     dsa_sign = pvalid[SSL_PKEY_DSA_SIGN] & CERT_PKEY_SIGN;
 #ifndef OPENSSL_NO_EC
     have_ecc_cert = pvalid[SSL_PKEY_ECC] & CERT_PKEY_VALID;
@@ -2855,8 +2855,6 @@ static int ssl_get_server_cert_index(const SSL *s)
     }
 
     idx = ssl_cipher_get_cert_index(s->s3->tmp.new_cipher);
-    if (idx == SSL_PKEY_RSA_ENC && !s->cert->pkeys[SSL_PKEY_RSA_ENC].x509)
-        idx = SSL_PKEY_RSA_SIGN;
     if (idx == SSL_PKEY_GOST_EC) {
         if (s->cert->pkeys[SSL_PKEY_GOST12_512].x509)
             idx = SSL_PKEY_GOST12_512;
@@ -2902,15 +2900,12 @@ EVP_PKEY *ssl_get_sign_pkey(SSL *s, const SSL_CIPHER *cipher,
     alg_a = cipher->algorithm_auth;
     c = s->cert;
 
-    if ((alg_a & SSL_aDSS) && (c->pkeys[SSL_PKEY_DSA_SIGN].privatekey != NULL))
+    if (alg_a & SSL_aDSS && c->pkeys[SSL_PKEY_DSA_SIGN].privatekey != NULL)
         idx = SSL_PKEY_DSA_SIGN;
-    else if (alg_a & SSL_aRSA) {
-        if (c->pkeys[SSL_PKEY_RSA_SIGN].privatekey != NULL)
-            idx = SSL_PKEY_RSA_SIGN;
-        else if (c->pkeys[SSL_PKEY_RSA_ENC].privatekey != NULL)
-            idx = SSL_PKEY_RSA_ENC;
-    } else if ((alg_a & SSL_aECDSA) &&
-               (c->pkeys[SSL_PKEY_ECC].privatekey != NULL))
+    else if (alg_a & SSL_aRSA && c->pkeys[SSL_PKEY_RSA].privatekey != NULL)
+            idx = SSL_PKEY_RSA;
+    else if (alg_a & SSL_aECDSA &&
+             c->pkeys[SSL_PKEY_ECC].privatekey != NULL)
         idx = SSL_PKEY_ECC;
     if (idx == -1) {
         SSLerr(SSL_F_SSL_GET_SIGN_PKEY, ERR_R_INTERNAL_ERROR);
diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h
index b868813e0e..2a23007393 100644
--- a/ssl/ssl_locl.h
+++ b/ssl/ssl_locl.h
@@ -398,14 +398,13 @@
 # define SSL_USE_ETM(s) (s->s3->flags & TLS1_FLAGS_ENCRYPT_THEN_MAC)
 
 /* Mostly for SSLv3 */
-# define SSL_PKEY_RSA_ENC        0
-# define SSL_PKEY_RSA_SIGN       1
-# define SSL_PKEY_DSA_SIGN       2
-# define SSL_PKEY_ECC            3
-# define SSL_PKEY_GOST01         4
-# define SSL_PKEY_GOST12_256     5
-# define SSL_PKEY_GOST12_512     6
-# define SSL_PKEY_NUM            7
+# define SSL_PKEY_RSA            0
+# define SSL_PKEY_DSA_SIGN       1
+# define SSL_PKEY_ECC            2
+# define SSL_PKEY_GOST01         3
+# define SSL_PKEY_GOST12_256     4
+# define SSL_PKEY_GOST12_512     5
+# define SSL_PKEY_NUM            6
 /*
  * Pseudo-constant. GOST cipher suites can use different certs for 1
  * SSL_CIPHER. So let's see which one we have in fact.
@@ -413,10 +412,10 @@
 # define SSL_PKEY_GOST_EC SSL_PKEY_NUM+1
 
 /*
- * TODO(TLS1.3) for now use RSA_SIGN keys for PSS
+ * TODO(TLS1.3) for now use SSL_PKEY_RSA keys for PSS
  */
 
-#define SSL_PKEY_RSA_PSS_SIGN SSL_PKEY_RSA_SIGN
+#define SSL_PKEY_RSA_PSS_SIGN SSL_PKEY_RSA
 
 /*-
  * SSL_kRSA <- RSA_ENC
diff --git a/ssl/statem/statem_lib.c b/ssl/statem/statem_lib.c
index 16c765fabf..8e7245b856 100644
--- a/ssl/statem/statem_lib.c
+++ b/ssl/statem/statem_lib.c
@@ -1071,7 +1071,7 @@ int ssl_cert_type(const X509 *x, const EVP_PKEY *pk)
     default:
         return -1;
     case EVP_PKEY_RSA:
-        return SSL_PKEY_RSA_ENC;
+        return SSL_PKEY_RSA;
     case EVP_PKEY_DSA:
         return SSL_PKEY_DSA_SIGN;
 #ifndef OPENSSL_NO_EC
diff --git a/ssl/statem/statem_srvr.c b/ssl/statem/statem_srvr.c
index 20ea684906..de0fcc0bbd 100644
--- a/ssl/statem/statem_srvr.c
+++ b/ssl/statem/statem_srvr.c
@@ -2465,7 +2465,7 @@ static int tls_process_cke_rsa(SSL *s, PACKET *pkt, int *al)
     unsigned char *rsa_decrypt = NULL;
     int ret = 0;
 
-    rsa = EVP_PKEY_get0_RSA(s->cert->pkeys[SSL_PKEY_RSA_ENC].privatekey);
+    rsa = EVP_PKEY_get0_RSA(s->cert->pkeys[SSL_PKEY_RSA].privatekey);
     if (rsa == NULL) {
         *al = SSL_AD_HANDSHAKE_FAILURE;
         SSLerr(SSL_F_TLS_PROCESS_CKE_RSA, SSL_R_MISSING_RSA_CERTIFICATE);
diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index d6a841a82f..7c8244d6a8 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -730,16 +730,16 @@ static const SIGALG_LOOKUP sigalg_lookup_tbl[] = {
      NID_sha512, SSL_MD_SHA512_IDX, EVP_PKEY_RSA_PSS, SSL_PKEY_RSA_PSS_SIGN,
      NID_undef, NID_undef},
     {"rsa_pkcs1_sha256", TLSEXT_SIGALG_rsa_pkcs1_sha256,
-     NID_sha256, SSL_MD_SHA256_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN,
+     NID_sha256, SSL_MD_SHA256_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA,
      NID_sha256WithRSAEncryption, NID_undef},
     {"rsa_pkcs1_sha384", TLSEXT_SIGALG_rsa_pkcs1_sha384,
-     NID_sha384, SSL_MD_SHA384_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN,
+     NID_sha384, SSL_MD_SHA384_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA,
      NID_sha384WithRSAEncryption, NID_undef},
     {"rsa_pkcs1_sha512", TLSEXT_SIGALG_rsa_pkcs1_sha512,
-     NID_sha512, SSL_MD_SHA512_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN,
+     NID_sha512, SSL_MD_SHA512_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA,
      NID_sha512WithRSAEncryption, NID_undef},
     {"rsa_pkcs1_sha1", TLSEXT_SIGALG_rsa_pkcs1_sha1,
-     NID_sha1, SSL_MD_SHA1_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA_SIGN,
+     NID_sha1, SSL_MD_SHA1_IDX, EVP_PKEY_RSA, SSL_PKEY_RSA,
      NID_sha1WithRSAEncryption, NID_undef},
 #ifndef OPENSSL_NO_DSA
     {NULL, TLSEXT_SIGALG_dsa_sha256,
@@ -1022,10 +1022,9 @@ void ssl_set_default_md(SSL *s)
 #endif
 #ifndef OPENSSL_NO_RSA
     if (SSL_USE_SIGALGS(s))
-        pmd[SSL_PKEY_RSA_SIGN] = ssl_md(SSL_MD_SHA1_IDX);
+        pmd[SSL_PKEY_RSA] = ssl_md(SSL_MD_SHA1_IDX);
     else
-        pmd[SSL_PKEY_RSA_SIGN] = ssl_md(SSL_MD_MD5_SHA1_IDX);
-    pmd[SSL_PKEY_RSA_ENC] = pmd[SSL_PKEY_RSA_SIGN];
+        pmd[SSL_PKEY_RSA] = ssl_md(SSL_MD_MD5_SHA1_IDX);
 #endif
 #ifndef OPENSSL_NO_EC
     pmd[SSL_PKEY_ECC] = ssl_md(SSL_MD_SHA1_IDX);
@@ -1358,13 +1357,13 @@ static int tls12_get_pkey_idx(int sig_nid)
     switch (sig_nid) {
 #ifndef OPENSSL_NO_RSA
     case EVP_PKEY_RSA:
-        return SSL_PKEY_RSA_SIGN;
+        return SSL_PKEY_RSA;
     /*
      * For now return RSA key for PSS. When we support PSS only keys
      * this will need to be updated.
      */
     case EVP_PKEY_RSA_PSS:
-        return SSL_PKEY_RSA_SIGN;
+        return SSL_PKEY_RSA;
 #endif
 #ifndef OPENSSL_NO_DSA
     case EVP_PKEY_DSA:
@@ -1605,10 +1604,6 @@ int tls1_process_sigalgs(SSL *s)
             md = ssl_md(sigptr->hash_idx);
             pmd[idx] = md;
             pvalid[idx] = CERT_PKEY_EXPLICIT_SIGN;
-            if (idx == SSL_PKEY_RSA_SIGN) {
-                pvalid[SSL_PKEY_RSA_ENC] = CERT_PKEY_EXPLICIT_SIGN;
-                pmd[SSL_PKEY_RSA_ENC] = md;
-            }
         }
     }
     /*
@@ -1626,9 +1621,8 @@ int tls1_process_sigalgs(SSL *s)
             pmd[SSL_PKEY_DSA_SIGN] = EVP_sha1();
 #endif
 #ifndef OPENSSL_NO_RSA
-        if (pmd[SSL_PKEY_RSA_SIGN] == NULL) {
-            pmd[SSL_PKEY_RSA_SIGN] = EVP_sha1();
-            pmd[SSL_PKEY_RSA_ENC] = EVP_sha1();
+        if (pmd[SSL_PKEY_RSA] == NULL) {
+            pmd[SSL_PKEY_RSA] = EVP_sha1();
         }
 #endif
 #ifndef OPENSSL_NO_EC
@@ -1945,8 +1939,7 @@ int tls1_check_chain(SSL *s, X509 *x, EVP_PKEY *pk, STACK_OF(X509) *chain,
         /* If no sigalgs extension use defaults from RFC5246 */
         else {
             switch (idx) {
-            case SSL_PKEY_RSA_ENC:
-            case SSL_PKEY_RSA_SIGN:
+            case SSL_PKEY_RSA:
                 rsign = EVP_PKEY_RSA;
                 default_nid = NID_sha1WithRSAEncryption;
                 break;
@@ -2133,8 +2126,7 @@ int tls1_check_chain(SSL *s, X509 *x, EVP_PKEY *pk, STACK_OF(X509) *chain,
 /* Set validity of certificates in an SSL structure */
 void tls1_set_cert_validity(SSL *s)
 {
-    tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA_ENC);
-    tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA_SIGN);
+    tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_RSA);
     tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_DSA_SIGN);
     tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_ECC);
     tls1_check_chain(s, NULL, NULL, NULL, SSL_PKEY_GOST01);
@@ -2299,14 +2291,8 @@ int tls_choose_sigalg(SSL *s)
                 continue;
             idx = lu->sig_idx;
             c = &s->cert->pkeys[idx];
-            if (c->x509 == NULL || c->privatekey == NULL) {
-                if (idx != SSL_PKEY_RSA_SIGN)
-                    continue;
-                idx = SSL_PKEY_RSA_ENC;
-                c = s->cert->pkeys + idx;
-                if (c->x509 == NULL || c->privatekey == NULL)
+            if (c->x509 == NULL || c->privatekey == NULL)
                     continue;
-            }
             if (lu->sig == EVP_PKEY_EC) {
 #ifndef OPENSSL_NO_EC
                 if (curve == -1) {