Check that the public key OID matches the sig alg

Using the rsa_pss_rsae_sha256 sig alg should imply that the key OID is
rsaEncryption. Similarly rsa_pss_pss_sha256 implies the key OID is
rsassaPss. However we did not check this and incorrectly tolerated a key
OID that did not match the sig alg sent by the peer.

Fixes #6611

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/6732)

1 files changed, 22 insertions, 9 deletions

diff --git a/ssl/ssl_cert.c b/ssl/ssl_cert.c
index b2b342767c..df5cff79c9 100644
--- a/ssl/ssl_cert.c
+++ b/ssl/ssl_cert.c
@@ -995,22 +995,35 @@ int ssl_ctx_security(const SSL_CTX *ctx, int op, int bits, int nid, void *other)
                              ctx->cert->sec_ex);
 }
 
-const SSL_CERT_LOOKUP *ssl_cert_lookup_by_pkey(const EVP_PKEY *pk, size_t *pidx)
+int ssl_cert_lookup_by_nid(int nid, size_t *pidx)
 {
-    int nid = EVP_PKEY_id(pk);
     size_t i;
 
-    if (nid == NID_undef)
-        return NULL;
-
     for (i = 0; i < OSSL_NELEM(ssl_cert_info); i++) {
         if (ssl_cert_info[i].nid == nid) {
-            if (pidx != NULL)
-                *pidx = i;
-            return &ssl_cert_info[i];
+            *pidx = i;
+            return 1;
         }
     }
-    return NULL;
+
+    return 0;
+}
+
+const SSL_CERT_LOOKUP *ssl_cert_lookup_by_pkey(const EVP_PKEY *pk, size_t *pidx)
+{
+    int nid = EVP_PKEY_id(pk);
+    size_t tmpidx;
+
+    if (nid == NID_undef)
+        return NULL;
+
+    if (!ssl_cert_lookup_by_nid(nid, &tmpidx))
+        return NULL;
+
+    if (pidx != NULL)
+        *pidx = tmpidx;
+
+    return &ssl_cert_info[tmpidx];
 }
 
 const SSL_CERT_LOOKUP *ssl_cert_lookup_by_idx(size_t idx)