Fix bogus check for EVP_PKEY mandatory digest in check_cert_usable()

In commit 6aca8d1a5 ("Honour mandatory digest on private key in
has_usable_cert()") I added two checks for the capabilities of the
EVP_PKEY being used. One of them was wrong, as it should only be
checking the signature of the X.509 cert (by its issuer) against the
sigalgs given in a TLS v1.3 signature_algorithms_cert extension.

Remove it.

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Ben Kaduk <kaduk@mit.edu>
Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/9705)

1 files changed, 9 insertions, 13 deletions

diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index e5e77023cc..52d58ddd4f 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -2563,27 +2563,23 @@ static int check_cert_usable(SSL *s, const SIGALG_LOOKUP *sig, X509 *x,
 {
     const SIGALG_LOOKUP *lu;
     int mdnid, pknid, default_mdnid;
-    int mandatory_md = 0;
     size_t i;
 
     /* If the EVP_PKEY reports a mandatory digest, allow nothing else. */
     ERR_set_mark();
-    switch (EVP_PKEY_get_default_digest_nid(pkey, &default_mdnid)) {
-    case 2:
-        mandatory_md = 1;
-        break;
-    case 1:
-    default: /* If it didn't report a mandatory NID, for whatever reasons,
-              * just clear the error and allow all hashes to be used. */
-        break;
-    }
+    if (EVP_PKEY_get_default_digest_nid(pkey, &default_mdnid) == 2 &&
+        sig->hash != default_mdnid)
+            return 0;
+
+    /* If it didn't report a mandatory NID, for whatever reasons,
+     * just clear the error and allow all hashes to be used. */
     ERR_pop_to_mark();
+
     if (s->s3->tmp.peer_cert_sigalgs != NULL) {
         for (i = 0; i < s->s3->tmp.peer_cert_sigalgslen; i++) {
             lu = tls1_lookup_sigalg(s->s3->tmp.peer_cert_sigalgs[i]);
             if (lu == NULL
-                || !X509_get_signature_info(x, &mdnid, &pknid, NULL, NULL)
-                || (mandatory_md && mdnid != default_mdnid))
+                || !X509_get_signature_info(x, &mdnid, &pknid, NULL, NULL))
                 continue;
             /*
              * TODO this does not differentiate between the
@@ -2596,7 +2592,7 @@ static int check_cert_usable(SSL *s, const SIGALG_LOOKUP *sig, X509 *x,
         }
         return 0;
     }
-    return !mandatory_md || sig->hash == default_mdnid;
+    return 1;
 }
 
 /*