Teach ASN1_item_verify_ctx() how to handle provided keys

We need to special case RSA-PSS because that uses X509_ALGOR style
parameters and we have no support for this on the provider side at this
stage.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15527)

1 files changed, 44 insertions, 17 deletions

diff --git a/crypto/asn1/a_verify.c b/crypto/asn1/a_verify.c
index b1adaf7a8f..9be231b043 100644
--- a/crypto/asn1/a_verify.c
+++ b/crypto/asn1/a_verify.c
@@ -20,6 +20,7 @@
 #include <openssl/evp.h>
 #include "crypto/asn1.h"
 #include "crypto/evp.h"
+#include "crypto/rsa.h"
 
 #ifndef OPENSSL_NO_DEPRECATED_3_0
 
@@ -136,7 +137,7 @@ int ASN1_item_verify_ctx(const ASN1_ITEM *it, const X509_ALGOR *alg,
         goto err;
     }
 
-    if (mdnid == NID_undef) {
+    if (mdnid == NID_undef && evp_pkey_is_legacy(pkey)) {
         if (pkey->ameth == NULL || pkey->ameth->item_verify == NULL) {
             ERR_raise(ERR_LIB_ASN1, ASN1_R_UNKNOWN_SIGNATURE_ALGORITHM);
             goto err;
@@ -153,23 +154,49 @@ int ASN1_item_verify_ctx(const ASN1_ITEM *it, const X509_ALGOR *alg,
         if (ret <= 1)
             goto err;
     } else {
-        const EVP_MD *type = EVP_get_digestbynid(mdnid);
+        const EVP_MD *type = NULL;
 
-        if (type == NULL) {
-            ERR_raise(ERR_LIB_ASN1, ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM);
-            goto err;
-        }
-
-        /* Check public key OID matches public key type */
-        if (!EVP_PKEY_is_a(pkey, OBJ_nid2sn(pknid))) {
-            ERR_raise(ERR_LIB_ASN1, ASN1_R_WRONG_PUBLIC_KEY_TYPE);
-            goto err;
-        }
-
-        if (!EVP_DigestVerifyInit(ctx, NULL, type, NULL, pkey)) {
-            ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB);
-            ret = 0;
-            goto err;
+        /*
+         * We don't yet have the ability for providers to be able to handle
+         * X509_ALGOR style parameters. Fortunately the only one that needs this
+         * so far is RSA-PSS, so we just special case this for now. In some
+         * future version of OpenSSL we should push this to the provider.
+         */
+        if (mdnid == NID_undef && pknid == EVP_PKEY_RSA_PSS) {
+            if (!EVP_PKEY_is_a(pkey, "RSA") && !EVP_PKEY_is_a(pkey, "RSA-PSS")) {
+                ERR_raise(ERR_LIB_ASN1, ASN1_R_WRONG_PUBLIC_KEY_TYPE);
+                goto err;
+            }
+            /* This function also calls EVP_DigestVerifyInit */
+            if (ossl_rsa_pss_to_ctx(ctx, NULL, alg, pkey) <= 0) {
+                ERR_raise(ERR_LIB_ASN1, ERR_R_INTERNAL_ERROR);
+                goto err;
+            }
+        } else {
+            /* Check public key OID matches public key type */
+            if (!EVP_PKEY_is_a(pkey, OBJ_nid2sn(pknid))) {
+                ERR_raise(ERR_LIB_ASN1, ASN1_R_WRONG_PUBLIC_KEY_TYPE);
+                goto err;
+            }
+
+            if (mdnid != NID_undef) {
+                type = EVP_get_digestbynid(mdnid);
+                if (type == NULL) {
+                    ERR_raise(ERR_LIB_ASN1,
+                              ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM);
+                    goto err;
+                }
+            }
+
+            /*
+             * Note that some algorithms (notably Ed25519 and Ed448) may allow
+             * a NULL digest value.
+             */
+            if (!EVP_DigestVerifyInit(ctx, NULL, type, NULL, pkey)) {
+                ERR_raise(ERR_LIB_ASN1, ERR_R_EVP_LIB);
+                ret = 0;
+                goto err;
+            }
         }
     }