EVP: add EVP_PKEY_is_a() and EVP_PKEY_can_sign()

EVP_PKEY_is_a() is the provider side key checking function corresponding
to checking EVP_PKEY_id() or an EVP_PKEY against macros like EVP_PKEY_EC.
It also works with legacy internal keys.

We also add a warning indoc/man3/EVP_PKEY_set1_RSA.pod regarding the
reliability of certain functions that only understand legacy keys.

Finally, we take the opportunity to clean up doc/man3/EVP_PKEY_set1_RSA.pod
to better conform with man-page layout norms, see man-pages(7) on Linux.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/11358)

5 files changed, 177 insertions, 16 deletions

diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c
index b176f100e8..c3a7fbe692 100644
--- a/crypto/evp/p_lib.c
+++ b/crypto/evp/p_lib.c
@@ -24,6 +24,7 @@
 #include <openssl/rsa.h>
 #include <openssl/dsa.h>
 #include <openssl/dh.h>
+#include <openssl/ec.h>
 #include <openssl/cmac.h>
 #include <openssl/engine.h>
 #include <openssl/params.h>
@@ -35,6 +36,11 @@
 #include "internal/provider.h"
 #include "evp_local.h"
 
+#include "crypto/ec.h"
+
+/* TODO remove this when the EVP_PKEY_is_a() #legacy support hack is removed */
+#include "e_os.h"                /* strcasecmp on Windows */
+
 static int pkey_set_type(EVP_PKEY *pkey, ENGINE *e, int type, const char *str,
                          int len, EVP_KEYMGMT *keymgmt);
 static void evp_pkey_free_it(EVP_PKEY *key);
@@ -732,6 +738,77 @@ int EVP_PKEY_base_id(const EVP_PKEY *pkey)
     return EVP_PKEY_type(pkey->type);
 }
 
+int EVP_PKEY_is_a(const EVP_PKEY *pkey, const char *name)
+{
+#ifndef FIPS_MODE
+    if (pkey->keymgmt == NULL) {
+        /*
+         * These hard coded cases are pure hackery to get around the fact
+         * that names in crypto/objects/objects.txt are a mess.  There is
+         * no "EC", and "RSA" leads to the NID for 2.5.8.1.1, an OID that's
+         * fallen out in favor of { pkcs-1 1 }, i.e. 1.2.840.113549.1.1.1,
+         * the NID of which is used for EVP_PKEY_RSA.  Strangely enough,
+         * "DSA" is accurate...  but still, better be safe and hard-code
+         * names that we know.
+         * TODO Clean this away along with all other #legacy support.
+         */
+        int type;
+
+        if (strcasecmp(name, "RSA") == 0)
+            type = EVP_PKEY_RSA;
+#ifndef OPENSSL_NO_EC
+        else if (strcasecmp(name, "EC") == 0)
+            type = EVP_PKEY_EC;
+#endif
+#ifndef OPENSSL_NO_DSA
+        else if (strcasecmp(name, "DSA") == 0)
+            type = EVP_PKEY_DSA;
+#endif
+        else
+            type = EVP_PKEY_type(OBJ_sn2nid(name));
+        return EVP_PKEY_type(pkey->type) == type;
+    }
+#endif
+    return EVP_KEYMGMT_is_a(pkey->keymgmt, name);
+}
+
+int EVP_PKEY_can_sign(const EVP_PKEY *pkey)
+{
+    if (pkey->keymgmt == NULL) {
+        switch (EVP_PKEY_base_id(pkey)) {
+        case EVP_PKEY_RSA:
+            return 1;
+#ifndef OPENSSL_NO_DSA
+        case EVP_PKEY_DSA:
+            return 1;
+#endif
+#ifndef OPENSSL_NO_EC
+        case EVP_PKEY_ED25519:
+        case EVP_PKEY_ED448:
+            return 1;
+        case EVP_PKEY_EC:        /* Including SM2 */
+            return EC_KEY_can_sign(pkey->pkey.ec);
+#endif
+        default:
+            break;
+        }
+    } else {
+        const OSSL_PROVIDER *prov = EVP_KEYMGMT_provider(pkey->keymgmt);
+        OPENSSL_CTX *libctx = ossl_provider_library_context(prov);
+        const char *supported_sig =
+            pkey->keymgmt->query_operation_name != NULL
+            ? pkey->keymgmt->query_operation_name(OSSL_OP_SIGNATURE)
+            : evp_first_name(prov, pkey->keymgmt->name_id);
+        EVP_SIGNATURE *signature = NULL;
+
+        signature = EVP_SIGNATURE_fetch(libctx, supported_sig, NULL);
+        if (signature != NULL) {
+            EVP_SIGNATURE_free(signature);
+            return 1;
+        }
+    }
+    return 0;
+}
 
 static int print_reset_indent(BIO **out, int pop_f_prefix, long saved_indent)
 {
diff --git a/doc/man3/EVP_PKEY_is_a.pod b/doc/man3/EVP_PKEY_is_a.pod
new file mode 100644
index 0000000000..cfce3de5da
--- /dev/null
+++ b/doc/man3/EVP_PKEY_is_a.pod
@@ -0,0 +1,72 @@
+=pod
+
+=head1 NAME
+
+EVP_PKEY_is_a, EVP_PKEY_can_sign
+- key type and capabilities functions
+
+=head1 SYNOPSIS
+
+ #include <openssl/evp.h>
+
+ int EVP_PKEY_is_a(const EVP_PKEY *pkey, const char *name);
+ int EVP_PKEY_can_sign(const EVP_PKEY *pkey);
+
+=head1 DESCRIPTION
+
+EVP_PKEY_is_a() checks if the key type of I<pkey> is I<name>.
+
+EVP_PKEY_can_sign() checks if the functionality for the key type of
+I<pkey> supports signing.  No other check is done, such as whether
+I<pkey> contains a private key.
+
+=head1 RETURN VALUES
+
+EVP_PKEY_is_a() returns 1 if I<pkey> has the key type I<name>,
+otherwise 0.
+
+EVP_PKEY_can_sign() returns 1 if the I<pkey> key type functionality
+supports signing, otherwise 0.
+
+=head1 EXAMPLES
+
+=head2 EVP_PKEY_is_a()
+
+The loaded providers and what key types they support will ultimately
+determine what I<name> is possible to use with EVP_PKEY_is_a().  We do know
+that the default provider supports RSA, DH, DSA and EC keys, so we can use
+this as an crude example:
+
+ #include <openssl/evp.h>
+
+ ...
+     /* |pkey| is an EVP_PKEY* */
+     if (EVP_PKEY_is_a(pkey, "RSA")) {
+         BIGNUM *modulus = NULL;
+         if (EVP_PKEY_get_bn_param(pkey, "n", &modulus))
+             /* do whatever with the modulus */
+         BN_free(modulus);
+     }
+
+=head2 EVP_PKEY_can_sign()
+
+ #include <openssl/evp.h>
+
+ ...
+     /* |pkey| is an EVP_PKEY* */
+     if (!EVP_PKEY_can_sign(pkey)) {
+         fprintf(stderr, "Not a signing key!");
+         exit(1);
+     }
+     /* Sign something... */
+
+=head1 COPYRIGHT
+
+Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut
diff --git a/doc/man3/EVP_PKEY_set1_RSA.pod b/doc/man3/EVP_PKEY_set1_RSA.pod
index 8423a0a3b8..598cda4221 100644
--- a/doc/man3/EVP_PKEY_set1_RSA.pod
+++ b/doc/man3/EVP_PKEY_set1_RSA.pod
@@ -51,52 +51,61 @@ EVP_PKEY_set1_engine, EVP_PKEY_get0_engine - EVP_PKEY assignment functions
 =head1 DESCRIPTION
 
 EVP_PKEY_set1_RSA(), EVP_PKEY_set1_DSA(), EVP_PKEY_set1_DH() and
-EVP_PKEY_set1_EC_KEY() set the key referenced by B<pkey> to B<key>.
+EVP_PKEY_set1_EC_KEY() set the key referenced by I<pkey> to I<key>.
 
 EVP_PKEY_get1_RSA(), EVP_PKEY_get1_DSA(), EVP_PKEY_get1_DH() and
-EVP_PKEY_get1_EC_KEY() return the referenced key in B<pkey> or
-B<NULL> if the key is not of the correct type.
+EVP_PKEY_get1_EC_KEY() return the referenced key in I<pkey> or
+NULL if the key is not of the correct type.
 
 EVP_PKEY_get0_hmac(), EVP_PKEY_get0_poly1305(), EVP_PKEY_get0_siphash(),
 EVP_PKEY_get0_RSA(), EVP_PKEY_get0_DSA(), EVP_PKEY_get0_DH()
-and EVP_PKEY_get0_EC_KEY() also return the referenced key in B<pkey> or B<NULL>
+and EVP_PKEY_get0_EC_KEY() also return the referenced key in I<pkey> or NULL
 if the key is not of the correct type but the reference count of the
 returned key is B<not> incremented and so must not be freed up after use.
 
 EVP_PKEY_assign_RSA(), EVP_PKEY_assign_DSA(), EVP_PKEY_assign_DH(),
 EVP_PKEY_assign_EC_KEY(), EVP_PKEY_assign_POLY1305() and
-EVP_PKEY_assign_SIPHASH() also set the referenced key to B<key>
-however these use the supplied B<key> internally and so B<key>
-will be freed when the parent B<pkey> is freed.
+EVP_PKEY_assign_SIPHASH() also set the referenced key to I<key>
+however these use the supplied I<key> internally and so I<key>
+will be freed when the parent I<pkey> is freed.
 
-EVP_PKEY_base_id() returns the type of B<pkey>. For example
+EVP_PKEY_base_id() returns the type of I<pkey>. For example
 an RSA key will return B<EVP_PKEY_RSA>.
 
-EVP_PKEY_id() returns the actual OID associated with B<pkey>. Historically keys
+EVP_PKEY_id() returns the actual OID associated with I<pkey>. Historically keys
 using the same algorithm could use different OIDs. For example an RSA key could
 use the OIDs corresponding to the NIDs B<NID_rsaEncryption> (equivalent to
 B<EVP_PKEY_RSA>) or B<NID_rsa> (equivalent to B<EVP_PKEY_RSA2>). The use of
 alternative non-standard OIDs is now rare so B<EVP_PKEY_RSA2> et al are not
 often seen in practice.
 
-EVP_PKEY_type() returns the underlying type of the NID B<type>. For example
+EVP_PKEY_type() returns the underlying type of the NID I<type>. For example
 EVP_PKEY_type(EVP_PKEY_RSA2) will return B<EVP_PKEY_RSA>.
 
-EVP_PKEY_get0_engine() returns a reference to the ENGINE handling B<pkey>.
+EVP_PKEY_get0_engine() returns a reference to the ENGINE handling I<pkey>.
 
-EVP_PKEY_set1_engine() sets the ENGINE handling B<pkey> to B<engine>. It
+EVP_PKEY_set1_engine() sets the ENGINE handling I<pkey> to I<engine>. It
 must be called after the key algorithm and components are set up.
-If B<engine> does not include an B<EVP_PKEY_METHOD> for B<pkey> an
+If I<engine> does not include an B<EVP_PKEY_METHOD> for I<pkey> an
 error occurs.
 
 EVP_PKEY_set_alias_type() allows modifying a EVP_PKEY to use a
 different set of algorithms than the default.
 
+=head1 WARNINGS
+
+The following functions are only reliable with B<EVP_PKEY>s that have
+been assigned an internal key with EVP_PKEY_assign_*():
+
+EVP_PKEY_id(), EVP_PKEY_base_id(), EVP_PKEY_type(), EVP_PKEY_set_alias_type()
+
+For EVP_PKEY key type checking purposes, L<EVP_PKEY_is_a(3)> is more generic.
+
 =head1 NOTES
 
 In accordance with the OpenSSL naming convention the key obtained
-from or assigned to the B<pkey> using the B<1> functions must be
-freed as well as B<pkey>.
+from or assigned to the I<pkey> using the B<1> functions must be
+freed as well as I<pkey>.
 
 EVP_PKEY_assign_RSA(), EVP_PKEY_assign_DSA(), EVP_PKEY_assign_DH(),
 EVP_PKEY_assign_EC_KEY(), EVP_PKEY_assign_POLY1305()
@@ -129,7 +138,7 @@ EVP_PKEY_set1_RSA(), EVP_PKEY_set1_DSA(), EVP_PKEY_set1_DH() and
 EVP_PKEY_set1_EC_KEY() return 1 for success or 0 for failure.
 
 EVP_PKEY_get1_RSA(), EVP_PKEY_get1_DSA(), EVP_PKEY_get1_DH() and
-EVP_PKEY_get1_EC_KEY() return the referenced key or B<NULL> if
+EVP_PKEY_get1_EC_KEY() return the referenced key or NULL if
 an error occurred.
 
 EVP_PKEY_assign_RSA(), EVP_PKEY_assign_DSA(), EVP_PKEY_assign_DH(),
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 4903fc5f42..4545bf7170 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -1104,6 +1104,7 @@ DEPRECATEDIN_3_0(int EVP_PKEY_decrypt_old(unsigned char *dec_key,
 DEPRECATEDIN_3_0(int EVP_PKEY_encrypt_old(unsigned char *enc_key,
                                           const unsigned char *key,
                                           int key_len, EVP_PKEY *pub_key))
+int EVP_PKEY_is_a(const EVP_PKEY *pkey, const char *name);
 int EVP_PKEY_type(int type);
 int EVP_PKEY_id(const EVP_PKEY *pkey);
 int EVP_PKEY_base_id(const EVP_PKEY *pkey);
diff --git a/util/libcrypto.num b/util/libcrypto.num
index 12761e4adc..e29234aaf9 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -5037,3 +5037,5 @@ EVP_PKEY_get_size_t_param               ?	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_get_bn_param                   ?	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_get_utf8_string_param          ?	3_0_0	EXIST::FUNCTION:
 EVP_PKEY_get_octet_string_param         ?	3_0_0	EXIST::FUNCTION:
+EVP_PKEY_is_a                           ?	3_0_0	EXIST::FUNCTION:
+EVP_PKEY_can_sign                       ?	3_0_0	EXIST::FUNCTION: