Add easy to digest selector macros for EVP_PKEYs

These are meant to be used with functions like
OSSL_ENCODER_CTX_new_by_EVP_PKEY()

The OSSL_ENCODER_CTX_new_by_EVP_PKEY() manual is also expanded on the
topics of output types and selections.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/13189)

6 files changed, 86 insertions, 25 deletions

diff --git a/crypto/asn1/i2d_pr.c b/crypto/asn1/i2d_pr.c
index 80cfde9a22..7185abef45 100644
--- a/crypto/asn1/i2d_pr.c
+++ b/crypto/asn1/i2d_pr.c
@@ -34,9 +34,7 @@ int i2d_PrivateKey(const EVP_PKEY *a, unsigned char **pp)
     if (evp_pkey_is_provided(a)) {
         /* |*pp| is unbounded, so we need an upper limit */
         size_t length = INT_MAX;
-        /* The private key includes everything */
-        int selection =
-            OSSL_KEYMGMT_SELECT_ALL_PARAMETERS | OSSL_KEYMGMT_SELECT_KEYPAIR;
+        int selection = EVP_PKEY_KEYPAIR;
         int ret = -1;
         OSSL_ENCODER_CTX *ctx;
 
diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c
index 1ec36d6cbc..4eeb95e413 100644
--- a/crypto/evp/p_lib.c
+++ b/crypto/evp/p_lib.c
@@ -1209,10 +1209,7 @@ static int print_pkey(const EVP_PKEY *pkey, BIO *out, int indent,
 int EVP_PKEY_print_public(BIO *out, const EVP_PKEY *pkey,
                           int indent, ASN1_PCTX *pctx)
 {
-    return print_pkey(pkey, out, indent,
-                      OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS
-                      | OSSL_KEYMGMT_SELECT_PUBLIC_KEY,
-                      NULL, NULL,
+    return print_pkey(pkey, out, indent, EVP_PKEY_PUBLIC_KEY, NULL, NULL,
                       (pkey->ameth != NULL ? pkey->ameth->pub_print : NULL),
                       pctx);
 }
@@ -1220,10 +1217,7 @@ int EVP_PKEY_print_public(BIO *out, const EVP_PKEY *pkey,
 int EVP_PKEY_print_private(BIO *out, const EVP_PKEY *pkey,
                            int indent, ASN1_PCTX *pctx)
 {
-    return print_pkey(pkey, out, indent,
-                      OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS
-                      | OSSL_KEYMGMT_SELECT_KEYPAIR,
-                      NULL, NULL,
+    return print_pkey(pkey, out, indent, EVP_PKEY_KEYPAIR, NULL, NULL,
                       (pkey->ameth != NULL ? pkey->ameth->priv_print : NULL),
                       pctx);
 }
@@ -1231,8 +1225,7 @@ int EVP_PKEY_print_private(BIO *out, const EVP_PKEY *pkey,
 int EVP_PKEY_print_params(BIO *out, const EVP_PKEY *pkey,
                           int indent, ASN1_PCTX *pctx)
 {
-    return print_pkey(pkey, out, indent, OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS,
-                      NULL, NULL,
+    return print_pkey(pkey, out, indent, EVP_PKEY_KEY_PARAMETERS, NULL, NULL,
                       (pkey->ameth != NULL ? pkey->ameth->param_print : NULL),
                       pctx);
 }
diff --git a/crypto/pem/pem_local.h b/crypto/pem/pem_local.h
index 39dc462e54..f9575d4988 100644
--- a/crypto/pem/pem_local.h
+++ b/crypto/pem/pem_local.h
@@ -28,11 +28,9 @@
  * decoding operation can return an object that contains what libcrypto
  * expects.
  */
-# define PEM_SELECTION_PUBKEY                                           \
-    (OSSL_KEYMGMT_SELECT_ALL_PARAMETERS | OSSL_KEYMGMT_SELECT_PUBLIC_KEY)
-# define PEM_SELECTION_PrivateKey                                       \
-    (OSSL_KEYMGMT_SELECT_ALL_PARAMETERS | OSSL_KEYMGMT_SELECT_KEYPAIR)
-# define PEM_SELECTION_Parameters OSSL_KEYMGMT_SELECT_ALL_PARAMETERS
+# define PEM_SELECTION_PUBKEY           EVP_PKEY_PUBLIC_KEY
+# define PEM_SELECTION_PrivateKey       EVP_PKEY_KEYPAIR
+# define PEM_SELECTION_Parameters       EVP_PKEY_KEY_PARAMETERS
 
 /* Alternative IMPLEMENT macros for provided encoders */
 
diff --git a/crypto/x509/x_pubkey.c b/crypto/x509/x_pubkey.c
index 7d39254685..b7dd04838c 100644
--- a/crypto/x509/x_pubkey.c
+++ b/crypto/x509/x_pubkey.c
@@ -103,10 +103,8 @@ int X509_PUBKEY_set(X509_PUBKEY **x, EVP_PKEY *pkey)
         OSSL_LIB_CTX *libctx = ossl_provider_libctx(pkprov);
         unsigned char *der = NULL;
         size_t derlen = 0;
-        int selection = (OSSL_KEYMGMT_SELECT_PUBLIC_KEY
-                         | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS);
         OSSL_ENCODER_CTX *ectx =
-            OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, "DER", selection,
+            OSSL_ENCODER_CTX_new_by_EVP_PKEY(pkey, "DER", EVP_PKEY_PUBLIC_KEY,
                                              libctx, NULL);
 
         if (OSSL_ENCODER_to_data(ectx, &der, &derlen)) {
@@ -310,10 +308,9 @@ int i2d_PUBKEY(const EVP_PKEY *a, unsigned char **pp)
     } else if (a->keymgmt != NULL) {
         const OSSL_PROVIDER *pkprov = EVP_KEYMGMT_provider(a->keymgmt);
         OSSL_LIB_CTX *libctx = ossl_provider_libctx(pkprov);
-        int selection = (OSSL_KEYMGMT_SELECT_PUBLIC_KEY
-                         | OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS);
         OSSL_ENCODER_CTX *ctx =
-            OSSL_ENCODER_CTX_new_by_EVP_PKEY(a, "DER", selection, libctx, NULL);
+            OSSL_ENCODER_CTX_new_by_EVP_PKEY(a, "DER", EVP_PKEY_PUBLIC_KEY,
+                                             libctx, NULL);
         BIO *out = BIO_new(BIO_s_mem());
         BUF_MEM *buf = NULL;
 
diff --git a/doc/man3/OSSL_ENCODER_CTX_new_by_EVP_PKEY.pod b/doc/man3/OSSL_ENCODER_CTX_new_by_EVP_PKEY.pod
index b47dc453f8..f90d13d551 100644
--- a/doc/man3/OSSL_ENCODER_CTX_new_by_EVP_PKEY.pod
+++ b/doc/man3/OSSL_ENCODER_CTX_new_by_EVP_PKEY.pod
@@ -39,7 +39,11 @@ OSSL_ENCODER_CTX_set_passphrase_ui
 OSSL_ENCODER_CTX_new_by_EVP_PKEY() is a utility function that creates a
 B<OSSL_ENCODER_CTX>, finds all applicable encoder implementations and sets
 them up, so almost all the caller has to do next is call functions like
-L<OSSL_ENCODER_to_bio(3)>.
+L<OSSL_ENCODER_to_bio(3)>.  I<output_type> determines the final output
+encoding, and I<selection> can be used to select what parts of the I<pkey>
+should be included in the output.  I<output_type> is further discussed in
+L</Output types> below, and I<selection> is further described in
+L</Selections>.
 
 Internally, OSSL_ENCODER_CTX_new_by_EVP_PKEY() uses the names from the
 L<EVP_KEYMGMT(3)> implementation associated with I<pkey> to build a list of
@@ -75,6 +79,69 @@ implementation can use to prompt for a pass phrase, giving the caller the
 choice of prefered pass phrase callback form.  These are called indirectly,
 through an internal B<OSSL_PASSPHRASE_CALLBACK> function.
 
+=head2 Output types
+
+The possible B<EVP_PKEY> output types depends on the available
+implementations.
+
+OpenSSL has built in implementations for the following output types:
+
+=over 4
+
+=item C<TEXT>
+
+The output is a human readable description of the key.
+L<EVP_PKEY_print_private(3)>, L<EVP_PKEY_print_public(3)> and
+L<EVP_PKEY_print_params(3)> use this for their output.
+
+=item C<DER>
+
+The output is the DER encoding of the I<selection> of the I<pkey>.
+
+=item C<PEM>
+
+The output is the I<selection> of the I<pkey> in PEM format.
+
+=back
+
+=head2 Selections
+
+=begin comment
+
+These constants should really be documented among the EVP manuals, but this
+will have to do for now.
+
+=end comment
+
+The following constants can be used for standard I<selection>:
+
+=over 4
+
+=item B<EVP_PKEY_KEY_PARAMETERS>
+
+Indicates that only the key parameters should be included in the output.
+Where it matters, the data type in the output will indicate that the data is
+parameters, not a key.
+
+=item B<EVP_PKEY_PUBLIC_KEY>
+
+Indicates that the public key and eventual key parameters will be included
+in the output.  Where it matters, the data type in the output will indicate
+that the data is a public key.
+
+=item B<EVP_PKEY_KEYPAIR>
+
+Indicates that the private key, the public key and eventual key parameters
+should be included in the output.  Where it matters, the data type in the
+output will indicate that the data is a private key.
+
+=back
+
+These are only indications, the encoder implementations are free to
+determine what makes sense to include in the output, and this may depend on
+the desired output.  For example, an EC key in a PKCS#8 structure doesn't
+usually include the public key.
+
 =head1 RETURN VALUES
 
 OSSL_ENCODER_CTX_new_by_EVP_PKEY() returns a pointer to a
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 4472bcf50e..ba13960d6a 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -75,6 +75,14 @@
 /* Special indicator that the object is uniquely provider side */
 # define EVP_PKEY_KEYMGMT -1
 
+/* Easy to use macros for EVP_PKEY related selections */
+# define EVP_PKEY_KEY_PARAMETERS                                            \
+    ( OSSL_KEYMGMT_SELECT_ALL_PARAMETERS )
+# define EVP_PKEY_PUBLIC_KEY                                                \
+    ( EVP_PKEY_KEY_PARAMETERS | OSSL_KEYMGMT_SELECT_PUBLIC_KEY )
+# define EVP_PKEY_KEYPAIR                                                   \
+    ( EVP_PKEY_PUBLIC_KEY | OSSL_KEYMGMT_SELECT_PRIVATE_KEY )
+
 #ifdef  __cplusplus
 extern "C" {
 #endif