EVP: Clarify the states of an EVP_PKEY

EVP_PKEY is rather complex, even before provider side keys entered the
stage.
You could have untyped / unassigned keys (pk->type == EVP_PKEY_NONE),
keys that had been assigned a type but no data (pk->pkey.ptr == NULL),
and fully assigned keys (pk->type != EVP_PKEY_NONE && pk->pkey.ptr != NULL).

For provider side keys, the corresponding states weren't well defined,
and the code didn't quite account for all the possibilities.

We also guard most of the legacy fields in EVP_PKEY with FIPS_MODE, so
they don't exist at all in the FIPS module.

Most of all, code needs to adapt to the case where an EVP_PKEY's
|keymgmt| is non-NULL, but its |keydata| is NULL.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/11375)

1 files changed, 28 insertions, 10 deletions

diff --git a/include/crypto/evp.h b/include/crypto/evp.h
index 2e0322fa98..e5f9aad010 100644
--- a/include/crypto/evp.h
+++ b/include/crypto/evp.h
@@ -502,14 +502,31 @@ const EVP_CIPHER *EVP_##cname##_ecb(void) { return &cname##_ecb; }
                              cipher##_init_key, NULL, NULL, NULL, NULL)
 
 /*
- * Type needs to be a bit field Sub-type needs to be for variations on the
- * method, as in, can it do arbitrary encryption....
+ * An EVP_PKEY can have the following states:
+ *
+ * untyped & empty:
+ *
+ *     type == EVP_PKEY_NONE && keymgmt == NULL
+ *
+ * typed & empty:
+ *
+ *     (type != EVP_PKEY_NONE && pkey.ptr == NULL)      ## legacy (libcrypto only)
+ *     || (keymgmt != NULL && keydata == NULL)          ## provider side
+ *
+ * fully assigned:
+ *
+ *     (type != EVP_PKEY_NONE && pkey.ptr != NULL)      ## legacy (libcrypto only)
+ *     || (keymgmt != NULL && keydata != NULL)          ## provider side
+ *
+ * The easiest way to detect a legacy key is:           type != EVP_PKEY_NONE
+ * The easiest way to detect a provider side key is:    keymgmt != NULL
  */
 struct evp_pkey_st {
     /* == Legacy attributes == */
     int type;
     int save_type;
 
+# ifndef FIPS_MODE
     /*
      * Legacy key "origin" is composed of a pointer to an EVP_PKEY_ASN1_METHOD,
      * a pointer to a low level key and possibly a pointer to an engine.
@@ -519,20 +536,21 @@ struct evp_pkey_st {
     ENGINE *pmeth_engine; /* If not NULL public key ENGINE to use */
     union {
         void *ptr;
-# ifndef OPENSSL_NO_RSA
+#  ifndef OPENSSL_NO_RSA
         struct rsa_st *rsa;     /* RSA */
-# endif
-# ifndef OPENSSL_NO_DSA
+#  endif
+#  ifndef OPENSSL_NO_DSA
         struct dsa_st *dsa;     /* DSA */
-# endif
-# ifndef OPENSSL_NO_DH
+#  endif
+#  ifndef OPENSSL_NO_DH
         struct dh_st *dh;       /* DH */
-# endif
-# ifndef OPENSSL_NO_EC
+#  endif
+#  ifndef OPENSSL_NO_EC
         struct ec_key_st *ec;   /* ECC */
         ECX_KEY *ecx;           /* X25519, X448, Ed25519, Ed448 */
-# endif
+#  endif
     } pkey;
+# endif
 
     /* == Common attributes == */
     CRYPTO_REF_COUNT references;