EVP: Clarify the states of an EVP_PKEY

EVP_PKEY is rather complex, even before provider side keys entered the
stage.
You could have untyped / unassigned keys (pk->type == EVP_PKEY_NONE),
keys that had been assigned a type but no data (pk->pkey.ptr == NULL),
and fully assigned keys (pk->type != EVP_PKEY_NONE && pk->pkey.ptr != NULL).

For provider side keys, the corresponding states weren't well defined,
and the code didn't quite account for all the possibilities.

We also guard most of the legacy fields in EVP_PKEY with FIPS_MODE, so
they don't exist at all in the FIPS module.

Most of all, code needs to adapt to the case where an EVP_PKEY's
|keymgmt| is non-NULL, but its |keydata| is NULL.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/11375)

1 files changed, 33 insertions, 16 deletions

diff --git a/crypto/evp/pmeth_check.c b/crypto/evp/pmeth_check.c
index c02353d5ea..587e8ae12a 100644
--- a/crypto/evp/pmeth_check.c
+++ b/crypto/evp/pmeth_check.c
@@ -35,19 +35,24 @@ int EVP_PKEY_public_check(EVP_PKEY_CTX *ctx)
         return evp_keymgmt_validate(keymgmt, key,
                                     OSSL_KEYMGMT_SELECT_PUBLIC_KEY);
 
+    if (pkey->type == EVP_PKEY_NONE)
+        goto not_supported;
+
+#ifndef FIPS_MODE
     /* legacy */
     /* call customized public key check function first */
     if (ctx->pmeth->public_check != NULL)
         return ctx->pmeth->public_check(pkey);
 
     /* use default public key check function in ameth */
-    if (pkey->ameth == NULL || pkey->ameth->pkey_public_check == NULL) {
-        EVPerr(EVP_F_EVP_PKEY_PUBLIC_CHECK,
-               EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
-    }
+    if (pkey->ameth == NULL || pkey->ameth->pkey_public_check == NULL)
+        goto not_supported;
 
     return pkey->ameth->pkey_public_check(pkey);
+#endif
+ not_supported:
+    EVPerr(0, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+    return -2;
 }
 
 int EVP_PKEY_param_check(EVP_PKEY_CTX *ctx)
@@ -68,19 +73,24 @@ int EVP_PKEY_param_check(EVP_PKEY_CTX *ctx)
         return evp_keymgmt_validate(keymgmt, key,
                                     OSSL_KEYMGMT_SELECT_ALL_PARAMETERS);
 
+    if (pkey->type == EVP_PKEY_NONE)
+        goto not_supported;
+
+#ifndef FIPS_MODE
+    /* legacy */
     /* call customized param check function first */
     if (ctx->pmeth->param_check != NULL)
         return ctx->pmeth->param_check(pkey);
 
-    /* legacy */
     /* use default param check function in ameth */
-    if (pkey->ameth == NULL || pkey->ameth->pkey_param_check == NULL) {
-        EVPerr(EVP_F_EVP_PKEY_PARAM_CHECK,
-               EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
-    }
+    if (pkey->ameth == NULL || pkey->ameth->pkey_param_check == NULL)
+        goto not_supported;
 
     return pkey->ameth->pkey_param_check(pkey);
+#endif
+ not_supported:
+    EVPerr(0, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+    return -2;
 }
 
 int EVP_PKEY_private_check(EVP_PKEY_CTX *ctx)
@@ -101,6 +111,7 @@ int EVP_PKEY_private_check(EVP_PKEY_CTX *ctx)
         return evp_keymgmt_validate(keymgmt, key,
                                     OSSL_KEYMGMT_SELECT_PRIVATE_KEY);
     /* not supported for legacy keys */
+    EVPerr(0, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
     return -2;
 }
 
@@ -121,6 +132,7 @@ int EVP_PKEY_pairwise_check(EVP_PKEY_CTX *ctx)
     if (key != NULL && keymgmt != NULL)
         return evp_keymgmt_validate(keymgmt, key, OSSL_KEYMGMT_SELECT_KEYPAIR);
     /* not supported for legacy keys */
+    EVPerr(0, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
     return -2;
 }
 
@@ -141,18 +153,23 @@ int EVP_PKEY_check(EVP_PKEY_CTX *ctx)
     if (key != NULL && keymgmt != NULL)
         return evp_keymgmt_validate(keymgmt, key, OSSL_KEYMGMT_SELECT_ALL);
 
+    if (pkey->type == EVP_PKEY_NONE)
+        goto not_supported;
+
+#ifndef FIPS_MODE
     /* legacy */
     /* call customized check function first */
     if (ctx->pmeth->check != NULL)
         return ctx->pmeth->check(pkey);
 
     /* use default check function in ameth */
-    if (pkey->ameth == NULL || pkey->ameth->pkey_check == NULL) {
-        EVPerr(EVP_F_EVP_PKEY_CHECK,
-               EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
-        return -2;
-    }
+    if (pkey->ameth == NULL || pkey->ameth->pkey_check == NULL)
+        goto not_supported;
 
     return pkey->ameth->pkey_check(pkey);
+#endif
+ not_supported:
+    EVPerr(0, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE);
+    return -2;
 }