Implement EVP_PKEY_param_check_quick() and use it in libssl

The low level DH API has two functions for checking parameters:
DH_check_ex() and DH_check_params_ex(). The former does a "full" check,
while the latter does a "quick" check. Most importantly it skips the
check for a safe prime. We're ok without using safe primes here because
we're doing ephemeral DH.

Now that libssl is fully using the EVP API, we need a way to specify that
we want a quick check instead of a full check. Therefore we introduce
EVP_PKEY_param_check_quick() and use it.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/14146)

5 files changed, 20 insertions, 11 deletions

diff --git a/providers/implementations/keymgmt/dh_kmgmt.c b/providers/implementations/keymgmt/dh_kmgmt.c
index 1691f66f44..007ab6a5b5 100644
--- a/providers/implementations/keymgmt/dh_kmgmt.c
+++ b/providers/implementations/keymgmt/dh_kmgmt.c
@@ -366,7 +366,7 @@ static int dh_validate_private(const DH *dh)
     return dh_check_priv_key(dh, priv_key, &status);;
 }
 
-static int dh_validate(const void *keydata, int selection)
+static int dh_validate(const void *keydata, int selection, int checktype)
 {
     const DH *dh = keydata;
     int ok = 0;
@@ -377,8 +377,17 @@ static int dh_validate(const void *keydata, int selection)
     if ((selection & DH_POSSIBLE_SELECTIONS) != 0)
         ok = 1;
 
-    if ((selection & OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS) != 0)
-        ok = ok && DH_check_ex(dh);
+    if ((selection & OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS) != 0) {
+        /*
+         * Both of these functions check parameters. DH_check_params_ex()
+         * performs a lightweight check (e.g. it does not check that p is a
+         * safe prime)
+         */
+        if (checktype == OSSL_KEYMGMT_VALIDATE_QUICK_CHECK)
+            ok = ok && DH_check_params_ex(dh);
+        else
+            ok = ok && DH_check_ex(dh);
+    }
 
     if ((selection & OSSL_KEYMGMT_SELECT_PUBLIC_KEY) != 0)
         ok = ok && dh_validate_public(dh);
diff --git a/providers/implementations/keymgmt/dsa_kmgmt.c b/providers/implementations/keymgmt/dsa_kmgmt.c
index bc4591b1d6..28e8409aa2 100644
--- a/providers/implementations/keymgmt/dsa_kmgmt.c
+++ b/providers/implementations/keymgmt/dsa_kmgmt.c
@@ -338,7 +338,7 @@ static int dsa_validate_private(const DSA *dsa)
     return dsa_check_priv_key(dsa, priv_key, &status);
 }
 
-static int dsa_validate(const void *keydata, int selection)
+static int dsa_validate(const void *keydata, int selection, int checktype)
 {
     const DSA *dsa = keydata;
     int ok = 0;
diff --git a/providers/implementations/keymgmt/ec_kmgmt.c b/providers/implementations/keymgmt/ec_kmgmt.c
index bb479181c3..33abdc8692 100644
--- a/providers/implementations/keymgmt/ec_kmgmt.c
+++ b/providers/implementations/keymgmt/ec_kmgmt.c
@@ -833,7 +833,7 @@ const OSSL_PARAM *sm2_settable_params(ossl_unused void *provctx)
 }
 
 static
-int sm2_validate(const void *keydata, int selection)
+int sm2_validate(const void *keydata, int selection, int checktype)
 {
     const EC_KEY *eck = keydata;
     int ok = 0;
@@ -868,7 +868,7 @@ int sm2_validate(const void *keydata, int selection)
 #endif
 
 static
-int ec_validate(const void *keydata, int selection)
+int ec_validate(const void *keydata, int selection, int checktype)
 {
     const EC_KEY *eck = keydata;
     int ok = 0;
diff --git a/providers/implementations/keymgmt/ecx_kmgmt.c b/providers/implementations/keymgmt/ecx_kmgmt.c
index 076e59eafe..3c057f3da4 100644
--- a/providers/implementations/keymgmt/ecx_kmgmt.c
+++ b/providers/implementations/keymgmt/ecx_kmgmt.c
@@ -726,22 +726,22 @@ static int ecx_validate(const void *keydata, int selection, int type, size_t key
     return ok;
 }
 
-static int x25519_validate(const void *keydata, int selection)
+static int x25519_validate(const void *keydata, int selection, int checktype)
 {
     return ecx_validate(keydata, selection, ECX_KEY_TYPE_X25519, X25519_KEYLEN);
 }
 
-static int x448_validate(const void *keydata, int selection)
+static int x448_validate(const void *keydata, int selection, int checktype)
 {
     return ecx_validate(keydata, selection, ECX_KEY_TYPE_X448, X448_KEYLEN);
 }
 
-static int ed25519_validate(const void *keydata, int selection)
+static int ed25519_validate(const void *keydata, int selection, int checktype)
 {
     return ecx_validate(keydata, selection, ECX_KEY_TYPE_ED25519, ED25519_KEYLEN);
 }
 
-static int ed448_validate(const void *keydata, int selection)
+static int ed448_validate(const void *keydata, int selection, int checktype)
 {
     return ecx_validate(keydata, selection, ECX_KEY_TYPE_ED448, ED448_KEYLEN);
 }
diff --git a/providers/implementations/keymgmt/rsa_kmgmt.c b/providers/implementations/keymgmt/rsa_kmgmt.c
index 64779ca6be..e4e10084b8 100644
--- a/providers/implementations/keymgmt/rsa_kmgmt.c
+++ b/providers/implementations/keymgmt/rsa_kmgmt.c
@@ -359,7 +359,7 @@ static const OSSL_PARAM *rsa_gettable_params(void *provctx)
     return rsa_params;
 }
 
-static int rsa_validate(const void *keydata, int selection)
+static int rsa_validate(const void *keydata, int selection, int checktype)
 {
     const RSA *rsa = keydata;
     int ok = 0;