Support public key and param check in EVP interface

EVP_PKEY_public_check() and EVP_PKEY_param_check()

Doc and test cases are added

Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/4647)

3 files changed, 109 insertions, 2 deletions

diff --git a/crypto/dh/dh_ameth.c b/crypto/dh/dh_ameth.c
index e504690fa3..05a1d4227e 100644
--- a/crypto/dh/dh_ameth.c
+++ b/crypto/dh/dh_ameth.c
@@ -509,6 +509,25 @@ static int dh_pkey_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
 
 }
 
+static int dh_pkey_public_check(const EVP_PKEY *pkey)
+{
+    DH *dh = pkey->pkey.dh;
+
+    if (dh->pub_key == NULL) {
+        DHerr(DH_F_DH_PKEY_PUBLIC_CHECK, DH_R_MISSING_PUBKEY);
+        return 0;
+    }
+
+    return DH_check_pub_key_ex(dh, dh->pub_key);
+}
+
+static int dh_pkey_param_check(const EVP_PKEY *pkey)
+{
+    DH *dh = pkey->pkey.dh;
+
+    return DH_check_ex(dh);
+}
+
 const EVP_PKEY_ASN1_METHOD dh_asn1_meth = {
     EVP_PKEY_DH,
     EVP_PKEY_DH,
@@ -539,7 +558,13 @@ const EVP_PKEY_ASN1_METHOD dh_asn1_meth = {
     0,
 
     int_dh_free,
-    0
+    0,
+
+    0, 0, 0, 0, 0,
+
+    0,
+    dh_pkey_public_check,
+    dh_pkey_param_check
 };
 
 const EVP_PKEY_ASN1_METHOD dhx_asn1_meth = {
@@ -572,7 +597,13 @@ const EVP_PKEY_ASN1_METHOD dhx_asn1_meth = {
     0,
 
     int_dh_free,
-    dh_pkey_ctrl
+    dh_pkey_ctrl,
+
+    0, 0, 0, 0, 0,
+
+    0,
+    dh_pkey_public_check,
+    dh_pkey_param_check
 };
 
 #ifndef OPENSSL_NO_CMS
diff --git a/crypto/dh/dh_check.c b/crypto/dh/dh_check.c
index 066bf83336..fc45577101 100644
--- a/crypto/dh/dh_check.c
+++ b/crypto/dh/dh_check.c
@@ -18,6 +18,19 @@
  * p is odd
  * 1 < g < p - 1
  */
+int DH_check_params_ex(const DH *dh)
+{
+    int errflags = 0;
+
+    (void)DH_check_params(dh, &errflags);
+
+    if ((errflags & DH_CHECK_P_NOT_PRIME) != 0)
+        DHerr(DH_F_DH_CHECK_PARAMS_EX, DH_R_CHECK_P_NOT_PRIME);
+    if ((errflags & DH_NOT_SUITABLE_GENERATOR) != 0)
+        DHerr(DH_F_DH_CHECK_PARAMS_EX, DH_R_NOT_SUITABLE_GENERATOR);
+
+    return errflags == 0;
+}
 
 int DH_check_params(const DH *dh, int *ret)
 {
@@ -61,6 +74,29 @@ int DH_check_params(const DH *dh, int *ret)
  * for 5, p mod 10 == 3 or 7
  * should hold.
  */
+int DH_check_ex(const DH *dh)
+{
+    int errflags = 0;
+
+    (void)DH_check(dh, &errflags);
+
+    if ((errflags & DH_NOT_SUITABLE_GENERATOR) != 0)
+        DHerr(DH_F_DH_CHECK_EX, DH_R_NOT_SUITABLE_GENERATOR);
+    if ((errflags & DH_CHECK_Q_NOT_PRIME) != 0)
+        DHerr(DH_F_DH_CHECK_EX, DH_R_CHECK_Q_NOT_PRIME);
+    if ((errflags & DH_CHECK_INVALID_Q_VALUE) != 0)
+        DHerr(DH_F_DH_CHECK_EX, DH_R_CHECK_INVALID_Q_VALUE);
+    if ((errflags & DH_CHECK_INVALID_J_VALUE) != 0)
+        DHerr(DH_F_DH_CHECK_EX, DH_R_CHECK_INVALID_J_VALUE);
+    if ((errflags & DH_UNABLE_TO_CHECK_GENERATOR) != 0)
+        DHerr(DH_F_DH_CHECK_EX, DH_R_UNABLE_TO_CHECK_GENERATOR);
+    if ((errflags & DH_CHECK_P_NOT_PRIME) != 0)
+        DHerr(DH_F_DH_CHECK_EX, DH_R_CHECK_P_NOT_PRIME);
+    if ((errflags & DH_CHECK_P_NOT_SAFE_PRIME) != 0)
+        DHerr(DH_F_DH_CHECK_EX, DH_R_CHECK_P_NOT_SAFE_PRIME);
+
+    return errflags == 0;
+}
 
 int DH_check(const DH *dh, int *ret)
 {
@@ -142,6 +178,22 @@ int DH_check(const DH *dh, int *ret)
     return ok;
 }
 
+int DH_check_pub_key_ex(const DH *dh, const BIGNUM *pub_key)
+{
+    int errflags = 0;
+
+    (void)DH_check(dh, &errflags);
+
+    if ((errflags & DH_CHECK_PUBKEY_TOO_SMALL) != 0)
+        DHerr(DH_F_DH_CHECK_PUB_KEY_EX, DH_R_CHECK_PUBKEY_TOO_SMALL);
+    if ((errflags & DH_CHECK_PUBKEY_TOO_LARGE) != 0)
+        DHerr(DH_F_DH_CHECK_PUB_KEY_EX, DH_R_CHECK_PUBKEY_TOO_LARGE);
+    if ((errflags & DH_CHECK_PUBKEY_INVALID) != 0)
+        DHerr(DH_F_DH_CHECK_PUB_KEY_EX, DH_R_CHECK_PUBKEY_INVALID);
+
+    return errflags == 0;
+}
+
 int DH_check_pub_key(const DH *dh, const BIGNUM *pub_key, int *ret)
 {
     int ok = 0;
diff --git a/crypto/dh/dh_err.c b/crypto/dh/dh_err.c
index aae66fa0b9..bbedab593e 100644
--- a/crypto/dh/dh_err.c
+++ b/crypto/dh/dh_err.c
@@ -18,6 +18,9 @@ static const ERR_STRING_DATA DH_str_functs[] = {
     {ERR_PACK(ERR_LIB_DH, DH_F_DHPARAMS_PRINT_FP, 0), "DHparams_print_fp"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_BUILTIN_GENPARAMS, 0),
      "dh_builtin_genparams"},
+    {ERR_PACK(ERR_LIB_DH, DH_F_DH_CHECK_EX, 0), "DH_check_ex"},
+    {ERR_PACK(ERR_LIB_DH, DH_F_DH_CHECK_PARAMS_EX, 0), "DH_check_params_ex"},
+    {ERR_PACK(ERR_LIB_DH, DH_F_DH_CHECK_PUB_KEY_EX, 0), "DH_check_pub_key_ex"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_CMS_DECRYPT, 0), "dh_cms_decrypt"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_CMS_SET_PEERKEY, 0), "dh_cms_set_peerkey"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_CMS_SET_SHARED_INFO, 0),
@@ -28,6 +31,8 @@ static const ERR_STRING_DATA DH_str_functs[] = {
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_NEW_BY_NID, 0), "DH_new_by_nid"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_NEW_METHOD, 0), "DH_new_method"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_PARAM_DECODE, 0), "dh_param_decode"},
+    {ERR_PACK(ERR_LIB_DH, DH_F_DH_PKEY_PUBLIC_CHECK, 0),
+     "dh_pkey_public_check"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_PRIV_DECODE, 0), "dh_priv_decode"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_PRIV_ENCODE, 0), "dh_priv_encode"},
     {ERR_PACK(ERR_LIB_DH, DH_F_DH_PUB_DECODE, 0), "dh_pub_decode"},
@@ -44,6 +49,20 @@ static const ERR_STRING_DATA DH_str_reasons[] = {
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_BAD_GENERATOR), "bad generator"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_BN_DECODE_ERROR), "bn decode error"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_BN_ERROR), "bn error"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_CHECK_INVALID_J_VALUE),
+    "check invalid j value"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_CHECK_INVALID_Q_VALUE),
+    "check invalid q value"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_CHECK_PUBKEY_INVALID),
+    "check pubkey invalid"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_CHECK_PUBKEY_TOO_LARGE),
+    "check pubkey too large"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_CHECK_PUBKEY_TOO_SMALL),
+    "check pubkey too small"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_CHECK_P_NOT_PRIME), "check p not prime"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_CHECK_P_NOT_SAFE_PRIME),
+    "check p not safe prime"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_CHECK_Q_NOT_PRIME), "check q not prime"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_DECODE_ERROR), "decode error"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_INVALID_PARAMETER_NAME),
     "invalid parameter name"},
@@ -52,13 +71,18 @@ static const ERR_STRING_DATA DH_str_reasons[] = {
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_INVALID_PUBKEY), "invalid public key"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_KDF_PARAMETER_ERROR), "kdf parameter error"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_KEYS_NOT_SET), "keys not set"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_MISSING_PUBKEY), "missing pubkey"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_MODULUS_TOO_LARGE), "modulus too large"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_NOT_SUITABLE_GENERATOR),
+    "not suitable generator"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_NO_PARAMETERS_SET), "no parameters set"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_NO_PRIVATE_VALUE), "no private value"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_PARAMETER_ENCODING_ERROR),
     "parameter encoding error"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_PEER_KEY_ERROR), "peer key error"},
     {ERR_PACK(ERR_LIB_DH, 0, DH_R_SHARED_INFO_ERROR), "shared info error"},
+    {ERR_PACK(ERR_LIB_DH, 0, DH_R_UNABLE_TO_CHECK_GENERATOR),
+    "unable to check generator"},
     {0, NULL}
 };