Fixes related to separation of DH and DHX types

Fix dh_rfc5114 option in genpkey.

Fixes #14145
Fixes #13956
Fixes #13952
Fixes #13871
Fixes #14054
Fixes #14444

Updated documentation for app to indicate what options are available for
DH and DHX keys.

DH and DHX now have different keymanager gen_set_params() methods.

Added CHANGES entry to indicate the breaking change.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14883)

1 files changed, 11 insertions, 32 deletions

diff --git a/crypto/dh/dh_pmeth.c b/crypto/dh/dh_pmeth.c
index affe40a53c..78d46aba22 100644
--- a/crypto/dh/dh_pmeth.c
+++ b/crypto/dh/dh_pmeth.c
@@ -35,7 +35,6 @@ typedef struct {
     int pad;
     /* message digest used for parameter generation */
     const EVP_MD *md;
-    int rfc5114_param;
     int param_nid;
     /* Keygen callback info */
     int gentmp[2];
@@ -98,7 +97,6 @@ static int pkey_dh_copy(EVP_PKEY_CTX *dst, const EVP_PKEY_CTX *src)
     dctx->paramgen_type = sctx->paramgen_type;
     dctx->pad = sctx->pad;
     dctx->md = sctx->md;
-    dctx->rfc5114_param = sctx->rfc5114_param;
     dctx->param_nid = sctx->param_nid;
 
     dctx->kdf_type = sctx->kdf_type;
@@ -156,11 +154,11 @@ static int pkey_dh_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
     case EVP_PKEY_CTRL_DH_RFC5114:
         if (p1 < 1 || p1 > 3 || dctx->param_nid != NID_undef)
             return -2;
-        dctx->rfc5114_param = p1;
+        dctx->param_nid = p1;
         return 1;
 
     case EVP_PKEY_CTRL_DH_NID:
-        if (p1 <= 0 || dctx->rfc5114_param != 0)
+        if (p1 <= 0 || dctx->param_nid != NID_undef)
             return -2;
         dctx->param_nid = p1;
         return 1;
@@ -233,11 +231,12 @@ static int pkey_dh_ctrl_str(EVP_PKEY_CTX *ctx,
     }
     if (strcmp(type, "dh_rfc5114") == 0) {
         DH_PKEY_CTX *dctx = ctx->data;
-        int len;
-        len = atoi(value);
-        if (len < 0 || len > 3)
+        int id;
+
+        id = atoi(value);
+        if (id < 0 || id > 3)
             return -2;
-        dctx->rfc5114_param = len;
+        dctx->param_nid = id;
         return 1;
     }
     if (strcmp(type, "dh_param") == 0) {
@@ -331,36 +330,16 @@ static int pkey_dh_paramgen(EVP_PKEY_CTX *ctx,
     /*
      * Look for a safe prime group for key establishment. Which uses
      * either RFC_3526 (modp_XXXX) or RFC_7919 (ffdheXXXX).
+     * RFC_5114 is also handled here for param_nid = (1..3)
      */
     if (dctx->param_nid != NID_undef) {
+        int type = dctx->param_nid <= 3 ? EVP_PKEY_DHX : EVP_PKEY_DH;
+
         if ((dh = DH_new_by_nid(dctx->param_nid)) == NULL)
             return 0;
-        EVP_PKEY_assign(pkey, EVP_PKEY_DH, dh);
-        return 1;
-    }
-
-#ifndef FIPS_MODULE
-    if (dctx->rfc5114_param) {
-        switch (dctx->rfc5114_param) {
-        case 1:
-            dh = DH_get_1024_160();
-            break;
-
-        case 2:
-            dh = DH_get_2048_224();
-            break;
-
-        case 3:
-            dh = DH_get_2048_256();
-            break;
-
-        default:
-            return -2;
-        }
-        EVP_PKEY_assign(pkey, EVP_PKEY_DHX, dh);
+        EVP_PKEY_assign(pkey, type, dh);
         return 1;
     }
-#endif /* FIPS_MODULE */
 
     if (ctx->pkey_gencb != NULL) {
         pcb = BN_GENCB_new();