Add multiple fixes for ffc key generation using invalid p,q,g parameters.

Fixes #11864

- The dsa keygen assumed valid p, q, g values were being passed. If this is not correct then it is
  possible that dsa keygen can either hang or segfault.
  The fix was to do a partial validation of p, q, and g inside the keygen.
- Fixed a potential double free in the dsa keypair test in the case when in failed (It should never fail!).
  It freed internal object members without setting them to NULL.
- Changed the FFC key validation to accept 1024 bit keys in non fips mode.
- Added tests that use both the default provider & fips provider to test these cases.

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
(Merged from https://github.com/openssl/openssl/pull/12176)

2 files changed, 36 insertions, 1 deletions

diff --git a/crypto/ffc/ffc_params_generate.c b/crypto/ffc/ffc_params_generate.c
index 325eb6768f..8a0b77e7f8 100644
--- a/crypto/ffc/ffc_params_generate.c
+++ b/crypto/ffc/ffc_params_generate.c
@@ -39,6 +39,11 @@
  */
 static int ffc_validate_LN(size_t L, size_t N, int type)
 {
+#ifndef FIPS_MODULE
+    if (L == 1024 && N == 160)
+        return 80;
+#endif
+
     if (type == FFC_PARAM_TYPE_DH) {
         /* Valid DH L,N parameters from SP800-56Ar3 5.5.1 Table 1 */
         if (L == 2048 && (N == 224 || N == 256))
@@ -498,6 +503,7 @@ int ffc_params_FIPS186_4_gen_verify(OPENSSL_CTX *libctx, FFC_PARAMS *params,
     EVP_MD *md = NULL;
     int verify = (mode == FFC_PARAM_MODE_VERIFY);
     unsigned int flags = verify ? params->flags : 0;
+    const char *def_name;
 
     *res = 0;
 
@@ -506,7 +512,10 @@ int ffc_params_FIPS186_4_gen_verify(OPENSSL_CTX *libctx, FFC_PARAMS *params,
     } else {
         if (N == 0)
             N = (L >= 2048 ? SHA256_DIGEST_LENGTH : SHA_DIGEST_LENGTH) * 8;
-        md = EVP_MD_fetch(libctx, default_mdname(N), NULL);
+        def_name = default_mdname(N);
+        if (def_name == NULL)
+            goto err;
+        md = EVP_MD_fetch(libctx, def_name, NULL);
     }
     if (md == NULL)
         goto err;
diff --git a/crypto/ffc/ffc_params_validate.c b/crypto/ffc/ffc_params_validate.c
index f3df0c2b39..821ff3e88a 100644
--- a/crypto/ffc/ffc_params_validate.c
+++ b/crypto/ffc/ffc_params_validate.c
@@ -78,3 +78,29 @@ int ffc_params_FIPS186_2_validate(OPENSSL_CTX *libctx, const FFC_PARAMS *params,
                                            FFC_PARAM_MODE_VERIFY, type,
                                            L, N, res, cb);
 }
+
+/*
+ * This does a simple check of L and N and partial g.
+ * It makes no attempt to do a full validation of p, q or g since these require
+ * extra parameters such as the digest and seed, which may not be available for
+ * this test.
+ */
+int ffc_params_simple_validate(OPENSSL_CTX *libctx, FFC_PARAMS *params, int type)
+{
+    int ret, res = 0;
+    int save_gindex;
+    unsigned int save_flags;
+
+    if (params == NULL)
+        return 0;
+
+    save_flags = params->flags;
+    save_gindex = params->gindex;
+    params->flags = FFC_PARAM_FLAG_VALIDATE_G;
+    params->gindex = FFC_UNVERIFIABLE_GINDEX;
+
+    ret = ffc_params_FIPS186_4_validate(libctx, params, type, &res, NULL);
+    params->flags = save_flags;
+    params->gindex = save_gindex;
+    return ret != FFC_PARAM_RET_STATUS_FAILED;
+}