Allow arbitrary digests with ECDSA and DSA

Unless the FIPS security check is enabled we allow arbitrary digests
with ECDSA and DSA.

Fixes #14696

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15220)

8 files changed, 16 insertions, 15 deletions

diff --git a/providers/common/digest_to_nid.c b/providers/common/digest_to_nid.c
index 96c5e4e38b..49af04ad2a 100644
--- a/providers/common/digest_to_nid.c
+++ b/providers/common/digest_to_nid.c
@@ -34,7 +34,7 @@ int ossl_digest_md_to_nid(const EVP_MD *md, const OSSL_ITEM *it, size_t it_len)
 }
 
 /*
- * Retrieve one of the FIPs approved hash algorithms by nid.
+ * Retrieve one of the FIPS approved hash algorithms by nid.
  * See FIPS 180-4 "Secure Hash Standard" and FIPS 202 - SHA-3.
  */
 int ossl_digest_get_approved_nid(const EVP_MD *md)
diff --git a/providers/common/include/prov/securitycheck.h b/providers/common/include/prov/securitycheck.h
index 7635c24973..4a7f85f711 100644
--- a/providers/common/include/prov/securitycheck.h
+++ b/providers/common/include/prov/securitycheck.h
@@ -16,6 +16,7 @@ int ossl_dsa_check_key(OSSL_LIB_CTX *ctx, const DSA *dsa, int sign);
 int ossl_dh_check_key(OSSL_LIB_CTX *ctx, const DH *dh);
 
 int ossl_digest_is_allowed(OSSL_LIB_CTX *ctx, const EVP_MD *md);
+/* With security check enabled it can return -1 to indicate disallowed md */
 int ossl_digest_get_approved_nid_with_sha1(OSSL_LIB_CTX *ctx, const EVP_MD *md,
                                            int sha1_allowed);
 
diff --git a/providers/common/securitycheck.c b/providers/common/securitycheck.c
index 4f36ce4593..699ada7c52 100644
--- a/providers/common/securitycheck.c
+++ b/providers/common/securitycheck.c
@@ -231,8 +231,8 @@ int ossl_digest_get_approved_nid_with_sha1(OSSL_LIB_CTX *ctx, const EVP_MD *md,
 
 # if !defined(OPENSSL_NO_FIPS_SECURITYCHECKS)
     if (ossl_securitycheck_enabled(ctx)) {
-        if (mdnid == NID_sha1 && !sha1_allowed)
-            mdnid = NID_undef;
+        if (mdnid == NID_undef || (mdnid == NID_sha1 && !sha1_allowed))
+            mdnid = -1; /* disallowed by security checks */
     }
 # endif /* OPENSSL_NO_FIPS_SECURITYCHECKS */
     return mdnid;
diff --git a/providers/fips-sources.checksums b/providers/fips-sources.checksums
index 13b1d901ca..dd8ae28a44 100644
--- a/providers/fips-sources.checksums
+++ b/providers/fips-sources.checksums
@@ -323,7 +323,7 @@ d447cd774869da68a2cc0bbb19c547ee6ed4858c7aee1f3d5bba7796f97823a9  providers/comm
 71c3fbb9bd80f5e7a217cf8005df61f96a645fbdd9daca9949ceef6d33a1feb0  providers/common/provider_err.c
 9eae3e2cac89c7b63d091fdca1b6d80c5c5d52aa79c8ba4ce0158c5437ad62f3  providers/common/provider_seeding.c
 eec462d685dd3b4764b076a3c18ecd9dd254350a0b78ddc2f8a60587829e1ce3  providers/common/provider_util.c
-494723d55bc6ecdb70f59499a2c42260cabc5fa30681ac3b48267dfa242158b3  providers/common/securitycheck.c
+ce6731be4da709c753bd2c04e88d51d567c955c651e7575bb1410968e6c7620e  providers/common/securitycheck.c
 50a0e01e877ae818cf874f4515a130db0e869d4e9e8ce882bff1255695aba789  providers/common/securitycheck_fips.c
 5c31ba4eedb31e2509288be50280e0df58faa86fe4b5e99a1167a53fd6f3bd0f  providers/fips/fipsprov.c
 c69e60c29711d55cd5672dab9ff051f3c093d54e63a0ec575baa899e6bbf9c2b  providers/fips/self_test.c
@@ -388,10 +388,10 @@ bf30274dd6b528ae913984775bd8f29c6c48c0ef06d464d0f738217727b7aa5c  providers/impl
 c36937930bcaecd6d5131d0317b9162a96cc956df164848dc53f423af838d04a  providers/implementations/rands/drbg_hash.c
 531c0ce4212570474b59a1b039e61a97ee5504e56e2f10de1f36578f1bca79d3  providers/implementations/rands/drbg_hmac.c
 888a671934abef4225956f9931cff842f245f90660e11f23a55228edca962e16  providers/implementations/rands/test_rng.c
-a7f16a6480f5051d1197b992e042a73535d0922bdd3c962d2a96af780994e858  providers/implementations/signature/dsa_sig.c
-1edce687e950bec7c289cdac7c4c455e195942ccddfc38af0344277421afcc0f  providers/implementations/signature/ecdsa_sig.c
+3a9dfbf5dcb9e1955f12f71f1ca086dded771b262d6d61bab2874f48260f702a  providers/implementations/signature/dsa_sig.c
+0ff792c30ba26f2d8f4d1c14b999f7183dcd928537f950a23573f0b65359b2f4  providers/implementations/signature/ecdsa_sig.c
 8074854e90be6a8266cc81ad722ef12213e9fc1360891822f109bfb03791f18e  providers/implementations/signature/eddsa_sig.c
 1cb6ec2efb7b2bb131622aa95e245273f5967065eb0018392ed4ced50d0813b7  providers/implementations/signature/mac_legacy_sig.c
-25fe1a61578d54c3e67b60646f3fd3d0a47ff1d4cd620ef1f1fca3341f2662a2  providers/implementations/signature/rsa_sig.c
+40322e8782474a35f02fa350b43439a56124e680a1d24556b2a66310ed2e9e2e  providers/implementations/signature/rsa_sig.c
 53a1e913fcc4a4e8e84009229cba60b9e29c7dc6536182fd290478331fad44b4  ssl/record/tls_pad.c
 85a9701b05ab8dfea42550fbc5e4d9f4011d08ccc64829648fc12091cc1133f5  ssl/s3_cbc.c
diff --git a/providers/fips.checksum b/providers/fips.checksum
index 99a3468e9b..642611c889 100644
--- a/providers/fips.checksum
+++ b/providers/fips.checksum
@@ -1 +1 @@
-b388b982a3a40d326d76d89e0776aefea46084f936f1aed03293e057d5f2a6de  providers/fips-sources.checksums
+90d4616e33b95990f96dd2cb1798cae41e6591d5cb55a4f589307908fa699587  providers/fips-sources.checksums
diff --git a/providers/implementations/signature/dsa_sig.c b/providers/implementations/signature/dsa_sig.c
index dde689903d..23e000db4c 100644
--- a/providers/implementations/signature/dsa_sig.c
+++ b/providers/implementations/signature/dsa_sig.c
@@ -131,11 +131,11 @@ static int dsa_setup_md(PROV_DSA_CTX *ctx,
                                                             sha1_allowed);
         size_t mdname_len = strlen(mdname);
 
-        if (md == NULL || md_nid == NID_undef) {
+        if (md == NULL || md_nid < 0) {
             if (md == NULL)
                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
                                "%s could not be fetched", mdname);
-            if (md_nid == NID_undef)
+            if (md_nid < 0)
                 ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
                                "digest=%s", mdname);
             if (mdname_len >= sizeof(ctx->mdname))
diff --git a/providers/implementations/signature/ecdsa_sig.c b/providers/implementations/signature/ecdsa_sig.c
index 8c4648106f..a4297d1903 100644
--- a/providers/implementations/signature/ecdsa_sig.c
+++ b/providers/implementations/signature/ecdsa_sig.c
@@ -227,7 +227,7 @@ static int ecdsa_setup_md(PROV_ECDSA_CTX *ctx, const char *mdname,
     sha1_allowed = (ctx->operation != EVP_PKEY_OP_SIGN);
     md_nid = ossl_digest_get_approved_nid_with_sha1(ctx->libctx, md,
                                                     sha1_allowed);
-    if (md_nid == NID_undef) {
+    if (md_nid < 0) {
         ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
                        "digest=%s", mdname);
         EVP_MD_free(md);
diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index 16025bffc0..abd3b1a77b 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -289,13 +289,13 @@ static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
         size_t mdname_len = strlen(mdname);
 
         if (md == NULL
-            || md_nid == NID_undef
+            || md_nid <= 0
             || !rsa_check_padding(ctx, mdname, NULL, md_nid)
             || mdname_len >= sizeof(ctx->mdname)) {
             if (md == NULL)
                 ERR_raise_data(ERR_LIB_PROV, PROV_R_INVALID_DIGEST,
                                "%s could not be fetched", mdname);
-            if (md_nid == NID_undef)
+            if (md_nid <= 0)
                 ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
                                "digest=%s", mdname);
             if (mdname_len >= sizeof(ctx->mdname))
@@ -344,9 +344,9 @@ static int rsa_setup_mgf1_md(PROV_RSA_CTX *ctx, const char *mdname,
         return 0;
     }
     /* The default for mgf1 is SHA1 - so allow SHA1 */
-    if ((mdnid = ossl_digest_rsa_sign_get_md_nid(ctx->libctx, md, 1)) == NID_undef
+    if ((mdnid = ossl_digest_rsa_sign_get_md_nid(ctx->libctx, md, 1)) <= 0
         || !rsa_check_padding(ctx, NULL, mdname, mdnid)) {
-        if (mdnid == NID_undef)
+        if (mdnid <= 0)
             ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
                            "digest=%s", mdname);
         EVP_MD_free(md);