Signature algos: allow having identical digest in params

The flag_allow_md prevents setting a digest in params however
this is unnecessarily strict. If the digest is the same as the
one already set, we do not return an error.

Fixes #16071

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16076)

3 files changed, 43 insertions, 17 deletions

diff --git a/providers/implementations/signature/dsa_sig.c b/providers/implementations/signature/dsa_sig.c
index 138fbce5e9..2acab0b481 100644
--- a/providers/implementations/signature/dsa_sig.c
+++ b/providers/implementations/signature/dsa_sig.c
@@ -145,6 +145,17 @@ static int dsa_setup_md(PROV_DSA_CTX *ctx,
             return 0;
         }
 
+        if (!ctx->flag_allow_md) {
+            if (ctx->mdname[0] != '\0' && !EVP_MD_is_a(md, ctx->mdname)) {
+                ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
+                               "digest %s != %s", mdname, ctx->mdname);
+                EVP_MD_free(md);
+                return 0;
+            }
+            EVP_MD_free(md);
+            return 1;
+        }
+
         EVP_MD_CTX_free(ctx->mdctx);
         EVP_MD_free(ctx->md);
 
@@ -260,13 +271,13 @@ static int dsa_digest_signverify_init(void *vpdsactx, const char *mdname,
     if (!ossl_prov_is_running())
         return 0;
 
-    pdsactx->flag_allow_md = 0;
     if (!dsa_signverify_init(vpdsactx, vdsa, params, operation))
         return 0;
 
     if (!dsa_setup_md(pdsactx, mdname, NULL))
         return 0;
 
+    pdsactx->flag_allow_md = 0;
     pdsactx->mdctx = EVP_MD_CTX_new();
     if (pdsactx->mdctx == NULL)
         goto error;
@@ -463,9 +474,6 @@ static int dsa_set_ctx_params(void *vpdsactx, const OSSL_PARAM params[])
         return 1;
 
     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_DIGEST);
-    /* Not allowed during certain operations */
-    if (p != NULL && !pdsactx->flag_allow_md)
-        return 0;
     if (p != NULL) {
         char mdname[OSSL_MAX_NAME_SIZE] = "", *pmdname = mdname;
         char mdprops[OSSL_MAX_PROPQUERY_SIZE] = "", *pmdprops = mdprops;
diff --git a/providers/implementations/signature/ecdsa_sig.c b/providers/implementations/signature/ecdsa_sig.c
index c32641f1eb..64be0657c3 100644
--- a/providers/implementations/signature/ecdsa_sig.c
+++ b/providers/implementations/signature/ecdsa_sig.c
@@ -234,6 +234,17 @@ static int ecdsa_setup_md(PROV_ECDSA_CTX *ctx, const char *mdname,
         return 0;
     }
 
+    if (!ctx->flag_allow_md) {
+        if (ctx->mdname[0] != '\0' && !EVP_MD_is_a(md, ctx->mdname)) {
+            ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
+                           "digest %s != %s", mdname, ctx->mdname);
+            EVP_MD_free(md);
+            return 0;
+        }
+        EVP_MD_free(md);
+        return 1;
+    }
+
     EVP_MD_CTX_free(ctx->mdctx);
     EVP_MD_free(ctx->md);
 
@@ -263,11 +274,11 @@ static int ecdsa_digest_signverify_init(void *vctx, const char *mdname,
     if (!ossl_prov_is_running())
         return 0;
 
-    ctx->flag_allow_md = 0;
     if (!ecdsa_signverify_init(vctx, ec, params, operation)
         || !ecdsa_setup_md(ctx, mdname, NULL))
         return 0;
 
+    ctx->flag_allow_md = 0;
     ctx->mdctx = EVP_MD_CTX_new();
     if (ctx->mdctx == NULL)
         goto error;
@@ -452,6 +463,7 @@ static int ecdsa_set_ctx_params(void *vctx, const OSSL_PARAM params[])
 {
     PROV_ECDSA_CTX *ctx = (PROV_ECDSA_CTX *)vctx;
     const OSSL_PARAM *p;
+    size_t mdsize = 0;
 
     if (ctx == NULL)
         return 0;
@@ -465,9 +477,6 @@ static int ecdsa_set_ctx_params(void *vctx, const OSSL_PARAM params[])
 #endif
 
     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_DIGEST);
-    /* Not allowed during certain operations */
-    if (p != NULL && !ctx->flag_allow_md)
-        return 0;
     if (p != NULL) {
         char mdname[OSSL_MAX_NAME_SIZE] = "", *pmdname = mdname;
         char mdprops[OSSL_MAX_PROPQUERY_SIZE] = "", *pmdprops = mdprops;
@@ -485,10 +494,12 @@ static int ecdsa_set_ctx_params(void *vctx, const OSSL_PARAM params[])
     }
 
     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_DIGEST_SIZE);
-    if (p != NULL
-        && (!ctx->flag_allow_md
-            || !OSSL_PARAM_get_size_t(p, &ctx->mdsize)))
-        return 0;
+    if (p != NULL) {
+        if (!OSSL_PARAM_get_size_t(p, &mdsize)
+            || (!ctx->flag_allow_md && mdsize != ctx->mdsize))
+            return 0;
+        ctx->mdsize = mdsize;
+    }
 
     return 1;
 }
diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index 30fd43e0e5..40a97c0165 100644
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -305,6 +305,17 @@ static int rsa_setup_md(PROV_RSA_CTX *ctx, const char *mdname,
             return 0;
         }
 
+        if (!ctx->flag_allow_md) {
+            if (ctx->mdname[0] != '\0' && !EVP_MD_is_a(md, ctx->mdname)) {
+                ERR_raise_data(ERR_LIB_PROV, PROV_R_DIGEST_NOT_ALLOWED,
+                               "digest %s != %s", mdname, ctx->mdname);
+                EVP_MD_free(md);
+                return 0;
+            }
+            EVP_MD_free(md);
+            return 1;
+        }
+
         if (!ctx->mgf1_md_set) {
             if (!EVP_MD_up_ref(md)) {
                 EVP_MD_free(md);
@@ -826,8 +837,6 @@ static int rsa_digest_signverify_init(void *vprsactx, const char *mdname,
     if (!ossl_prov_is_running())
         return 0;
 
-    if (prsactx != NULL)
-        prsactx->flag_allow_md = 0;
     if (!rsa_signverify_init(vprsactx, vrsa, params, operation))
         return 0;
     if (mdname != NULL
@@ -836,6 +845,7 @@ static int rsa_digest_signverify_init(void *vprsactx, const char *mdname,
         && !rsa_setup_md(prsactx, mdname, prsactx->propq))
         return 0;
 
+    prsactx->flag_allow_md = 0;
     prsactx->mdctx = EVP_MD_CTX_new();
     if (prsactx->mdctx == NULL) {
         ERR_raise(ERR_LIB_PROV, ERR_R_MALLOC_FAILURE);
@@ -1141,9 +1151,6 @@ static int rsa_set_ctx_params(void *vprsactx, const OSSL_PARAM params[])
     saltlen = prsactx->saltlen;
 
     p = OSSL_PARAM_locate_const(params, OSSL_SIGNATURE_PARAM_DIGEST);
-    /* Not allowed during certain operations */
-    if (p != NULL && !prsactx->flag_allow_md)
-        return 0;
     if (p != NULL) {
         const OSSL_PARAM *propsp =
             OSSL_PARAM_locate_const(params,