Replace provider digest flags with separate param fields

Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/13830)

8 files changed, 44 insertions, 52 deletions

diff --git a/crypto/evp/digest.c b/crypto/evp/digest.c
index e89b591978..40aedae47b 100644
--- a/crypto/evp/digest.c
+++ b/crypto/evp/digest.c
@@ -830,23 +830,27 @@ static void set_legacy_nid(const char *name, void *vlegacy_nid)
 
 static int evp_md_cache_constants(EVP_MD *md)
 {
-    int ok;
+    int ok, xof = 0, algid_absent = 0;
     size_t blksz = 0;
     size_t mdsize = 0;
-    unsigned long flags = 0;
-    OSSL_PARAM params[4];
+    OSSL_PARAM params[5];
 
     params[0] = OSSL_PARAM_construct_size_t(OSSL_DIGEST_PARAM_BLOCK_SIZE, &blksz);
     params[1] = OSSL_PARAM_construct_size_t(OSSL_DIGEST_PARAM_SIZE, &mdsize);
-    params[2] = OSSL_PARAM_construct_ulong(OSSL_DIGEST_PARAM_FLAGS, &flags);
-    params[3] = OSSL_PARAM_construct_end();
+    params[2] = OSSL_PARAM_construct_int(OSSL_DIGEST_PARAM_XOF, &xof);
+    params[3] = OSSL_PARAM_construct_int(OSSL_DIGEST_PARAM_ALGID_ABSENT,
+                                         &algid_absent);
+    params[4] = OSSL_PARAM_construct_end();
     ok = evp_do_md_getparams(md, params);
     if (mdsize > INT_MAX || blksz > INT_MAX)
         ok = 0;
     if (ok) {
         md->block_size = (int)blksz;
         md->md_size = (int)mdsize;
-        md->flags = flags;
+        if (xof)
+            md->flags |= EVP_MD_FLAG_XOF;
+        if (algid_absent)
+            md->flags |= EVP_MD_FLAG_DIGALGID_ABSENT;
     }
     return ok;
 }
diff --git a/crypto/evp/evp_lib.c b/crypto/evp/evp_lib.c
index 3237683797..427ffc813a 100644
--- a/crypto/evp/evp_lib.c
+++ b/crypto/evp/evp_lib.c
@@ -731,30 +731,6 @@ unsigned long EVP_MD_flags(const EVP_MD *md)
     return md->flags;
 }
 
-int evp_md_cache_constants(EVP_MD *md)
-{
-    int ok, xof = 0, algid_absent = 0;
-    size_t sz = 0, blksz = 0;
-    OSSL_PARAM params[5];
-
-    params[0] = OSSL_PARAM_construct_size_t(OSSL_DIGEST_PARAM_BLOCK_SIZE, &blksz);
-    params[1] = OSSL_PARAM_construct_size_t(OSSL_DIGEST_PARAM_SIZE, &sz);
-    params[2] = OSSL_PARAM_construct_int(OSSL_DIGEST_PARAM_XOF, &xof);
-    params[3] = OSSL_PARAM_construct_int(OSSL_DIGEST_PARAM_ALGID_ABSENT,
-                                         &algid_absent);
-    params[4] = OSSL_PARAM_construct_end();
-    ok = evp_do_md_getparams(md, params);
-    if (ok) {
-        md->block_size = blksz;
-        md->md_size = sz;
-        if (xof)
-            md->flags |= EVP_MD_FLAG_XOF;
-        if (algid_absent)
-            md->flags |= EVP_MD_FLAG_DIGALGID_ABSENT;
-    }
-    return ok;
-}
-
 EVP_MD *EVP_MD_meth_new(int md_type, int pkey_type)
 {
     EVP_MD *md = evp_md_new();
diff --git a/doc/man3/EVP_DigestInit.pod b/doc/man3/EVP_DigestInit.pod
index 3a17243976..28572f23b3 100644
--- a/doc/man3/EVP_DigestInit.pod
+++ b/doc/man3/EVP_DigestInit.pod
@@ -393,13 +393,13 @@ EVP_MD_CTX_set_params() can be used with the following OSSL_PARAM keys:
 
 =over 4
 
-=item "xoflen" (B<OSSL_PARAM_DIGEST_KEY_XOFLEN>) <unsigned integer>
+=item "xoflen" (B<OSSL_DIGEST_PARAM_XOFLEN>) <unsigned integer>
 
 Sets the digest length for extendable output functions.
 It is used by the SHAKE algorithm and should not exceed what can be given
 using a B<size_t>.
 
-=item "pad_type" (B<OSSL_PARAM_DIGEST_KEY_PAD_TYPE>) <integer>
+=item "pad-type" (B<OSSL_DIGEST_PARAM_PAD_TYPE>) <unsigned integer>
 
 Sets the padding type.
 It is used by the MDC2 algorithm.
diff --git a/doc/man7/EVP_MD-MDC2.pod b/doc/man7/EVP_MD-MDC2.pod
index 516e19da19..53069557ea 100644
--- a/doc/man7/EVP_MD-MDC2.pod
+++ b/doc/man7/EVP_MD-MDC2.pod
@@ -25,7 +25,7 @@ settable for an B<EVP_MD_CTX> with L<EVP_MD_CTX_set_params(3)>:
 
 =over 4
 
-=item "pad_type" (B<OSSL_DIGEST_PARAM_PAD_TYPE>) <unsigned integer>
+=item "pad-type" (B<OSSL_DIGEST_PARAM_PAD_TYPE>) <unsigned integer>
 
 Sets the padding type to be used.
 Normally the final MDC2 block is padded with zeros.
diff --git a/providers/implementations/digests/digestcommon.c b/providers/implementations/digests/digestcommon.c
index 6d926713c8..b8e7efde60 100644
--- a/providers/implementations/digests/digestcommon.c
+++ b/providers/implementations/digests/digestcommon.c
@@ -26,8 +26,15 @@ int digest_default_get_params(OSSL_PARAM params[], size_t blksz, size_t paramsz,
         ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER);
         return 0;
     }
-    p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_FLAGS);
-    if (p != NULL && !OSSL_PARAM_set_ulong(p, flags)) {
+    p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_XOF);
+    if (p != NULL
+        && !OSSL_PARAM_set_int(p, (flags & PROV_DIGEST_FLAG_XOF) != 0)) {
+        ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER);
+        return 0;
+    }
+    p = OSSL_PARAM_locate(params, OSSL_DIGEST_PARAM_ALGID_ABSENT);
+    if (p != NULL
+        && !OSSL_PARAM_set_int(p, (flags & PROV_DIGEST_FLAG_ALGID_ABSENT) != 0)) {
         ERR_raise(ERR_LIB_PROV, PROV_R_FAILED_TO_SET_PARAMETER);
         return 0;
     }
@@ -37,7 +44,8 @@ int digest_default_get_params(OSSL_PARAM params[], size_t blksz, size_t paramsz,
 static const OSSL_PARAM digest_default_known_gettable_params[] = {
     OSSL_PARAM_size_t(OSSL_DIGEST_PARAM_BLOCK_SIZE, NULL),
     OSSL_PARAM_size_t(OSSL_DIGEST_PARAM_SIZE, NULL),
-    OSSL_PARAM_ulong(OSSL_DIGEST_PARAM_FLAGS, NULL),
+    OSSL_PARAM_int(OSSL_DIGEST_PARAM_XOF, NULL),
+    OSSL_PARAM_int(OSSL_DIGEST_PARAM_ALGID_ABSENT, NULL),
     OSSL_PARAM_END
 };
 const OSSL_PARAM *digest_default_gettable_params(void *provctx)
diff --git a/providers/implementations/digests/sha2_prov.c b/providers/implementations/digests/sha2_prov.c
index 2f01149ad9..4cff62131c 100644
--- a/providers/implementations/digests/sha2_prov.c
+++ b/providers/implementations/digests/sha2_prov.c
@@ -24,6 +24,8 @@
 #include "prov/implementations.h"
 #include "crypto/sha.h"
 
+#define SHA2_FLAGS PROV_DIGEST_FLAG_ALGID_ABSENT
+
 static OSSL_FUNC_digest_set_ctx_params_fn sha1_set_ctx_params;
 static OSSL_FUNC_digest_settable_ctx_params_fn sha1_settable_ctx_params;
 
@@ -53,43 +55,37 @@ static int sha1_set_ctx_params(void *vctx, const OSSL_PARAM params[])
 
 /* ossl_sha1_functions */
 IMPLEMENT_digest_functions_with_settable_ctx(
-    sha1, SHA_CTX, SHA_CBLOCK, SHA_DIGEST_LENGTH, EVP_MD_FLAG_DIGALGID_ABSENT,
+    sha1, SHA_CTX, SHA_CBLOCK, SHA_DIGEST_LENGTH, SHA2_FLAGS,
     SHA1_Init, SHA1_Update, SHA1_Final,
     sha1_settable_ctx_params, sha1_set_ctx_params)
 
 /* ossl_sha224_functions */
 IMPLEMENT_digest_functions(sha224, SHA256_CTX,
-                           SHA256_CBLOCK, SHA224_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA256_CBLOCK, SHA224_DIGEST_LENGTH, SHA2_FLAGS,
                            SHA224_Init, SHA224_Update, SHA224_Final)
 
 /* ossl_sha256_functions */
 IMPLEMENT_digest_functions(sha256, SHA256_CTX,
-                           SHA256_CBLOCK, SHA256_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA256_CBLOCK, SHA256_DIGEST_LENGTH, SHA2_FLAGS,
                            SHA256_Init, SHA256_Update, SHA256_Final)
 
 /* ossl_sha384_functions */
 IMPLEMENT_digest_functions(sha384, SHA512_CTX,
-                           SHA512_CBLOCK, SHA384_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA512_CBLOCK, SHA384_DIGEST_LENGTH, SHA2_FLAGS,
                            SHA384_Init, SHA384_Update, SHA384_Final)
 
 /* ossl_sha512_functions */
 IMPLEMENT_digest_functions(sha512, SHA512_CTX,
-                           SHA512_CBLOCK, SHA512_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA512_CBLOCK, SHA512_DIGEST_LENGTH, SHA2_FLAGS,
                            SHA512_Init, SHA512_Update, SHA512_Final)
 
 /* ossl_sha512_224_functions */
 IMPLEMENT_digest_functions(sha512_224, SHA512_CTX,
-                           SHA512_CBLOCK, SHA224_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA512_CBLOCK, SHA224_DIGEST_LENGTH, SHA2_FLAGS,
                            sha512_224_init, SHA512_Update, SHA512_Final)
 
 /* ossl_sha512_256_functions */
 IMPLEMENT_digest_functions(sha512_256, SHA512_CTX,
-                           SHA512_CBLOCK, SHA256_DIGEST_LENGTH,
-                           EVP_MD_FLAG_DIGALGID_ABSENT,
+                           SHA512_CBLOCK, SHA256_DIGEST_LENGTH, SHA2_FLAGS,
                            sha512_256_init, SHA512_Update, SHA512_Final)
 
diff --git a/providers/implementations/digests/sha3_prov.c b/providers/implementations/digests/sha3_prov.c
index 6b44792529..6e731fd842 100644
--- a/providers/implementations/digests/sha3_prov.c
+++ b/providers/implementations/digests/sha3_prov.c
@@ -18,6 +18,10 @@
 #include "prov/implementations.h"
 #include "prov/providercommonerr.h"
 
+#define SHA3_FLAGS PROV_DIGEST_FLAG_ALGID_ABSENT
+#define SHAKE_FLAGS PROV_DIGEST_FLAG_XOF
+#define KMAC_FLAGS PROV_DIGEST_FLAG_XOF
+
 /*
  * Forward declaration of any unique methods implemented here. This is not strictly
  * necessary for the compiler, but provides an assurance that the signatures
@@ -286,18 +290,18 @@ static int shake_set_ctx_params(void *vctx, const OSSL_PARAM params[])
     SHA3_newctx(sha3, SHA3_##bitlen, sha3_##bitlen, bitlen, '\x06')            \
     PROV_FUNC_SHA3_DIGEST(sha3_##bitlen, bitlen,                               \
                           SHA3_BLOCKSIZE(bitlen), SHA3_MDSIZE(bitlen),         \
-                          EVP_MD_FLAG_DIGALGID_ABSENT)
+                          SHA3_FLAGS)
 
 #define IMPLEMENT_SHAKE_functions(bitlen)                                      \
     SHA3_newctx(shake, SHAKE_##bitlen, shake_##bitlen, bitlen, '\x1f')         \
     PROV_FUNC_SHAKE_DIGEST(shake_##bitlen, bitlen,                             \
                           SHA3_BLOCKSIZE(bitlen), SHA3_MDSIZE(bitlen),         \
-                          EVP_MD_FLAG_XOF)
+                          SHAKE_FLAGS)
 #define IMPLEMENT_KMAC_functions(bitlen)                                       \
     KMAC_newctx(keccak_kmac_##bitlen, bitlen, '\x04')                          \
     PROV_FUNC_SHAKE_DIGEST(keccak_kmac_##bitlen, bitlen,                       \
                            SHA3_BLOCKSIZE(bitlen), KMAC_MDSIZE(bitlen),        \
-                           EVP_MD_FLAG_XOF)
+                           KMAC_FLAGS)
 
 /* ossl_sha3_224_functions */
 IMPLEMENT_SHA3_functions(224)
diff --git a/providers/implementations/include/prov/digestcommon.h b/providers/implementations/include/prov/digestcommon.h
index 99004731fa..f1164c5a1a 100644
--- a/providers/implementations/include/prov/digestcommon.h
+++ b/providers/implementations/include/prov/digestcommon.h
@@ -15,6 +15,10 @@
 # include <openssl/params.h>
 # include "prov/providercommon.h"
 
+/* Internal flags that can be queried */
+#define PROV_DIGEST_FLAG_XOF             0x0001
+#define PROV_DIGEST_FLAG_ALGID_ABSENT    0x0002
+
 # ifdef __cplusplus
 extern "C" {
 # endif