Support pmeth->digest_custom

Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7113)

5 files changed, 36 insertions, 0 deletions

diff --git a/crypto/evp/m_sigver.c b/crypto/evp/m_sigver.c
index 2377944f66..4a0e5d5c55 100644
--- a/crypto/evp/m_sigver.c
+++ b/crypto/evp/m_sigver.c
@@ -75,6 +75,14 @@ static int do_sigver_init(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx,
         return 1;
     if (!EVP_DigestInit_ex(ctx, type, e))
         return 0;
+    if (ctx->pctx->pmeth->flags & EVP_PKEY_FLAG_DIGEST_CUSTOM) {
+        /*
+         * This indicates the current algorithm requires
+         * special treatment before hashing the tbs-message.
+         */
+        if (ctx->pctx->pmeth->digest_custom)
+            return ctx->pctx->pmeth->digest_custom(ctx->pctx, ctx);
+    }
     return 1;
 }
 
diff --git a/crypto/evp/pmeth_lib.c b/crypto/evp/pmeth_lib.c
index ef923fdc5e..7e6388e8f5 100644
--- a/crypto/evp/pmeth_lib.c
+++ b/crypto/evp/pmeth_lib.c
@@ -655,6 +655,13 @@ void EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
     pmeth->param_check = check;
 }
 
+void EVP_PKEY_meth_set_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (*digest_custom) (EVP_PKEY_CTX *ctx,
+                                                           EVP_MD_CTX *mctx))
+{
+    pmeth->digest_custom = digest_custom;
+}
+
 void EVP_PKEY_meth_get_init(const EVP_PKEY_METHOD *pmeth,
                             int (**pinit) (EVP_PKEY_CTX *ctx))
 {
@@ -842,3 +849,11 @@ void EVP_PKEY_meth_get_param_check(const EVP_PKEY_METHOD *pmeth,
     if (*pcheck)
         *pcheck = pmeth->param_check;
 }
+
+void EVP_PKEY_meth_get_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (**pdigest_custom) (EVP_PKEY_CTX *ctx,
+                                                             EVP_MD_CTX *mctx))
+{
+    if (*pdigest_custom)
+        *pdigest_custom = pmeth->digest_custom;
+}
diff --git a/crypto/include/internal/evp_int.h b/crypto/include/internal/evp_int.h
index daf2e459e2..3264b3df71 100644
--- a/crypto/include/internal/evp_int.h
+++ b/crypto/include/internal/evp_int.h
@@ -79,6 +79,8 @@ struct evp_pkey_method_st {
     int (*check) (EVP_PKEY *pkey);
     int (*public_check) (EVP_PKEY *pkey);
     int (*param_check) (EVP_PKEY *pkey);
+
+    int (*digest_custom) (EVP_PKEY_CTX *ctx, EVP_MD_CTX *mctx);
 } /* EVP_PKEY_METHOD */ ;
 
 DEFINE_STACK_OF_CONST(EVP_PKEY_METHOD)
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index d72c2104bc..b7c0243857 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -1325,6 +1325,8 @@ void EVP_PKEY_asn1_set_security_bits(EVP_PKEY_ASN1_METHOD *ameth,
  * Method handles all operations: don't assume any digest related defaults.
  */
 # define EVP_PKEY_FLAG_SIGCTX_CUSTOM     4
+/* Do a customized hashing process */
+# define EVP_PKEY_FLAG_DIGEST_CUSTOM     8
 
 const EVP_PKEY_METHOD *EVP_PKEY_meth_find(int type);
 EVP_PKEY_METHOD *EVP_PKEY_meth_new(int id, int flags);
@@ -1523,6 +1525,10 @@ void EVP_PKEY_meth_set_public_check(EVP_PKEY_METHOD *pmeth,
 void EVP_PKEY_meth_set_param_check(EVP_PKEY_METHOD *pmeth,
                                    int (*check) (EVP_PKEY *pkey));
 
+void EVP_PKEY_meth_set_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (*digest_custom) (EVP_PKEY_CTX *ctx,
+                                                           EVP_MD_CTX *mctx));
+
 void EVP_PKEY_meth_get_init(const EVP_PKEY_METHOD *pmeth,
                             int (**pinit) (EVP_PKEY_CTX *ctx));
 
@@ -1624,6 +1630,9 @@ void EVP_PKEY_meth_get_public_check(const EVP_PKEY_METHOD *pmeth,
 void EVP_PKEY_meth_get_param_check(const EVP_PKEY_METHOD *pmeth,
                                    int (**pcheck) (EVP_PKEY *pkey));
 
+void EVP_PKEY_meth_get_digest_custom(EVP_PKEY_METHOD *pmeth,
+                                     int (**pdigest_custom) (EVP_PKEY_CTX *ctx,
+                                                             EVP_MD_CTX *mctx));
 void EVP_add_alg_module(void);
 
 
diff --git a/util/libcrypto.num b/util/libcrypto.num
index c6c881f4c2..c4460c98fa 100644
--- a/util/libcrypto.num
+++ b/util/libcrypto.num
@@ -4575,3 +4575,5 @@ EC_GROUP_get_curve                      4528	1_1_1	EXIST::FUNCTION:EC
 OCSP_resp_get0_tbs_sigalg               4529	1_1_0j	EXIST::FUNCTION:OCSP
 OCSP_resp_get0_respdata                 4530	1_1_0j	EXIST::FUNCTION:OCSP
 EVP_MD_CTX_set_pkey_ctx                 4531	1_1_1	EXIST::FUNCTION:
+EVP_PKEY_meth_set_digest_custom         4532	1_1_1	EXIST::FUNCTION:
+EVP_PKEY_meth_get_digest_custom         4533	1_1_1	EXIST::FUNCTION: