EVP: add internal evp_keymgmt_util_get_deflt_digest_name() and use it

evp_keymgmt_util_get_deflt_digest_name() is a refactor of the provider
side key part of EVP_PKEY_get_default_digest_name(), that takes
EVP_KEYMGMT and provider keydata pointers instead of an EVP_PKEY
pointer.

We also ensure that it uses SN_undef as the default name if the
provider implementation gave us an empty string, since this is what
EVP_PKEY_get_default_digest_name() responds when getting the digest
name via a EVP_PKEY_ASN1_METHOD ctrl call that returns NID_undef.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/11576)

2 files changed, 52 insertions, 22 deletions

diff --git a/crypto/evp/keymgmt_lib.c b/crypto/evp/keymgmt_lib.c
index 6c66bfa72d..9cf8ccbddc 100644
--- a/crypto/evp/keymgmt_lib.c
+++ b/crypto/evp/keymgmt_lib.c
@@ -412,3 +412,51 @@ void *evp_keymgmt_util_gen(EVP_PKEY *target, EVP_KEYMGMT *keymgmt,
 
     return keydata;
 }
+
+/*
+ * Returns the same numbers as EVP_PKEY_get_default_digest_name()
+ * When the string from the EVP_KEYMGMT implementation is "", we use
+ * SN_undef, since that corresponds to what EVP_PKEY_get_default_nid()
+ * returns for no digest.
+ */
+int evp_keymgmt_util_get_deflt_digest_name(EVP_KEYMGMT *keymgmt,
+                                           void *keydata,
+                                           char *mdname, size_t mdname_sz)
+{
+    OSSL_PARAM params[3];
+    char mddefault[100] = "";
+    char mdmandatory[100] = "";
+    char *result = NULL;
+    int rv = -2;
+
+    params[0] =
+        OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_DEFAULT_DIGEST,
+                                         mddefault, sizeof(mddefault));
+    params[0].return_size = sizeof(mddefault) + 1;
+    params[1] =
+        OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_MANDATORY_DIGEST,
+                                         mdmandatory,
+                                         sizeof(mdmandatory));
+    params[1].return_size = sizeof(mdmandatory) + 1;
+    params[2] = OSSL_PARAM_construct_end();
+
+    if (!evp_keymgmt_get_params(keymgmt, keydata, params))
+        return 0;
+
+    if (params[1].return_size != sizeof(mdmandatory) + 1) {
+        if (params[1].return_size == 1) /* Only a NUL byte */
+            result = SN_undef;
+        else
+            result = mdmandatory;
+        rv = 2;
+    } else if (params[0].return_size != sizeof(mddefault) + 1) {
+        if (params[0].return_size == 1) /* Only a NUL byte */
+            result = SN_undef;
+        else
+            result = mddefault;
+        rv = 1;
+    }
+    if (rv > 0)
+        OPENSSL_strlcpy(mdname, result, mdname_sz);
+    return rv;
+}
diff --git a/crypto/evp/p_lib.c b/crypto/evp/p_lib.c
index fa166958f0..2d41dafccb 100644
--- a/crypto/evp/p_lib.c
+++ b/crypto/evp/p_lib.c
@@ -1007,28 +1007,10 @@ int EVP_PKEY_get_default_digest_nid(EVP_PKEY *pkey, int *pnid)
 int EVP_PKEY_get_default_digest_name(EVP_PKEY *pkey,
                                      char *mdname, size_t mdname_sz)
 {
-    if (pkey->ameth == NULL) {
-        OSSL_PARAM params[3];
-        char mddefault[100] = "";
-        char mdmandatory[100] = "";
-
-        params[0] =
-            OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_DEFAULT_DIGEST,
-                                             mddefault, sizeof(mddefault));
-        params[1] =
-            OSSL_PARAM_construct_utf8_string(OSSL_PKEY_PARAM_MANDATORY_DIGEST,
-                                             mdmandatory,
-                                             sizeof(mdmandatory));
-        params[2] = OSSL_PARAM_construct_end();
-        if (!evp_keymgmt_get_params(pkey->keymgmt, pkey->keydata, params))
-            return 0;
-        if (mdmandatory[0] != '\0') {
-            OPENSSL_strlcpy(mdname, mdmandatory, mdname_sz);
-            return 2;
-        }
-        OPENSSL_strlcpy(mdname, mddefault, mdname_sz);
-        return 1;
-    }
+    if (pkey->ameth == NULL)
+        return evp_keymgmt_util_get_deflt_digest_name(pkey->keymgmt,
+                                                      pkey->keydata,
+                                                      mdname, mdname_sz);
 
     {
         int nid = NID_undef;