Improve the usability of the ca app using EdDSA

Previously you had to supply "null" as the digest to use EdDSA. This changes
things so that any digest is ignored.

Reviewed-by: Viktor Dukhovni <viktor@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/6901)

1 files changed, 13 insertions, 9 deletions

diff --git a/apps/ca.c b/apps/ca.c
index 558809ee30..48f7cd1973 100644
--- a/apps/ca.c
+++ b/apps/ca.c
@@ -255,7 +255,7 @@ int ca_main(int argc, char **argv)
     int batch = 0, default_op = 1, doupdatedb = 0, ext_copy = EXT_COPY_NONE;
     int keyformat = FORMAT_PEM, multirdn = 0, notext = 0, output_der = 0;
     int ret = 1, email_dn = 1, req = 0, verbose = 0, gencrl = 0, dorevoke = 0;
-    int rand_ser = 0, i, j, selfsign = 0;
+    int rand_ser = 0, i, j, selfsign = 0, def_nid, def_ret;
     long crldays = 0, crlhours = 0, crlsec = 0, days = 0;
     unsigned long chtype = MBSTRING_ASC, certopt = 0;
     X509 *x509 = NULL, *x509p = NULL, *x = NULL;
@@ -728,24 +728,28 @@ end_of_options:
         }
     }
 
-    if (md == NULL && (md = lookup_conf(conf, section, ENV_DEFAULT_MD)) == NULL)
-        goto end;
-
-    if (strcmp(md, "null") == 0) {
+    def_ret = EVP_PKEY_get_default_digest_nid(pkey, &def_nid);
+    /*
+     * EVP_PKEY_get_default_digest_nid() returns 2 if the digest is
+     * mandatory for this algorithm.
+     */
+    if (def_ret == 2 && def_nid == NID_undef) {
+        /* The signing algorithm requires there to be no digest */
         dgst = EVP_md_null();
+    } else if (md == NULL
+               && (md = lookup_conf(conf, section, ENV_DEFAULT_MD)) == NULL) {
+        goto end;
     } else {
         if (strcmp(md, "default") == 0) {
-            int def_nid;
-            if (EVP_PKEY_get_default_digest_nid(pkey, &def_nid) <= 0) {
+            if (def_ret <= 0) {
                 BIO_puts(bio_err, "no default digest\n");
                 goto end;
             }
             md = (char *)OBJ_nid2sn(def_nid);
         }
 
-        if (!opt_md(md, &dgst)) {
+        if (!opt_md(md, &dgst))
             goto end;
-        }
     }
 
     if (req) {