Improve the usability of the ca app using EdDSA

Previously you had to supply "null" as the digest to use EdDSA. This changes
things so that any digest is ignored.

Reviewed-by: Viktor Dukhovni <viktor@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/6901)

4 files changed, 33 insertions, 17 deletions

diff --git a/apps/ca.c b/apps/ca.c
index 558809ee30..48f7cd1973 100644
--- a/apps/ca.c
+++ b/apps/ca.c
@@ -255,7 +255,7 @@ int ca_main(int argc, char **argv)
     int batch = 0, default_op = 1, doupdatedb = 0, ext_copy = EXT_COPY_NONE;
     int keyformat = FORMAT_PEM, multirdn = 0, notext = 0, output_der = 0;
     int ret = 1, email_dn = 1, req = 0, verbose = 0, gencrl = 0, dorevoke = 0;
-    int rand_ser = 0, i, j, selfsign = 0;
+    int rand_ser = 0, i, j, selfsign = 0, def_nid, def_ret;
     long crldays = 0, crlhours = 0, crlsec = 0, days = 0;
     unsigned long chtype = MBSTRING_ASC, certopt = 0;
     X509 *x509 = NULL, *x509p = NULL, *x = NULL;
@@ -728,24 +728,28 @@ end_of_options:
         }
     }
 
-    if (md == NULL && (md = lookup_conf(conf, section, ENV_DEFAULT_MD)) == NULL)
-        goto end;
-
-    if (strcmp(md, "null") == 0) {
+    def_ret = EVP_PKEY_get_default_digest_nid(pkey, &def_nid);
+    /*
+     * EVP_PKEY_get_default_digest_nid() returns 2 if the digest is
+     * mandatory for this algorithm.
+     */
+    if (def_ret == 2 && def_nid == NID_undef) {
+        /* The signing algorithm requires there to be no digest */
         dgst = EVP_md_null();
+    } else if (md == NULL
+               && (md = lookup_conf(conf, section, ENV_DEFAULT_MD)) == NULL) {
+        goto end;
     } else {
         if (strcmp(md, "default") == 0) {
-            int def_nid;
-            if (EVP_PKEY_get_default_digest_nid(pkey, &def_nid) <= 0) {
+            if (def_ret <= 0) {
                 BIO_puts(bio_err, "no default digest\n");
                 goto end;
             }
             md = (char *)OBJ_nid2sn(def_nid);
         }
 
-        if (!opt_md(md, &dgst)) {
+        if (!opt_md(md, &dgst))
             goto end;
-        }
     }
 
     if (req) {
diff --git a/crypto/ec/ecx_meth.c b/crypto/ec/ecx_meth.c
index e75e07b052..b76bfdb6dc 100644
--- a/crypto/ec/ecx_meth.c
+++ b/crypto/ec/ecx_meth.c
@@ -331,8 +331,18 @@ static int ecx_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
         }
         return 0;
 
+    default:
+        return -2;
+
+    }
+}
+
+static int ecd_ctrl(EVP_PKEY *pkey, int op, long arg1, void *arg2)
+{
+    switch (op) {
     case ASN1_PKEY_CTRL_DEFAULT_MD_NID:
-        *(int *)arg2 = NID_sha256;
+        /* We currently only support Pure EdDSA which takes no digest */
+        *(int *)arg2 = NID_undef;
         return 2;
 
     default:
@@ -579,7 +589,7 @@ const EVP_PKEY_ASN1_METHOD ed25519_asn1_meth = {
     0, 0,
 
     ecx_free,
-    0,
+    ecd_ctrl,
     NULL,
     NULL,
     ecd_item_verify,
@@ -621,7 +631,7 @@ const EVP_PKEY_ASN1_METHOD ed448_asn1_meth = {
     0, 0,
 
     ecx_free,
-    0,
+    ecd_ctrl,
     NULL,
     NULL,
     ecd_item_verify,
diff --git a/doc/man1/ca.pod b/doc/man1/ca.pod
index ebd8a4386a..9b282e6479 100644
--- a/doc/man1/ca.pod
+++ b/doc/man1/ca.pod
@@ -184,9 +184,9 @@ The number of days to certify the certificate for.
 =item B<-md alg>
 
 The message digest to use.
-Any digest supported by the OpenSSL B<dgst> command can be used. If the signing
-key is using Ed25519 or Ed448 then you should specify "null" for the digest.
-This option also applies to CRLs.
+Any digest supported by the OpenSSL B<dgst> command can be used. For signing
+algorithms that do not support a digest (i.e. Ed25519 and Ed448) any message
+digest that is set is ignored. This option also applies to CRLs.
 
 =item B<-policy arg>
 
@@ -453,7 +453,8 @@ least one of these must be present to generate a CRL.
 
 =item B<default_md>
 
-The same as the B<-md> option. Mandatory.
+The same as the B<-md> option. Mandatory except where the signing algorithm does
+not require a digest (i.e. Ed25519 and Ed448).
 
 =item B<database>
 
diff --git a/doc/man3/EVP_PKEY_get_default_digest_nid.pod b/doc/man3/EVP_PKEY_get_default_digest_nid.pod
index 3dce5c59a8..6113115aec 100644
--- a/doc/man3/EVP_PKEY_get_default_digest_nid.pod
+++ b/doc/man3/EVP_PKEY_get_default_digest_nid.pod
@@ -13,7 +13,8 @@ EVP_PKEY_get_default_digest_nid - get default signature digest
 
 The EVP_PKEY_get_default_digest_nid() function sets B<pnid> to the default
 message digest NID for the public key signature operations associated with key
-B<pkey>.
+B<pkey>. Note that some signature algorithms (i.e. Ed25519 and Ed448) do not use
+a digest during signing. In this case B<pnid> will be set to NID_undef.
 
 =head1 NOTES