Skip CN DNS name constraint checks when not needed

Only check the CN against DNS name contraints if the
`X509_CHECK_FLAG_NEVER_CHECK_SUBJECT` flag is not set, and either the
certificate has no DNS subject alternative names or the
`X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT` flag is set.

Add pertinent documentation, and touch up some stale text about
name checks and DANE.

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tim Hudson <tjh@openssl.org>

5 files changed, 59 insertions, 30 deletions

diff --git a/crypto/x509/x509_vfy.c b/crypto/x509/x509_vfy.c
index 447f5e06d3..3a60d412da 100644
--- a/crypto/x509/x509_vfy.c
+++ b/crypto/x509/x509_vfy.c
@@ -560,6 +560,27 @@ static int check_chain_extensions(X509_STORE_CTX *ctx)
     return 1;
 }
 
+static int has_san_id(X509 *x, int gtype)
+{
+    int i;
+    int ret = 0;
+    GENERAL_NAMES *gs = X509_get_ext_d2i(x, NID_subject_alt_name, NULL, NULL);
+
+    if (gs == NULL)
+        return 0;
+
+    for (i = 0; i < sk_GENERAL_NAME_num(gs); i++) {
+        GENERAL_NAME *g = sk_GENERAL_NAME_value(gs, i);
+
+        if (g->type == gtype) {
+            ret = 1;
+            break;
+        }
+    }
+    GENERAL_NAMES_free(gs);
+    return ret;
+}
+
 static int check_name_constraints(X509_STORE_CTX *ctx)
 {
     int i;
@@ -658,7 +679,12 @@ static int check_name_constraints(X509_STORE_CTX *ctx)
                 int rv = NAME_CONSTRAINTS_check(x, nc);
 
                 /* If EE certificate check commonName too */
-                if (rv == X509_V_OK && i == 0)
+                if (rv == X509_V_OK && i == 0
+                    && (ctx->param->hostflags
+                        & X509_CHECK_FLAG_NEVER_CHECK_SUBJECT) == 0
+                    && ((ctx->param->hostflags
+                         & X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT) != 0
+                        || !has_san_id(x, GEN_DNS)))
                     rv = NAME_CONSTRAINTS_check_CN(x, nc);
 
                 switch (rv) {
diff --git a/crypto/x509v3/v3_ncons.c b/crypto/x509v3/v3_ncons.c
index 77edcd4dfb..dea6c6c8af 100644
--- a/crypto/x509v3/v3_ncons.c
+++ b/crypto/x509v3/v3_ncons.c
@@ -299,9 +299,9 @@ int NAME_CONSTRAINTS_check(X509 *x, NAME_CONSTRAINTS *nc)
 
 static int cn2dnsid(ASN1_STRING *cn, unsigned char **dnsid, size_t *idlen)
 {
-    int utf8_length;    /* Return type of ASN1_STRING_to_UTF8 */
-    int i;
+    int utf8_length;
     unsigned char *utf8_value;
+    int i;
     int isdnsname = 0;
 
     /* Don't leave outputs uninitialized */
@@ -337,8 +337,10 @@ static int cn2dnsid(ASN1_STRING *cn, unsigned char **dnsid, size_t *idlen)
         --utf8_length;
 
     /* Reject *embedded* NULs */
-    if ((size_t)utf8_length != strlen((char *)utf8_value))
-        return X509_V_ERR_UNSPECIFIED;
+    if ((size_t)utf8_length != strlen((char *)utf8_value)) {
+        OPENSSL_free(utf8_value);
+        return X509_V_ERR_UNSUPPORTED_NAME_SYNTAX;
+    }
 
     /*
      * XXX: Deviation from strict DNS name syntax, also check names with '_'
@@ -389,14 +391,12 @@ static int cn2dnsid(ASN1_STRING *cn, unsigned char **dnsid, size_t *idlen)
 }
 
 /*
- * Check CN against DNS-ID name constraints, provided no DNS-ID
- * subjectAlternativeName values are present in the certificate.
+ * Check CN against DNS-ID name constraints.
  */
 int NAME_CONSTRAINTS_check_CN(X509 *x, NAME_CONSTRAINTS *nc)
 {
     int r, i;
-    GENERAL_NAMES *gens = NULL;
-    X509_NAME *nm;
+    X509_NAME *nm = X509_get_subject_name(x);
     ASN1_STRING stmp;
     GENERAL_NAME gntmp;
 
@@ -405,21 +405,6 @@ int NAME_CONSTRAINTS_check_CN(X509 *x, NAME_CONSTRAINTS *nc)
     gntmp.type = GEN_DNS;
     gntmp.d.dNSName = &stmp;
 
-    gens = X509_get_ext_d2i(x, NID_subject_alt_name, NULL, NULL);
-    if (gens != NULL) {
-        for (i = 0; i < sk_GENERAL_NAME_num(gens); i++) {
-            GENERAL_NAME *gen = sk_GENERAL_NAME_value(gens, i);
-
-            if (gen->type == GEN_DNS) {
-                GENERAL_NAMES_free(gens);
-                return X509_V_OK;
-            }
-        }
-        GENERAL_NAMES_free(gens);
-    }
-
-    nm = X509_get_subject_name(x);
-
     /* Process any commonName attributes in subject name */
 
     for (i = -1;;) {
diff --git a/doc/man3/SSL_set1_host.pod b/doc/man3/SSL_set1_host.pod
index 9e0210d062..b4d7d5e37c 100644
--- a/doc/man3/SSL_set1_host.pod
+++ b/doc/man3/SSL_set1_host.pod
@@ -56,7 +56,7 @@ is cleared or freed, or a renegotiation takes place.  Applications
 must not free the return value.
 
 SSL clients are advised to use these functions in preference to
-explicitly calling L<X509_check_host(3)>.  Hostname checks are out
+explicitly calling L<X509_check_host(3)>.  Hostname checks may be out
 of scope with the RFC7671 DANE-EE(3) certificate usage, and the
 internal check will be suppressed as appropriate when DANE is
 enabled.
diff --git a/doc/man3/X509_VERIFY_PARAM_set_flags.pod b/doc/man3/X509_VERIFY_PARAM_set_flags.pod
index 7765029766..61b52efeea 100644
--- a/doc/man3/X509_VERIFY_PARAM_set_flags.pod
+++ b/doc/man3/X509_VERIFY_PARAM_set_flags.pod
@@ -133,14 +133,29 @@ B<name> clearing any previously specified host name or names.  If
 B<name> is NULL, or empty the list of hostnames is cleared, and
 name checks are not performed on the peer certificate.  If B<name>
 is NUL-terminated, B<namelen> may be zero, otherwise B<namelen>
-must be set to the length of B<name>.  When a hostname is specified,
+must be set to the length of B<name>.
+
+When a hostname is specified,
 certificate verification automatically invokes L<X509_check_host(3)>
 with flags equal to the B<flags> argument given to
 X509_VERIFY_PARAM_set_hostflags() (default zero).  Applications
 are strongly advised to use this interface in preference to explicitly
-calling L<X509_check_host(3)>, hostname checks are out of scope
+calling L<X509_check_host(3)>, hostname checks may be out of scope
 with the DANE-EE(3) certificate usage, and the internal check will
-be suppressed as appropriate when DANE support is added to OpenSSL.
+be suppressed as appropriate when DANE verification is enabled.
+
+When the subject CommonName will not be ignored, whether as a result of the
+B<X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT> host flag, or because no DNS subject
+alternative names are present in the certificate, any DNS name constraints in
+issuer certificates apply to the subject CommonName as well as the subject
+alternative name extension.
+
+When the subject CommonName will be ignored, whether as a result of the
+B<X509_CHECK_FLAG_NEVER_CHECK_SUBJECT> host flag, or because some DNS subject
+alternative names are present in the certificate, DNS name constraints in
+issuer certificates will not be applied to the subject DN.
+As described in X509_check_host(3) the B<X509_CHECK_FLAG_NEVER_CHECK_SUBJECT>
+flag takes precendence over the B<X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT> flag.
 
 X509_VERIFY_PARAM_get_hostflags() returns any host flags previously set via a
 call to X509_VERIFY_PARAM_set_hostflags().
diff --git a/doc/man3/X509_check_host.pod b/doc/man3/X509_check_host.pod
index 7b96b7c80b..dba6a6976e 100644
--- a/doc/man3/X509_check_host.pod
+++ b/doc/man3/X509_check_host.pod
@@ -93,6 +93,9 @@ consider the subject DN even if the certificate contains no subject alternative
 names of the right type (DNS name or email address as appropriate); the default
 is to use the subject DN when no corresponding subject alternative names are
 present.
+If both B<X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT> and
+B<X509_CHECK_FLAG_NEVER_CHECK_SUBJECT> are specified, the latter takes
+precedence and the subject DN is not checked for matching names.
 
 If set, B<X509_CHECK_FLAG_NO_WILDCARDS> disables wildcard
 expansion; this only applies to B<X509_check_host>.
@@ -128,9 +131,9 @@ NULs.
 
 Applications are encouraged to use X509_VERIFY_PARAM_set1_host()
 rather than explicitly calling L<X509_check_host(3)>. Host name
-checks are out of scope with the DANE-EE(3) certificate usage,
+checks may be out of scope with the DANE-EE(3) certificate usage,
 and the internal checks will be suppressed as appropriate when
-DANE support is added to OpenSSL.
+DANE support is enabled.
 
 =head1 SEE ALSO