Compat self-signed trust with reject-only aux data

When auxiliary data contains only reject entries, continue to trust
self-signed objects just as when no auxiliary data is present.

This makes it possible to reject specific uses without changing
what's accepted (and thus overring the underlying EKU).

Added new supported certs and doubled test count from 38 to 76.

Reviewed-by: Dr. Stephen Henson <steve@openssl.org>

2 files changed, 50 insertions, 29 deletions

diff --git a/crypto/x509/x509_trs.c b/crypto/x509/x509_trs.c
index c81c725ea1..4c5281a718 100644
--- a/crypto/x509/x509_trs.c
+++ b/crypto/x509/x509_trs.c
@@ -117,13 +117,9 @@ int X509_check_trust(X509 *x, int id, int flags)
     int idx;
 
     /* We get this as a default value */
-    if (id == 0) {
-        int rv;
-        rv = obj_trust(NID_anyExtendedKeyUsage, x, 0);
-        if (rv != X509_TRUST_UNTRUSTED)
-            return rv;
-        return trust_compat(NULL, x, 0);
-    }
+    if (id == X509_TRUST_DEFAULT)
+        return obj_trust(NID_anyExtendedKeyUsage, x,
+                         flags | X509_TRUST_DO_SS_COMPAT);
     idx = X509_TRUST_get_by_id(id);
     if (idx == -1)
         return default_trust(id, x, flags);
@@ -265,20 +261,25 @@ int X509_TRUST_get_trust(X509_TRUST *xp)
 
 static int trust_1oidany(X509_TRUST *trust, X509 *x, int flags)
 {
-    if (x->aux && (x->aux->trust || x->aux->reject))
-        return obj_trust(trust->arg1, x, flags);
     /*
-     * we don't have any trust settings: for compatibility we return trusted
-     * if it is self signed
+     * Declare the chain verified if the desired trust OID is not rejected in
+     * any auxiliary trust info for this certificate, and the OID is either
+     * expressly trusted, or else either "anyEKU" is trusted, or the
+     * certificate is self-signed.
      */
-    return trust_compat(trust, x, flags);
+    flags |= X509_TRUST_DO_SS_COMPAT | X509_TRUST_OK_ANY_EKU;
+    return obj_trust(trust->arg1, x, flags);
 }
 
 static int trust_1oid(X509_TRUST *trust, X509 *x, int flags)
 {
-    if (x->aux && (x->aux->trust || x->aux->reject))
-        return obj_trust(trust->arg1, x, flags);
-    return X509_TRUST_UNTRUSTED;
+    /*
+     * Declare the chain verified only if the desired trust OID is not
+     * rejected and is expressly trusted.  Neither "anyEKU" nor "compat"
+     * trust in self-signed certificates apply.
+     */
+    flags &= ~(X509_TRUST_DO_SS_COMPAT | X509_TRUST_OK_ANY_EKU);
+    return obj_trust(trust->arg1, x, flags);
 }
 
 static int trust_compat(X509_TRUST *trust, X509 *x, int flags)
@@ -296,23 +297,24 @@ static int obj_trust(int id, X509 *x, int flags)
     X509_CERT_AUX *ax = x->aux;
     int i;
 
-    if (!ax)
-        return X509_TRUST_UNTRUSTED;
-    if (ax->reject) {
+    if (ax && ax->reject) {
         for (i = 0; i < sk_ASN1_OBJECT_num(ax->reject); i++) {
             ASN1_OBJECT *obj = sk_ASN1_OBJECT_value(ax->reject, i);
             int nid = OBJ_obj2nid(obj);
 
-            if (nid == id || nid == NID_anyExtendedKeyUsage)
+            if (nid == id || (nid == NID_anyExtendedKeyUsage &&
+                (flags & X509_TRUST_OK_ANY_EKU)))
                 return X509_TRUST_REJECTED;
         }
     }
-    if (ax->trust) {
+
+    if (ax && ax->trust) {
         for (i = 0; i < sk_ASN1_OBJECT_num(ax->trust); i++) {
             ASN1_OBJECT *obj = sk_ASN1_OBJECT_value(ax->trust, i);
             int nid = OBJ_obj2nid(obj);
 
-            if (nid == id || nid == NID_anyExtendedKeyUsage)
+            if (nid == id || (nid == NID_anyExtendedKeyUsage &&
+                (flags & X509_TRUST_OK_ANY_EKU)))
                 return X509_TRUST_TRUSTED;
         }
         /*
@@ -331,5 +333,12 @@ static int obj_trust(int id, X509 *x, int flags)
          */
         return X509_TRUST_REJECTED;
     }
-    return X509_TRUST_UNTRUSTED;
+
+    if ((flags & X509_TRUST_DO_SS_COMPAT) == 0)
+        return X509_TRUST_UNTRUSTED;
+
+    /*
+     * Not rejected, and there is no list of accepted uses, try compat.
+     */
+    return trust_compat(NULL, x, flags);
 }
diff --git a/crypto/x509/x509_vfy.c b/crypto/x509/x509_vfy.c
index 14d6a8d74e..1f3b2b9dab 100644
--- a/crypto/x509/x509_vfy.c
+++ b/crypto/x509/x509_vfy.c
@@ -369,12 +369,11 @@ static STACK_OF(X509) *lookup_certs_sk(X509_STORE_CTX *ctx, X509_NAME *nm)
 static int check_purpose(X509_STORE_CTX *ctx, X509 *x, int purpose, int depth,
                          int must_be_ca)
 {
-    int pu_ok = X509_check_purpose(x, purpose, must_be_ca > 0);
     int tr_ok = X509_TRUST_UNTRUSTED;
 
     /*
      * For trusted certificates we want to see whether any auxiliary trust
-     * settings override the purpose constraints we failed to meet above.
+     * settings trump the purpose constraints.
      *
      * This is complicated by the fact that the trust ordinals in
      * ctx->param->trust are entirely independent of the purpose ordinals in
@@ -388,15 +387,28 @@ static int check_purpose(X509_STORE_CTX *ctx, X509 *x, int purpose, int depth,
      *
      * Therefore, we can only check for trust overrides when the purpose we're
      * checking is the same as ctx->param->purpose and ctx->param->trust is
-     * also set, or can be inferred from the purpose.
+     * also set.
      */
     if (depth >= ctx->num_untrusted && purpose == ctx->param->purpose)
         tr_ok = X509_check_trust(x, ctx->param->trust, X509_TRUST_NO_SS_COMPAT);
 
-    if (tr_ok != X509_TRUST_REJECTED &&
-        (pu_ok == 1 ||
-         (pu_ok != 0 && (ctx->param->flags & X509_V_FLAG_X509_STRICT) == 0)))
+    switch (tr_ok) {
+    case X509_TRUST_TRUSTED:
         return 1;
+    case X509_TRUST_REJECTED:
+        break;
+    default:
+        switch (X509_check_purpose(x, purpose, must_be_ca > 0)) {
+        case 1:
+            return 1;
+        case 0:
+            break;
+        default:
+            if ((ctx->param->flags & X509_V_FLAG_X509_STRICT) == 0)
+                return 1;
+        }
+        break;
+    }
 
     ctx->error = X509_V_ERR_INVALID_PURPOSE;
     ctx->error_depth = depth;
@@ -493,7 +505,7 @@ static int check_chain_extensions(X509_STORE_CTX *ctx)
         if (ret == 0) {
             ctx->error_depth = i;
             ctx->current_cert = x;
-            if (! ctx->verify_cb(0, ctx))
+            if (!ctx->verify_cb(0, ctx))
                 return 0;
         }
         if (purpose > 0) {