Add X509_self_signed(), extending and improving documenation and tests

Reviewed-by: Viktor Dukhovni <viktor@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/10587)

2 files changed, 37 insertions, 20 deletions

diff --git a/crypto/cmp/cmp_util.c b/crypto/cmp/cmp_util.c
index 570e14cd24..d1128d7e66 100644
--- a/crypto/cmp/cmp_util.c
+++ b/crypto/cmp/cmp_util.c
@@ -218,7 +218,7 @@ int ossl_cmp_sk_X509_add1_cert(STACK_OF(X509) *sk, X509 *cert,
 }
 
 int ossl_cmp_sk_X509_add1_certs(STACK_OF(X509) *sk, STACK_OF(X509) *certs,
-                                int no_self_issued, int no_dups, int prepend)
+                                int no_self_signed, int no_dups, int prepend)
 /* compiler would allow 'const' for the list of certs, yet they are up-ref'ed */
 {
     int i;
@@ -230,7 +230,7 @@ int ossl_cmp_sk_X509_add1_certs(STACK_OF(X509) *sk, STACK_OF(X509) *certs,
     for (i = 0; i < sk_X509_num(certs); i++) { /* certs may be NULL */
         X509 *cert = sk_X509_value(certs, i);
 
-        if (!no_self_issued || X509_check_issued(cert, cert) != X509_V_OK) {
+        if (!no_self_signed || X509_self_signed(cert, 0) != 1) {
             if (!ossl_cmp_sk_X509_add1_cert(sk, cert, no_dups, prepend))
                 return 0;
         }
@@ -239,7 +239,7 @@ int ossl_cmp_sk_X509_add1_certs(STACK_OF(X509) *sk, STACK_OF(X509) *certs,
 }
 
 int ossl_cmp_X509_STORE_add1_certs(X509_STORE *store, STACK_OF(X509) *certs,
-                                   int only_self_issued)
+                                   int only_self_signed)
 {
     int i;
 
@@ -252,7 +252,7 @@ int ossl_cmp_X509_STORE_add1_certs(X509_STORE *store, STACK_OF(X509) *certs,
     for (i = 0; i < sk_X509_num(certs); i++) {
         X509 *cert = sk_X509_value(certs, i);
 
-        if (!only_self_issued || X509_check_issued(cert, cert) == X509_V_OK)
+        if (!only_self_signed || X509_self_signed(cert, 0) == 1)
             if (!X509_STORE_add_cert(store, cert)) /* ups cert ref counter */
                 return 0;
     }
diff --git a/crypto/x509/x509_vfy.c b/crypto/x509/x509_vfy.c
index 122d6f8a3b..1f17c71bc1 100644
--- a/crypto/x509/x509_vfy.c
+++ b/crypto/x509/x509_vfy.c
@@ -111,25 +111,43 @@ static int null_callback(int ok, X509_STORE_CTX *e)
     return ok;
 }
 
-/*
+/*-
  * Return 1 if given cert is considered self-signed, 0 if not, or -1 on error.
- * This does not verify self-signedness but relies on x509v3_cache_extensions()
- * matching issuer and subject names (i.e., the cert being self-issued) and any
- * present authority key identifier matching the subject key identifier, etc.
+ * This actually verifies self-signedness only if requested.
+ * It calls X509v3_cache_extensions()
+ * to match issuer and subject names (i.e., the cert being self-issued) and any
+ * present authority key identifier to match the subject key identifier, etc.
  */
-static int cert_self_signed(X509_STORE_CTX *ctx, X509 *x)
+static int x509_self_signed_ex(X509 *cert, int verify_signature,
+                               OPENSSL_CTX *libctx, const char *propq)
 {
-    if (!X509v3_cache_extensions(x, ctx->libctx, ctx->propq))
-        return -1;
+    EVP_PKEY *pkey;
 
-    if (x->ex_flags & EXFLAG_SS)
-        return 1;
-    else
+    if ((pkey = X509_get0_pubkey(cert)) == NULL) { /* handles cert == NULL */
+        X509err(0, X509_R_UNABLE_TO_GET_CERTS_PUBLIC_KEY);
+        return -1;
+    }
+    if (!X509v3_cache_extensions(cert, libctx, propq))
+        return -1;
+    if ((cert->ex_flags & EXFLAG_SS) == 0)
         return 0;
+    if (!verify_signature)
+        return 1;
+    return X509_verify_ex(cert, pkey, libctx, propq);
 }
 
-/* Given a certificate try and find an exact match in the store */
+/* wrapper for internal use */
+static int cert_self_signed(X509_STORE_CTX *ctx, X509 *x, int verify_signature)
+{
+    return x509_self_signed_ex(x, verify_signature, ctx->libctx, ctx->propq);
+}
+
+int X509_self_signed(X509 *cert, int verify_signature)
+{
+    return x509_self_signed_ex(cert, verify_signature, NULL, NULL);
+}
 
+/* Given a certificate try and find an exact match in the store */
 static X509 *lookup_cert_match(X509_STORE_CTX *ctx, X509 *x)
 {
     STACK_OF(X509) *certs;
@@ -365,7 +383,6 @@ static int check_issued(X509_STORE_CTX *ctx, X509 *x, X509 *issuer)
 }
 
 /* Alternative lookup method: look from a STACK stored in other_ctx */
-
 static int get_issuer_sk(X509 **issuer, X509_STORE_CTX *ctx, X509 *x)
 {
     *issuer = find_issuer(ctx, ctx->other_ctx, x);
@@ -2954,7 +2971,7 @@ static int build_chain(X509_STORE_CTX *ctx)
     SSL_DANE *dane = ctx->dane;
     int num = sk_X509_num(ctx->chain);
     X509 *cert = sk_X509_value(ctx->chain, num - 1);
-    int self_signed = cert_self_signed(ctx, cert);
+    int self_signed;
     STACK_OF(X509) *sktmp = NULL;
     unsigned int search;
     int may_trusted = 0;
@@ -2972,7 +2989,7 @@ static int build_chain(X509_STORE_CTX *ctx)
         return 0;
     }
 
-    self_signed = cert_self_signed(ctx, cert);
+    self_signed = cert_self_signed(ctx, cert, 0);
     if (self_signed < 0) {
         ctx->error = X509_V_ERR_UNSPECIFIED;
         return 0;
@@ -3150,7 +3167,7 @@ static int build_chain(X509_STORE_CTX *ctx)
                         search = 0;
                         continue;
                     }
-                    self_signed = cert_self_signed(ctx, x);
+                    self_signed = cert_self_signed(ctx, x, 0);
                     if (self_signed < 0) {
                         ctx->error = X509_V_ERR_UNSPECIFIED;
                         return 0;
@@ -3276,7 +3293,7 @@ static int build_chain(X509_STORE_CTX *ctx)
 
             x = xtmp;
             ++ctx->num_untrusted;
-            self_signed = cert_self_signed(ctx, xtmp);
+            self_signed = cert_self_signed(ctx, xtmp, 0);
             if (self_signed < 0) {
                 sk_X509_free(sktmp);
                 ctx->error = X509_V_ERR_UNSPECIFIED;