X509_cmp(): Fix comparison in case x509v3_cache_extensions() failed to due to invalid cert

This is the backport of #13755 to v1.1.1.
Fixes #13698

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/13756)

8 files changed, 62 insertions, 26 deletions

diff --git a/crypto/x509/x509_cmp.c b/crypto/x509/x509_cmp.c
index ad620af0af..c9d8933640 100644
--- a/crypto/x509/x509_cmp.c
+++ b/crypto/x509/x509_cmp.c
@@ -133,19 +133,21 @@ unsigned long X509_subject_name_hash_old(X509 *x)
  */
 int X509_cmp(const X509 *a, const X509 *b)
 {
-    int rv;
+    int rv = 0;
 
     if (a == b) /* for efficiency */
         return 0;
-    /* ensure hash is valid */
-    if (X509_check_purpose((X509 *)a, -1, 0) != 1)
-        return -2;
-    if (X509_check_purpose((X509 *)b, -1, 0) != 1)
-        return -2;
-
-    rv = memcmp(a->sha1_hash, b->sha1_hash, SHA_DIGEST_LENGTH);
-    if (rv)
+
+    /* try to make sure hash is valid */
+    (void)X509_check_purpose((X509 *)a, -1, 0);
+    (void)X509_check_purpose((X509 *)b, -1, 0);
+
+    if ((a->ex_flags & EXFLAG_NO_FINGERPRINT) == 0
+            && (b->ex_flags & EXFLAG_NO_FINGERPRINT) == 0)
+        rv = memcmp(a->sha1_hash, b->sha1_hash, SHA_DIGEST_LENGTH);
+    if (rv != 0)
         return rv;
+
     /* Check for match against stored encoding too */
     if (!a->cert_info.enc.modified && !b->cert_info.enc.modified) {
         if (a->cert_info.enc.len < b->cert_info.enc.len)
diff --git a/crypto/x509/x_all.c b/crypto/x509/x_all.c
index aa5ccba448..bec850af57 100644
--- a/crypto/x509/x_all.c
+++ b/crypto/x509/x_all.c
@@ -363,7 +363,7 @@ int X509_digest(const X509 *data, const EVP_MD *type, unsigned char *md,
                 unsigned int *len)
 {
     if (type == EVP_sha1() && (data->ex_flags & EXFLAG_SET) != 0
-            && (data->ex_flags & EXFLAG_INVALID) == 0) {
+            && (data->ex_flags & EXFLAG_NO_FINGERPRINT) == 0) {
         /* Asking for SHA1 and we already computed it. */
         if (len != NULL)
             *len = sizeof(data->sha1_hash);
diff --git a/crypto/x509v3/v3_purp.c b/crypto/x509v3/v3_purp.c
index 2b06dba053..93b5ca4d42 100644
--- a/crypto/x509v3/v3_purp.c
+++ b/crypto/x509v3/v3_purp.c
@@ -391,7 +391,8 @@ static void x509v3_cache_extensions(X509 *x)
     }
 
     if (!X509_digest(x, EVP_sha1(), x->sha1_hash, NULL))
-        x->ex_flags |= EXFLAG_INVALID;
+        x->ex_flags |= (EXFLAG_NO_FINGERPRINT | EXFLAG_INVALID);
+
     /* V1 should mean no extensions ... */
     if (!X509_get_version(x))
         x->ex_flags |= EXFLAG_V1;
diff --git a/doc/man3/X509_get_extension_flags.pod b/doc/man3/X509_get_extension_flags.pod
index 43c9c952c6..cca72c71fc 100644
--- a/doc/man3/X509_get_extension_flags.pod
+++ b/doc/man3/X509_get_extension_flags.pod
@@ -78,12 +78,17 @@ The certificate contains an unhandled critical extension.
 
 =item B<EXFLAG_INVALID>
 
-Some certificate extension values are invalid or inconsistent. The
-certificate should be rejected.
+Some certificate extension values are invalid or inconsistent.
+The certificate should be rejected.
 This bit may also be raised after an out-of-memory error while
 processing the X509 object, so it may not be related to the processed
 ASN1 object itself.
 
+=item B<EXFLAG_NO_FINGERPRINT>
+
+Failed to compute the internal SHA1 hash value of the certificate.
+This may be due to malloc failure or because no SHA1 implementation was found.
+
 =item B<EXFLAG_INVALID_POLICY>
 
 The NID_certificate_policies certificate extension is invalid or
diff --git a/include/openssl/x509v3.h b/include/openssl/x509v3.h
index 6c6eca38a5..b9a8943273 100644
--- a/include/openssl/x509v3.h
+++ b/include/openssl/x509v3.h
@@ -364,8 +364,9 @@ struct ISSUING_DIST_POINT_st {
 
 # define EXFLAG_INVALID_POLICY   0x800
 # define EXFLAG_FRESHEST         0x1000
-/* Self signed */
-# define EXFLAG_SS               0x2000
+# define EXFLAG_SS               0x2000 /* cert is apparently self-signed */
+
+# define EXFLAG_NO_FINGERPRINT   0x100000
 
 # define KU_DIGITAL_SIGNATURE    0x0080
 # define KU_NON_REPUDIATION      0x0040
diff --git a/test/certs/invalid-cert.pem b/test/certs/invalid-cert.pem
new file mode 100644
index 0000000000..a8951305a3
--- /dev/null
+++ b/test/certs/invalid-cert.pem
@@ -0,0 +1,19 @@
+-----BEGIN TRUSTED CERTIFICATE-----
+MIIDJTCCAg2gAwIBAgIUEUSW5o7qpgNCWyXic9Fc9tCLS0gwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIUGVyc29TaW0wHhcNMjAxMjE2MDY1NjM5WhcNMzAxMjE2
+MDY1NjM5WjATMREwDwYDVQQDDAhQZXJzb1NpbTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAMsgRKnnZbQtG9bB9Hn+CoOOsanmnRELSlGq521qi/eBgs2w
+SdHYM6rsJFwY89RvINLGeUZh/pu7c+ODtTafAWE3JkynG01d2Zrvp1V1r97+FGyD
+f+b1hAggxBy70bTRyr1gAoKQTAm74U/1lj13EpWz7zshgXJ/Pn/hUyTmpNW+fTRE
+xaifN0jkl5tZUURGA6w3+BRhVDQtt92vLihqUGaEFpL8yqqFnN44AoQ5+lgMafWi
+UyYMHcK75ZB8WWklq8zjRP3xC1h56k01rT6KJO6i+BxMcADerYsn5qTlcUiKcpRU
+b6RzLvCUwj91t1aX6npDI3BzSP+wBUUANBfuHEMCAwEAAaNxMG8wFwYDVR0OBBA8
+yBBnvz1Zt6pHm2GwBaRyMBcGA1UdIwQQPMgQZ789WbeqR5thsAWkcjAPBgNVHRMB
+Af8EBTADAQH/MAsGA1UdDwQEAwIChDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB
+BQUHAwIwDQYJKoZIhvcNAQELBQADggEBAIEzVbttOUc7kK4aY+74TANFZK/qtBQ7
+94a/P30TGWSRUq2HnDsR8Vo4z8xm5oKeC+SIi6NGzviWYquuzpJ7idcbr0MIuSyD
++Vg6n1sG64DxWNdGO9lR5c4mWFdIajShczS2+4QIRB/lFZCf7GhPMtIcbP1o9ckY
+2vyv5ZAEU9Z5n0PY+abrKsj0XyvJwdycEsUTywa36fuv6hP3UboLtvK6naXLMrTj
+WtSA6PXjHy7h8h0NC8XLk64mc0lcRC4WM+xJ/C+NHglpmBqBxnStpnZykMZYD1Vy
+JJ1wNc+Y3e2uMBDxZviH3dIPIgqP1Vpi2TWfqr3DTBNCRf4dl/wwNU8=
+-----END TRUSTED CERTIFICATE-----
diff --git a/test/recipes/80-test_x509aux.t b/test/recipes/80-test_x509aux.t
index 65ba5fcf52..30adf25257 100644
--- a/test/recipes/80-test_x509aux.t
+++ b/test/recipes/80-test_x509aux.t
@@ -14,14 +14,17 @@ use OpenSSL::Test::Utils;
 
 setup("test_x509aux");
 
+my @path = qw(test certs);
+
 plan skip_all => "test_dane uses ec which is not supported by this OpenSSL build"
     if disabled("ec");
 
 plan tests => 1;                # The number of tests being performed
 
 ok(run(test(["x509aux", 
-                srctop_file("test", "certs", "roots.pem"),
-                srctop_file("test", "certs", "root+anyEKU.pem"),
-                srctop_file("test", "certs", "root-anyEKU.pem"),
-                srctop_file("test", "certs", "root-cert.pem")]
-        )), "x509aux tests");
+             srctop_file(@path, "roots.pem"),
+             srctop_file(@path, "root+anyEKU.pem"),
+             srctop_file(@path, "root-anyEKU.pem"),
+             srctop_file(@path, "root-cert.pem"),
+             srctop_file(@path, "invalid-cert.pem"),
+            ])), "x509aux tests");
diff --git a/test/x509aux.c b/test/x509aux.c
index e41f1f6809..78013f23ae 100644
--- a/test/x509aux.c
+++ b/test/x509aux.c
@@ -30,17 +30,16 @@ static int test_certs(int num)
     typedef int (*i2d_X509_t)(X509 *, unsigned char **);
     int err = 0;
     BIO *fp = BIO_new_file(test_get_argument(num), "r");
-    X509 *reuse = NULL;
 
     if (!TEST_ptr(fp))
         return 0;
 
     for (c = 0; !err && PEM_read_bio(fp, &name, &header, &data, &len); ++c) {
         const int trusted = (strcmp(name, PEM_STRING_X509_TRUSTED) == 0);
-
         d2i_X509_t d2i = trusted ? d2i_X509_AUX : d2i_X509;
         i2d_X509_t i2d = trusted ? i2d_X509_AUX : i2d_X509;
         X509 *cert = NULL;
+        X509 *reuse = NULL;
         const unsigned char *p = data;
         unsigned char *buf = NULL;
         unsigned char *bufp;
@@ -93,9 +92,15 @@ static int test_certs(int num)
             goto next;
         }
         p = buf;
-        reuse = d2i(&reuse, &p, enclen);
-        if (reuse == NULL || X509_cmp (reuse, cert)) {
-            TEST_error("X509_cmp does not work with %s", name);
+        reuse = d2i(NULL, &p, enclen);
+        if (reuse == NULL) {
+            TEST_error("second d2i call failed for %s", name);
+            err = 1;
+            goto next;
+        }
+        err = X509_cmp(reuse, cert);
+        if (err != 0) {
+            TEST_error("X509_cmp for %s resulted in %d", name, err);
             err = 1;
             goto next;
         }
@@ -141,13 +146,13 @@ static int test_certs(int num)
          */
     next:
         X509_free(cert);
+        X509_free(reuse);
         OPENSSL_free(buf);
         OPENSSL_free(name);
         OPENSSL_free(header);
         OPENSSL_free(data);
     }
     BIO_free(fp);
-    X509_free(reuse);
 
     if (ERR_GET_REASON(ERR_peek_last_error()) == PEM_R_NO_START_LINE) {
         /* Reached end of PEM file */