X509_cmp(): Fix comparison in case x509v3_cache_extensions() failed to due to invalid cert

This is the backport of #13755 to v1.1.1.
Fixes #13698

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/13756)

3 files changed, 38 insertions, 11 deletions

diff --git a/test/certs/invalid-cert.pem b/test/certs/invalid-cert.pem
new file mode 100644
index 0000000000..a8951305a3
--- /dev/null
+++ b/test/certs/invalid-cert.pem
@@ -0,0 +1,19 @@
+-----BEGIN TRUSTED CERTIFICATE-----
+MIIDJTCCAg2gAwIBAgIUEUSW5o7qpgNCWyXic9Fc9tCLS0gwDQYJKoZIhvcNAQEL
+BQAwEzERMA8GA1UEAwwIUGVyc29TaW0wHhcNMjAxMjE2MDY1NjM5WhcNMzAxMjE2
+MDY1NjM5WjATMREwDwYDVQQDDAhQZXJzb1NpbTCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAMsgRKnnZbQtG9bB9Hn+CoOOsanmnRELSlGq521qi/eBgs2w
+SdHYM6rsJFwY89RvINLGeUZh/pu7c+ODtTafAWE3JkynG01d2Zrvp1V1r97+FGyD
+f+b1hAggxBy70bTRyr1gAoKQTAm74U/1lj13EpWz7zshgXJ/Pn/hUyTmpNW+fTRE
+xaifN0jkl5tZUURGA6w3+BRhVDQtt92vLihqUGaEFpL8yqqFnN44AoQ5+lgMafWi
+UyYMHcK75ZB8WWklq8zjRP3xC1h56k01rT6KJO6i+BxMcADerYsn5qTlcUiKcpRU
+b6RzLvCUwj91t1aX6npDI3BzSP+wBUUANBfuHEMCAwEAAaNxMG8wFwYDVR0OBBA8
+yBBnvz1Zt6pHm2GwBaRyMBcGA1UdIwQQPMgQZ789WbeqR5thsAWkcjAPBgNVHRMB
+Af8EBTADAQH/MAsGA1UdDwQEAwIChDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYB
+BQUHAwIwDQYJKoZIhvcNAQELBQADggEBAIEzVbttOUc7kK4aY+74TANFZK/qtBQ7
+94a/P30TGWSRUq2HnDsR8Vo4z8xm5oKeC+SIi6NGzviWYquuzpJ7idcbr0MIuSyD
++Vg6n1sG64DxWNdGO9lR5c4mWFdIajShczS2+4QIRB/lFZCf7GhPMtIcbP1o9ckY
+2vyv5ZAEU9Z5n0PY+abrKsj0XyvJwdycEsUTywa36fuv6hP3UboLtvK6naXLMrTj
+WtSA6PXjHy7h8h0NC8XLk64mc0lcRC4WM+xJ/C+NHglpmBqBxnStpnZykMZYD1Vy
+JJ1wNc+Y3e2uMBDxZviH3dIPIgqP1Vpi2TWfqr3DTBNCRf4dl/wwNU8=
+-----END TRUSTED CERTIFICATE-----
diff --git a/test/recipes/80-test_x509aux.t b/test/recipes/80-test_x509aux.t
index 65ba5fcf52..30adf25257 100644
--- a/test/recipes/80-test_x509aux.t
+++ b/test/recipes/80-test_x509aux.t
@@ -14,14 +14,17 @@ use OpenSSL::Test::Utils;
 
 setup("test_x509aux");
 
+my @path = qw(test certs);
+
 plan skip_all => "test_dane uses ec which is not supported by this OpenSSL build"
     if disabled("ec");
 
 plan tests => 1;                # The number of tests being performed
 
 ok(run(test(["x509aux", 
-                srctop_file("test", "certs", "roots.pem"),
-                srctop_file("test", "certs", "root+anyEKU.pem"),
-                srctop_file("test", "certs", "root-anyEKU.pem"),
-                srctop_file("test", "certs", "root-cert.pem")]
-        )), "x509aux tests");
+             srctop_file(@path, "roots.pem"),
+             srctop_file(@path, "root+anyEKU.pem"),
+             srctop_file(@path, "root-anyEKU.pem"),
+             srctop_file(@path, "root-cert.pem"),
+             srctop_file(@path, "invalid-cert.pem"),
+            ])), "x509aux tests");
diff --git a/test/x509aux.c b/test/x509aux.c
index e41f1f6809..78013f23ae 100644
--- a/test/x509aux.c
+++ b/test/x509aux.c
@@ -30,17 +30,16 @@ static int test_certs(int num)
     typedef int (*i2d_X509_t)(X509 *, unsigned char **);
     int err = 0;
     BIO *fp = BIO_new_file(test_get_argument(num), "r");
-    X509 *reuse = NULL;
 
     if (!TEST_ptr(fp))
         return 0;
 
     for (c = 0; !err && PEM_read_bio(fp, &name, &header, &data, &len); ++c) {
         const int trusted = (strcmp(name, PEM_STRING_X509_TRUSTED) == 0);
-
         d2i_X509_t d2i = trusted ? d2i_X509_AUX : d2i_X509;
         i2d_X509_t i2d = trusted ? i2d_X509_AUX : i2d_X509;
         X509 *cert = NULL;
+        X509 *reuse = NULL;
         const unsigned char *p = data;
         unsigned char *buf = NULL;
         unsigned char *bufp;
@@ -93,9 +92,15 @@ static int test_certs(int num)
             goto next;
         }
         p = buf;
-        reuse = d2i(&reuse, &p, enclen);
-        if (reuse == NULL || X509_cmp (reuse, cert)) {
-            TEST_error("X509_cmp does not work with %s", name);
+        reuse = d2i(NULL, &p, enclen);
+        if (reuse == NULL) {
+            TEST_error("second d2i call failed for %s", name);
+            err = 1;
+            goto next;
+        }
+        err = X509_cmp(reuse, cert);
+        if (err != 0) {
+            TEST_error("X509_cmp for %s resulted in %d", name, err);
             err = 1;
             goto next;
         }
@@ -141,13 +146,13 @@ static int test_certs(int num)
          */
     next:
         X509_free(cert);
+        X509_free(reuse);
         OPENSSL_free(buf);
         OPENSSL_free(name);
         OPENSSL_free(header);
         OPENSSL_free(data);
     }
     BIO_free(fp);
-    X509_free(reuse);
 
     if (ERR_GET_REASON(ERR_peek_last_error()) == PEM_R_NO_START_LINE) {
         /* Reached end of PEM file */