X509_LOOKUP_store: new X509_LOOKUP_METHOD that works by OSSL_STORE URI

This is a wrapper around OSSL_STORE.

This also adds necessary support functions:

- X509_STORE_load_file
- X509_STORE_load_path
- X509_STORE_load_store
- SSL_add_store_cert_subjects_to_stack
- SSL_CTX_set_default_verify_store
- SSL_CTX_load_verify_file
- SSL_CTX_load_verify_dir
- SSL_CTX_load_verify_store

and deprecates X509_STORE_load_locations and SSL_CTX_load_verify_locations,
as they aren't extensible.

Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8442)

4 files changed, 278 insertions, 18 deletions

diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index 27aad6a6ce..abbf232133 100644
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -1828,6 +1828,7 @@ X509_F_COMMON_VERIFY_SM2:165:common_verify_sm2
 X509_F_DANE_I2D:107:dane_i2d
 X509_F_DIR_CTRL:102:dir_ctrl
 X509_F_GET_CERT_BY_SUBJECT:103:get_cert_by_subject
+X509_F_CACHE_OBJECTS:163:cache_objects
 X509_F_I2D_X509_AUX:151:i2d_X509_AUX
 X509_F_LOOKUP_CERTS_SK:152:lookup_certs_sk
 X509_F_NETSCAPE_SPKI_B64_DECODE:129:NETSCAPE_SPKI_b64_decode
diff --git a/crypto/x509/build.info b/crypto/x509/build.info
index bee9f80961..ca7bb2a03f 100644
--- a/crypto/x509/build.info
+++ b/crypto/x509/build.info
@@ -5,7 +5,7 @@ SOURCE[../../libcrypto]=\
         x509_set.c x509cset.c x509rset.c x509_err.c \
         x509name.c x509_v3.c x509_ext.c x509_att.c \
         x509type.c x509_meth.c x509_lu.c x_all.c x509_txt.c \
-        x509_trs.c by_file.c by_dir.c x509_vpm.c \
+        x509_trs.c by_file.c by_dir.c by_store.c x509_vpm.c \
         x_crl.c t_crl.c x_req.c t_req.c x_x509.c t_x509.c \
         x_pubkey.c x_x509a.c x_attrib.c x_exten.c x_name.c \
         v3_bcons.c v3_bitst.c v3_conf.c v3_extku.c v3_ia5.c v3_lib.c \
diff --git a/crypto/x509/by_store.c b/crypto/x509/by_store.c
new file mode 100644
index 0000000000..b2264d7123
--- /dev/null
+++ b/crypto/x509/by_store.c
@@ -0,0 +1,227 @@
+/*
+ * Copyright 2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License 2.0 (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#include <openssl/store.h>
+#include "internal/cryptlib.h"
+#include "crypto/x509.h"
+#include "x509_local.h"
+
+/* Generic object loader, given expected type and criterion */
+static int cache_objects(X509_LOOKUP *lctx, const char *uri,
+                         const OSSL_STORE_SEARCH *criterion,
+                         int depth)
+{
+    int ok = 0;
+    OSSL_STORE_CTX *ctx = NULL;
+    X509_STORE *xstore = X509_LOOKUP_get_store(lctx);
+
+    if ((ctx = OSSL_STORE_open(uri, NULL, NULL, NULL, NULL)) == NULL)
+        return 0;
+
+    /*
+     * We try to set the criterion, but don't care if it was valid or not.
+     * For a OSSL_STORE, it merely serves as an optimization, the expectation
+     * being that if the criterion couldn't be used, we will get *everything*
+     * from the container that the URI represents rather than the subset that
+     * the criterion indicates, so the biggest harm is that we cache more
+     * objects certs and CRLs than we may expect, but that's ok.
+     *
+     * Specifically for OpenSSL's own file: scheme, the only workable
+     * criterion is the BY_NAME one, which it can only apply on directories,
+     * but it's possible that the URI is a single file rather than a directory,
+     * and in that case, the BY_NAME criterion is pointless.
+     *
+     * We could very simply not apply any criterion at all here, and just let
+     * the code that selects certs and CRLs from the cached objects do its job,
+     * but it's a nice optimization when it can be applied (such as on an
+     * actual directory with a thousand CA certs).
+     */
+    if (criterion != NULL)
+        OSSL_STORE_find(ctx, criterion);
+
+    for (;;) {
+        OSSL_STORE_INFO *info = OSSL_STORE_load(ctx);
+        int infotype;
+
+        /* NULL means error or "end of file".  Either way, we break. */
+        if (info == NULL)
+            break;
+
+        infotype = OSSL_STORE_INFO_get_type(info);
+        ok = 0;
+
+        if (infotype == OSSL_STORE_INFO_NAME) {
+            /*
+             * This is an entry in the "directory" represented by the current
+             * uri.  if |depth| allows, dive into it.
+             */
+            if (depth > 0)
+                ok = cache_objects(lctx, OSSL_STORE_INFO_get0_NAME(info),
+                                   criterion, depth - 1);
+        } else {
+            /*
+             * We know that X509_STORE_add_{cert|crl} increments the object's
+             * refcount, so we can safely use OSSL_STORE_INFO_get0_{cert,crl}
+             * to get them.
+             */
+            switch (infotype) {
+            case OSSL_STORE_INFO_CERT:
+                ok = X509_STORE_add_cert(xstore,
+                                         OSSL_STORE_INFO_get0_CERT(info));
+                break;
+            case OSSL_STORE_INFO_CRL:
+                ok = X509_STORE_add_crl(xstore,
+                                        OSSL_STORE_INFO_get0_CRL(info));
+                break;
+            }
+        }
+
+        OSSL_STORE_INFO_free(info);
+        if (!ok)
+            break;
+    }
+    OSSL_STORE_close(ctx);
+
+    return ok;
+}
+
+
+/* Because OPENSSL_free is a macro and for C type match */
+static void free_uri(OPENSSL_STRING data)
+{
+    OPENSSL_free(data);
+}
+
+static void by_store_free(X509_LOOKUP *ctx)
+{
+    STACK_OF(OPENSSL_STRING) *uris = X509_LOOKUP_get_method_data(ctx);
+    sk_OPENSSL_STRING_pop_free(uris, free_uri);
+}
+
+static int by_store_ctrl(X509_LOOKUP *ctx, int cmd,
+                         const char *argp, long argl,
+                         char **retp)
+{
+    switch (cmd) {
+    case X509_L_ADD_STORE:
+        /* If no URI is given, use the default cert dir as default URI */
+        if (argp == NULL)
+            argp = ossl_safe_getenv(X509_get_default_cert_dir_env());
+        if (argp == NULL)
+            argp = X509_get_default_cert_dir();
+
+        {
+            STACK_OF(OPENSSL_STRING) *uris = X509_LOOKUP_get_method_data(ctx);
+
+            if (uris == NULL) {
+                uris = sk_OPENSSL_STRING_new_null();
+                X509_LOOKUP_set_method_data(ctx, uris);
+            }
+            return sk_OPENSSL_STRING_push(uris, OPENSSL_strdup(argp)) > 0;
+        }
+    case X509_L_LOAD_STORE:
+        /* This is a shortcut for quick loading of specific containers */
+        return cache_objects(ctx, argp, NULL, 0);
+    }
+
+    return 0;
+}
+
+static int by_store(X509_LOOKUP *ctx, X509_LOOKUP_TYPE type,
+                    const OSSL_STORE_SEARCH *criterion, X509_OBJECT *ret)
+{
+    STACK_OF(OPENSSL_STRING) *uris = X509_LOOKUP_get_method_data(ctx);
+    int i;
+    int ok = 0;
+
+    for (i = 0; i < sk_OPENSSL_STRING_num(uris); i++) {
+        ok = cache_objects(ctx, sk_OPENSSL_STRING_value(uris, i), criterion,
+                           1 /* depth */);
+
+        if (ok)
+            break;
+    }
+    return ok;
+}
+
+static int by_store_subject(X509_LOOKUP *ctx, X509_LOOKUP_TYPE type,
+                            X509_NAME *name, X509_OBJECT *ret)
+{
+    OSSL_STORE_SEARCH *criterion = OSSL_STORE_SEARCH_by_name(name);
+    int ok = by_store(ctx, type, criterion, ret);
+    STACK_OF(X509_OBJECT) *store_objects =
+        X509_STORE_get0_objects(X509_LOOKUP_get_store(ctx));
+    X509_OBJECT *tmp = NULL;
+
+    OSSL_STORE_SEARCH_free(criterion);
+
+    if (ok)
+        tmp = X509_OBJECT_retrieve_by_subject(store_objects, type, name);
+
+    ok = 0;
+    if (tmp != NULL) {
+        /*
+         * This could also be done like this:
+         *
+         *     if (tmp != NULL) {
+         *         *ret = *tmp;
+         *         ok = 1;
+         *     }
+         *
+         * However, we want to exercise the documented API to the max, so
+         * we do it the hard way.
+         *
+         * To be noted is that X509_OBJECT_set1_* increment the refcount,
+         * but so does X509_STORE_CTX_get_by_subject upon return of this
+         * function, so we must ensure the the refcount is decremented
+         * before we return, or we will get a refcount leak.  We cannot do
+         * this with X509_OBJECT_free(), though, as that will free a bit
+         * too much.
+         */
+        switch (type) {
+        case X509_LU_X509:
+            ok = X509_OBJECT_set1_X509(ret, tmp->data.x509);
+            if (ok)
+                X509_free(tmp->data.x509);
+            break;
+        case X509_LU_CRL:
+            ok = X509_OBJECT_set1_X509_CRL(ret, tmp->data.crl);
+            if (ok)
+                X509_CRL_free(tmp->data.crl);
+            break;
+        case X509_LU_NONE:
+            break;
+        }
+    }
+    return ok;
+}
+
+/*
+ * We lack the implementations for get_by_issuer_serial, get_by_fingerprint
+ * and get_by_alias.  There's simply not enough support in the X509_LOOKUP
+ * or X509_STORE APIs.
+ */
+
+static X509_LOOKUP_METHOD x509_store_lookup = {
+    "Load certs from STORE URIs",
+    NULL,                        /* new_item */
+    by_store_free,               /* free */
+    NULL,                        /* init */
+    NULL,                        /* shutdown */
+    by_store_ctrl,               /* ctrl */
+    by_store_subject,            /* get_by_subject */
+    NULL,                        /* get_by_issuer_serial */
+    NULL,                        /* get_by_fingerprint */
+    NULL,                        /* get_by_alias */
+};
+
+X509_LOOKUP_METHOD *X509_LOOKUP_store(void)
+{
+    return &x509_store_lookup;
+}
diff --git a/crypto/x509/x509_d2.c b/crypto/x509/x509_d2.c
index 70d57f5274..5beb7034a7 100644
--- a/crypto/x509/x509_d2.c
+++ b/crypto/x509/x509_d2.c
@@ -26,32 +26,64 @@ int X509_STORE_set_default_paths(X509_STORE *ctx)
         return 0;
     X509_LOOKUP_add_dir(lookup, NULL, X509_FILETYPE_DEFAULT);
 
+    lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_store());
+    if (lookup == NULL)
+        return 0;
+    X509_LOOKUP_add_store(lookup, NULL);
+
     /* clear any errors */
     ERR_clear_error();
 
     return 1;
 }
 
-int X509_STORE_load_locations(X509_STORE *ctx, const char *file,
-                              const char *path)
+int X509_STORE_load_file(X509_STORE *ctx, const char *file)
+{
+    X509_LOOKUP *lookup;
+
+    if (file == NULL
+        || (lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_file())) == NULL
+        || X509_LOOKUP_load_file(lookup, file, X509_FILETYPE_PEM) == 0)
+        return 0;
+
+    return 1;
+}
+
+int X509_STORE_load_path(X509_STORE *ctx, const char *path)
 {
     X509_LOOKUP *lookup;
 
-    if (file != NULL) {
-        lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_file());
-        if (lookup == NULL)
-            return 0;
-        if (X509_LOOKUP_load_file(lookup, file, X509_FILETYPE_PEM) != 1)
-            return 0;
-    }
-    if (path != NULL) {
-        lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_hash_dir());
-        if (lookup == NULL)
-            return 0;
-        if (X509_LOOKUP_add_dir(lookup, path, X509_FILETYPE_PEM) != 1)
-            return 0;
-    }
-    if ((path == NULL) && (file == NULL))
+    if (path == NULL
+        || (lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_hash_dir())) == NULL
+        || X509_LOOKUP_add_dir(lookup, path, X509_FILETYPE_PEM) == 0)
+        return 0;
+
+    return 1;
+}
+
+int X509_STORE_load_store(X509_STORE *ctx, const char *uri)
+{
+    X509_LOOKUP *lookup;
+
+    if (uri == NULL
+        || (lookup = X509_STORE_add_lookup(ctx, X509_LOOKUP_store())) == NULL
+        || X509_LOOKUP_add_store(lookup, uri) == 0)
+        return 0;
+
+    return 1;
+}
+
+/* Deprecated */
+#if OPENSSL_API_LEVEL < 3
+int X509_STORE_load_locations(X509_STORE *ctx, const char *file,
+                              const char *path)
+{
+    if (file == NULL && path == NULL)
+        return 0;
+    if (file != NULL && !X509_STORE_load_file(ctx, file))
+        return 0;
+    if (path != NULL && !X509_STORE_load_path(ctx, path))
         return 0;
     return 1;
 }
+#endif