Add X509 related libctx changes.

- In order to not add many X509_XXXX_with_libctx() functions the libctx and propq may be stored in the X509 object via a call to X509_new_with_libctx().
- Loading via PEM_read_bio_X509() or d2i_X509() should pass in a created cert using X509_new_with_libctx().
- Renamed some XXXX_ex() to XXX_with_libctx() for X509 API's.
- Removed the extra parameters in check_purpose..
- X509_digest() has been modified so that it expects a const EVP_MD object() and then internally it does the fetch when it needs to (via ASN1_item_digest_with_libctx()).
- Added API's that set the libctx when they load such as X509_STORE_new_with_libctx() so that the cert chains can be verified.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12153)

64 files changed, 1081 insertions, 544 deletions

diff --git a/apps/ca.c b/apps/ca.c
index d0309ae15c..e001a34190 100644
--- a/apps/ca.c
+++ b/apps/ca.c
@@ -1643,7 +1643,7 @@ static int do_body(X509 **xret, EVP_PKEY *pkey, X509 *x509,
         BIO_printf(bio_err,
                    "Everything appears to be ok, creating and signing the certificate\n");
 
-    if ((ret = X509_new()) == NULL)
+    if ((ret = X509_new_with_libctx(app_get0_libctx(), app_get0_propq())) == NULL)
         goto end;
 
 #ifdef X509_V3
diff --git a/apps/include/apps.h b/apps/include/apps.h
index 9a76dcd339..e91cdcdb8f 100644
--- a/apps/include/apps.h
+++ b/apps/include/apps.h
@@ -299,4 +299,7 @@ void app_params_free(OSSL_PARAM *params);
 int app_provider_load(OPENSSL_CTX *libctx, const char *provider_name);
 void app_providers_cleanup(void);
 
+OPENSSL_CTX *app_get0_libctx(void);
+const char *app_get0_propq(void);
+
 #endif
diff --git a/apps/lib/apps.c b/apps/lib/apps.c
index ba40e9bc7e..cf99ca0ebf 100644
--- a/apps/lib/apps.c
+++ b/apps/lib/apps.c
@@ -342,6 +342,12 @@ OPENSSL_CTX *app_get0_libctx(void)
     return app_libctx;
 }
 
+/* TODO(3.0): Make this an environment variable if required */
+const char *app_get0_propq(void)
+{
+    return NULL;
+}
+
 OPENSSL_CTX *app_create_libctx(void)
 {
     /*
@@ -657,9 +663,11 @@ static int load_certs_crls(const char *file, int format,
     if (bio == NULL)
         return 0;
 
-    xis = PEM_X509_INFO_read_bio(bio, NULL,
-                                 (pem_password_cb *)password_callback,
-                                 &cb_data);
+    xis = PEM_X509_INFO_read_bio_with_libctx(bio, NULL,
+                                             (pem_password_cb *)password_callback,
+                                             &cb_data,
+                                             app_get0_libctx(),
+                                             app_get0_propq());
 
     BIO_free(bio);
 
@@ -765,6 +773,8 @@ int load_key_cert_crl(const char *uri, int maybe_stdin,
 {
     PW_CB_DATA uidata;
     OSSL_STORE_CTX *ctx = NULL;
+    OPENSSL_CTX *libctx = app_get0_libctx();
+    const char *propq = app_get0_propq();
     int ret = 0;
     /* TODO make use of the engine reference 'eng' when loading pkeys */
 
@@ -791,11 +801,12 @@ int load_key_cert_crl(const char *uri, int maybe_stdin,
         unbuffer(stdin);
         bio = BIO_new_fp(stdin, 0);
         if (bio != NULL)
-            ctx = OSSL_STORE_attach(bio, NULL, "file", NULL,
+            ctx = OSSL_STORE_attach(bio, "file", libctx, propq,
                                     get_ui_method(), &uidata, NULL, NULL);
         uri = "<stdin>";
     } else {
-        ctx = OSSL_STORE_open(uri, get_ui_method(), &uidata, NULL, NULL);
+        ctx = OSSL_STORE_open_with_libctx(uri, libctx, propq, get_ui_method(),
+                                          &uidata, NULL, NULL);
     }
     if (ctx == NULL) {
         BIO_printf(bio_err, "Could not open file or uri %s for loading %s\n",
@@ -1099,6 +1110,8 @@ X509_STORE *setup_verify(const char *CAfile, int noCAfile,
 {
     X509_STORE *store = X509_STORE_new();
     X509_LOOKUP *lookup;
+    OPENSSL_CTX *libctx = app_get0_libctx();
+    const char *propq = app_get0_propq();
 
     if (store == NULL)
         goto end;
@@ -1108,12 +1121,16 @@ X509_STORE *setup_verify(const char *CAfile, int noCAfile,
         if (lookup == NULL)
             goto end;
         if (CAfile != NULL) {
-            if (!X509_LOOKUP_load_file(lookup, CAfile, X509_FILETYPE_PEM)) {
+            if (!X509_LOOKUP_load_file_with_libctx(lookup, CAfile,
+                                                   X509_FILETYPE_PEM,
+                                                   libctx, propq)) {
                 BIO_printf(bio_err, "Error loading file %s\n", CAfile);
                 goto end;
             }
         } else {
-            X509_LOOKUP_load_file(lookup, NULL, X509_FILETYPE_DEFAULT);
+            X509_LOOKUP_load_file_with_libctx(lookup, NULL,
+                                              X509_FILETYPE_DEFAULT,
+                                              libctx, propq);
         }
     }
 
@@ -1135,7 +1152,7 @@ X509_STORE *setup_verify(const char *CAfile, int noCAfile,
         lookup = X509_STORE_add_lookup(store, X509_LOOKUP_store());
         if (lookup == NULL)
             goto end;
-        if (!X509_LOOKUP_add_store(lookup, CAstore)) {
+        if (!X509_LOOKUP_add_store_with_libctx(lookup, CAstore, libctx, propq)) {
             if (CAstore != NULL)
                 BIO_printf(bio_err, "Error loading store URI %s\n", CAstore);
             goto end;
diff --git a/apps/req.c b/apps/req.c
index 4ae828cd45..bee0329b24 100644
--- a/apps/req.c
+++ b/apps/req.c
@@ -742,7 +742,8 @@ int req_main(int argc, char **argv)
         if (x509) {
             EVP_PKEY *tmppkey;
             X509V3_CTX ext_ctx;
-            if ((x509ss = X509_new()) == NULL)
+            if ((x509ss = X509_new_with_libctx(app_get0_libctx(),
+                                               app_get0_propq())) == NULL)
                 goto end;
 
             /* Set version to V3 */
diff --git a/apps/storeutl.c b/apps/storeutl.c
index 87e8fcc9e6..95af277260 100644
--- a/apps/storeutl.c
+++ b/apps/storeutl.c
@@ -19,7 +19,7 @@
 static int process(const char *uri, const UI_METHOD *uimeth, PW_CB_DATA *uidata,
                    int expected, int criterion, OSSL_STORE_SEARCH *search,
                    int text, int noout, int recursive, int indent, BIO *out,
-                   const char *prog);
+                   const char *prog, OPENSSL_CTX *libctx, const char *propq);
 
 typedef enum OPTION_choice {
     OPT_ERR = -1, OPT_EOF = 0, OPT_HELP, OPT_ENGINE, OPT_OUT, OPT_PASSIN,
@@ -84,6 +84,8 @@ int storeutl_main(int argc, char *argv[])
     char *alias = NULL;
     OSSL_STORE_SEARCH *search = NULL;
     const EVP_MD *digest = NULL;
+    OPENSSL_CTX *libctx = app_get0_libctx();
+    const char *propq = app_get0_propq();
 
     while ((o = opt_next()) != OPT_EOF) {
         switch (o) {
@@ -322,7 +324,7 @@ int storeutl_main(int argc, char *argv[])
 
     ret = process(argv[0], get_ui_method(), &pw_cb_data,
                   expected, criterion, search,
-                  text, noout, recursive, 0, out, prog);
+                  text, noout, recursive, 0, out, prog, libctx, propq);
 
  end:
     OPENSSL_free(fingerprint);
@@ -353,12 +355,13 @@ static int indent_printf(int indent, BIO *bio, const char *format, ...)
 static int process(const char *uri, const UI_METHOD *uimeth, PW_CB_DATA *uidata,
                    int expected, int criterion, OSSL_STORE_SEARCH *search,
                    int text, int noout, int recursive, int indent, BIO *out,
-                   const char *prog)
+                   const char *prog, OPENSSL_CTX *libctx, const char *propq)
 {
     OSSL_STORE_CTX *store_ctx = NULL;
     int ret = 1, items = 0;
 
-    if ((store_ctx = OSSL_STORE_open(uri, uimeth, uidata, NULL, NULL))
+    if ((store_ctx = OSSL_STORE_open_with_libctx(uri, libctx, propq,
+                                                 uimeth, uidata, NULL, NULL))
         == NULL) {
         BIO_printf(bio_err, "Couldn't open file or uri %s\n", uri);
         ERR_print_errors(bio_err);
@@ -439,7 +442,8 @@ static int process(const char *uri, const UI_METHOD *uimeth, PW_CB_DATA *uidata,
                 const char *suburi = OSSL_STORE_INFO_get0_NAME(info);
                 ret += process(suburi, uimeth, uidata,
                                expected, criterion, search,
-                               text, noout, recursive, indent + 2, out, prog);
+                               text, noout, recursive, indent + 2, out, prog,
+                               libctx, propq);
             }
             break;
         case OSSL_STORE_INFO_PARAMS:
diff --git a/apps/ts.c b/apps/ts.c
index da4584ab53..09c586b44f 100644
--- a/apps/ts.c
+++ b/apps/ts.c
@@ -957,6 +957,8 @@ static X509_STORE *create_cert_store(const char *CApath, const char *CAfile,
 {
     X509_STORE *cert_ctx = NULL;
     X509_LOOKUP *lookup = NULL;
+    OPENSSL_CTX *libctx = app_get0_libctx();
+    const char *propq = app_get0_propq();
 
     cert_ctx = X509_STORE_new();
     X509_STORE_set_verify_cb(cert_ctx, verify_cb);
@@ -978,7 +980,9 @@ static X509_STORE *create_cert_store(const char *CApath, const char *CAfile,
             BIO_printf(bio_err, "memory allocation failure\n");
             goto err;
         }
-        if (!X509_LOOKUP_load_file(lookup, CAfile, X509_FILETYPE_PEM)) {
+        if (!X509_LOOKUP_load_file_with_libctx(lookup, CAfile,
+                                               X509_FILETYPE_PEM,
+                                               libctx, propq)) {
             BIO_printf(bio_err, "Error loading file %s\n", CAfile);
             goto err;
         }
@@ -990,7 +994,7 @@ static X509_STORE *create_cert_store(const char *CApath, const char *CAfile,
             BIO_printf(bio_err, "memory allocation failure\n");
             goto err;
         }
-        if (!X509_LOOKUP_load_store(lookup, CAstore)) {
+        if (!X509_LOOKUP_load_store_with_libctx(lookup, CAstore, libctx, propq)) {
             BIO_printf(bio_err, "Error loading store URI %s\n", CAstore);
             goto err;
         }
diff --git a/apps/x509.c b/apps/x509.c
index c09bca37bc..d8f69c08eb 100644
--- a/apps/x509.c
+++ b/apps/x509.c
@@ -510,7 +510,8 @@ int x509_main(int argc, char **argv)
         goto end;
     }
 
-    if (!X509_STORE_set_default_paths(ctx)) {
+    if (!X509_STORE_set_default_paths_with_libctx(ctx, app_get0_libctx(),
+                                                  app_get0_propq())) {
         ERR_print_errors(bio_err);
         goto end;
     }
@@ -607,7 +608,7 @@ int x509_main(int argc, char **argv)
                        "We need a private key to sign with, use -signkey or -CAkey or -CA <file> with private key\n");
             goto end;
         }
-        if ((x = X509_new()) == NULL)
+        if ((x = X509_new_with_libctx(app_get0_libctx(), app_get0_propq())) == NULL)
             goto end;
 
         if (sno == NULL) {
diff --git a/crypto/asn1/a_digest.c b/crypto/asn1/a_digest.c
index caf2f6c34f..c0c1cda272 100644
--- a/crypto/asn1/a_digest.c
+++ b/crypto/asn1/a_digest.c
@@ -7,16 +7,21 @@
  * https://www.openssl.org/source/license.html
  */
 
+/* We need to use some engine deprecated APIs */
+#define OPENSSL_SUPPRESS_DEPRECATED
+
 #include <stdio.h>
 #include <time.h>
 #include <sys/types.h>
 
 #include "internal/cryptlib.h"
 
+#include <openssl/engine.h>
 #include <openssl/err.h>
 #include <openssl/evp.h>
 #include <openssl/buffer.h>
 #include <openssl/x509.h>
+#include "crypto/x509.h"
 
 #ifndef OPENSSL_NO_DEPRECATED_3_0
 
@@ -48,20 +53,39 @@ int ASN1_digest(i2d_of_void *i2d, const EVP_MD *type, char *data,
 
 #endif
 
-int ASN1_item_digest(const ASN1_ITEM *it, const EVP_MD *type, void *asn,
-                     unsigned char *md, unsigned int *len)
+int asn1_item_digest_with_libctx(const ASN1_ITEM *it, const EVP_MD *md,
+                                 void *asn, unsigned char *data,
+                                 unsigned int *len, OPENSSL_CTX *libctx,
+                                 const char *propq)
 {
-    int i;
+    int i, ret = 0;
     unsigned char *str = NULL;
+    EVP_MD *fetched_md = (EVP_MD *)md;
 
     i = ASN1_item_i2d(asn, &str, it);
-    if (!str)
+    if (str == NULL)
         return 0;
 
-    if (!EVP_Digest(str, i, md, len, type, NULL)) {
-        OPENSSL_free(str);
-        return 0;
+    if (EVP_MD_provider(md) == NULL) {
+#if !defined(OPENSSL_NO_ENGINE)
+        if (ENGINE_get_digest_engine(EVP_MD_type(md)) == NULL)
+#endif
+            fetched_md = EVP_MD_fetch(libctx, EVP_MD_name(md), propq);
     }
+     if (fetched_md == NULL)
+         goto err;
+
+    ret = EVP_Digest(str, i, data, len, fetched_md, NULL);
+err:
     OPENSSL_free(str);
-    return 1;
+    if (fetched_md != md)
+        EVP_MD_free(fetched_md);
+    return ret;
 }
+
+int ASN1_item_digest(const ASN1_ITEM *it, const EVP_MD *md, void *asn,
+                     unsigned char *data, unsigned int *len)
+{
+    return asn1_item_digest_with_libctx(it, md, asn, data, len, NULL, NULL);
+}
+
diff --git a/crypto/ess/ess_lib.c b/crypto/ess/ess_lib.c
index 4a7a2632ba..ad0d6f332c 100644
--- a/crypto/ess/ess_lib.c
+++ b/crypto/ess/ess_lib.c
@@ -12,6 +12,7 @@
 #include <openssl/err.h>
 #include <openssl/ess.h>
 #include "crypto/ess.h"
+#include "crypto/x509.h"
 
 DEFINE_STACK_OF(ESS_CERT_ID)
 DEFINE_STACK_OF(ESS_CERT_ID_V2)
@@ -61,7 +62,7 @@ static ESS_CERT_ID *ESS_CERT_ID_new_init(X509 *cert, int issuer_needed)
     unsigned char cert_sha1[SHA_DIGEST_LENGTH];
 
     /* Call for side-effect of computing hash and caching extensions */
-    if (!X509v3_cache_extensions(cert, NULL, NULL))
+    if (!x509v3_cache_extensions(cert))
         return NULL;
 
     if ((cid = ESS_CERT_ID_new()) == NULL)
@@ -304,7 +305,7 @@ int ess_find_cert(const STACK_OF(ESS_CERT_ID) *cert_ids, X509 *cert)
         return -1;
 
     /* Recompute SHA1 hash of certificate if necessary (side effect). */
-    if (!X509v3_cache_extensions(cert, NULL, NULL))
+    if (!x509v3_cache_extensions(cert))
         return -1;
 
     /* TODO(3.0): fetch sha1 algorithm from providers */
diff --git a/crypto/pem/pem_info.c b/crypto/pem/pem_info.c
index f6a5dedc48..a3981c9dda 100644
--- a/crypto/pem/pem_info.c
+++ b/crypto/pem/pem_info.c
@@ -26,25 +26,35 @@
 DEFINE_STACK_OF(X509_INFO)
 
 #ifndef OPENSSL_NO_STDIO
-STACK_OF(X509_INFO) *PEM_X509_INFO_read(FILE *fp, STACK_OF(X509_INFO) *sk,
-                                        pem_password_cb *cb, void *u)
+STACK_OF(X509_INFO)
+*PEM_X509_INFO_read_with_libctx(FILE *fp, STACK_OF(X509_INFO) *sk,
+                                pem_password_cb *cb, void *u,
+                                OPENSSL_CTX *libctx, const char *propq)
 {
     BIO *b;
     STACK_OF(X509_INFO) *ret;
 
     if ((b = BIO_new(BIO_s_file())) == NULL) {
-        PEMerr(PEM_F_PEM_X509_INFO_READ, ERR_R_BUF_LIB);
+        PEMerr(0, ERR_R_BUF_LIB);
         return 0;
     }
     BIO_set_fp(b, fp, BIO_NOCLOSE);
-    ret = PEM_X509_INFO_read_bio(b, sk, cb, u);
+    ret = PEM_X509_INFO_read_bio_with_libctx(b, sk, cb, u, libctx, propq);
     BIO_free(b);
     return ret;
 }
+
+STACK_OF(X509_INFO) *PEM_X509_INFO_read(FILE *fp, STACK_OF(X509_INFO) *sk,
+                                        pem_password_cb *cb, void *u)
+{
+    return PEM_X509_INFO_read_with_libctx(fp, sk, cb, u, NULL, NULL);
+}
 #endif
 
-STACK_OF(X509_INFO) *PEM_X509_INFO_read_bio(BIO *bp, STACK_OF(X509_INFO) *sk,
-                                            pem_password_cb *cb, void *u)
+STACK_OF(X509_INFO)
+*PEM_X509_INFO_read_bio_with_libctx(BIO *bp, STACK_OF(X509_INFO) *sk,
+                                    pem_password_cb *cb, void *u,
+                                    OPENSSL_CTX *libctx, const char *propq)
 {
     X509_INFO *xi = NULL;
     char *name = NULL, *header = NULL;
@@ -59,7 +69,7 @@ STACK_OF(X509_INFO) *PEM_X509_INFO_read_bio(BIO *bp, STACK_OF(X509_INFO) *sk,
 
     if (sk == NULL) {
         if ((ret = sk_X509_INFO_new_null()) == NULL) {
-            PEMerr(PEM_F_PEM_X509_INFO_READ_BIO, ERR_R_MALLOC_FAILURE);
+            PEMerr(0, ERR_R_MALLOC_FAILURE);
             goto err;
         }
     } else
@@ -90,6 +100,9 @@ STACK_OF(X509_INFO) *PEM_X509_INFO_read_bio(BIO *bp, STACK_OF(X509_INFO) *sk,
                     goto err;
                 goto start;
             }
+            xi->x509 = X509_new_with_libctx(libctx, propq);
+            if (xi->x509 == NULL)
+                goto err;
             pp = &(xi->x509);
         } else if ((strcmp(name, PEM_STRING_X509_TRUSTED) == 0)) {
             d2i = (D2I_OF(void)) d2i_X509_AUX;
@@ -100,6 +113,9 @@ STACK_OF(X509_INFO) *PEM_X509_INFO_read_bio(BIO *bp, STACK_OF(X509_INFO) *sk,
                     goto err;
                 goto start;
             }
+            xi->x509 = X509_new_with_libctx(libctx, propq);
+            if (xi->x509 == NULL)
+                goto err;
             pp = &(xi->x509);
         } else if (strcmp(name, PEM_STRING_X509_CRL) == 0) {
             d2i = (D2I_OF(void)) d2i_X509_CRL;
@@ -197,11 +213,11 @@ STACK_OF(X509_INFO) *PEM_X509_INFO_read_bio(BIO *bp, STACK_OF(X509_INFO) *sk,
                 p = data;
                 if (ptype) {
                     if (!d2i_PrivateKey(ptype, pp, &p, len)) {
-                        PEMerr(PEM_F_PEM_X509_INFO_READ_BIO, ERR_R_ASN1_LIB);
+                        PEMerr(0, ERR_R_ASN1_LIB);
                         goto err;
                     }
                 } else if (d2i(pp, &p, len) == NULL) {
-                    PEMerr(PEM_F_PEM_X509_INFO_READ_BIO, ERR_R_ASN1_LIB);
+                    PEMerr(0, ERR_R_ASN1_LIB);
                     goto err;
                 }
             } else {            /* encrypted RSA data */
@@ -251,6 +267,12 @@ STACK_OF(X509_INFO) *PEM_X509_INFO_read_bio(BIO *bp, STACK_OF(X509_INFO) *sk,
     return ret;
 }
 
+STACK_OF(X509_INFO) *PEM_X509_INFO_read_bio(BIO *bp, STACK_OF(X509_INFO) *sk,
+                                            pem_password_cb *cb, void *u)
+{
+    return PEM_X509_INFO_read_bio_with_libctx(bp, sk, cb, u, NULL, NULL);
+}
+
 /* A TJH addition */
 int PEM_X509_INFO_write_bio(BIO *bp, const X509_INFO *xi, EVP_CIPHER *enc,
                             const unsigned char *kstr, int klen,
diff --git a/crypto/pem/pem_pkey.c b/crypto/pem/pem_pkey.c
index ee9b6764a6..c60eed97c0 100644
--- a/crypto/pem/pem_pkey.c
+++ b/crypto/pem/pem_pkey.c
@@ -39,7 +39,7 @@ EVP_PKEY *PEM_read_bio_PrivateKey_ex(BIO *bp, EVP_PKEY **x, pem_password_cb *cb,
     if ((ui_method = UI_UTIL_wrap_read_pem_callback(cb, 0)) == NULL)
         return NULL;
 
-    if ((ctx = OSSL_STORE_attach(bp, libctx, "file", propq, ui_method, u,
+    if ((ctx = OSSL_STORE_attach(bp, "file", libctx, propq, ui_method, u,
                                  NULL, NULL)) == NULL)
         goto err;
 #ifndef OPENSSL_NO_SECURE_HEAP
@@ -50,7 +50,8 @@ EVP_PKEY *PEM_read_bio_PrivateKey_ex(BIO *bp, EVP_PKEY **x, pem_password_cb *cb,
     }
 #endif
 
-    while (!OSSL_STORE_eof(ctx) && (info = OSSL_STORE_load(ctx)) != NULL) {
+    while (!OSSL_STORE_eof(ctx)
+           && (info = OSSL_STORE_load(ctx)) != NULL) {
         if (OSSL_STORE_INFO_get_type(info) == OSSL_STORE_INFO_PKEY) {
             ret = OSSL_STORE_INFO_get1_PKEY(info);
             break;
@@ -106,7 +107,7 @@ EVP_PKEY *PEM_read_bio_Parameters(BIO *bp, EVP_PKEY **x)
     OSSL_STORE_CTX *ctx = NULL;
     OSSL_STORE_INFO *info = NULL;
 
-    if ((ctx = OSSL_STORE_attach(bp, NULL, "file", NULL, UI_null(), NULL,
+    if ((ctx = OSSL_STORE_attach(bp, "file", NULL, NULL, UI_null(), NULL,
                                  NULL, NULL)) == NULL)
         goto err;
 
@@ -201,7 +202,7 @@ DH *PEM_read_bio_DHparams(BIO *bp, DH **x, pem_password_cb *cb, void *u)
     if ((ui_method = UI_UTIL_wrap_read_pem_callback(cb, 0)) == NULL)
         return NULL;
 
-    if ((ctx = OSSL_STORE_attach(bp, NULL, "file", NULL, ui_method, u,
+    if ((ctx = OSSL_STORE_attach(bp, "file", NULL, NULL, ui_method, u,
                                  NULL, NULL)) == NULL)
         goto err;
 
diff --git a/crypto/store/loader_file.c b/crypto/store/loader_file.c
index 30f4e6ecaf..5ff93e33ab 100644
--- a/crypto/store/