Add X509 related libctx changes.

- In order to not add many X509_XXXX_with_libctx() functions the libctx and propq may be stored in the X509 object via a call to X509_new_with_libctx().
- Loading via PEM_read_bio_X509() or d2i_X509() should pass in a created cert using X509_new_with_libctx().
- Renamed some XXXX_ex() to XXX_with_libctx() for X509 API's.
- Removed the extra parameters in check_purpose..
- X509_digest() has been modified so that it expects a const EVP_MD object() and then internally it does the fetch when it needs to (via ASN1_item_digest_with_libctx()).
- Added API's that set the libctx when they load such as X509_STORE_new_with_libctx() so that the cert chains can be verified.

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/12153)

1 files changed, 77 insertions, 48 deletions

diff --git a/ssl/ssl_rsa.c b/ssl/ssl_rsa.c
index 3df32b725b..144dd2c374 100644
--- a/ssl/ssl_rsa.c
+++ b/ssl/ssl_rsa.c
@@ -34,10 +34,7 @@ int SSL_use_certificate(SSL *ssl, X509 *x)
         SSLerr(SSL_F_SSL_USE_CERTIFICATE, ERR_R_PASSED_NULL_PARAMETER);
         return 0;
     }
-    if (!X509v3_cache_extensions(x, ssl->ctx->libctx, ssl->ctx->propq)) {
-        SSLerr(0, ERR_LIB_X509);
-        return 0;
-    }
+
     rv = ssl_security_cert(ssl, NULL, x, 0, 1);
     if (rv != 1) {
         SSLerr(SSL_F_SSL_USE_CERTIFICATE, rv);
@@ -52,7 +49,7 @@ int SSL_use_certificate_file(SSL *ssl, const char *file, int type)
     int j;
     BIO *in;
     int ret = 0;
-    X509 *x = NULL;
+    X509 *cert = NULL, *x = NULL;
 
     in = BIO_new(BIO_s_file());
     if (in == NULL) {
@@ -64,19 +61,29 @@ int SSL_use_certificate_file(SSL *ssl, const char *file, int type)
         SSLerr(SSL_F_SSL_USE_CERTIFICATE_FILE, ERR_R_SYS_LIB);
         goto end;
     }
+
+    if (type != SSL_FILETYPE_ASN1 && type != SSL_FILETYPE_PEM) {
+        SSLerr(SSL_F_SSL_USE_CERTIFICATE_FILE, SSL_R_BAD_SSL_FILETYPE);
+        goto end;
+    }
+    x = X509_new_with_libctx(ssl->ctx->libctx, ssl->ctx->propq);
+    if (x == NULL) {
+        SSLerr(0, ERR_R_MALLOC_FAILURE);
+        goto end;
+    }
     if (type == SSL_FILETYPE_ASN1) {
         j = ERR_R_ASN1_LIB;
-        x = d2i_X509_bio(in, NULL);
+        cert = d2i_X509_bio(in, &x);
     } else if (type == SSL_FILETYPE_PEM) {
         j = ERR_R_PEM_LIB;
-        x = PEM_read_bio_X509(in, NULL, ssl->default_passwd_callback,
-                              ssl->default_passwd_callback_userdata);
+        cert = PEM_read_bio_X509(in, &x, ssl->default_passwd_callback,
+                                 ssl->default_passwd_callback_userdata);
     } else {
         SSLerr(SSL_F_SSL_USE_CERTIFICATE_FILE, SSL_R_BAD_SSL_FILETYPE);
         goto end;
     }
 
-    if (x == NULL) {
+    if (cert == NULL) {
         SSLerr(SSL_F_SSL_USE_CERTIFICATE_FILE, j);
         goto end;
     }
@@ -93,8 +100,14 @@ int SSL_use_certificate_ASN1(SSL *ssl, const unsigned char *d, int len)
     X509 *x;
     int ret;
 
-    x = d2i_X509(NULL, &d, (long)len);
+    x = X509_new_with_libctx(ssl->ctx->libctx, ssl->ctx->propq);
     if (x == NULL) {
+        SSLerr(0, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
+
+    if (d2i_X509(&x, &d, (long)len)== NULL) {
+        X509_free(x);
         SSLerr(SSL_F_SSL_USE_CERTIFICATE_ASN1, ERR_R_ASN1_LIB);
         return 0;
     }
@@ -316,10 +329,7 @@ int SSL_CTX_use_certificate(SSL_CTX *ctx, X509 *x)
         SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE, ERR_R_PASSED_NULL_PARAMETER);
         return 0;
     }
-    if (!X509v3_cache_extensions(x, ctx->libctx, ctx->propq)) {
-        SSLerr(0, ERR_LIB_X509);
-        return 0;
-    }
+
     rv = ssl_security_cert(NULL, ctx, x, 0, 1);
     if (rv != 1) {
         SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE, rv);
@@ -390,10 +400,10 @@ static int ssl_set_cert(CERT *c, X509 *x)
 
 int SSL_CTX_use_certificate_file(SSL_CTX *ctx, const char *file, int type)
 {
-    int j;
+    int j = SSL_R_BAD_VALUE;
     BIO *in;
     int ret = 0;
-    X509 *x = NULL;
+    X509 *x = NULL, *cert = NULL;
 
     in = BIO_new(BIO_s_file());
     if (in == NULL) {
@@ -405,19 +415,24 @@ int SSL_CTX_use_certificate_file(SSL_CTX *ctx, const char *file, int type)
         SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE_FILE, ERR_R_SYS_LIB);
         goto end;
     }
+    if (type != SSL_FILETYPE_ASN1 && type != SSL_FILETYPE_PEM) {
+        SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE_FILE, SSL_R_BAD_SSL_FILETYPE);
+        goto end;
+    }
+    x = X509_new_with_libctx(ctx->libctx, ctx->propq);
+    if (x == NULL) {
+        SSLerr(0, ERR_R_MALLOC_FAILURE);
+        goto end;
+    }
     if (type == SSL_FILETYPE_ASN1) {
         j = ERR_R_ASN1_LIB;
-        x = d2i_X509_bio(in, NULL);
+        cert = d2i_X509_bio(in, &x);
     } else if (type == SSL_FILETYPE_PEM) {
         j = ERR_R_PEM_LIB;
-        x = PEM_read_bio_X509(in, NULL, ctx->default_passwd_callback,
-                              ctx->default_passwd_callback_userdata);
-    } else {
-        SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE_FILE, SSL_R_BAD_SSL_FILETYPE);
-        goto end;
+        cert = PEM_read_bio_X509(in, &x, ctx->default_passwd_callback,
+                                 ctx->default_passwd_callback_userdata);
     }
-
-    if (x == NULL) {
+    if (cert == NULL) {
         SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE_FILE, j);
         goto end;
     }
@@ -434,8 +449,14 @@ int SSL_CTX_use_certificate_ASN1(SSL_CTX *ctx, int len, const unsigned char *d)
     X509 *x;
     int ret;
 
-    x = d2i_X509(NULL, &d, (long)len);
+    x = X509_new_with_libctx(ctx->libctx, ctx->propq);
     if (x == NULL) {
+        SSLerr(0, ERR_R_MALLOC_FAILURE);
+        return 0;
+    }
+
+    if (d2i_X509(&x, &d, (long)len) == NULL) {
+        X509_free(x);
         SSLerr(SSL_F_SSL_CTX_USE_CERTIFICATE_ASN1, ERR_R_ASN1_LIB);
         return 0;
     }
@@ -610,6 +631,7 @@ static int use_certificate_chain_file(SSL_CTX *ctx, SSL *ssl, const char *file)
     X509 *x = NULL;
     pem_password_cb *passwd_callback;
     void *passwd_callback_userdata;
+    SSL_CTX *real_ctx = (ssl == NULL) ? ctx : ssl->ctx;
 
     ERR_clear_error();          /* clear error stack for
                                  * SSL_CTX_use_certificate() */
@@ -633,9 +655,13 @@ static int use_certificate_chain_file(SSL_CTX *ctx, SSL *ssl, const char *file)
         goto end;
     }
 
-    x = PEM_read_bio_X509_AUX(in, NULL, passwd_callback,
-                              passwd_callback_userdata);
+    x = X509_new_with_libctx(real_ctx->libctx, real_ctx->propq);
     if (x == NULL) {
+        SSLerr(SSL_F_USE_CERTIFICATE_CHAIN_FILE, ERR_R_MALLOC_FAILURE);
+        goto end;
+    }
+    if (PEM_read_bio_X509_AUX(in, &x, passwd_callback,
+                              passwd_callback_userdata) == NULL) {
         SSLerr(SSL_F_USE_CERTIFICATE_CHAIN_FILE, ERR_R_PEM_LIB);
         goto end;
     }
@@ -667,23 +693,32 @@ static int use_certificate_chain_file(SSL_CTX *ctx, SSL *ssl, const char *file)
             goto end;
         }
 
-        while ((ca = PEM_read_bio_X509(in, NULL, passwd_callback,
-                                       passwd_callback_userdata))
-               != NULL) {
-            if (ctx)
-                r = SSL_CTX_add0_chain_cert(ctx, ca);
-            else
-                r = SSL_add0_chain_cert(ssl, ca);
-            /*
-             * Note that we must not free ca if it was successfully added to
-             * the chain (while we must free the main certificate, since its
-             * reference count is increased by SSL_CTX_use_certificate).
-             */
-            if (!r) {
-                X509_free(ca);
-                ret = 0;
+        while (1) {
+            ca = X509_new_with_libctx(real_ctx->libctx, real_ctx->propq);
+            if (ca == NULL) {
+                SSLerr(SSL_F_USE_CERTIFICATE_CHAIN_FILE, ERR_R_MALLOC_FAILURE);
                 goto end;
             }
+            if (PEM_read_bio_X509(in, &ca, passwd_callback,
+                                  passwd_callback_userdata) != NULL) {
+                if (ctx)
+                    r = SSL_CTX_add0_chain_cert(ctx, ca);
+                else
+                    r = SSL_add0_chain_cert(ssl, ca);
+                /*
+                 * Note that we must not free ca if it was successfully added to
+                 * the chain (while we must free the main certificate, since its
+                 * reference count is increased by SSL_CTX_use_certificate).
+                 */
+                if (!r) {
+                    X509_free(ca);
+                    ret = 0;
+                    goto end;
+                }
+            } else {
+                X509_free(ca);
+                break;
+            }
         }
         /* When the while loop ends, it's usually just EOF. */
         err = ERR_peek_last_error();
@@ -1063,15 +1098,9 @@ static int ssl_set_cert_and_key(SSL *ssl, SSL_CTX *ctx, X509 *x509, EVP_PKEY *pr
     int j;
     int rv;
     CERT *c = ssl != NULL ? ssl->cert : ctx->cert;
-    SSL_CTX *actualctx = ssl == NULL ? ctx : ssl->ctx;
     STACK_OF(X509) *dup_chain = NULL;
     EVP_PKEY *pubkey = NULL;
 
-    if (!X509v3_cache_extensions(x509, actualctx->libctx, actualctx->propq)) {
-        SSLerr(0, ERR_R_X509_LIB);
-        goto out;
-    }
-
     /* Do all security checks before anything else */
     rv = ssl_security_cert(ssl, ctx, x509, 0, 1);
     if (rv != 1) {