Generalize the HTTP client so far implemented mostly in crypto/ocsp/ocsp_ht.c

The new client has become an independent libcrpyto module in crypto/http/ and
* can handle any types of requests and responses (ASN.1-encoded and plain)
* does not include potentially busy loops when waiting for responses but
* makes use of a new timeout mechanism integrated with socket-based BIO
* supports the use of HTTP proxies and TLS, including HTTPS over proxies
* supports HTTP redirection via codes 301 and 302 for GET requests
* returns more useful diagnostics in various error situations
Also adapts - and strongly simplifies - hitherto uses of HTTP in crypto/ocsp/,
crypto/x509/x_all.c, apps/lib/apps.c, and apps/{ocsp,s_client,s_server}.c

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: David von Oheimb <david.von.oheimb@siemens.com>
(Merged from https://github.com/openssl/openssl/pull/10667)

1 files changed, 210 insertions, 0 deletions

diff --git a/doc/man3/OSSL_HTTP_transfer.pod b/doc/man3/OSSL_HTTP_transfer.pod
new file mode 100644
index 0000000000..68010cb6bd
--- /dev/null
+++ b/doc/man3/OSSL_HTTP_transfer.pod
@@ -0,0 +1,210 @@
+=pod
+
+=head1 NAME
+
+OSSL_HTTP_get,
+OSSL_HTTP_get_asn1,
+OSSL_HTTP_post_asn1,
+OSSL_HTTP_transfer,
+OSSL_HTTP_bio_cb_t,
+OSSL_HTTP_proxy_connect,
+OSSL_HTTP_parse_url
+- http client functions
+
+=head1 SYNOPSIS
+
+ #include <openssl/http.h>
+
+ typedef BIO *(*OSSL_HTTP_bio_cb_t)(BIO *bio, void *arg,
+                                    int connect, int detail);
+ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *proxy_port,
+                    BIO *bio, BIO *rbio,
+                    OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                    const STACK_OF(CONF_VALUE) *headers,
+                    int maxline, unsigned long max_resp_len, int timeout,
+                    const char *expected_content_type, int expect_asn1);
+ ASN1_VALUE *OSSL_HTTP_get_asn1(const char *url,
+                                const char *proxy, const char *proxy_port,
+                                BIO *bio, BIO *rbio,
+                                OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                                const STACK_OF(CONF_VALUE) *headers,
+                                int maxline, unsigned long max_resp_len,
+                                int timeout, const char *expected_content_type,
+                                const ASN1_ITEM *it);
+ ASN1_VALUE *OSSL_HTTP_post_asn1(const char *server, const char *port,
+                                 const char *path, int use_ssl,
+                                 const char *proxy, const char *proxy_port,
+                                 BIO *bio, BIO *rbio,
+                                 OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                                 const STACK_OF(CONF_VALUE) *headers,
+                                 const char *content_type,
+                                 ASN1_VALUE *req, const ASN1_ITEM *req_it,
+                                 int maxline, unsigned long max_resp_len,
+                                 int timeout, const char *expected_ct,
+                                 const ASN1_ITEM *rsp_it);
+ BIO *OSSL_HTTP_transfer(const char *server, const char *port, const char *path,
+                         int use_ssl, const char *proxy, const char *proxy_port,
+                         BIO *bio, BIO *rbio,
+                         OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                         const STACK_OF(CONF_VALUE) *headers,
+                         const char *content_type, BIO *req_mem,
+                         int maxline, unsigned long max_resp_len, int timeout,
+                         const char *expected_ct, int expect_asn1,
+                         char **redirection_url);
+ int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port,
+                             const char *proxyuser, const char *proxypass,
+                             int timeout, BIO *bio_err, const char *prog);
+ int OSSL_HTTP_parse_url(const char *url, char **phost, char **pport,
+                         char **ppath, int *pssl);
+
+=head1 DESCRIPTION
+
+OSSL_HTTP_get() uses HTTP GET to obtain data (of any type) from the given B<url>
+and returns it as a memory BIO.
+
+OSSL_HTTP_get_asn1() uses HTTP GET to obtain an ASN.1-encoded value
+(e.g., an X.509 certificate) with the expected structure specified by B<it>
+(e.g., I<ASN1_ITEM_rptr(X509)>) from the given B<url>
+and returns it on success as a pointer to I<ASN1_VALUE>.
+
+OSSL_HTTP_post_asn1() uses the HTTP POST method to send a request B<req>
+with the ASN.1 structure defined in B<req_it> and the given B<content_type> to
+the given B<server> and optional B<port> and B<path>, which defaults to "/".
+If B<use_ssl> is nonzero a TLS connection is requested and the B<bio_update_fn>
+parameter, described below, must be provided.
+The optional list B<headers> may contain additional custom HTTP header lines.
+The expected structure of the response is specified by B<rsp_it>.
+On success it returns the response as a pointer to B<ASN1_VALUE>.
+
+OSSL_HTTP_transfer() exchanges an HTTP request and response with
+the given B<server> and optional B<port> and B<path>, which defaults to "/".
+If B<use_ssl> is nonzero a TLS connection is requested and the B<bio_update_fn>
+parameter, described below, must be provided.
+If B<req_mem> is NULL it uses the HTTP GET method, else it uses HTTP POST to
+send a request with the contents of the memory BIO and optional B<content_type>.
+The optional list B<headers> may contain additional custom HTTP header lines.
+If B<req_mem> is NULL (i.e., the HTTP method is GET) and B<redirection_url>
+is not NULL the latter pointer is used to provide any new location that
+the server may return with HTTP code 301 (MOVED_PERMANENTLY) or 302 (FOUND).
+In this case the caller is responsible for deallocating this URL with
+L<OPENSSL_free(3)>.
+
+The above functions have the following parameters in common.
+
+If the B<proxy> parameter is not NULL the HTTP client functions connect
+via the given proxy and the optionally given B<proxy_port>.
+Proxying plain HTTP is supported directly,
+while using a proxy for HTTPS connections requires a suitable callback function
+such as OSSL_HTTP_proxy_connect(), described below.
+
+Typically the B<bio> and B<rbio> parameters are NULL and the client creates a
+network BIO internally for connecting to the given server and port (optionally
+via a proxy and its port), and uses it for exchanging the request and response.
+If B<bio> is given and B<rbio> is NULL then the client uses this BIO instead.
+If both B<bio> and B<rbio> are given (which may be memory BIOs for instance)
+then no explicit connection is attempted,
+B<bio> is used for writing the request, and B<rbio> for reading the response.
+As soon as the client has flushed B<bio> the server must be ready to provide
+a response or indicate a waiting condition via B<rbio>.
+
+The B<maxline> parameter specifies the response header maximum line length,
+where 0 indicates the default value, which currently is 4k.
+The B<max_resp_len> parameter specifies the maximum response length,
+where 0 indicates the default value, which currently is 100k.
+
+An ASN.1-encoded response is expected by OSSL_HTTP_get_asn1() and
+OSSL_HTTP_post_asn1(), while for OSSL_HTTP_get() or OSSL_HTTP_transfer()
+this is only the case if the B<expect_asn1> parameter is nonzero.
+If the response header contains one or more Content-Length header lines and/or
+an ASN.1-encoded response is expected, which should include a total length,
+the length indications received are checked for consistency
+and for not exceeding the maximum response length.
+
+If the parameter B<expected_content_type> (or B<expected_ct>, respectively)
+is not NULL then the HTTP client checks that the given content type string
+is included in the HTTP header of the response and returns an error if not.
+
+If the B<timeout> parameter is > 0 this indicates the maximum number of seconds
+to wait until the transfer is complete.
+A value of 0 enables waiting indefinitely,
+while a value < 0 immediately leads to a timeout condition.
+
+The optional parameter B<bio_update_fn> with its optional argument B<arg> may
+be used to modify the connection BIO used by the HTTP client (and cannot be
+used when both B<bio> and B<rbio> are given).
+B<bio_update_fn> is a BIO connect/disconnect callback function with prototype
+
+ BIO *(*OSSL_HTTP_bio_cb_t)(BIO *bio, void *arg, int connect, int detail)
+
+The callback may modify the HTTP BIO provided in the B<bio> argument,
+whereby it may make use of a custom defined argument B<arg>,
+which may for instance refer to an I<SSL_CTX> structure.
+During connection establishment, just after calling BIO_connect_retry(),
+the function is invoked with the B<connect> argument being 1 and the B<detail>
+argument being 1 if HTTPS is requested, i.e., SSL/TLS should be enabled.
+On disconnect B<connect> is 0 and B<detail> is 1 if no error occurred, else 0.
+For instance, on connect the function may prepend a TLS BIO to implement HTTPS;
+after disconnect it may do some diagnostic output and/or specific cleanup.
+The function should return NULL to indicate failure.
+Here is a simple example that supports TLS connections (but not via a proxy):
+
+ BIO *http_tls_cb(BIO *hbio, void *arg, int connect, int detail)
+ {
+     SSL_CTX *ctx = (SSL_CTX *)arg;
+
+     if (connect && detail) { /* connecting with TLS */
+         BIO *sbio = BIO_new_ssl(ctx, 1);
+         hbio = sbio != NULL ? BIO_push(sbio, hbio) : NULL;
+     } else if (!connect && !detail) { /* disconnecting after error */
+         /* optionally add diagnostics here */
+     }
+     return hbio;
+ }
+
+After disconnect the modified BIO will be deallocated using BIO_free_all().
+
+OSSL_HTTP_proxy_connect() may be used by an above BIO connect callback function
+to set up an SSL/TLS connection via an HTTP proxy.
+It promotes the given BIO B<bio> representing a connection
+pre-established with a TLS proxy using the HTTP CONNECT method,
+optionally using proxy client credentials B<proxyuser> and B<proxypass>,
+to connect with TLS protection ultimately to B<server> and B<port>.
+The B<timeout> parameter is used as described above.
+Since this function is typically called by appplications such as
+L<openssl-s_client(1)> it uses the B<bio_err> and B<prog> parameters (unless
+NULL) to print additional diagnostic information in a user-oriented way.
+
+OSSL_HTTP_parse_url() parses its input string B<url> as a URL and splits it up
+into host, port and path components and a flag whether it begins with 'https'.
+The host component may be a DNS name or an IPv4 or an IPv6 address.
+The port component is optional and defaults to "443" for HTTPS, else "80".
+The path component is also optional and defaults to "/".
+As far as the result pointer arguments are not NULL it assigns via
+them copies of the respective string components.
+The strings returned this way must be deallocated by the caller using
+L<OPENSSL_free(3)> unless they are NULL, which is their default value on error.
+
+=head1 RETURN VALUES
+
+OSSL_HTTP_get(), OSSL_HTTP_get_asn1(), OSSL_HTTP_post_asn1(), and
+OSSL_HTTP_transfer() return on success the data received via HTTP, else NULL.
+Error conditions include connection/transfer timeout, parse errors, etc.
+
+OSSL_HTTP_proxy_connect() and OSSL_HTTP_parse_url()
+return 1 on success, 0 on error.
+
+=head1 HISTORY
+
+OSSL_HTTP_get(), OSSL_HTTP_get_asn1(), OSSL_HTTP_post_asn1(),
+OSSL_HTTP_proxy_connect(), and OSSL_HTTP_parse_url() were added in OpenSSL 3.0.
+
+=head1 COPYRIGHT
+
+Copyright 2019-2020 The OpenSSL Project Authors. All Rights Reserved.
+
+Licensed under the Apache License 2.0 (the "License").  You may not use
+this file except in compliance with the License.  You can obtain a copy
+in the file LICENSE in the source distribution or at
+L<https://www.openssl.org/source/license.html>.
+
+=cut