HTTP client: Minimal changes that include the improved API

This is a minimal version of pull request #15053 including all the
proposed improvements to the HTTP client API and its documentation
but only those code adaptations strictly needed for it.

The proposed new features include
* support for persistent connections (keep-alive),
* generalization to arbitrary request and response types, and
* support for streaming BIOs for request and response data.

The related API changes include:
* Split the monolithic OSSL_HTTP_transfer() into OSSL_HTTP_open(),
  OSSL_HTTP_set_request(), a lean OSSL_HTTP_transfer(), and OSSL_HTTP_close().
* Split the timeout functionality accordingly and improve default behavior.
* Extract part of OSSL_HTTP_REQ_CTX_new() to OSSL_HTTP_REQ_CTX_set_expected().

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15147)

1 files changed, 153 insertions, 119 deletions

diff --git a/doc/man3/OSSL_HTTP_transfer.pod b/doc/man3/OSSL_HTTP_transfer.pod
index d2ff8eeebc..da84789472 100644
--- a/doc/man3/OSSL_HTTP_transfer.pod
+++ b/doc/man3/OSSL_HTTP_transfer.pod
@@ -2,13 +2,15 @@
 
 =head1 NAME
 
+OSSL_HTTP_open,
+OSSL_HTTP_bio_cb_t,
+OSSL_HTTP_proxy_connect,
+OSSL_HTTP_set_request,
+OSSL_HTTP_exchange,
 OSSL_HTTP_get,
-OSSL_HTTP_get_asn1,
-OSSL_HTTP_post_asn1,
 OSSL_HTTP_transfer,
-OSSL_HTTP_bio_cb_t,
-OSSL_HTTP_proxy_connect
-- http client functions
+OSSL_HTTP_close
+-  HTTP client high-level functions
 
 =head1 SYNOPSIS
 
@@ -16,91 +18,53 @@ OSSL_HTTP_proxy_connect
 
  typedef BIO *(*OSSL_HTTP_bio_cb_t)(BIO *bio, void *arg,
                                     int connect, int detail);
+ OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port,
+                                   const char *proxy, const char *no_proxy,
+                                   int use_ssl, BIO *bio, BIO *rbio,
+                                   OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                                   int buf_size, int overall_timeout);
+ int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port,
+                             const char *proxyuser, const char *proxypass,
+                             int timeout, BIO *bio_err, const char *prog);
+ int OSSL_HTTP_set_request(OSSL_HTTP_REQ_CTX *rctx, const char *path,
+                           const STACK_OF(CONF_VALUE) *headers,
+                           const char *content_type, BIO *req,
+                           const char *expected_content_type, int expect_asn1,
+                           size_t max_resp_len, int timeout, int keep_alive);
+ BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url);
  BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
                     BIO *bio, BIO *rbio,
                     OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                    const STACK_OF(CONF_VALUE) *headers,
-                    int maxline, unsigned long max_resp_len, int timeout,
-                    const char *expected_ct, int expect_asn1);
- ASN1_VALUE *OSSL_HTTP_get_asn1(const char *url,
-                                const char *proxy, const char *no_proxy,
-                                BIO *bio, BIO *rbio,
-                                OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                                const STACK_OF(CONF_VALUE) *headers,
-                                int maxline, unsigned long max_resp_len,
-                                int timeout, const char *expected_ct,
-                                const ASN1_ITEM *rsp_it);
- ASN1_VALUE *OSSL_HTTP_post_asn1(const char *server, const char *port,
-                                 const char *path, int use_ssl,
-                                 const char *proxy, const char *no_proxy,
-                                 BIO *bio, BIO *rbio,
-                                 OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                                 const STACK_OF(CONF_VALUE) *headers,
-                                 const char *content_type,
-                                 const ASN1_VALUE *req, const ASN1_ITEM *req_it,
-                                 int maxline, unsigned long max_resp_len,
-                                 int timeout, const char *expected_ct,
-                                 const ASN1_ITEM *rsp_it);
- BIO *OSSL_HTTP_transfer(const char *server, const char *port, const char *path,
-                         int use_ssl, const char *proxy, const char *no_proxy,
+                    int buf_size, const STACK_OF(CONF_VALUE) *headers,
+                    const char *expected_content_type, int expect_asn1,
+                    size_t max_resp_len, int timeout);
+ BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
+                         const char *server, const char *port,
+                         const char *path, int use_ssl,
+                         const char *proxy, const char *no_proxy,
                          BIO *bio, BIO *rbio,
                          OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                         const STACK_OF(CONF_VALUE) *headers,
-                         const char *content_type, BIO *req_mem,
-                         int maxline, unsigned long max_resp_len, int timeout,
-                         const char *expected_ct, int expect_asn1,
-                         char **redirection_url);
- int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port,
-                             const char *proxyuser, const char *proxypass,
-                             int timeout, BIO *bio_err, const char *prog);
+                         int buf_size, const STACK_OF(CONF_VALUE) *headers,
+                         const char *content_type, BIO *req,
+                         const char *expected_content_type, int expect_asn1,
+                         size_t max_resp_len, int timeout, int keep_alive);
+ int OSSL_HTTP_close(OSSL_HTTP_REQ_CTX *rctx, int ok);
 
 =head1 DESCRIPTION
 
-OSSL_HTTP_get() uses HTTP GET to obtain data (of any type) from the given I<url>
-and returns it as a memory BIO.
-If the schema component of the I<url> is C<https> a TLS connection is requested
-and the I<bio_update_fn> parameter, described below, must be provided.
-Any userinfo and fragment components in the I<url> are ignored.
-Any query component is handled as part of the path component.
-
-OSSL_HTTP_get_asn1() is like OSSL_HTTP_get() but in addition
-parses the received contents (e.g., an X.509 certificate)
-as an ASN.1 DER encoded value with the expected structure specified by I<rsp_it>
-and returns it on success as a pointer to I<ASN1_VALUE>.
-
-OSSL_HTTP_post_asn1() is like OSSL_HTTP_get_asn1() but uses the HTTP POST method
-to send a request I<req> with the ASN.1 structure defined in I<req_it> and the
-given I<content_type> to the given I<server> and optional I<port> and I<path>.
-If I<use_ssl> is nonzero a TLS connection is requested and the I<bio_update_fn>
-parameter, described below, must be provided.
+OSSL_HTTP_open() initiates an HTTP session using the I<bio> argument if not
+NULL, else by connecting to a given I<server> optionally via a I<proxy>.
 
-OSSL_HTTP_transfer() exchanges any form of HTTP request and response.
-It implements the core of the functions described above.
-If I<path> parameter is NULL it defaults to "/".
-If I<use_ssl> is nonzero a TLS connection is requested
-and the I<bio_update_fn> parameter, described below, must be provided.
-If I<req_mem> is NULL it uses the HTTP GET method, else it uses HTTP POST to
-send a request with the contents of the memory BIO and optional I<content_type>.
-The optional list I<headers> may contain additional custom HTTP header lines.
-If I<req_mem> is NULL (i.e., the HTTP method is GET) and I<redirection_url>
-is not NULL the latter pointer is used to provide any new location that
-the server may return with HTTP code 301 (MOVED_PERMANENTLY) or 302 (FOUND).
-In this case the caller is responsible for deallocating this URL with
-L<OPENSSL_free(3)>.
-
-The above functions have the following parameters in common.
-
-Typically the OpenSSL build supports sockets
-and the I<bio> and I<rbio> parameters are both NULL.
-In this case the client creates a network BIO internally
-for connecting to the given I<server>
-at the specified I<port> (if any, defaulting to 80 for HTTP or 443 for HTTPS),
-optionally via a I<proxy> (respecting I<no_proxy>) as described below.
-Then the client uses this internal BIO for exchanging the request and response.
-If I<bio> is given and I<rbio> is NULL then the client uses this I<bio> instead.
+Typically the OpenSSL build supports sockets and the I<bio> parameter is NULL.
+In this case I<rbio> must be NULL as well, and the
+library creates a network BIO internally for connecting to the given I<server>
+at the specified I<port> if any, defaulting to 80 for HTTP or 443 for HTTPS.
+Then this internal BIO is used for setting up a connection
+and for exchanging one or more request and response.
+If I<bio> is given and I<rbio> is NULL then this I<bio> is used instead.
 If both I<bio> and I<rbio> are given (which may be memory BIOs for instance)
-then no explicit connection is attempted,
-I<bio> is used for writing the request, and I<rbio> for reading the response.
+then no explicit connection is set up, but
+I<bio> is used for writing requests and I<rbio> for reading responses.
 As soon as the client has flushed I<bio> the server must be ready to provide
 a response or indicate a waiting condition via I<rbio>.
 
@@ -121,33 +85,12 @@ Proxying plain HTTP is supported directly,
 while using a proxy for HTTPS connections requires a suitable callback function
 such as OSSL_HTTP_proxy_connect(), described below.
 
-The I<maxline> parameter specifies the response header maximum line length,
-where a value <= 0 indicates that the B<HTTP_DEFAULT_MAX_LINE_LENGTH> of 4KiB
-should be used.
-This length is also used as the number of content bytes that are read at a time.
-The I<max_resp_len> parameter specifies the maximum response length,
-where 0 indicates B<HTTP_DEFAULT_MAX_RESP_LEN>, which currently is 100 KiB.
-
-An ASN.1-encoded response is expected by OSSL_HTTP_get_asn1() and
-OSSL_HTTP_post_asn1(), while for OSSL_HTTP_get() or OSSL_HTTP_transfer()
-this is only the case if the I<expect_asn1> parameter is nonzero.
-If the response header contains one or more "Content-Length" header lines and/or
-an ASN.1-encoded response is expected, which should include a total length,
-the length indications received are checked for consistency
-and for not exceeding the maximum response length.
-
-If the parameter I<expected_ct>
-is not NULL then the HTTP client checks that the given content type string
-is included in the HTTP header of the response and returns an error if not.
-
-If the I<timeout> parameter is > 0 this indicates the maximum number of seconds
-to wait until the transfer is complete.
-A value of 0 enables waiting indefinitely,
-while a value < 0 immediately leads to a timeout condition.
+If I<use_ssl> is nonzero a TLS connection is requested
+and the I<bio_update_fn> parameter must be provided.
 
-The optional parameter I<bio_update_fn> with its optional argument I<arg> may
-be used to modify the connection BIO used by the HTTP client (and cannot be
-used when both I<bio> and I<rbio> are given).
+The parameter I<bio_update_fn>, which is optional if I<use_ssl> is 0,
+may be used to modify the connection BIO used by the HTTP client,
+but cannot be used when both I<bio> and I<rbio> are given.
 I<bio_update_fn> is a BIO connect/disconnect callback function with prototype
 
  BIO *(*OSSL_HTTP_bio_cb_t)(BIO *bio, void *arg, int connect, int detail)
@@ -157,7 +100,7 @@ whereby it may make use of a custom defined argument I<arg>,
 which may for instance refer to an I<SSL_CTX> structure.
 During connection establishment, just after calling BIO_do_connect_retry(),
 the function is invoked with the I<connect> argument being 1 and the I<detail>
-argument being 1 if HTTPS is requested, i.e., SSL/TLS should be enabled.
+argument being 1 if HTTPS is requested, i.e., SSL/TLS should be enabled, else 0.
 On disconnect I<connect> is 0 and I<detail> is 1 if no error occurred, else 0.
 For instance, on connect the function may prepend a TLS BIO to implement HTTPS;
 after disconnect it may do some diagnostic output and/or specific cleanup.
@@ -166,10 +109,10 @@ Here is a simple example that supports TLS connections (but not via a proxy):
 
  BIO *http_tls_cb(BIO *hbio, void *arg, int connect, int detail)
  {
-     SSL_CTX *ctx = (SSL_CTX *)arg;
-
      if (connect && detail) { /* connecting with TLS */
+         SSL_CTX *ctx = (SSL_CTX *)arg;
          BIO *sbio = BIO_new_ssl(ctx, 1);
+
          hbio = sbio != NULL ? BIO_push(sbio, hbio) : NULL;
      } else if (!connect && !detail) { /* disconnecting after error */
          /* optionally add diagnostics here */
@@ -179,6 +122,16 @@ Here is a simple example that supports TLS connections (but not via a proxy):
 
 After disconnect the modified BIO will be deallocated using BIO_free_all().
 
+The I<buf_size> parameter specifies the response header maximum line length.
+A value <= 0 indicates that
+the B<HTTP_DEFAULT_MAX_LINE_LENGTH> of 4KiB should be used.
+I<buf_size> is also used as the number of content bytes that are read at a time.
+
+If the I<overall_timeout> parameter is > 0 this indicates the maximum number of
+seconds the overall HTTP transfer (i.e., connection setup if needed,
+sending requests, and receiving responses) is allowed to take until completion.
+A value <= 0 enables waiting indefinitely, i.e., no timeout.
+
 OSSL_HTTP_proxy_connect() may be used by an above BIO connect callback function
 to set up an SSL/TLS connection via an HTTPS proxy.
 It promotes the given BIO I<bio> representing a connection
@@ -186,11 +139,86 @@ pre-established with a TLS proxy using the HTTP CONNECT method,
 optionally using proxy client credentials I<proxyuser> and I<proxypass>,
 to connect with TLS protection ultimately to I<server> and I<port>.
 If the I<port> argument is NULL or the empty string it defaults to "443".
-The I<timeout> parameter is used as described above.
+If the I<timeout> parameter is > 0 this indicates the maximum number of
+seconds the connection setup is allowed to take.
+A value <= 0 enables waiting indefinitely, i.e., no timeout.
 Since this function is typically called by applications such as
 L<openssl-s_client(1)> it uses the I<bio_err> and I<prog> parameters (unless
 NULL) to print additional diagnostic information in a user-oriented way.
 
+OSSL_HTTP_set_request() sets up in I<rctx> the request header and content data
+and expectations on the response using the following parameters.
+If I<path> is NULL it defaults to "/".
+If I<req> is NULL the HTTP GET method will be used to send the request
+else HTTP POST with the contents of I<req> and optional I<content_type>, where
+the length of the data in I<req> does not need to be determined in advance: the
+BIO will be read on-the-fly while sending the request, which supports streaming.
+The optional list I<headers> may contain additional custom HTTP header lines.
+If the parameter I<expected_content_type>
+is not NULL then the client will check that the given content type string
+is included in the HTTP header of the response and return an error if not.
+If the I<expect_asn1> parameter is nonzero,
+a structure in ASN.1 encoding will be expected as response content.
+The I<max_resp_len> parameter specifies the maximum allowed
+response content length, where the value 0 indicates no limit.
+If the I<timeout> parameter is > 0 this indicates the maximum number of seconds
+the subsequent HTTP transfer (sending the request and receiving a response)
+is allowed to take.
+A value of 0 enables waiting indefinitely, i.e., no timeout.
+A value < 0 indicates that the I<overall_timeout> parameter value given
+when opening the HTTP transfer will be used instead.
+If I<keep_alive> is 0 the connection is not kept open
+after receiving a response, which is the default behavior for HTTP 1.0.
+If the value is 1 or 2 then a persistent connection is requested.
+If the value is 2 then a persistent connection is required,
+i.e., an error occurs in case the server does not grant it.
+
+OSSL_HTTP_exchange() exchanges any form of HTTP request and response
+as specified by I<rctx>, which must include both connection and request data,
+typically set up using OSSL_HTTP_open() and OSSL_HTTP_set_request().
+It implements the core of the functions described below.
+If the HTTP method is GET and I<redirection_url>
+is not NULL the latter pointer is used to provide any new location that
+the server may return with HTTP code 301 (MOVED_PERMANENTLY) or 302 (FOUND).
+In this case the function returns NULL and the caller is
+responsible for deallocating the URL with L<OPENSSL_free(3)>.
+If the response header contains one or more "Content-Length" header lines and/or
+an ASN.1-encoded response is expected, which should include a total length,
+the length indications received are checked for consistency
+and for not exceeding any given maximum response length.
+On receiving a response, the function returns the contents as a memory BIO,
+which does not support streaming, in case an ASN.1-encoded response is expected.
+Else it returns directly the read BIO that holds the response contents,
+which allows a response of indefinite length and may support streaming.
+
+OSSL_HTTP_get() uses HTTP GET to obtain data from I<bio> if non-NULL,
+else from the server contained in the I<url>, and returns it as a BIO.
+It supports redirection via HTTP status code 301 or 302.  It is meant for
+transfers with a single round trip, so does not support persistent connections.
+If I<bio> is non-NULL, any host and port components in the I<url> are not used
+for connecting but the hostname is used, as usual, for the C<Host> header.
+Any userinfo and fragment components in the I<url> are ignored.
+Any query component is handled as part of the path component.
+If the scheme component of the I<url> is C<https> a TLS connection is requested
+and the I<bio_update_fn>, as described for OSSL_HTTP_open(), must be provided.
+Also the remaining parameters are interpreted as described for OSSL_HTTP_open()
+and OSSL_HTTP_set_request(), respectively.
+
+OSSL_HTTP_transfer() exchanges an HTTP request and response
+over a connection managed via I<prctx> without supporting redirection.
+It combines OSSL_HTTP_open(), OSSL_HTTP_set_request(), OSSL_HTTP_exchange(),
+and OSSL_HTTP_close().
+If I<prctx> is not NULL it reuses any open connection represented by a non-NULL
+I<*prctx>.  It keeps the connection open if a persistent connection is requested
+or required and this was granted by the server, else it closes the connection
+and assigns NULL to I<*prctx>.
+The remaining parameters are interpreted as described for OSSL_HTTP_open()
+and OSSL_HTTP_set_request(), respectively.
+
+OSSL_HTTP_close() closes the connection and releases I<rctx>.
+The I<ok> parameter is passed to any BIO update function
+given during setup as described above for OSSL_HTTP_open().
+
 =head1 NOTES
 
 The names of the environment variables used by this implementation:
@@ -200,23 +228,29 @@ other HTTP client implementations such as wget, curl, and git.
 
 =head1 RETURN VALUES
 
-On success, OSSL_HTTP_get(), OSSL_HTTP_get_asn1(), OSSL_HTTP_post_asn1(), and
-OSSL_HTTP_transfer() return a memory BIO containing the data received via HTTP.
-This must be freed by the caller. On failure, NULL is returned.
+OSSL_HTTP_open() returns on success a B<OSSL_HTTP_REQ_CTX>, else NULL.
+
+OSSL_HTTP_proxy_connect() and OSSL_HTTP_set_request()
+return 1 on success, 0 on error.
+
+On success, OSSL_HTTP_exchange(), OSSL_HTTP_get(), and OSSL_HTTP_transfer()
+return a memory BIO containing the data received if an ASN.1-encoded response
+is expected, else a BIO that may support streaming.
+The BIO must be freed by the caller.
+On failure, they return NULL.
 Failure conditions include connection/transfer timeout, parse errors, etc.
 
-OSSL_HTTP_proxy_connect() returns 1 on success, 0 on error.
+OSSL_HTTP_close() returns 0 if anything went wrong while disconnecting, else 1.
 
 =head1 SEE ALSO
 
-L<OSSL_HTTP_parse_url(3)>
-L<BIO_set_conn_port(3)>
+L<OSSL_HTTP_parse_url(3)>, L<BIO_set_conn_port(3)>
+L<ASN1_item_i2d_mem_bio(3)>, L<ASN1_item_d2i_bio(3)>,
+L<OSSL_HTTP_is_alive(3)>
 
 =head1 HISTORY
 
-OSSL_HTTP_get(), OSSL_HTTP_get_asn1(), OSSL_HTTP_post_asn1(),
-OSSL_HTTP_transfer(), and OSSL_HTTP_proxy_connect()
-were added in OpenSSL 3.0.
+All the functions described here were added in OpenSSL 3.0.
 
 =head1 COPYRIGHT