HTTP client API: Generalize to arbitrary request and response contents

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15053)

8 files changed, 45 insertions, 20 deletions

diff --git a/NEWS.md b/NEWS.md
index 3193ce6149..78d0772b9a 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -53,8 +53,10 @@ OpenSSL 3.0
     also covering CRMF (RFC 4211) and HTTP transfer (RFC 6712).
     It is part of the crypto lib and adds a 'cmp' app with a demo configuration.
     All widely used CMP features are supported for both clients and servers.
-  * Added a proper HTTP(S) client to libcrypto supporting GET and POST,
-    redirection, plain and ASN.1-encoded contents, proxies, and timeouts.
+  * Added a proper HTTP client supporting GET with optional redirection, POST,
+    arbitrary request and response content types, TLS, persistent connections,
+    connections via HTTP(s) proxies, connections and exchange via user-defined
+    BIOs (allowing implicit connections), and timeout checks.
   * Added util/check-format.pl for checking adherence to the coding guidelines.
   * Added OSSL_ENCODER, a generic encoder API.
   * Added OSSL_PARAM_BLD, an easier to use API to OSSL_PARAM.
diff --git a/apps/include/apps.h b/apps/include/apps.h
index 207ed41bc7..41178a6e22 100644
--- a/apps/include/apps.h
+++ b/apps/include/apps.h
@@ -285,6 +285,7 @@ ASN1_VALUE *app_http_post_asn1(const char *host, const char *port,
                                const STACK_OF(CONF_VALUE) *headers,
                                const char *content_type,
                                ASN1_VALUE *req, const ASN1_ITEM *req_it,
+                               const char *expected_content_type,
                                long timeout, const ASN1_ITEM *rsp_it);
 # endif
 
diff --git a/apps/lib/apps.c b/apps/lib/apps.c
index dafcf419bf..d32f6c5490 100644
--- a/apps/lib/apps.c
+++ b/apps/lib/apps.c
@@ -2521,6 +2521,7 @@ ASN1_VALUE *app_http_post_asn1(const char *host, const char *port,
                                const STACK_OF(CONF_VALUE) *headers,
                                const char *content_type,
                                ASN1_VALUE *req, const ASN1_ITEM *req_it,
+                               const char *expected_content_type,
                                long timeout, const ASN1_ITEM *rsp_it)
 {
     APP_HTTP_TLS_INFO info;
@@ -2538,7 +2539,7 @@ ASN1_VALUE *app_http_post_asn1(const char *host, const char *port,
                              proxy, no_proxy, NULL /* bio */, NULL /* rbio */,
                              app_http_tls_cb, &info,
                              0 /* buf_size */, headers, content_type, req_mem,
-                             NULL /* expected_ct */, 1 /* expect_asn1 */,
+                             expected_content_type, 1 /* expect_asn1 */,
                              HTTP_DEFAULT_MAX_RESP_LEN, timeout,
                              0 /* keep_alive */);
     BIO_free(req_mem);
diff --git a/apps/ocsp.c b/apps/ocsp.c
index 694855fe09..dd816c4221 100644
--- a/apps/ocsp.c
+++ b/apps/ocsp.c
@@ -1214,6 +1214,7 @@ OCSP_RESPONSE *process_responder(OCSP_REQUEST *req,
         app_http_post_asn1(host, port, path, NULL, NULL /* no proxy used */,
                            ctx, headers, "application/ocsp-request",
                            (ASN1_VALUE *)req, ASN1_ITEM_rptr(OCSP_REQUEST),
+                           "application/ocsp-response",
                            req_timeout, ASN1_ITEM_rptr(OCSP_RESPONSE));
 
     if (resp == NULL)
diff --git a/crypto/http/http_client.c b/crypto/http/http_client.c
index ee97f64ef6..077159cab6 100644
--- a/crypto/http/http_client.c
+++ b/crypto/http/http_client.c
@@ -796,6 +796,7 @@ static BIO *HTTP_new_bio(const char *server /* optionally includes ":port" */,
 }
 #endif /* OPENSSL_NO_SOCK */
 
+/* Exchange request and response via HTTP on (non-)blocking BIO */
 BIO *OSSL_HTTP_REQ_CTX_exchange(OSSL_HTTP_REQ_CTX *rctx)
 {
     int rv;
@@ -856,6 +857,10 @@ OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port,
 
     if (bio != NULL) {
         cbio = bio;
+        if (proxy != NULL || no_proxy != NULL) {
+            ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_INVALID_ARGUMENT);
+            return NULL;
+        }
     } else {
 #ifndef OPENSSL_NO_SOCK
         char *proxy_host = NULL, *proxy_port = NULL;
@@ -1064,7 +1069,7 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
                                        NULL /* content_type */,
                                        NULL /* req_mem */,
                                        expected_ct, expect_asn1,
-                                       -1 /* use same max time */,
+                                       -1 /* use same max time (timeout) */,
                                        0 /* no keep_alive */))
                 OSSL_HTTP_REQ_CTX_free(rctx);
             else
@@ -1100,6 +1105,7 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
     return resp;
 }
 
+/* Exchange request and response over a connection managed via |prctx| */
 BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
                         const char *server, const char *port,
                         const char *path, int use_ssl,
@@ -1122,11 +1128,14 @@ BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
     }
     if (rctx != NULL) {
         if (OSSL_HTTP_set_request(rctx, path, headers, content_type, req,
-                                  expected_ct, expect_asn1, timeout, keep_alive))
+                                  expected_ct, expect_asn1,
+                                  timeout, keep_alive))
             resp = OSSL_HTTP_exchange(rctx, NULL);
         if (resp == NULL || !OSSL_HTTP_is_alive(rctx)) {
-            if (!OSSL_HTTP_close(rctx, resp != NULL))
+            if (!OSSL_HTTP_close(rctx, resp != NULL)) {
+                BIO_free(resp);
                 resp = NULL;
+            }
             rctx = NULL;
         }
     }
diff --git a/doc/man3/OSSL_HTTP_REQ_CTX.pod b/doc/man3/OSSL_HTTP_REQ_CTX.pod
index 52e5d441f3..f5f70584df 100644
--- a/doc/man3/OSSL_HTTP_REQ_CTX.pod
+++ b/doc/man3/OSSL_HTTP_REQ_CTX.pod
@@ -123,6 +123,7 @@ and to gather the response via HTTP, using the I<wbio> and I<rbio>
 that were given when calling OSSL_HTTP_REQ_CTX_new().
 If successful, the contents of the internal memory B<BIO> contains
 the contents of the HTTP response, without the response headers.
+This does not support streaming.
 The function may need to be called again if its result is -1, which indicates
 L<BIO_should_retry(3)>.  In such a case it is advisable to sleep a little in
 between, using L<BIO_wait(3)> on the read BIO to prevent a busy loop.
@@ -221,7 +222,7 @@ OSSL_HTTP_REQ_CTX_nbio() and OSSL_HTTP_REQ_CTX_nbio_d2i()
 return 1 for success, 0 on error or redirection, -1 if retry is needed.
 
 OSSL_HTTP_REQ_CTX_exchange() and OSSL_HTTP_REQ_CTX_get0_mem_bio()
-returns a pointer to a B<BIO> on success and NULL on failure.
+return a pointer to a B<BIO> on success and NULL on failure.
 
 OSSL_HTTP_REQ_CTX_get_resp_len() returns the size of the response contents
 or 0 if not available or an error occurred.
diff --git a/doc/man3/OSSL_HTTP_transfer.pod b/doc/man3/OSSL_HTTP_transfer.pod
index 9745932e37..71294913e5 100644
--- a/doc/man3/OSSL_HTTP_transfer.pod
+++ b/doc/man3/OSSL_HTTP_transfer.pod
@@ -30,7 +30,7 @@ OSSL_HTTP_close
                            const STACK_OF(CONF_VALUE) *headers,
                            const char *content_type, BIO *req,
                            const char *expected_content_type, int expect_asn1,
-                           size_t max_resp_len, int timeout, int keep_alive);
+                           int timeout, int keep_alive);
  BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url);
  BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
                     BIO *bio, BIO *rbio,
@@ -52,8 +52,8 @@ OSSL_HTTP_close
 
 =head1 DESCRIPTION
 
-OSSL_HTTP_open() initiates an HTTP session using I<bio> if not NULL, else
-by connecting to a given I<server> optionally via a I<proxy>.
+OSSL_HTTP_open() initiates an HTTP session using the I<bio> argument if not
+NULL, else by connecting to a given I<server> optionally via a I<proxy>.
 
 Typically the OpenSSL build supports sockets and the I<bio> parameter is NULL.
 In this case I<rbio> must be NULL as well, and the
@@ -148,12 +148,12 @@ Since this function is typically called by applications such as
 L<openssl-s_client(1)> it uses the I<bio_err> and I<prog> parameters (unless
 NULL) to print additional diagnostic information in a user-oriented way.
 
-
 OSSL_HTTP_set_request() sets up in I<rctx> the request header and content data
 and expectations on the response using the following parameters.
 If I<path> is NULL it defaults to "/".
-If I<req_mem> is NULL the HTTP GET method will be used to send the request
-else HTTP POST with the contents of I<req_mem> and optional I<content_type>.
+If I<req> is NULL the HTTP GET method will be used to send the request
+else HTTP POST with the contents of I<req> and optional I<content_type>, where
+I<req> must contain data with determined length; streaming is not supported.
 The optional list I<headers> may contain additional custom HTTP header lines.
 If the parameter I<expected_content_type>
 is not NULL then the client will check that the given content type string
@@ -172,15 +172,15 @@ If the value is 1 or 2 then a persistent connection is requested.
 If the value is 2 then a persistent connection is required,
 i.e., an error occurs in case the server does not grant it.
 
-OSSL_HTTP_transfer() exchanges any form of HTTP request and response
+OSSL_HTTP_exchange() exchanges any form of HTTP request and response
 as specified by I<rctx>, which must include both connection and request data,
 typically set up using OSSL_HTTP_open() and OSSL_HTTP_set_request().
 It implements the core of the functions described below.
 If the HTTP method is GET and I<redirection_url>
 is not NULL the latter pointer is used to provide any new location that
 the server may return with HTTP code 301 (MOVED_PERMANENTLY) or 302 (FOUND).
-In this case the caller is responsible for deallocating this URL with
-L<OPENSSL_free(3)>.
+In this case the function returns NULL and the caller is
+responsible for deallocating the URL with L<OPENSSL_free(3)>.
 If the response header contains one or more "Content-Length" header lines and/or
 an ASN.1-encoded response is expected, which should include a total length,
 the length indications received are checked for consistency
@@ -203,6 +203,17 @@ and the I<bio_update_fn>, as described for OSSL_HTTP_open(), must be provided.
 Also the remaining parameters are interpreted as described for OSSL_HTTP_open()
 and OSSL_HTTP_set_request(), respectively.
 
+OSSL_HTTP_transfer() exchanges an HTTP request and response
+over a connection managed via I<prctx> without supporting redirection.
+It combines OSSL_HTTP_open(), OSSL_HTTP_set_request(), OSSL_HTTP_exchange(),
+and OSSL_HTTP_close().
+If I<prctx> is not NULL it reuses any open connection represented by a non-NULL
+I<*prctx>.  It keeps the connection open if a persistent connection is requested
+or required and this was granted by the server, else it closes the connection
+and assigns NULL to I<*prctx>.
+The remaining parameters are interpreted as described for OSSL_HTTP_open()
+and OSSL_HTTP_set_request(), respectively.
+
 OSSL_HTTP_close() closes the connection and releases I<rctx>.
 The I<ok> parameter is passed to any BIO update function
 given during setup as described above for OSSL_HTTP_open().
@@ -222,9 +233,8 @@ OSSL_HTTP_proxy_connect() and OSSL_HTTP_set_request()
 return 1 on success, 0 on error.
 
 On success, OSSL_HTTP_exchange(), OSSL_HTTP_get(), and OSSL_HTTP_transfer()
-return a memory BIO containing the data received if an ASN.1-encoded response
-is expected, else a BIO that may support streaming.
-The BIO must be freed by the caller.
+return a memory BIO containing the data received.
+This must be freed by the caller.
 On failure, they return NULL.
 Failure conditions include connection/transfer timeout, parse errors, etc.
 
diff --git a/include/openssl/http.h b/include/openssl/http.h
index e3bd9d7579..7552b2f42e 100644
--- a/include/openssl/http.h
+++ b/include/openssl/http.h
@@ -72,7 +72,7 @@ int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port,
                             int timeout, BIO *bio_err, const char *prog);
 int OSSL_HTTP_set_request(OSSL_HTTP_REQ_CTX *rctx, const char *path,
                           const STACK_OF(CONF_VALUE) *headers,
-                          const char *content_type, BIO *req_mem,
+                          const char *content_type, BIO *req,
                           const char *expected_content_type, int expect_asn1,
                           int timeout, int keep_alive);
 BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url);