HTTP client: Minimal changes that include the improved API

This is a minimal version of pull request #15053 including all the
proposed improvements to the HTTP client API and its documentation
but only those code adaptations strictly needed for it.

The proposed new features include
* support for persistent connections (keep-alive),
* generalization to arbitrary request and response types, and
* support for streaming BIOs for request and response data.

The related API changes include:
* Split the monolithic OSSL_HTTP_transfer() into OSSL_HTTP_open(),
  OSSL_HTTP_set_request(), a lean OSSL_HTTP_transfer(), and OSSL_HTTP_close().
* Split the timeout functionality accordingly and improve default behavior.
* Extract part of OSSL_HTTP_REQ_CTX_new() to OSSL_HTTP_REQ_CTX_set_expected().

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15147)

23 files changed, 600 insertions, 488 deletions

diff --git a/CHANGES.md b/CHANGES.md
index 80a7bc7075..e4e33e4e88 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -358,18 +358,20 @@ OpenSSL 3.0
  * Deprecated the type OCSP_REQ_CTX and the functions OCSP_REQ_CTX_new(),
    OCSP_REQ_CTX_free(), OCSP_REQ_CTX_http(), OCSP_REQ_CTX_add1_header(),
    OCSP_REQ_CTX_i2d() and its special form OCSP_REQ_CTX_set1_req(),
-   OCSP_REQ_CTX_nbio(), OCSP_REQ_CTX_nbio_d2i(),
+   OCSP_REQ_CTX_nbio(),
+   OCSP_REQ_CTX_nbio_d2i() and its special form OCSP_sendreq_nbio(),
    OCSP_REQ_CTX_get0_mem_bio() and OCSP_set_max_response_length().  These
    were used to collect all necessary data to form a HTTP request, and to
    perform the HTTP transfer with that request.  With OpenSSL 3.0, the
    type is OSSL_HTTP_REQ_CTX, and the deprecated functions are replaced
    with OSSL_HTTP_REQ_CTX_new(), OSSL_HTTP_REQ_CTX_free(),
    OSSL_HTTP_REQ_CTX_set_request_line(), OSSL_HTTP_REQ_CTX_add1_header(),
-   OSSL_HTTP_REQ_CTX_i2d(), OSSL_HTTP_REQ_CTX_nbio(),
-   OSSL_HTTP_REQ_CTX_sendreq_d2i(), OSSL_HTTP_REQ_CTX_get0_mem_bio() and
+   OSSL_HTTP_REQ_CTX_set1_req(),
+   OSSL_HTTP_REQ_CTX_nbio(), OSSL_HTTP_REQ_CTX_nbio_d2i(),
+   OSSL_HTTP_REQ_CTX_get0_mem_bio(), and
    OSSL_HTTP_REQ_CTX_set_max_response_length().
 
-   *Rich Salz and Richard Levitte*
+   *Rich Salz, Richard Levitte, and David von Oheimb*
 
  * Deprecated `X509_http_nbio()` and `X509_CRL_http_nbio()`,
    which are superseded by `X509_load_http()` and `X509_CRL_load_http()`.
@@ -812,8 +814,12 @@ OpenSSL 3.0
    *David von Oheimb, Martin Peylo*
 
  * Generalized the HTTP client code from `crypto/ocsp/` into `crpyto/http/`.
-   The legacy OCSP-focused and only partly documented API is retained for
-   backward compatibility. See L<OSSL_CMP_MSG_http_perform(3)> etc. for details.
+   It supports arbitrary request and response content types, GET redirection,
+   TLS, connections via HTTP(S) proxies, connections and exchange via
+   user-defined BIOs (allowing implicit connections), persistent connections,
+   and timeout checks.  See L<OSSL_HTTP_transfer(3)> etc. for details.
+   The legacy OCSP-focused (and only partly documented) API
+   is retained for backward compatibility, while most of it is deprecated.
 
    *David von Oheimb*
 
diff --git a/apps/lib/apps.c b/apps/lib/apps.c
index f0a9ffc93a..67e089bcd4 100644
--- a/apps/lib/apps.c
+++ b/apps/lib/apps.c
@@ -2479,6 +2479,7 @@ ASN1_VALUE *app_http_get_asn1(const char *url, const char *proxy,
     char *server;
     char *port;
     int use_ssl;
+    BIO *mem;
     ASN1_VALUE *resp = NULL;
 
     if (url == NULL || it == NULL) {
@@ -2500,10 +2501,13 @@ ASN1_VALUE *app_http_get_asn1(const char *url, const char *proxy,
     info.use_proxy = proxy != NULL;
     info.timeout = timeout;
     info.ssl_ctx = ssl_ctx;
-    resp = OSSL_HTTP_get_asn1(url, proxy, no_proxy,
-                              NULL, NULL, app_http_tls_cb, &info,
-                              headers, 0 /* maxline */, 0 /* max_resp_len */,
-                              timeout, expected_content_type, it);
+    mem = OSSL_HTTP_get(url, proxy, no_proxy, NULL /* bio */, NULL /* rbio */,
+                        app_http_tls_cb, &info, 0 /* buf_size */, headers,
+                        expected_content_type, 1 /* expect_asn1 */,
+                        HTTP_DEFAULT_MAX_RESP_LEN, timeout);
+    resp = ASN1_item_d2i_bio(it, mem, NULL);
+    BIO_free(mem);
+
  end:
     OPENSSL_free(server);
     OPENSSL_free(port);
@@ -2520,18 +2524,27 @@ ASN1_VALUE *app_http_post_asn1(const char *host, const char *port,
                                long timeout, const ASN1_ITEM *rsp_it)
 {
     APP_HTTP_TLS_INFO info;
+    BIO *rsp, *req_mem = ASN1_item_i2d_mem_bio(req_it, req);
+    ASN1_VALUE *res;
 
+    if (req_mem == NULL)
+        return NULL;
     info.server = host;
     info.port = port;
     info.use_proxy = proxy != NULL;
     info.timeout = timeout;
     info.ssl_ctx = ssl_ctx;
-    return OSSL_HTTP_post_asn1(host, port, path, ssl_ctx != NULL,
-                               proxy, no_proxy,
-                               NULL, NULL, app_http_tls_cb, &info,
-                               headers, content_type, req, req_it,
-                               0 /* maxline */,
-                               0 /* max_resp_len */, timeout, NULL, rsp_it);
+    rsp = OSSL_HTTP_transfer(NULL, host, port, path, ssl_ctx != NULL,
+                             proxy, no_proxy, NULL /* bio */, NULL /* rbio */,
+                             app_http_tls_cb, &info,
+                             0 /* buf_size */, headers, content_type, req_mem,
+                             NULL /* expected_ct */, 1 /* expect_asn1 */,
+                             HTTP_DEFAULT_MAX_RESP_LEN, timeout,
+                             0 /* keep_alive */);
+    BIO_free(req_mem);
+    res = ASN1_item_d2i_bio(rsp_it, rsp, NULL);
+    BIO_free(rsp);
+    return res;
 }
 
 #endif
diff --git a/crypto/cmp/cmp_http.c b/crypto/cmp/cmp_http.c
index 215c47c7c5..a358622feb 100644
--- a/crypto/cmp/cmp_http.c
+++ b/crypto/cmp/cmp_http.c
@@ -37,9 +37,11 @@ OSSL_CMP_MSG *OSSL_CMP_MSG_http_perform(OSSL_CMP_CTX *ctx,
 {
     char server_port[32] = { '\0' };
     STACK_OF(CONF_VALUE) *headers = NULL;
-    const char *const content_type_pkix = "application/pkixcmp";
+    const char content_type_pkix[] = "application/pkixcmp";
     int tls_used;
-    OSSL_CMP_MSG *res;
+    const ASN1_ITEM *it = ASN1_ITEM_rptr(OSSL_CMP_MSG);
+    BIO *req_mem, *rsp;
+    OSSL_CMP_MSG *res = NULL;
 
     if (ctx == NULL || req == NULL) {
         ERR_raise(ERR_LIB_CMP, CMP_R_NULL_ARGUMENT);
@@ -48,6 +50,8 @@ OSSL_CMP_MSG *OSSL_CMP_MSG_http_perform(OSSL_CMP_CTX *ctx,
 
     if (!X509V3_add_value("Pragma", "no-cache", &headers))
         return NULL;
+    if ((req_mem = ASN1_item_i2d_mem_bio(it, (const ASN1_VALUE *)req)) == NULL)
+        goto err;
 
     if (ctx->serverPort != 0)
         BIO_snprintf(server_port, sizeof(server_port), "%d", ctx->serverPort);
@@ -55,15 +59,21 @@ OSSL_CMP_MSG *OSSL_CMP_MSG_http_perform(OSSL_CMP_CTX *ctx,
     tls_used = OSSL_CMP_CTX_get_http_cb_arg(ctx) != NULL;
     ossl_cmp_log2(DEBUG, ctx, "connecting to CMP server %s%s",
                   ctx->server, tls_used ? " using TLS" : "");
-    res = (OSSL_CMP_MSG *)
-        OSSL_HTTP_post_asn1(ctx->server, server_port, ctx->serverPath,
-                            tls_used, ctx->proxy, ctx->no_proxy, NULL, NULL,
-                            ctx->http_cb, OSSL_CMP_CTX_get_http_cb_arg(ctx),
-                            headers, content_type_pkix, (const ASN1_VALUE *)req,
-                            ASN1_ITEM_rptr(OSSL_CMP_MSG),
-                            0, 0, ctx->msg_timeout, content_type_pkix,
-                            ASN1_ITEM_rptr(OSSL_CMP_MSG));
+    rsp = OSSL_HTTP_transfer(NULL, ctx->server, server_port,
+                             ctx->serverPath, tls_used,
+                             ctx->proxy, ctx->no_proxy,
+                             NULL /* bio */, NULL /* rbio */,
+                             ctx->http_cb, OSSL_CMP_CTX_get_http_cb_arg(ctx),
+                             0 /* buf_size */, headers,
+                             content_type_pkix, req_mem,
+                             content_type_pkix, 1 /* expect_asn1 */,
+                             HTTP_DEFAULT_MAX_RESP_LEN,
+                             ctx->msg_timeout, 0 /* keep_alive */);
+    BIO_free(req_mem);
+    res = (OSSL_CMP_MSG *)ASN1_item_d2i_bio(it, rsp, NULL);
+    BIO_free(rsp);
     ossl_cmp_debug(ctx, "disconnected from CMP server");
+ err:
     sk_CONF_VALUE_pop_free(headers, X509V3_conf_free);
     return res;
 }
diff --git a/crypto/http/http_client.c b/crypto/http/http_client.c
index bf2e3b54c7..c32b352137 100644
--- a/crypto/http/http_client.c
+++ b/crypto/http/http_client.c
@@ -48,12 +48,14 @@ struct ossl_http_req_ctx_st {
     BIO *rbio;                  /* BIO to read response from */
     BIO *mem;                   /* Memory BIO response is built into */
     int method_POST;            /* HTTP method is "POST" (else "GET") */
-    const char *expected_ct;    /* expected Content-Type, or NULL */
+    char *expected_ct;          /* expected Content-Type, or NULL */
     int expect_asn1;            /* response must be ASN.1-encoded */
     long len_to_send;           /* number of bytes in request still to send */
     unsigned long resp_len;     /* length of response */
     unsigned long max_resp_len; /* Maximum length of response */
-    time_t max_time;            /* Maximum end time of the transfer, or 0 */
+    int keep_alive;             /* Persistent conn. 0=no, 1=prefer, 2=require */
+    time_t max_time;            /* Maximum end time of current transfer, or 0 */
+    time_t max_total_time;      /* Maximum end time of total transfer, or 0 */
     char *redirection_url;      /* Location given with HTTP status 301/302 */
 };
 
@@ -72,10 +74,7 @@ struct ossl_http_req_ctx_st {
 #define OHS_DONE           (8 | OHS_NOREAD) /* Completed */
 #define OHS_HTTP_HEADER    (9 | OHS_NOREAD) /* Headers set, w/o final \r\n */
 
-OSSL_HTTP_REQ_CTX *OSSL_HTTP_REQ_CTX_new(BIO *wbio, BIO *rbio,
-                                         int maxline, unsigned long max_resp_len,
-                                         int timeout, const char *expected_ct,
-                                         int expect_asn1)
+OSSL_HTTP_REQ_CTX *OSSL_HTTP_REQ_CTX_new(BIO *wbio, BIO *rbio, int maxline)
 {
     OSSL_HTTP_REQ_CTX *rctx;
 
@@ -95,11 +94,8 @@ OSSL_HTTP_REQ_CTX *OSSL_HTTP_REQ_CTX_new(BIO *wbio, BIO *rbio,
         OPENSSL_free(rctx);
         return NULL;
     }
-    rctx->expected_ct = expected_ct;
-    rctx->expect_asn1 = expect_asn1;
     rctx->resp_len = 0;
-    OSSL_HTTP_REQ_CTX_set_max_response_length(rctx, max_resp_len);
-    rctx->max_time = timeout > 0 ? time(NULL) + timeout : 0;
+    rctx->max_resp_len = HTTP_DEFAULT_MAX_RESP_LEN;
     /* everything else is 0, e.g. rctx->len_to_send, or NULL, e.g. rctx->mem  */
     return rctx;
 }
@@ -110,6 +106,7 @@ void OSSL_HTTP_REQ_CTX_free(OSSL_HTTP_REQ_CTX *rctx)
         return;
     BIO_free(rctx->mem); /* this may indirectly call ERR_clear_error() */
     OPENSSL_free(rctx->readbuf);
+    OPENSSL_free(rctx->expected_ct);
     OPENSSL_free(rctx);
 }
 
@@ -122,6 +119,15 @@ BIO *OSSL_HTTP_REQ_CTX_get0_mem_bio(const OSSL_HTTP_REQ_CTX *rctx)
     return rctx->mem;
 }
 
+size_t OSSL_HTTP_REQ_CTX_get_resp_len(const OSSL_HTTP_REQ_CTX *rctx)
+{
+    if (rctx == NULL) {
+        ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+    return rctx->resp_len;
+}
+
 void OSSL_HTTP_REQ_CTX_set_max_response_length(OSSL_HTTP_REQ_CTX *rctx,
                                                unsigned long len)
 {
@@ -201,6 +207,36 @@ int OSSL_HTTP_REQ_CTX_add1_header(OSSL_HTTP_REQ_CTX *rctx,
     return 1;
 }
 
+int OSSL_HTTP_REQ_CTX_set_expected(OSSL_HTTP_REQ_CTX *rctx,
+                                   const char *content_type, int asn1,
+                                   int timeout, int keep_alive)
+{
+    if (rctx == NULL) {
+        ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+    if (keep_alive != 0
+            && rctx->state != OHS_ERROR && rctx->state != OHS_HEADERS) {
+        /* Cannot anymore set keep-alive in request header */
+        ERR_raise(ERR_LIB_HTTP, ERR_R_SHOULD_NOT_HAVE_BEEN_CALLED);
+        return 0;
+    }
+
+    OPENSSL_free(rctx->expected_ct);
+    rctx->expected_ct = NULL;
+    if (content_type != NULL
+            && (rctx->expected_ct = OPENSSL_strdup(content_type)) == NULL)
+        return 0;
+
+    rctx->expect_asn1 = asn1;
+    if (timeout >= 0)
+        rctx->max_time = timeout > 0 ? time(NULL) + timeout : 0;
+    else
+        rctx->max_time = rctx->max_total_time;
+    rctx->keep_alive = keep_alive;
+    return 1;
+}
+
 static int ossl_http_req_ctx_set_content(OSSL_HTTP_REQ_CTX *rctx,
                                          const char *content_type, BIO *req_mem)
 {
@@ -228,26 +264,8 @@ static int ossl_http_req_ctx_set_content(OSSL_HTTP_REQ_CTX *rctx,
         && BIO_write(rctx->mem, req, req_len) == (int)req_len;
 }
 
-BIO *ossl_http_asn1_item2bio(const ASN1_ITEM *it, const ASN1_VALUE *val)
-{
-    BIO *res;
-
-    if (it == NULL || val == NULL) {
-        ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER);
-        return NULL;
-    }
-
-    if ((res = BIO_new(BIO_s_mem())) == NULL)
-        return NULL;
-    if (ASN1_item_i2d_bio(it, res, val) <= 0) {
-        BIO_free(res);
-        res = NULL;
-    }
-    return res;
-}
-
 int OSSL_HTTP_REQ_CTX_set1_req(OSSL_HTTP_REQ_CTX *rctx, const char *content_type,
-                               const ASN1_ITEM *it, ASN1_VALUE *req)
+                               const ASN1_ITEM *it, const ASN1_VALUE *req)
 {
     BIO *mem;
     int res;
@@ -257,7 +275,7 @@ int OSSL_HTTP_REQ_CTX_set1_req(OSSL_HTTP_REQ_CTX *rctx, const char *content_type
         return 0;
     }
 
-    res = (mem = ossl_http_asn1_item2bio(it, req)) != NULL
+    res = (mem = ASN1_item_i2d_mem_bio(it, req)) != NULL
         && ossl_http_req_ctx_set_content(rctx, content_type, mem);
     BIO_free(mem);
     return res;
@@ -289,14 +307,13 @@ static int OSSL_HTTP_REQ_CTX_add1_headers(OSSL_HTTP_REQ_CTX *rctx,
  * If !use_http_proxy then the 'server' and 'port' parameters are ignored.
  * If req_mem == NULL then use GET and ignore content_type, else POST.
  */
-OSSL_HTTP_REQ_CTX
+static OSSL_HTTP_REQ_CTX
 *ossl_http_req_ctx_new(BIO *wbio, BIO *rbio, int use_http_proxy,
                        const char *server, const char *port,
                        const char *path,
                        const STACK_OF(CONF_VALUE) *headers,
                        const char *content_type, BIO *req_mem,
-                       int maxline, unsigned long max_resp_len,
-                       int timeout,
+                       int maxline, int timeout,
                        const char *expected_ct, int expect_asn1)
 {
     OSSL_HTTP_REQ_CTX *rctx;
@@ -307,14 +324,14 @@ OSSL_HTTP_REQ_CTX
     }
     /* remaining parameters are checked indirectly by the functions called */
 
-    if ((rctx = OSSL_HTTP_REQ_CTX_new(wbio, rbio, maxline, max_resp_len, timeout,
-                                      expected_ct, expect_asn1))
+    if ((rctx = OSSL_HTTP_REQ_CTX_new(wbio, rbio, maxline))
         == NULL)
         return NULL;
-
     if (OSSL_HTTP_REQ_CTX_set_request_line(rctx, req_mem != NULL,
                                            use_http_proxy ? server : NULL, port,
                                            path)
+        && OSSL_HTTP_REQ_CTX_set_expected(rctx, expected_ct, expect_asn1,
+                                          timeout, 0)
         && OSSL_HTTP_REQ_CTX_add1_headers(rctx, headers, server)
         && (req_mem == NULL
             || ossl_http_req_ctx_set_content(rctx, content_type, req_mem)))
@@ -588,6 +605,7 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx)
                                    rctx->expected_ct, value);
                     return 0;
                 }
+                OPENSSL_free(rctx->expected_ct);
                 rctx->expected_ct = NULL; /* content-type has been found */
             }
             if (strcasecmp(key, "Content-Length") == 0) {
@@ -688,6 +706,20 @@ int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx)
     }
 }
 
+int OSSL_HTTP_REQ_CTX_nbio_d2i(OSSL_HTTP_REQ_CTX *rctx,
+                               ASN1_VALUE **pval, const ASN1_ITEM *it)
+{
+    const unsigned char *p;
+    int rv;
+
+    *pval = NULL;
+    if ((rv = OSSL_HTTP_REQ_CTX_nbio(rctx)) != 1)
+        return rv;
+    *pval = ASN1_item_d2i(NULL, &p, BIO_get_mem_data(rctx->mem, &p), it);
+    return *pval != NULL;
+
+}
+
 #ifndef OPENSSL_NO_SOCK
 
 /* set up a new connection BIO, to HTTP server or to HTTP(S) proxy if given */
@@ -723,20 +755,12 @@ static BIO *HTTP_new_bio(const char *server /* optionally includes ":port" */,
 }
 #endif /* OPENSSL_NO_SOCK */
 
-static ASN1_VALUE *BIO_mem_d2i(BIO *mem, const ASN1_ITEM *it)
+int OSSL_HTTP_is_alive(const OSSL_HTTP_REQ_CTX *rctx)
 {
-    const unsigned char *p;
-    ASN1_VALUE *resp;
-
-    if (mem == NULL)
-        return NULL;
-
-    if ((resp = ASN1_item_d2i(NULL, &p, BIO_get_mem_data(mem, &p), it)) == NULL)
-        ERR_raise(ERR_LIB_HTTP, HTTP_R_RESPONSE_PARSE_ERROR);
-    return resp;
+    return rctx != NULL && rctx->keep_alive != 0;
 }
 
-static BIO *ossl_http_req_ctx_transfer(OSSL_HTTP_REQ_CTX *rctx)
+BIO *OSSL_HTTP_REQ_CTX_exchange(OSSL_HTTP_REQ_CTX *rctx)
 {
     int rv;
 
@@ -767,17 +791,6 @@ static BIO *ossl_http_req_ctx_transfer(OSSL_HTTP_REQ_CTX *rctx)
     return rctx->mem;
 }
 
-/* Exchange ASN.1-encoded request and response via HTTP on (non-)blocking BIO */
-ASN1_VALUE *OSSL_HTTP_REQ_CTX_sendreq_d2i(OSSL_HTTP_REQ_CTX *rctx,
-                                          const ASN1_ITEM *it)
-{
-    if (rctx == NULL || it == NULL) {
-        ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER);
-        return NULL;
-    }
-    return BIO_mem_d2i(ossl_http_req_ctx_transfer(rctx), it);
-}
-
 static int update_timeout(int timeout, time_t start_time)
 {
     long elapsed_time;
@@ -788,6 +801,15 @@ static int update_timeout(int timeout, time_t start_time)
     return timeout <= elapsed_time ? -1 : timeout - elapsed_time;
 }
 
+OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port,
+                                  const char *proxy, const char *no_proxy,
+                                  int use_ssl, BIO *bio, BIO *rbio,
+                                  OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                                  int buf_size, int overall_timeout)
+{
+    return NULL; /* TODO(3.0) expand */
+}
+
 /*-
  * Exchange HTTP request and response with the given server.
  * If req_mem == NULL then use GET and ignore content_type, else POST.
@@ -815,16 +837,31 @@ static int update_timeout(int timeout, time_t start_time)
  * The function should return NULL to indicate failure.
  * After disconnect the modified BIO will be deallocated using BIO_free_all().
  */
-BIO *OSSL_HTTP_transfer(const char *server, const char *port, const char *path,
+int OSSL_HTTP_set_request(OSSL_HTTP_REQ_CTX *rctx, const char *path,
+                          const STACK_OF(CONF_VALUE) *headers,
+                          const char *content_type, BIO *req,
+                          const char *expected_content_type, int expect_asn1,
+                          size_t max_resp_len, int timeout, int keep_alive)
+{
+    return 0; /* TODO(3.0) expand */
+}
+
+BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url)
+{
+    return NULL; /* TODO(3.0) expand */
+}
+
+BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
+                        const char *server, const char *port, const char *path,
                         int use_ssl, const char *proxy, const char *no_proxy,
                         BIO *bio, BIO *rbio,
                         OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                        const STACK_OF(CONF_VALUE) *headers,
+                        int maxline, const STACK_OF(CONF_VALUE) *headers,
                         const char *content_type, BIO *req_mem,
-                        int maxline, unsigned long max_resp_len, int timeout,
                         const char *expected_ct, int expect_asn1,
-                        char **redirection_url)
+                        size_t max_resp_len, int timeout, int keep_alive)
 {
+    char **redirection_url = (char **)prctx; /* TODO(3.0) fix when API approved */
     time_t start_time = timeout > 0 ? time(NULL) : 0;
     BIO *cbio; /* = bio if present, used as connection BIO if rbio is NULL */
     OSSL_HTTP_REQ_CTX *rctx;
@@ -892,12 +929,12 @@ BIO *OSSL_HTTP_transfer(const char *server, const char *port, const char *path,
     rctx = ossl_http_req_ctx_new(cbio, rbio != NULL ? rbio : cbio,
                                  !use_ssl && proxy != NULL, server, port, path,
                                  headers, content_type, req_mem, maxline,
-                                 max_resp_len, update_timeout(timeout, start_time),
+                                 update_timeout(timeout, start_time),
                                  expected_ct, expect_asn1);
     if (rctx == NULL)
         goto end;
 
-    resp = ossl_http_req_ctx_transfer(rctx);
+    resp = OSSL_HTTP_REQ_CTX_exchange(rctx);
     if (resp == NULL) {
         if (rctx->redirection_url != NULL) {
             if (redirection_url == NULL)
@@ -981,12 +1018,12 @@ static int redirection_ok(int n_redir, const char *old_url, const char *new_url)
 BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
                    BIO *bio, BIO *rbio,
                    OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                   const STACK_OF(CONF_VALUE) *headers,
-                   int maxline, unsigned long max_resp_len, int timeout,
-                   const char *expected_ct, int expect_asn1)
+                   int maxline, const STACK_OF(CONF_VALUE) *headers,
+                   const char *expected_ct, int expect_asn1,
+                   unsigned long max_resp_len, int timeout)
 {
     time_t start_time = timeout > 0 ? time(NULL) : 0;
-    char *current_url, *redirection_url;
+    char *current_url, *redirection_url = NULL;
     int n_redirs = 0;
     char *host;
     char *port;
@@ -1007,13 +1044,13 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
             break;
 
      new_rpath:
-        resp = OSSL_HTTP_transfer(host, port, path, use_ssl, proxy, no_proxy,
+        resp = OSSL_HTTP_transfer((OSSL_HTTP_REQ_CTX **)&redirection_url, /* TODO(3.0) fix when API approved */
+                                  host, port, path, use_ssl, proxy, no_proxy,
                                   bio, rbio,
-                                  bio_update_fn, arg, headers, NULL, NULL,
-                                  maxline, max_resp_len,
-                                  update_timeout(timeout, start_time),
+                                  bio_update_fn, arg, maxline, headers, NULL, NULL,
                                   expected_ct, expect_asn1,
-                                  &redirection_url);
+                                  max_resp_len,
+                                  update_timeout(timeout, start_time), 0);
         OPENSSL_free(path);
         if (resp == NULL && redirection_url != NULL) {
             if (redirection_ok(++n_redirs, current_url, redirection_url)) {
@@ -1038,65 +1075,9 @@ BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
     return resp;
 }
 
-/* Get ASN.1-encoded data via HTTP from server at given URL */
-ASN1_VALUE *OSSL_HTTP_get_asn1(const char *url,
-                               const char *proxy, const char *no_proxy,
-                               BIO *bio, BIO *rbio,
-                               OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                               const STACK_OF(CONF_VALUE) *headers,
-                               int maxline, unsigned long max_resp_len,
-                               int timeout, const char *expected_ct,
-                               const ASN1_ITEM *rsp_it)
+int OSSL_HTTP_close(OSSL_HTTP_REQ_CTX *rctx, int ok)
 {
-    BIO *mem;
-    ASN1_VALUE *resp = NULL;
-
-    if (url == NULL || rsp_it == NULL) {
-        ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER);
-        return NULL;
-    }
-    mem = OSSL_HTTP_get(url, proxy, no_proxy, bio, rbio, bio_update_fn,
-                        arg, headers, maxline, max_resp_len, timeout,
-                        expected_ct, 1 /* expect_asn1 */);
-    resp = BIO_mem_d2i(mem /* may be NULL */, rsp_it);
-    BIO_free(mem);
-    return resp;
-}
-
-/* Post ASN.1-encoded request via HTTP to server return ASN.1 response */
-ASN1_VALUE *OSSL_HTTP_post_asn1(const char *server, const char *port,
-                                const char *path, int use_ssl,
-                                const char *proxy, const char *no_proxy,
-                                BIO *bio, BIO *rbio,
-                                OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                                const STACK_OF(CONF_VALUE) *headers,
-                                const char *content_type,
-                                const ASN1_VALUE *req, const ASN1_ITEM *req_it,
-                                int maxline, unsigned long max_resp_len,
-                                int timeout, const char *expected_ct,
-                                const ASN1_ITEM *rsp_it)
-{
-    BIO *req_mem;
-    BIO *res_mem;
-    ASN1_VALUE *resp = NULL;
-
-    if (req == NULL) {
-        ERR_raise(ERR_LIB_HTTP, ERR_R_PASSED_NULL_PARAMETER);
-        return NULL;
-    }
-    /* remaining parameters are checked indirectly */
-
-    req_mem = ossl_http_asn1_item2bio(req_it, req);
-    res_mem = OSSL_HTTP_transfer(server, port, path, use_ssl, proxy, no_proxy,
-                                 bio, rbio,
-                                 bio_update_fn, arg, headers, content_type,
-                                 req_mem /* may be NULL */, maxline,
-                                 max_resp_len, timeout,
-                                 expected_ct, 1 /* expect_asn1 */, NULL);
-    BIO_free(req_mem);
-    resp = BIO_mem_d2i(res_mem /* may be NULL */, rsp_it);
-    BIO_free(res_mem);
-    return resp;
+    return 0; /* TODO(3.0) expand */
 }
 
 /* BASE64 encoder used for encoding basic proxy authentication credentials */
diff --git a/crypto/http/http_local.h b/crypto/http/http_local.h
index 3164f62a77..16f7f7c8a5 100644
--- a/crypto/http/http_local.h
+++ b/crypto/http/http_local.h
@@ -11,21 +11,6 @@
 #ifndef OSSL_CRYPTO_HTTP_LOCAL_H
 # define OSSL_CRYPTO_HTTP_LOCAL_H
 
-# include <openssl/ocsp.h>
-
-BIO *ossl_http_asn1_item2bio(const ASN1_ITEM *it, const ASN1_VALUE *val);
-
-OSSL_HTTP_REQ_CTX
-*ossl_http_req_ctx_new(BIO *wbio, BIO *rbio, int use_http_proxy,
-                       const char *server, const char *port,
-                       const char *path,
-                       const STACK_OF(CONF_VALUE) *headers,
-                       const char *content_type, BIO *req_mem,
-                       int maxline, unsigned long max_resp_len,
-                       int timeout,
-                       const char *expected_content_type,
-                       int expect_asn1);
-
 int ossl_http_use_proxy(const char *no_proxy, const char *server);
 const char *ossl_http_adapt_proxy(const char *proxy, const char *no_proxy,
                                   const char *server, int use_ssl);
diff --git a/crypto/ocsp/ocsp_http.c b/crypto/ocsp/ocsp_http.c
index 7a3c19c860..8cf816e53f 100644
--- a/crypto/ocsp/ocsp_http.c
+++ b/crypto/ocsp/ocsp_http.c
@@ -16,17 +16,18 @@
 OSSL_HTTP_REQ_CTX *OCSP_sendreq_new(BIO *io, const char *path,
                                     const OCSP_REQUEST *req, int maxline)
 {
-    OSSL_HTTP_REQ_CTX *rctx = NULL;
+    OSSL_HTTP_REQ_CTX *rctx = OSSL_HTTP_REQ_CTX_new(io, io, maxline);
 
-    if ((rctx = OSSL_HTTP_REQ_CTX_new(io, io,
-                                      maxline, 0 /* default max_resp_len */,
-                                      0 /* no timeout, blocking indefinitely */,
-                                      NULL, 1 /* expect_asn1 */)) == NULL)
+    if (rctx == NULL)
         return NULL;
 
     if (!OSSL_HTTP_REQ_CTX_set_request_line(rctx, 1 /* POST */, NULL, NULL, path))
         goto err;
 
+    if (!OSSL_HTTP_REQ_CTX_set_expected(rctx,
+                                        NULL /* content_type */, 1 /* asn1 */,
+                                        0 /* timeout */, 0 /* keep_alive */))
+        goto err;
     if (req != NULL
         && !OSSL_HTTP_REQ_CTX_set1_req(rctx, "application/ocsp-request",
                                        ASN1_ITEM_rptr(OCSP_REQUEST),
@@ -40,23 +41,19 @@ OSSL_HTTP_REQ_CTX *OCSP_sendreq_new(BIO *io, const char *path,
     return NULL;
 }
 
-int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OSSL_HTTP_REQ_CTX *rctx)
-{
-    *presp = (OCSP_RESPONSE *)
-        OSSL_HTTP_REQ_CTX_sendreq_d2i(rctx, ASN1_ITEM_rptr(OCSP_RESPONSE));
-    return *presp != NULL;
-}
-
 OCSP_RESPONSE *OCSP_sendreq_bio(BIO *b, const char *path, OCSP_REQUEST *req)
 {
     OCSP_RESPONSE *resp = NULL;
     OSSL_HTTP_REQ_CTX *ctx;
+    BIO *mem;
 
     ctx = OCSP_sendreq_new(b, path, req, -1 /* default max resp line length */);
     if (ctx == NULL)
         return NULL;
-
-    OCSP_sendreq_nbio(&resp, ctx);
+    mem = OSSL_HTTP_REQ_CTX_exchange(ctx);
+    resp = (OCSP_RESPONSE *)
+        ASN1_item_d2i_bio(ASN1_ITEM_rptr(OCSP_RESPONSE), mem, NULL);
+    BIO_free(mem);
 
     /* this indirectly calls ERR_clear_error(): */
     OSSL_HTTP_REQ_CTX_free(ctx);
diff --git a/crypto/x509/x_all.c b/crypto/x509/x_all.c
index 9733597d37..1bd47ce654 100644
--- a/crypto/x509/x_all.c
+++ b/crypto/x509/x_all.c
@@ -75,11 +75,15 @@ int X509_sign_ctx(X509 *x, EVP_MD_CTX *ctx)
 static ASN1_VALUE *simple_get_asn1(const char *url, BIO *bio, BIO *rbio,
                                    int timeout, const ASN1_ITEM *it)
 {
-    return OSSL_HTTP_get_asn1(url, NULL, NULL /* no proxy used */, bio,
-                              rbio, NULL /* no callback for SSL/TLS */, NULL,
-                              NULL /* headers */, 1024 /* maxline */,
-                              0 /* max_resp_len */, timeout,
-                              NULL /* expected_content_type */, it);
+    BIO *mem = OSSL_HTTP_get(url, NULL /* proxy */, NULL /* no_proxy */,
+                             bio, rbio, NULL /* cb */ , NULL /* arg */,
+                             1024 /* buf_size */, NULL /* headers */,
+                             NULL /* expected_ct */, 1 /* expect_asn1 */,
+                             HTTP_DEFAULT_MAX_RESP_LEN, timeout);
+    ASN1_VALUE *res = ASN1_item_d2i_bio(it, mem, NULL);
+
+    BIO_free(mem);
+    return res;
 }
 
 X509 *X509_load_http(const char *url, BIO *bio, BIO *rbio, int timeout)
diff --git a/doc/man1/openssl-cmp.pod.in b/doc/man1/openssl-cmp.pod.in
index 28ea4ee6a5..49105ca315 100644
--- a/doc/man1/openssl-cmp.pod.in
+++ b/doc/man1/openssl-cmp.pod.in
@@ -52,6 +52,7 @@ Message transfer options:
 [B<-proxy> I<[http[s]://][userinfo@]host[:port][/path][?query][#fragment]>]
 [B<-no_proxy> I<addresses>]
 [B<-recipient> I<name>]
+[B<-keep_alive> I<value>]
 [B<-msg_timeout> I<seconds>]
 [B<-total_timeout> I<seconds>]
 
@@ -488,11 +489,20 @@ as far as any of those is present, else the NULL-DN as last resort.
 The argument must be formatted as I</type0=value0/type1=value1/type2=...>.
 For details see the description of the B<-subject> option.
 
+=item B<-keep_alive> I<value>
+
+If the given value is 0 then HTTP connections are not kept open
+after receiving a response, which is the default behavior for HTTP 1.0.
+If the value is 1 or 2 then persistent connections are requested.
+If the value is 2 then