HTTP client: Minimal changes that include the improved API

This is a minimal version of pull request #15053 including all the
proposed improvements to the HTTP client API and its documentation
but only those code adaptations strictly needed for it.

The proposed new features include
* support for persistent connections (keep-alive),
* generalization to arbitrary request and response types, and
* support for streaming BIOs for request and response data.

The related API changes include:
* Split the monolithic OSSL_HTTP_transfer() into OSSL_HTTP_open(),
  OSSL_HTTP_set_request(), a lean OSSL_HTTP_transfer(), and OSSL_HTTP_close().
* Split the timeout functionality accordingly and improve default behavior.
* Extract part of OSSL_HTTP_REQ_CTX_new() to OSSL_HTTP_REQ_CTX_set_expected().

Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/15147)

5 files changed, 78 insertions, 77 deletions

diff --git a/include/crypto/httperr.h b/include/crypto/httperr.h
index 648f55c691..c68ca3b0c4 100644
--- a/include/crypto/httperr.h
+++ b/include/crypto/httperr.h
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 2020-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
diff --git a/include/openssl/cmp.h.in b/include/openssl/cmp.h.in
index 2591963b6f..352ffcdb2f 100644
--- a/include/openssl/cmp.h.in
+++ b/include/openssl/cmp.h.in
@@ -262,25 +262,29 @@ void OSSL_CMP_MSG_free(OSSL_CMP_MSG *msg);
 OSSL_CMP_CTX *OSSL_CMP_CTX_new(OSSL_LIB_CTX *libctx, const char *propq);
 void OSSL_CMP_CTX_free(OSSL_CMP_CTX *ctx);
 int OSSL_CMP_CTX_reinit(OSSL_CMP_CTX *ctx);
-/* various CMP options: */
+/* CMP general options: */
 #  define OSSL_CMP_OPT_LOG_VERBOSITY 0
-#  define OSSL_CMP_OPT_MSG_TIMEOUT 1
-#  define OSSL_CMP_OPT_TOTAL_TIMEOUT 2
-#  define OSSL_CMP_OPT_VALIDITY_DAYS 3
-#  define OSSL_CMP_OPT_SUBJECTALTNAME_NODEFAULT 4
-#  define OSSL_CMP_OPT_SUBJECTALTNAME_CRITICAL 5
-#  define OSSL_CMP_OPT_POLICIES_CRITICAL 6
-#  define OSSL_CMP_OPT_POPO_METHOD 7
-#  define OSSL_CMP_OPT_DIGEST_ALGNID 8
-#  define OSSL_CMP_OPT_OWF_ALGNID 9
-#  define OSSL_CMP_OPT_MAC_ALGNID 10
-#  define OSSL_CMP_OPT_REVOCATION_REASON 11
-#  define OSSL_CMP_OPT_IMPLICIT_CONFIRM 12
-#  define OSSL_CMP_OPT_DISABLE_CONFIRM 13
-#  define OSSL_CMP_OPT_UNPROTECTED_SEND 14
-#  define OSSL_CMP_OPT_UNPROTECTED_ERRORS 15
-#  define OSSL_CMP_OPT_IGNORE_KEYUSAGE 16
-#  define OSSL_CMP_OPT_PERMIT_TA_IN_EXTRACERTS_FOR_IR 17
+/* CMP transfer options: */
+#  define OSSL_CMP_OPT_KEEP_ALIVE 10
+#  define OSSL_CMP_OPT_MSG_TIMEOUT 11
+#  define OSSL_CMP_OPT_TOTAL_TIMEOUT 12
+/* CMP request options: */
+#  define OSSL_CMP_OPT_VALIDITY_DAYS 20
+#  define OSSL_CMP_OPT_SUBJECTALTNAME_NODEFAULT 21
+#  define OSSL_CMP_OPT_SUBJECTALTNAME_CRITICAL 22
+#  define OSSL_CMP_OPT_POLICIES_CRITICAL 23
+#  define OSSL_CMP_OPT_POPO_METHOD 24
+#  define OSSL_CMP_OPT_IMPLICIT_CONFIRM 25
+#  define OSSL_CMP_OPT_DISABLE_CONFIRM 26
+#  define OSSL_CMP_OPT_REVOCATION_REASON 27
+/* CMP protection options: */
+#  define OSSL_CMP_OPT_UNPROTECTED_SEND 30
+#  define OSSL_CMP_OPT_UNPROTECTED_ERRORS 31
+#  define OSSL_CMP_OPT_OWF_ALGNID 32
+#  define OSSL_CMP_OPT_MAC_ALGNID 33
+#  define OSSL_CMP_OPT_DIGEST_ALGNID 34
+#  define OSSL_CMP_OPT_IGNORE_KEYUSAGE 35
+#  define OSSL_CMP_OPT_PERMIT_TA_IN_EXTRACERTS_FOR_IR 36
 int OSSL_CMP_CTX_set_option(OSSL_CMP_CTX *ctx, int opt, int val);
 int OSSL_CMP_CTX_get_option(const OSSL_CMP_CTX *ctx, int opt);
 /* CMP-specific callback for logging and outputting the error queue: */
diff --git a/include/openssl/http.h b/include/openssl/http.h
index 18d0f13b3e..2140d5d2f8 100644
--- a/include/openssl/http.h
+++ b/include/openssl/http.h
@@ -23,8 +23,6 @@
 extern "C" {
 # endif
 
-typedef BIO *(*OSSL_HTTP_bio_cb_t)(BIO *bio, void *arg, int connect, int detail);
-
 # define OSSL_HTTP_NAME "http"
 # define OSSL_HTTPS_NAME "https"
 # define OSSL_HTTP_PREFIX OSSL_HTTP_NAME"://"
@@ -38,63 +36,64 @@ typedef BIO *(*OSSL_HTTP_bio_cb_t)(BIO *bio, void *arg, int connect, int detail)
 #define HTTP_DEFAULT_MAX_LINE_LENGTH (4 * 1024)
 #define HTTP_DEFAULT_MAX_RESP_LEN (100 * 1024)
 
-OSSL_HTTP_REQ_CTX *OSSL_HTTP_REQ_CTX_new(BIO *wbio, BIO *rbio,
-                                         int maxline, unsigned long max_resp_len,
-                                         int timeout, const char *expected_ct,
-                                         int expect_asn1);
+/* Low-level HTTP API */
+OSSL_HTTP_REQ_CTX *OSSL_HTTP_REQ_CTX_new(BIO *wbio, BIO *rbio, int buf_size);
 void OSSL_HTTP_REQ_CTX_free(OSSL_HTTP_REQ_CTX *rctx);
 int OSSL_HTTP_REQ_CTX_set_request_line(OSSL_HTTP_REQ_CTX *rctx, int method_POST,
                                        const char *server, const char *port,
                                        const char *path);
 int OSSL_HTTP_REQ_CTX_add1_header(OSSL_HTTP_REQ_CTX *rctx,
                                   const char *name, const char *value);
+int OSSL_HTTP_REQ_CTX_set_expected(OSSL_HTTP_REQ_CTX *rctx,
+                                   const char *content_type, int asn1,
+                                   int timeout, int keep_alive);
 int OSSL_HTTP_REQ_CTX_set1_req(OSSL_HTTP_REQ_CTX *rctx, const char *content_type,
-                               const ASN1_ITEM *it, ASN1_VALUE *req);
+                               const ASN1_ITEM *it, const ASN1_VALUE *req);
 int OSSL_HTTP_REQ_CTX_nbio(OSSL_HTTP_REQ_CTX *rctx);
-ASN1_VALUE *OSSL_HTTP_REQ_CTX_sendreq_d2i(OSSL_HTTP_REQ_CTX *rctx,
-                                          const ASN1_ITEM *it);
+int OSSL_HTTP_REQ_CTX_nbio_d2i(OSSL_HTTP_REQ_CTX *rctx,
+                               ASN1_VALUE **pval, const ASN1_ITEM *it);
+BIO *OSSL_HTTP_REQ_CTX_exchange(OSSL_HTTP_REQ_CTX *rctx);
 BIO *OSSL_HTTP_REQ_CTX_get0_mem_bio(const OSSL_HTTP_REQ_CTX *rctx);
+size_t OSSL_HTTP_REQ_CTX_get_resp_len(const OSSL_HTTP_REQ_CTX *rctx);
 void OSSL_HTTP_REQ_CTX_set_max_response_length(OSSL_HTTP_REQ_CTX *rctx,
                                                unsigned long len);
+int OSSL_HTTP_is_alive(const OSSL_HTTP_REQ_CTX *rctx);
 
+/* High-level HTTP API */
+typedef BIO *(*OSSL_HTTP_bio_cb_t)(BIO *bio, void *arg, int connect, int detail);
+OSSL_HTTP_REQ_CTX *OSSL_HTTP_open(const char *server, const char *port,
+                                  const char *proxy, const char *no_proxy,
+                                  int use_ssl, BIO *bio, BIO *rbio,
+                                  OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
+                                  int buf_size, int overall_timeout);
+int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port,
+                            const char *proxyuser, const char *proxypass,
+                            int timeout, BIO *bio_err, const char *prog);
+int OSSL_HTTP_set_request(OSSL_HTTP_REQ_CTX *rctx, const char *path,
+                          const STACK_OF(CONF_VALUE) *headers,
+                          const char *content_type, BIO *req,
+                          const char *expected_content_type, int expect_asn1,
+                          size_t max_resp_len, int timeout, int keep_alive);
+BIO *OSSL_HTTP_exchange(OSSL_HTTP_REQ_CTX *rctx, char **redirection_url);
 BIO *OSSL_HTTP_get(const char *url, const char *proxy, const char *no_proxy,
                    BIO *bio, BIO *rbio,
                    OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                   const STACK_OF(CONF_VALUE) *headers,
-                   int maxline, unsigned long max_resp_len, int timeout,
-                   const char *expected_ct, int expect_asn1);
-ASN1_VALUE *OSSL_HTTP_get_asn1(const char *url,
-                               const char *proxy, const char *no_proxy,
-                               BIO *bio, BIO *rbio,
-                               OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                               const STACK_OF(CONF_VALUE) *headers,
-                               int maxline, unsigned long max_resp_len,
-                               int timeout, const char *expected_ct,
-                               const ASN1_ITEM *rsp_it);
-ASN1_VALUE *OSSL_HTTP_post_asn1(const char *server, const char *port,
-                                const char *path, int use_ssl,
-                                const char *proxy, const char *no_proxy,
-                                BIO *bio, BIO *rbio,
-                                OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                                const STACK_OF(CONF_VALUE) *headers,
-                                const char *content_type,
-                                const ASN1_VALUE *req, const ASN1_ITEM *req_it,
-                                int maxline, unsigned long max_resp_len,
-                                int timeout, const char *expected_ct,
-                                const ASN1_ITEM *rsp_it);
-BIO *OSSL_HTTP_transfer(const char *server, const char *port, const char *path,
-                        int use_ssl, const char *proxy, const char *no_proxy,
+                   int buf_size, const STACK_OF(CONF_VALUE) *headers,
+                   const char *expected_content_type, int expect_asn1,
+                   size_t max_resp_len, int timeout);
+BIO *OSSL_HTTP_transfer(OSSL_HTTP_REQ_CTX **prctx,
+                        const char *server, const char *port,
+                        const char *path, int use_ssl,
+                        const char *proxy, const char *no_proxy,
                         BIO *bio, BIO *rbio,
                         OSSL_HTTP_bio_cb_t bio_update_fn, void *arg,
-                        const STACK_OF(CONF_VALUE) *headers,
-                        const char *content_type, BIO *req_mem,
-                        int maxline, unsigned long max_resp_len, int timeout,
-                        const char *expected_ct, int expect_asn1,
-                        char **redirection_url);
-int OSSL_HTTP_proxy_connect(BIO *bio, const char *server, const char *port,
-                            const char *proxyuser, const char *proxypass,
-                            int timeout, BIO *bio_err, const char *prog);
+                        int buf_size, const STACK_OF(CONF_VALUE) *headers,
+                        const char *content_type, BIO *req,
+                        const char *expected_content_type, int expect_asn1,
+                        size_t max_resp_len, int timeout, int keep_alive);
+int OSSL_HTTP_close(OSSL_HTTP_REQ_CTX *rctx, int ok);
 
+/* Auxiliary functions */
 int OSSL_parse_url(const char *url, char **pscheme, char **puser, char **phost,
                    char **pport, int *pport_num,
                    char **ppath, char **pquery, char **pfrag);
diff --git a/include/openssl/httperr.h b/include/openssl/httperr.h
index af5717d3dc..b639ef0051 100644
--- a/include/openssl/httperr.h
+++ b/include/openssl/httperr.h
@@ -29,6 +29,7 @@
 # define HTTP_R_ERROR_RECEIVING                           103
 # define HTTP_R_ERROR_SENDING                             102
 # define HTTP_R_FAILED_READING_DATA                       128
+# define HTTP_R_HEADER_PARSE_ERROR                        126
 # define HTTP_R_INCONSISTENT_CONTENT_LENGTH               120
 # define HTTP_R_INVALID_PORT_NUMBER                       123
 # define HTTP_R_INVALID_URL_PATH                          125
@@ -43,6 +44,7 @@
 # define HTTP_R_REDIRECTION_NOT_ENABLED                   116
 # define HTTP_R_RESPONSE_LINE_TOO_LONG                    113
 # define HTTP_R_RESPONSE_PARSE_ERROR                      104
+# define HTTP_R_SERVER_CANCELED_CONNECTION                127
 # define HTTP_R_SOCK_NOT_SUPPORTED                        122
 # define HTTP_R_STATUS_CODE_UNSUPPORTED                   114
 # define HTTP_R_TLS_NOT_ENABLED                           107
diff --git a/include/openssl/ocsp.h.in b/include/openssl/ocsp.h.in
index 83c8a175fe..869c3ad415 100644
--- a/include/openssl/ocsp.h.in
+++ b/include/openssl/ocsp.h.in
@@ -170,34 +170,30 @@ typedef struct ocsp_service_locator_st OCSP_SERVICELOC;
 
 DECLARE_ASN1_DUP_FUNCTION(OCSP_CERTID)
 
-OCSP_RESPONSE *OCSP_sendreq_bio(BIO *b, const char *path, OCSP_REQUEST *req);
 OSSL_HTTP_REQ_CTX *OCSP_sendreq_new(BIO *io, const char *path,
-                                    const OCSP_REQUEST *req, int maxline);
-int OCSP_sendreq_nbio(OCSP_RESPONSE **presp, OSSL_HTTP_REQ_CTX *rctx);
+                                    const OCSP_REQUEST *req, int buf_size);
+OCSP_RESPONSE *OCSP_sendreq_bio(BIO *b, const char *path, OCSP_REQUEST *req);
 
 #  ifndef OPENSSL_NO_DEPRECATED_3_0
 typedef OSSL_HTTP_REQ_CTX OCSP_REQ_CTX;
-#   define OCSP_REQ_CTX_new(io, maxline) \
-        OSSL_HTTP_REQ_CTX_new(io, io, maxline, 0, 0, NULL, 1)
-#   define OCSP_REQ_CTX_free(r) \
-        OSSL_HTTP_REQ_CTX_free(r)
+#   define OCSP_REQ_CTX_new(io, buf_size) \
+        OSSL_HTTP_REQ_CTX_new(io, io, buf_size)
+#   define OCSP_REQ_CTX_free OSSL_HTTP_REQ_CTX_free
 #   define OCSP_REQ_CTX_http(rctx, op, path) \
         OSSL_HTTP_REQ_CTX_set_request_line(rctx, strcmp(op, "POST") == 0, \
                                            NULL, NULL, path)
-#   define OCSP_REQ_CTX_add1_header(r, n, v) \
-        OSSL_HTTP_REQ_CTX_add1_header(r, n, v)
+#   define OCSP_REQ_CTX_add1_header OSSL_HTTP_REQ_CTX_add1_header
 #   define OCSP_REQ_CTX_i2d(r, it, req) \
         OSSL_HTTP_REQ_CTX_set1_req(r, "application/ocsp-request", it, req)
 #   define OCSP_REQ_CTX_set1_req(r, req) \
         OCSP_REQ_CTX_i2d(r, ASN1_ITEM_rptr(OCSP_REQUEST), (ASN1_VALUE *)(req))
-#   define OCSP_REQ_CTX_nbio(r) \
-        OSSL_HTTP_REQ_CTX_nbio(r)
-#   define OCSP_REQ_CTX_nbio_d2i(r, p, i)        \
-        ((*(p) = OSSL_HTTP_REQ_CTX_sendreq_d2i(r, i)) != NULL)
-#   define OCSP_REQ_CTX_get0_mem_bio(r) \
-        OSSL_HTTP_REQ_CTX_get0_mem_bio(r)
-#   define OCSP_set_max_response_length(r, l) \
-        OSSL_HTTP_REQ_CTX_set_max_response_length(r, l)
+#   define OCSP_REQ_CTX_nbio OSSL_HTTP_REQ_CTX_nbio
+#   define OCSP_REQ_CTX_nbio_d2i OSSL_HTTP_REQ_CTX_nbio_d2i
+#   define OCSP_sendreq_nbio(r, p) \
+        OSSL_HTTP_REQ_CTX_nbio_d2i(r, (ASN1_VALUE **)(p), \
+                                   ASN1_ITEM_rptr(OCSP_RESPONSE))
+#   define OCSP_REQ_CTX_get0_mem_bio OSSL_HTTP_REQ_CTX_get0_mem_bio
+#   define OCSP_set_max_response_length OSSL_HTTP_REQ_CTX_set_max_response_length
 #  endif
 
 OCSP_CERTID *OCSP_cert_to_id(const EVP_MD *dgst, const X509 *subject,