Add TFO support to socket BIO and s_client/s_server

Supports Linux, MacOS and FreeBSD
Disabled by default, enabled via `enabled-tfo`
Some tests

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/8692)

4 files changed, 73 insertions, 12 deletions

diff --git a/apps/include/s_apps.h b/apps/include/s_apps.h
index 5b188b9892..3f00302721 100644
--- a/apps/include/s_apps.h
+++ b/apps/include/s_apps.h
@@ -23,8 +23,8 @@ void get_sock_info_address(int asock, char **hostname, char **service);
 int report_server_accept(BIO *out, int asock, int with_address, int with_pid);
 int do_server(int *accept_sock, const char *host, const char *port,
               int family, int type, int protocol, do_server_cb cb,
-              unsigned char *context, int naccept, BIO *bio_s_out);
-
+              unsigned char *context, int naccept, BIO *bio_s_out,
+              int tfo);
 int verify_callback(int ok, X509_STORE_CTX *ctx);
 
 int set_cert_stuff(SSL_CTX *ctx, char *cert_file, char *key_file);
@@ -36,7 +36,8 @@ int ssl_print_groups(BIO *out, SSL *s, int noshared);
 int ssl_print_tmp_key(BIO *out, SSL *s);
 int init_client(int *sock, const char *host, const char *port,
                 const char *bindhost, const char *bindport,
-                int family, int type, int protocol);
+                int family, int type, int protocol, int tfo,
+                BIO_ADDR **ba_ret);
 int should_retry(int i);
 void do_ssl_shutdown(SSL *ssl);
 
diff --git a/apps/lib/s_socket.c b/apps/lib/s_socket.c
index e5802d5054..1ed5a213af 100644
--- a/apps/lib/s_socket.c
+++ b/apps/lib/s_socket.c
@@ -64,6 +64,8 @@ BIO_ADDR *ourpeer = NULL;
  *  AF_UNSPEC
  * @type: socket type, must be SOCK_STREAM or SOCK_DGRAM
  * @protocol: socket protocol, e.g. IPPROTO_TCP or IPPROTO_UDP (or 0 for any)
+ * @tfo: flag to enable TCP Fast Open
+ * @ba_ret: BIO_ADDR that was connected to for TFO, to be freed by caller
  *
  * This will create a socket and use it to connect to a host:port, or if
  * family == AF_UNIX, to the path found in host.
@@ -76,7 +78,8 @@ BIO_ADDR *ourpeer = NULL;
  */
 int init_client(int *sock, const char *host, const char *port,
                 const char *bindhost, const char *bindport,
-                int family, int type, int protocol)
+                int family, int type, int protocol, int tfo,
+                BIO_ADDR **ba_ret)
 {
     BIO_ADDRINFO *res = NULL;
     BIO_ADDRINFO *bindaddr = NULL;
@@ -84,6 +87,10 @@ int init_client(int *sock, const char *host, const char *port,
     const BIO_ADDRINFO *bi = NULL;
     int found = 0;
     int ret;
+    int options = 0;
+
+    if (tfo && ba_ret != NULL)
+        *ba_ret = NULL;
 
     if (BIO_sock_init() != 1)
         return 0;
@@ -160,14 +167,22 @@ int init_client(int *sock, const char *host, const char *port,
             BIO_free(tmpbio);
         }
 #endif
+        if (BIO_ADDRINFO_protocol(ai) == IPPROTO_TCP) {
+            options |= BIO_SOCK_NODELAY;
+            if (tfo)
+                options |= BIO_SOCK_TFO;
+        }
 
-        if (!BIO_connect(*sock, BIO_ADDRINFO_address(ai),
-                         BIO_ADDRINFO_protocol(ai) == IPPROTO_TCP ? BIO_SOCK_NODELAY : 0)) {
+        if (!BIO_connect(*sock, BIO_ADDRINFO_address(ai), options)) {
             BIO_closesocket(*sock);
             *sock = INVALID_SOCKET;
             continue;
         }
 
+        /* Save the address */
+        if (tfo && ba_ret != NULL)
+            *ba_ret = BIO_ADDR_dup(BIO_ADDRINFO_address(ai));
+
         /* Success, don't try any more addresses */
         break;
     }
@@ -278,7 +293,8 @@ int report_server_accept(BIO *out, int asock, int with_address, int with_pid)
  */
 int do_server(int *accept_sock, const char *host, const char *port,
               int family, int type, int protocol, do_server_cb cb,
-              unsigned char *context, int naccept, BIO *bio_s_out)
+              unsigned char *context, int naccept, BIO *bio_s_out,
+              int tfo)
 {
     int asock = 0;
     int sock;
@@ -312,6 +328,8 @@ int do_server(int *accept_sock, const char *host, const char *port,
     sock_protocol = BIO_ADDRINFO_protocol(res);
     sock_address = BIO_ADDRINFO_address(res);
     next = BIO_ADDRINFO_next(res);
+    if (tfo && sock_type == SOCK_STREAM)
+        sock_options |= BIO_SOCK_TFO;
 #ifdef AF_INET6
     if (sock_family == AF_INET6)
         sock_options |= BIO_SOCK_V6_ONLY;
diff --git a/apps/s_client.c b/apps/s_client.c
index 105a243b8e..f78404c63f 100644
--- a/apps/s_client.c
+++ b/apps/s_client.c
@@ -456,6 +456,7 @@ typedef enum OPTION_choice {
     OPT_USE_SRTP, OPT_KEYMATEXPORT, OPT_KEYMATEXPORTLEN, OPT_PROTOHOST,
     OPT_MAXFRAGLEN, OPT_MAX_SEND_FRAG, OPT_SPLIT_SEND_FRAG, OPT_MAX_PIPELINES,
     OPT_READ_BUF, OPT_KEYLOG_FILE, OPT_EARLY_DATA, OPT_REQCAFILE,
+    OPT_TFO,
     OPT_V_ENUM,
     OPT_X_ENUM,
     OPT_S_ENUM, OPT_IGNORE_UNEXPECTED_EOF,
@@ -539,6 +540,9 @@ const OPTIONS s_client_options[] = {
      "Do not load certificates from the default certificates store"},
     {"requestCAfile", OPT_REQCAFILE, '<',
       "PEM format file of CA names to send to the server"},
+#if defined(TCP_FASTOPEN) && !defined(OPENSSL_NO_TFO)
+    {"tfo", OPT_TFO, '-', "Connect using TCP Fast Open"},
+#endif
     {"dane_tlsa_domain", OPT_DANE_TLSA_DOMAIN, 's', "DANE TLSA base domain"},
     {"dane_tlsa_rrdata", OPT_DANE_TLSA_RRDATA, 's',
      "DANE TLSA rrdata presentation form"},
@@ -899,6 +903,8 @@ int s_client_main(int argc, char **argv)
 #ifndef OPENSSL_NO_KTLS
     int enable_ktls = 0;
 #endif
+    int tfo = 0;
+    BIO_ADDR *tfo_addr = NULL;
 
     FD_ZERO(&readfds);
     FD_ZERO(&writefds);
@@ -1413,6 +1419,9 @@ int s_client_main(int argc, char **argv)
             if (!opt_pair(opt_arg(), services, &starttls_proto))
                 goto end;
             break;
+        case OPT_TFO:
+            tfo = 1;
+            break;
         case OPT_SERVERNAME:
             servername = opt_arg();
             break;
@@ -2035,10 +2044,18 @@ int s_client_main(int argc, char **argv)
                    "-dane_tlsa_domain option.\n", prog);
         goto end;
     }
+#ifndef OPENSSL_NO_DTLS
+    if (isdtls && tfo) {
+        BIO_printf(bio_err, "%s: DTLS does not support the -tfo option\n", prog);
+        goto end;
+    }
+#endif
 
+    if (tfo)
+        BIO_printf(bio_c_out, "Connecting via TFO\n");
  re_start:
     if (init_client(&sock, host, port, bindhost, bindport, socket_family,
-                    socket_type, protocol) == 0) {
+                    socket_type, protocol, tfo, &tfo_addr) == 0) {
         BIO_printf(bio_err, "connect:errno=%d\n", get_last_socket_error());
         BIO_closesocket(sock);
         goto end;
@@ -2120,6 +2137,12 @@ int s_client_main(int argc, char **argv)
         goto end;
     }
 
+    /* Now that we're using a BIO... */
+    if (tfo_addr != NULL)
+        (void)BIO_set_conn_address(sbio, tfo_addr);
+    if (tfo)
+        (void)BIO_set_tfo(sbio, 1);
+
     if (nbio_test) {
         BIO *test;
 
@@ -2909,9 +2932,12 @@ int s_client_main(int argc, char **argv)
 
             case SSL_ERROR_SYSCALL:
                 if ((k != 0) || (cbuf_len != 0)) {
-                    BIO_printf(bio_err, "write:errno=%d\n",
-                               get_last_socket_error());
-                    goto shut;
+                    int sockerr = get_last_socket_error();
+
+                    if (!tfo || sockerr != EISCONN) {
+                        BIO_printf(bio_err, "write:errno=%d\n", sockerr);
+                        goto shut;
+                    }
                 } else {
                     read_tty = 1;
                     write_ssl = 0;
@@ -3131,6 +3157,7 @@ int s_client_main(int argc, char **argv)
     OPENSSL_free(srp_arg.srppassin);
 #endif
     OPENSSL_free(sname_alloc);
+    BIO_ADDR_free(tfo_addr);
     OPENSSL_free(connectstr);
     OPENSSL_free(bindstr);
     OPENSSL_free(bindhost);
diff --git a/apps/s_server.c b/apps/s_server.c
index 2036d51795..736d8498d1 100644
--- a/apps/s_server.c
+++ b/apps/s_server.c
@@ -717,6 +717,7 @@ typedef enum OPTION_choice {
     OPT_KEYLOG_FILE, OPT_MAX_EARLY, OPT_RECV_MAX_EARLY, OPT_EARLY_DATA,
     OPT_S_NUM_TICKETS, OPT_ANTI_REPLAY, OPT_NO_ANTI_REPLAY, OPT_SCTP_LABEL_BUG,
     OPT_HTTP_SERVER_BINMODE, OPT_NOCANAMES, OPT_IGNORE_UNEXPECTED_EOF, OPT_KTLS,
+    OPT_TFO,
     OPT_R_ENUM,
     OPT_S_ENUM,
     OPT_V_ENUM,
@@ -747,6 +748,9 @@ const OPTIONS s_server_options[] = {
 #endif
     {"4", OPT_4, '-', "Use IPv4 only"},
     {"6", OPT_6, '-', "Use IPv6 only"},
+#if defined(TCP_FASTOPEN) && !defined(OPENSSL_NO_TFO)
+    {"tfo", OPT_TFO, '-', "Listen for TCP Fast Open connections"},
+#endif
 
     OPT_SECTION("Identity"),
     {"context", OPT_CONTEXT, 's', "Set session ID context"},
@@ -1057,6 +1061,7 @@ int s_server_main(int argc, char *argv[])
 #ifndef OPENSSL_NO_KTLS
     int enable_ktls = 0;
 #endif
+    int tfo = 0;
 
     /* Init of few remaining global variables */
     local_argc = argc;
@@ -1649,6 +1654,9 @@ int s_server_main(int argc, char *argv[])
         case OPT_IGNORE_UNEXPECTED_EOF:
             ignore_unexpected_eof = 1;
             break;
+        case OPT_TFO:
+            tfo = 1;
+            break;
         }
     }
 
@@ -1677,6 +1685,11 @@ int s_server_main(int argc, char *argv[])
     }
 #endif
 
+    if (tfo && socket_type != SOCK_STREAM) {
+        BIO_printf(bio_err, "Can only use -tfo with TLS\n");
+        goto end;
+    }
+
     if (stateless && socket_type != SOCK_STREAM) {
         BIO_printf(bio_err, "Can only use --stateless with TLS\n");
         goto end;
@@ -2240,8 +2253,10 @@ int s_server_main(int argc, char *argv[])
         && unlink_unix_path)
         unlink(host);
 #endif
+    if (tfo)
+        BIO_printf(bio_s_out, "Listening for TFO\n");
     do_server(&accept_socket, host, port, socket_family, socket_type, protocol,
-              server_cb, context, naccept, bio_s_out);
+              server_cb, context, naccept, bio_s_out, tfo);
     print_stats(bio_s_out, ctx);
     ret = 0;
  end: