Add support for KTLS zerocopy sendfile on Linux

TLS device offload allows to perform zerocopy sendfile transmissions.
FreeBSD provides this feature by default, and Linux 5.19 introduced it
as an opt-in. Zerocopy improves the TX rate significantly, but has a
side effect: if the underlying file is changed while being transmitted,
and a TCP retransmission happens, the receiver may get a TLS record
containing both new and old data, which leads to an authentication
failure and termination of connection. This effect is the reason Linux
makes a copy on sendfile by default.

This commit adds support for TLS zerocopy sendfile on Linux disabled by
default to avoid any unlikely backward compatibility issues on Linux,
although sacrificing consistency in OpenSSL's behavior on Linux and
FreeBSD. A new option called KTLSTxZerocopySendfile is added to enable
the new zerocopy behavior on Linux. This option should be used when the
the application guarantees that the file is not modified during
transmission, or it doesn't care about breaking the connection.

The related documentation is also added in this commit. The unit test
added doesn't test the actual functionality (it would require specific
hardware and a non-local peer), but solely checks that it's possible to
set the new option flag.

Signed-off-by: Maxim Mikityanskiy <maximmi@nvidia.com>
Reviewed-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Boris Pismenny <borisp@nvidia.com>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Todd Short <todd.short@me.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/18650)

14 files changed, 115 insertions, 5 deletions

diff --git a/CHANGES.md b/CHANGES.md
index 2c12daf151..ede13f7d79 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -226,6 +226,10 @@ OpenSSL 3.2
 
    *Tianjia Zhang*
 
+ * Zerocopy KTLS sendfile() support on Linux.
+
+   *Maxim Mikityanskiy*
+
 OpenSSL 3.0
 -----------
 
diff --git a/apps/s_server.c b/apps/s_server.c
index f519505ade..46af6b87da 100644
--- a/apps/s_server.c
+++ b/apps/s_server.c
@@ -96,6 +96,7 @@ static int keymatexportlen = 20;
 static int async = 0;
 
 static int use_sendfile = 0;
+static int use_zc_sendfile = 0;
 
 static const char *session_id_prefix = NULL;
 
@@ -716,6 +717,7 @@ typedef enum OPTION_choice {
     OPT_KEYLOG_FILE, OPT_MAX_EARLY, OPT_RECV_MAX_EARLY, OPT_EARLY_DATA,
     OPT_S_NUM_TICKETS, OPT_ANTI_REPLAY, OPT_NO_ANTI_REPLAY, OPT_SCTP_LABEL_BUG,
     OPT_HTTP_SERVER_BINMODE, OPT_NOCANAMES, OPT_IGNORE_UNEXPECTED_EOF, OPT_KTLS,
+    OPT_USE_ZC_SENDFILE,
     OPT_TFO, OPT_CERT_COMP,
     OPT_R_ENUM,
     OPT_S_ENUM,
@@ -966,6 +968,7 @@ const OPTIONS s_server_options[] = {
 #ifndef OPENSSL_NO_KTLS
     {"ktls", OPT_KTLS, '-', "Enable Kernel TLS for sending and receiving"},
     {"sendfile", OPT_SENDFILE, '-', "Use sendfile to response file with -WWW"},
+    {"zerocopy_sendfile", OPT_USE_ZC_SENDFILE, '-', "Use zerocopy mode of KTLS sendfile"},
 #endif
 
     OPT_R_OPTIONS,
@@ -1080,6 +1083,7 @@ int s_server_main(int argc, char *argv[])
     s_brief = 0;
     async = 0;
     use_sendfile = 0;
+    use_zc_sendfile = 0;
 
     port = OPENSSL_strdup(PORT);
     cctx = SSL_CONF_CTX_new();
@@ -1656,6 +1660,11 @@ int s_server_main(int argc, char *argv[])
             use_sendfile = 1;
 #endif
             break;
+        case OPT_USE_ZC_SENDFILE:
+#ifndef OPENSSL_NO_KTLS
+            use_zc_sendfile = 1;
+#endif
+            break;
         case OPT_IGNORE_UNEXPECTED_EOF:
             ignore_unexpected_eof = 1;
             break;
@@ -1728,6 +1737,11 @@ int s_server_main(int argc, char *argv[])
 #endif
 
 #ifndef OPENSSL_NO_KTLS
+    if (use_zc_sendfile && !use_sendfile) {
+        BIO_printf(bio_out, "Warning: -zerocopy_sendfile depends on -sendfile, enabling -sendfile now.\n");
+        use_sendfile = 1;
+    }
+
     if (use_sendfile && enable_ktls == 0) {
         BIO_printf(bio_out, "Warning: -sendfile depends on -ktls, enabling -ktls now.\n");
         enable_ktls = 1;
@@ -1933,6 +1947,8 @@ int s_server_main(int argc, char *argv[])
 #ifndef OPENSSL_NO_KTLS
     if (enable_ktls)
         SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS);
+    if (use_zc_sendfile)
+        SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE);
 #endif
 
     if (max_send_fragment > 0
diff --git a/crypto/bio/bss_conn.c b/crypto/bio/bss_conn.c
index 75bfe64566..f494b14000 100644
--- a/crypto/bio/bss_conn.c
+++ b/crypto/bio/bss_conn.c
@@ -598,6 +598,11 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr)
         BIO_clear_ktls_ctrl_msg_flag(b);
         ret = 0;
         break;
+    case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE:
+        ret = ktls_enable_tx_zerocopy_sendfile(b->num);
+        if (ret)
+            BIO_set_ktls_zerocopy_sendfile_flag(b);
+        break;
 # endif
     default:
         ret = 0;
diff --git a/crypto/bio/bss_sock.c b/crypto/bio/bss_sock.c
index 69dfd37bfe..f64eb8c843 100644
--- a/crypto/bio/bss_sock.c
+++ b/crypto/bio/bss_sock.c
@@ -235,6 +235,11 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr)
         BIO_clear_ktls_ctrl_msg_flag(b);
         ret = 0;
         break;
+    case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE:
+        ret = ktls_enable_tx_zerocopy_sendfile(b->num);
+        if (ret)
+            BIO_set_ktls_zerocopy_sendfile_flag(b);
+        break;
 # endif
     case BIO_CTRL_EOF:
         ret = (b->flags & BIO_FLAGS_IN_EOF) != 0;
diff --git a/doc/man1/openssl-s_server.pod.in b/doc/man1/openssl-s_server.pod.in
index 94f3b4b46c..a1e354908c 100644
--- a/doc/man1/openssl-s_server.pod.in
+++ b/doc/man1/openssl-s_server.pod.in
@@ -132,6 +132,7 @@ B<openssl> B<s_server>
 [B<-alpn> I<val>]
 [B<-ktls>]
 [B<-sendfile>]
+[B<-zerocopy_sendfile>]
 [B<-keylogfile> I<outfile>]
 [B<-recv_max_early_data> I<int>]
 [B<-max_early_data> I<int>]
@@ -792,6 +793,15 @@ instead of BIO_write() to send the HTTP response requested by a client.
 This option is only valid when B<-ktls> along with B<-WWW> or B<-HTTP>
 are specified.
 
+=item B<-zerocopy_sendfile>
+
+If this option is set, SSL_sendfile() will use the zerocopy TX mode, which gives
+a performance boost when used with KTLS hardware offload. Note that invalid
+TLS records might be transmitted if the file is changed while being sent.
+This option depends on B<-sendfile>; when used alone, B<-sendfile> is implied,
+and a warning is shown. Note that KTLS sendfile on FreeBSD always runs in the
+zerocopy mode.
+
 =item B<-keylogfile> I<outfile>
 
 Appends TLS secrets to the specified keylog file such that external programs
diff --git a/doc/man3/SSL_CONF_cmd.pod b/doc/man3/SSL_CONF_cmd.pod
index c20df37e3b..3717c202bd 100644
--- a/doc/man3/SSL_CONF_cmd.pod
+++ b/doc/man3/SSL_CONF_cmd.pod
@@ -561,6 +561,14 @@ B<RxCertificateCompression>: support receiving compressed certificates, enabled
 default. Inverse of B<SSL_OP_NO_RX_CERTIFICATE_COMPRESSION>: that is,
 B<-RxCertificateCompression> is the same as setting B<SSL_OP_NO_RX_CERTIFICATE_COMPRESSION>.
 
+B<KTLSTxZerocopySendfile>: use the zerocopy TX mode of sendfile(), which gives
+a performance boost when used with KTLS hardware offload. Note that invalid TLS
+records might be transmitted if the file is changed while being sent. This
+option has no effect if B<KTLS> is not enabled. Equivalent to
+B<SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE>. This option only applies to Linux.
+KTLS sendfile on FreeBSD doesn't offer an option to disable zerocopy and
+always runs in this mode.
+
 =item B<VerifyMode>
 
 The B<value> argument is a comma separated list of flags to set.
diff --git a/doc/man3/SSL_CTX_set_options.pod b/doc/man3/SSL_CTX_set_options.pod
index 1da057adb8..b72973f8d0 100644
--- a/doc/man3/SSL_CTX_set_options.pod
+++ b/doc/man3/SSL_CTX_set_options.pod
@@ -175,6 +175,16 @@ by the kernel directly and not via any available OpenSSL Providers. This might
 be undesirable if, for example, the application requires all cryptographic
 operations to be performed by the FIPS provider.
 
+=item SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE
+
+With this option, sendfile() will use the zerocopy mode, which gives a
+performance boost when used with KTLS hardware offload. Note that invalid TLS
+records might be transmitted if the file is changed while being sent. This
+option has no effect if B<SSL_OP_ENABLE_KTLS> is not enabled.
+
+This option only applies to Linux. KTLS sendfile on FreeBSD doesn't offer an
+option to disable zerocopy and always runs in this mode.
+
 =item SSL_OP_ENABLE_MIDDLEBOX_COMPAT
 
 If set then dummy Change Cipher Spec (CCS) messages are sent in TLSv1.3. This
diff --git a/include/internal/bio.h b/include/internal/bio.h
index 40218e1fb0..9481f4c985 100644
--- a/include/internal/bio.h
+++ b/include/internal/bio.h
@@ -43,16 +43,20 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
 # define BIO_CTRL_SET_KTLS                      72
 # define BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG     74
 # define BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG        75
+# define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90
 
 /*
  * This is used with socket BIOs:
  * BIO_FLAGS_KTLS_TX means we are using ktls with this BIO for sending.
  * BIO_FLAGS_KTLS_TX_CTRL_MSG means we are about to send a ctrl message next.
  * BIO_FLAGS_KTLS_RX means we are using ktls with this BIO for receiving.
+ * BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE means we are using the zerocopy mode with
+ * this BIO for sending using sendfile.
  */
 # define BIO_FLAGS_KTLS_TX_CTRL_MSG 0x1000
 # define BIO_FLAGS_KTLS_RX          0x2000
 # define BIO_FLAGS_KTLS_TX          0x4000
+# define BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE 0x8000
 
 /* KTLS related controls and flags */
 # define BIO_set_ktls_flag(b, is_tx) \
@@ -65,6 +69,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
     BIO_test_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG)
 # define BIO_clear_ktls_ctrl_msg_flag(b) \
     BIO_clear_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG)
+# define BIO_set_ktls_zerocopy_sendfile_flag(b) \
+    BIO_set_flags(b, BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE)
 
 # define BIO_set_ktls(b, keyblob, is_tx)   \
      BIO_ctrl(b, BIO_CTRL_SET_KTLS, is_tx, keyblob)
@@ -72,6 +78,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
      BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG, record_type, NULL)
 # define BIO_clear_ktls_ctrl_msg(b) \
      BIO_ctrl(b, BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG, 0, NULL)
+# define BIO_set_ktls_tx_zerocopy_sendfile(b) \
+     BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE, 0, NULL)
 
 /* Functions to allow the core to offer the CORE_BIO type to providers */
 OSSL_CORE_BIO *ossl_core_bio_new_from_bio(BIO *bio);
diff --git a/include/internal/ktls.h b/include/internal/ktls.h
index efa4451d7c..af27a32569 100644
--- a/include/internal/ktls.h
+++ b/include/internal/ktls.h
@@ -214,6 +214,13 @@ static ossl_inline ossl_ssize_t ktls_sendfile(int s, int fd, off_t off,
 #     warning "Skipping Compilation of KTLS receive data path"
 #    endif
 #   endif
+#   if LINUX_VERSION_CODE < KERNEL_VERSION(5, 19, 0)
+#    define OPENSSL_NO_KTLS_ZC_TX
+#    ifndef PEDANTIC
+#     warning "KTLS requires Kernel Headers >= 5.19.0 for zerocopy sendfile"
+#     warning "Skipping Compilation of KTLS zerocopy sendfile"
+#    endif
+#   endif
 #   define OPENSSL_KTLS_AES_GCM_128
 #   if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)
 #    define OPENSSL_KTLS_AES_GCM_256
@@ -293,6 +300,18 @@ static ossl_inline int ktls_start(int fd, ktls_crypto_info_t *crypto_info,
                       crypto_info, crypto_info->tls_crypto_info_len) ? 0 : 1;
 }
 
+static ossl_inline int ktls_enable_tx_zerocopy_sendfile(int fd)
+{
+#ifndef OPENSSL_NO_KTLS_ZC_TX
+    int enable = 1;
+
+    return setsockopt(fd, SOL_TLS, TLS_TX_ZEROCOPY_RO,
+                      &enable, sizeof(enable)) ? 0 : 1;
+#else
+    return 0;
+#endif
+}
+
 /*
  * Send a TLS record using the crypto_info provided in ktls_start and use
  * record_type instead of the default SSL3_RT_APPLICATION_DATA.
diff --git a/include/openssl/bio.h.in b/include/openssl/bio.h.in
index 6105c602fd..89c88c67e7 100644
--- a/include/openssl/bio.h.in
+++ b/include/openssl/bio.h.in
@@ -182,6 +182,11 @@ extern "C" {
 # define BIO_CTRL_DGRAM_GET_NO_TRUNC            88
 # define BIO_CTRL_DGRAM_SET_NO_TRUNC            89
 
+/*
+ * internal BIO:
+ * # define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90
+ */
+
 # define BIO_DGRAM_CAP_NONE                 0U
 # define BIO_DGRAM_CAP_HANDLES_SRC_ADDR     (1U << 0)
 # define BIO_DGRAM_CAP_HANDLES_DST_ADDR     (1U << 1)
@@ -225,7 +230,7 @@ extern "C" {
 # define BIO_FLAGS_NONCLEAR_RST  0x400
 # define BIO_FLAGS_IN_EOF        0x800
 
-/* the BIO FLAGS values 0x1000 to 0x4000 are reserved for internal KTLS flags */
+/* the BIO FLAGS values 0x1000 to 0x8000 are reserved for internal KTLS flags */
 
 typedef union bio_addr_st BIO_ADDR;
 typedef struct bio_addrinfo_st BIO_ADDRINFO;
diff --git a/include/openssl/ssl.h.in b/include/openssl/ssl.h.in
index 2224b3269b..871ad265c5 100644
--- a/include/openssl/ssl.h.in
+++ b/include/openssl/ssl.h.in
@@ -420,6 +420,8 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg);
  */
 # define SSL_OP_NO_TX_CERTIFICATE_COMPRESSION            SSL_OP_BIT(32)
 # define SSL_OP_NO_RX_CERTIFICATE_COMPRESSION            SSL_OP_BIT(33)
+    /* Enable KTLS TX zerocopy on Linux */
+# define SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE         SSL_OP_BIT(34)
 
 /*
  * Option "collections."
diff --git a/ssl/record/methods/ktls_meth.c b/ssl/record/methods/ktls_meth.c
index 5c94837dc0..2f9b11a512 100644
--- a/ssl/record/methods/ktls_meth.c
+++ b/ssl/record/methods/ktls_meth.c
@@ -334,6 +334,14 @@ static int ktls_set_crypto_state(OSSL_RECORD_LAYER *rl, int level,
     if (!BIO_set_ktls(rl->bio, &crypto_info, rl->direction))
         return OSSL_RECORD_RETURN_NON_FATAL_ERR;
 
+    if (rl->direction == OSSL_RECORD_DIRECTION_WRITE &&
+        (rl->options & SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE) != 0)
+        /* Ignore errors. The application opts in to using the zerocopy
+         * optimization. If the running kernel doesn't support it, just
+         * continue without the optimization.
+         */
+        BIO_set_ktls_tx_zerocopy_sendfile(rl->bio);
+
     return OSSL_RECORD_RETURN_SUCCESS;
 }
 
diff --git a/ssl/ssl_conf.c b/ssl/ssl_conf.c
index bebfc501a9..0bea29fb66 100644
--- a/ssl/ssl_conf.c
+++ b/ssl/ssl_conf.c
@@ -400,6 +400,7 @@ static int cmd_Options(SSL_CONF_CTX *cctx, const char *value)
         SSL_FLAG_TBL_CERT("StrictCertCheck", SSL_CERT_FLAG_TLS_STRICT),
         SSL_FLAG_TBL_INV("TxCertificateCompression", SSL_OP_NO_TX_CERTIFICATE_COMPRESSION),
         SSL_FLAG_TBL_INV("RxCertificateCompression", SSL_OP_NO_RX_CERTIFICATE_COMPRESSION),
+        SSL_FLAG_TBL("KTLSTxZerocopySendfile", SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE),
     };
     if (value == NULL)
         return -3;
diff --git a/test/sslapitest.c b/test/sslapitest.c
index 8f14381b56..a26f6286f3 100644
--- a/test/sslapitest.c
+++ b/test/sslapitest.c
@@ -1293,7 +1293,8 @@ end:
 #define SENDFILE_CHUNK                  (4 * 4096)
 #define min(a,b)                        ((a) > (b) ? (b) : (a))
 
-static int execute_test_ktls_sendfile(int tls_version, const char *cipher)
+static int execute_test_ktls_sendfile(int tls_version, const char *cipher,
+                                      int zerocopy)
 {
     SSL_CTX *cctx = NULL, *sctx = NULL;
     SSL *clientssl = NULL, *serverssl = NULL;
@@ -1350,6 +1351,12 @@ static int execute_test_ktls_sendfile(int tls_version, const char *cipher)
     if (!TEST_true(SSL_set_options(serverssl, SSL_OP_ENABLE_KTLS)))
         goto end;
 
+    if (zerocopy) {
+        if (!TEST_true(SSL_set_options(serverssl,
+                                       SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE)))
+            goto end;
+    }
+
     if (!TEST_true(create_ssl_connection(serverssl, clientssl,
                                          SSL_ERROR_NONE)))
         goto end;
@@ -1480,14 +1487,16 @@ static int test_ktls(int test)
                              cipher->cipher);
 }
 
-static int test_ktls_sendfile(int tst)
+static int test_ktls_sendfile(int test)
 {
     struct ktls_test_cipher *cipher;
+    int tst = test >> 1;
 
     OPENSSL_assert(tst < (int)NUM_KTLS_TEST_CIPHERS);
     cipher = &ktls_test_ciphers[tst];
 
-    return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher);
+    return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher,
+                                      test & 1);
 }
 #endif
 
@@ -10544,7 +10553,7 @@ int setup_tests(void)
 #if !defined(OPENSSL_NO_KTLS) && !defined(OPENSSL_NO_SOCK)
 # if !defined(OPENSSL_NO_TLS1_2) || !defined(OSSL_NO_USABLE_TLS1_3)
     ADD_ALL_TESTS(test_ktls, NUM_KTLS_TEST_CIPHERS * 4);
-    ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS);
+    ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS * 2);
 # endif
 #endif
     ADD_TEST(test_large_message_tls);