summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--CHANGES.md4
-rw-r--r--apps/s_server.c16
-rw-r--r--crypto/bio/bss_conn.c5
-rw-r--r--crypto/bio/bss_sock.c5
-rw-r--r--doc/man1/openssl-s_server.pod.in10
-rw-r--r--doc/man3/SSL_CONF_cmd.pod8
-rw-r--r--doc/man3/SSL_CTX_set_options.pod10
-rw-r--r--include/internal/bio.h8
-rw-r--r--include/internal/ktls.h19
-rw-r--r--include/openssl/bio.h.in7
-rw-r--r--include/openssl/ssl.h.in2
-rw-r--r--ssl/record/methods/ktls_meth.c8
-rw-r--r--ssl/ssl_conf.c1
-rw-r--r--test/sslapitest.c17
14 files changed, 115 insertions, 5 deletions
diff --git a/CHANGES.md b/CHANGES.md
index 2c12daf151..ede13f7d79 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -226,6 +226,10 @@ OpenSSL 3.2
*Tianjia Zhang*
+ * Zerocopy KTLS sendfile() support on Linux.
+
+ *Maxim Mikityanskiy*
+
OpenSSL 3.0
-----------
diff --git a/apps/s_server.c b/apps/s_server.c
index f519505ade..46af6b87da 100644
--- a/apps/s_server.c
+++ b/apps/s_server.c
@@ -96,6 +96,7 @@ static int keymatexportlen = 20;
static int async = 0;
static int use_sendfile = 0;
+static int use_zc_sendfile = 0;
static const char *session_id_prefix = NULL;
@@ -716,6 +717,7 @@ typedef enum OPTION_choice {
OPT_KEYLOG_FILE, OPT_MAX_EARLY, OPT_RECV_MAX_EARLY, OPT_EARLY_DATA,
OPT_S_NUM_TICKETS, OPT_ANTI_REPLAY, OPT_NO_ANTI_REPLAY, OPT_SCTP_LABEL_BUG,
OPT_HTTP_SERVER_BINMODE, OPT_NOCANAMES, OPT_IGNORE_UNEXPECTED_EOF, OPT_KTLS,
+ OPT_USE_ZC_SENDFILE,
OPT_TFO, OPT_CERT_COMP,
OPT_R_ENUM,
OPT_S_ENUM,
@@ -966,6 +968,7 @@ const OPTIONS s_server_options[] = {
#ifndef OPENSSL_NO_KTLS
{"ktls", OPT_KTLS, '-', "Enable Kernel TLS for sending and receiving"},
{"sendfile", OPT_SENDFILE, '-', "Use sendfile to response file with -WWW"},
+ {"zerocopy_sendfile", OPT_USE_ZC_SENDFILE, '-', "Use zerocopy mode of KTLS sendfile"},
#endif
OPT_R_OPTIONS,
@@ -1080,6 +1083,7 @@ int s_server_main(int argc, char *argv[])
s_brief = 0;
async = 0;
use_sendfile = 0;
+ use_zc_sendfile = 0;
port = OPENSSL_strdup(PORT);
cctx = SSL_CONF_CTX_new();
@@ -1656,6 +1660,11 @@ int s_server_main(int argc, char *argv[])
use_sendfile = 1;
#endif
break;
+ case OPT_USE_ZC_SENDFILE:
+#ifndef OPENSSL_NO_KTLS
+ use_zc_sendfile = 1;
+#endif
+ break;
case OPT_IGNORE_UNEXPECTED_EOF:
ignore_unexpected_eof = 1;
break;
@@ -1728,6 +1737,11 @@ int s_server_main(int argc, char *argv[])
#endif
#ifndef OPENSSL_NO_KTLS
+ if (use_zc_sendfile && !use_sendfile) {
+ BIO_printf(bio_out, "Warning: -zerocopy_sendfile depends on -sendfile, enabling -sendfile now.\n");
+ use_sendfile = 1;
+ }
+
if (use_sendfile && enable_ktls == 0) {
BIO_printf(bio_out, "Warning: -sendfile depends on -ktls, enabling -ktls now.\n");
enable_ktls = 1;
@@ -1933,6 +1947,8 @@ int s_server_main(int argc, char *argv[])
#ifndef OPENSSL_NO_KTLS
if (enable_ktls)
SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS);
+ if (use_zc_sendfile)
+ SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE);
#endif
if (max_send_fragment > 0
diff --git a/crypto/bio/bss_conn.c b/crypto/bio/bss_conn.c
index 75bfe64566..f494b14000 100644
--- a/crypto/bio/bss_conn.c
+++ b/crypto/bio/bss_conn.c
@@ -598,6 +598,11 @@ static long conn_ctrl(BIO *b, int cmd, long num, void *ptr)
BIO_clear_ktls_ctrl_msg_flag(b);
ret = 0;
break;
+ case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE:
+ ret = ktls_enable_tx_zerocopy_sendfile(b->num);
+ if (ret)
+ BIO_set_ktls_zerocopy_sendfile_flag(b);
+ break;
# endif
default:
ret = 0;
diff --git a/crypto/bio/bss_sock.c b/crypto/bio/bss_sock.c
index 69dfd37bfe..f64eb8c843 100644
--- a/crypto/bio/bss_sock.c
+++ b/crypto/bio/bss_sock.c
@@ -235,6 +235,11 @@ static long sock_ctrl(BIO *b, int cmd, long num, void *ptr)
BIO_clear_ktls_ctrl_msg_flag(b);
ret = 0;
break;
+ case BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE:
+ ret = ktls_enable_tx_zerocopy_sendfile(b->num);
+ if (ret)
+ BIO_set_ktls_zerocopy_sendfile_flag(b);
+ break;
# endif
case BIO_CTRL_EOF:
ret = (b->flags & BIO_FLAGS_IN_EOF) != 0;
diff --git a/doc/man1/openssl-s_server.pod.in b/doc/man1/openssl-s_server.pod.in
index 94f3b4b46c..a1e354908c 100644
--- a/doc/man1/openssl-s_server.pod.in
+++ b/doc/man1/openssl-s_server.pod.in
@@ -132,6 +132,7 @@ B<openssl> B<s_server>
[B<-alpn> I<val>]
[B<-ktls>]
[B<-sendfile>]
+[B<-zerocopy_sendfile>]
[B<-keylogfile> I<outfile>]
[B<-recv_max_early_data> I<int>]
[B<-max_early_data> I<int>]
@@ -792,6 +793,15 @@ instead of BIO_write() to send the HTTP response requested by a client.
This option is only valid when B<-ktls> along with B<-WWW> or B<-HTTP>
are specified.
+=item B<-zerocopy_sendfile>
+
+If this option is set, SSL_sendfile() will use the zerocopy TX mode, which gives
+a performance boost when used with KTLS hardware offload. Note that invalid
+TLS records might be transmitted if the file is changed while being sent.
+This option depends on B<-sendfile>; when used alone, B<-sendfile> is implied,
+and a warning is shown. Note that KTLS sendfile on FreeBSD always runs in the
+zerocopy mode.
+
=item B<-keylogfile> I<outfile>
Appends TLS secrets to the specified keylog file such that external programs
diff --git a/doc/man3/SSL_CONF_cmd.pod b/doc/man3/SSL_CONF_cmd.pod
index c20df37e3b..3717c202bd 100644
--- a/doc/man3/SSL_CONF_cmd.pod
+++ b/doc/man3/SSL_CONF_cmd.pod
@@ -561,6 +561,14 @@ B<RxCertificateCompression>: support receiving compressed certificates, enabled
default. Inverse of B<SSL_OP_NO_RX_CERTIFICATE_COMPRESSION>: that is,
B<-RxCertificateCompression> is the same as setting B<SSL_OP_NO_RX_CERTIFICATE_COMPRESSION>.
+B<KTLSTxZerocopySendfile>: use the zerocopy TX mode of sendfile(), which gives
+a performance boost when used with KTLS hardware offload. Note that invalid TLS
+records might be transmitted if the file is changed while being sent. This
+option has no effect if B<KTLS> is not enabled. Equivalent to
+B<SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE>. This option only applies to Linux.
+KTLS sendfile on FreeBSD doesn't offer an option to disable zerocopy and
+always runs in this mode.
+
=item B<VerifyMode>
The B<value> argument is a comma separated list of flags to set.
diff --git a/doc/man3/SSL_CTX_set_options.pod b/doc/man3/SSL_CTX_set_options.pod
index 1da057adb8..b72973f8d0 100644
--- a/doc/man3/SSL_CTX_set_options.pod
+++ b/doc/man3/SSL_CTX_set_options.pod
@@ -175,6 +175,16 @@ by the kernel directly and not via any available OpenSSL Providers. This might
be undesirable if, for example, the application requires all cryptographic
operations to be performed by the FIPS provider.
+=item SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE
+
+With this option, sendfile() will use the zerocopy mode, which gives a
+performance boost when used with KTLS hardware offload. Note that invalid TLS
+records might be transmitted if the file is changed while being sent. This
+option has no effect if B<SSL_OP_ENABLE_KTLS> is not enabled.
+
+This option only applies to Linux. KTLS sendfile on FreeBSD doesn't offer an
+option to disable zerocopy and always runs in this mode.
+
=item SSL_OP_ENABLE_MIDDLEBOX_COMPAT
If set then dummy Change Cipher Spec (CCS) messages are sent in TLSv1.3. This
diff --git a/include/internal/bio.h b/include/internal/bio.h
index 40218e1fb0..9481f4c985 100644
--- a/include/internal/bio.h
+++ b/include/internal/bio.h
@@ -43,16 +43,20 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
# define BIO_CTRL_SET_KTLS 72
# define BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG 74
# define BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG 75
+# define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90
/*
* This is used with socket BIOs:
* BIO_FLAGS_KTLS_TX means we are using ktls with this BIO for sending.
* BIO_FLAGS_KTLS_TX_CTRL_MSG means we are about to send a ctrl message next.
* BIO_FLAGS_KTLS_RX means we are using ktls with this BIO for receiving.
+ * BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE means we are using the zerocopy mode with
+ * this BIO for sending using sendfile.
*/
# define BIO_FLAGS_KTLS_TX_CTRL_MSG 0x1000
# define BIO_FLAGS_KTLS_RX 0x2000
# define BIO_FLAGS_KTLS_TX 0x4000
+# define BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE 0x8000
/* KTLS related controls and flags */
# define BIO_set_ktls_flag(b, is_tx) \
@@ -65,6 +69,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
BIO_test_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG)
# define BIO_clear_ktls_ctrl_msg_flag(b) \
BIO_clear_flags(b, BIO_FLAGS_KTLS_TX_CTRL_MSG)
+# define BIO_set_ktls_zerocopy_sendfile_flag(b) \
+ BIO_set_flags(b, BIO_FLAGS_KTLS_TX_ZEROCOPY_SENDFILE)
# define BIO_set_ktls(b, keyblob, is_tx) \
BIO_ctrl(b, BIO_CTRL_SET_KTLS, is_tx, keyblob)
@@ -72,6 +78,8 @@ int bread_conv(BIO *bio, char *data, size_t datal, size_t *read);
BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_SEND_CTRL_MSG, record_type, NULL)
# define BIO_clear_ktls_ctrl_msg(b) \
BIO_ctrl(b, BIO_CTRL_CLEAR_KTLS_TX_CTRL_MSG, 0, NULL)
+# define BIO_set_ktls_tx_zerocopy_sendfile(b) \
+ BIO_ctrl(b, BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE, 0, NULL)
/* Functions to allow the core to offer the CORE_BIO type to providers */
OSSL_CORE_BIO *ossl_core_bio_new_from_bio(BIO *bio);
diff --git a/include/internal/ktls.h b/include/internal/ktls.h
index efa4451d7c..af27a32569 100644
--- a/include/internal/ktls.h
+++ b/include/internal/ktls.h
@@ -214,6 +214,13 @@ static ossl_inline ossl_ssize_t ktls_sendfile(int s, int fd, off_t off,
# warning "Skipping Compilation of KTLS receive data path"
# endif
# endif
+# if LINUX_VERSION_CODE < KERNEL_VERSION(5, 19, 0)
+# define OPENSSL_NO_KTLS_ZC_TX
+# ifndef PEDANTIC
+# warning "KTLS requires Kernel Headers >= 5.19.0 for zerocopy sendfile"
+# warning "Skipping Compilation of KTLS zerocopy sendfile"
+# endif
+# endif
# define OPENSSL_KTLS_AES_GCM_128
# if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 1, 0)
# define OPENSSL_KTLS_AES_GCM_256
@@ -293,6 +300,18 @@ static ossl_inline int ktls_start(int fd, ktls_crypto_info_t *crypto_info,
crypto_info, crypto_info->tls_crypto_info_len) ? 0 : 1;
}
+static ossl_inline int ktls_enable_tx_zerocopy_sendfile(int fd)
+{
+#ifndef OPENSSL_NO_KTLS_ZC_TX
+ int enable = 1;
+
+ return setsockopt(fd, SOL_TLS, TLS_TX_ZEROCOPY_RO,
+ &enable, sizeof(enable)) ? 0 : 1;
+#else
+ return 0;
+#endif
+}
+
/*
* Send a TLS record using the crypto_info provided in ktls_start and use
* record_type instead of the default SSL3_RT_APPLICATION_DATA.
diff --git a/include/openssl/bio.h.in b/include/openssl/bio.h.in
index 6105c602fd..89c88c67e7 100644
--- a/include/openssl/bio.h.in
+++ b/include/openssl/bio.h.in
@@ -182,6 +182,11 @@ extern "C" {
# define BIO_CTRL_DGRAM_GET_NO_TRUNC 88
# define BIO_CTRL_DGRAM_SET_NO_TRUNC 89
+/*
+ * internal BIO:
+ * # define BIO_CTRL_SET_KTLS_TX_ZEROCOPY_SENDFILE 90
+ */
+
# define BIO_DGRAM_CAP_NONE 0U
# define BIO_DGRAM_CAP_HANDLES_SRC_ADDR (1U << 0)
# define BIO_DGRAM_CAP_HANDLES_DST_ADDR (1U << 1)
@@ -225,7 +230,7 @@ extern "C" {
# define BIO_FLAGS_NONCLEAR_RST 0x400
# define BIO_FLAGS_IN_EOF 0x800
-/* the BIO FLAGS values 0x1000 to 0x4000 are reserved for internal KTLS flags */
+/* the BIO FLAGS values 0x1000 to 0x8000 are reserved for internal KTLS flags */
typedef union bio_addr_st BIO_ADDR;
typedef struct bio_addrinfo_st BIO_ADDRINFO;
diff --git a/include/openssl/ssl.h.in b/include/openssl/ssl.h.in
index 2224b3269b..871ad265c5 100644
--- a/include/openssl/ssl.h.in
+++ b/include/openssl/ssl.h.in
@@ -420,6 +420,8 @@ typedef int (*SSL_async_callback_fn)(SSL *s, void *arg);
*/
# define SSL_OP_NO_TX_CERTIFICATE_COMPRESSION SSL_OP_BIT(32)
# define SSL_OP_NO_RX_CERTIFICATE_COMPRESSION SSL_OP_BIT(33)
+ /* Enable KTLS TX zerocopy on Linux */
+# define SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE SSL_OP_BIT(34)
/*
* Option "collections."
diff --git a/ssl/record/methods/ktls_meth.c b/ssl/record/methods/ktls_meth.c
index 5c94837dc0..2f9b11a512 100644
--- a/ssl/record/methods/ktls_meth.c
+++ b/ssl/record/methods/ktls_meth.c
@@ -334,6 +334,14 @@ static int ktls_set_crypto_state(OSSL_RECORD_LAYER *rl, int level,
if (!BIO_set_ktls(rl->bio, &crypto_info, rl->direction))
return OSSL_RECORD_RETURN_NON_FATAL_ERR;
+ if (rl->direction == OSSL_RECORD_DIRECTION_WRITE &&
+ (rl->options & SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE) != 0)
+ /* Ignore errors. The application opts in to using the zerocopy
+ * optimization. If the running kernel doesn't support it, just
+ * continue without the optimization.
+ */
+ BIO_set_ktls_tx_zerocopy_sendfile(rl->bio);
+
return OSSL_RECORD_RETURN_SUCCESS;
}
diff --git a/ssl/ssl_conf.c b/ssl/ssl_conf.c
index bebfc501a9..0bea29fb66 100644
--- a/ssl/ssl_conf.c
+++ b/ssl/ssl_conf.c
@@ -400,6 +400,7 @@ static int cmd_Options(SSL_CONF_CTX *cctx, const char *value)
SSL_FLAG_TBL_CERT("StrictCertCheck", SSL_CERT_FLAG_TLS_STRICT),
SSL_FLAG_TBL_INV("TxCertificateCompression", SSL_OP_NO_TX_CERTIFICATE_COMPRESSION),
SSL_FLAG_TBL_INV("RxCertificateCompression", SSL_OP_NO_RX_CERTIFICATE_COMPRESSION),
+ SSL_FLAG_TBL("KTLSTxZerocopySendfile", SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE),
};
if (value == NULL)
return -3;
diff --git a/test/sslapitest.c b/test/sslapitest.c
index 8f14381b56..a26f6286f3 100644
--- a/test/sslapitest.c
+++ b/test/sslapitest.c
@@ -1293,7 +1293,8 @@ end:
#define SENDFILE_CHUNK (4 * 4096)
#define min(a,b) ((a) > (b) ? (b) : (a))
-static int execute_test_ktls_sendfile(int tls_version, const char *cipher)
+static int execute_test_ktls_sendfile(int tls_version, const char *cipher,
+ int zerocopy)
{
SSL_CTX *cctx = NULL, *sctx = NULL;
SSL *clientssl = NULL, *serverssl = NULL;
@@ -1350,6 +1351,12 @@ static int execute_test_ktls_sendfile(int tls_version, const char *cipher)
if (!TEST_true(SSL_set_options(serverssl, SSL_OP_ENABLE_KTLS)))
goto end;
+ if (zerocopy) {
+ if (!TEST_true(SSL_set_options(serverssl,
+ SSL_OP_ENABLE_KTLS_TX_ZEROCOPY_SENDFILE)))
+ goto end;
+ }
+
if (!TEST_true(create_ssl_connection(serverssl, clientssl,
SSL_ERROR_NONE)))
goto end;
@@ -1480,14 +1487,16 @@ static int test_ktls(int test)
cipher->cipher);
}
-static int test_ktls_sendfile(int tst)
+static int test_ktls_sendfile(int test)
{
struct ktls_test_cipher *cipher;
+ int tst = test >> 1;
OPENSSL_assert(tst < (int)NUM_KTLS_TEST_CIPHERS);
cipher = &ktls_test_ciphers[tst];
- return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher);
+ return execute_test_ktls_sendfile(cipher->tls_version, cipher->cipher,
+ test & 1);
}
#endif
@@ -10544,7 +10553,7 @@ int setup_tests(void)
#if !defined(OPENSSL_NO_KTLS) && !defined(OPENSSL_NO_SOCK)
# if !defined(OPENSSL_NO_TLS1_2) || !defined(OSSL_NO_USABLE_TLS1_3)
ADD_ALL_TESTS(test_ktls, NUM_KTLS_TEST_CIPHERS * 4);
- ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS);
+ ADD_ALL_TESTS(test_ktls_sendfile, NUM_KTLS_TEST_CIPHERS * 2);
# endif
#endif
ADD_TEST(test_large_message_tls);
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-01 22:44:27 +0000