Disable middlebox for dtls

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22275)

4 files changed, 16 insertions, 11 deletions

diff --git a/ssl/ssl_local.h b/ssl/ssl_local.h
index 51f69f269c..0fb5cb15f0 100644
--- a/ssl/ssl_local.h
+++ b/ssl/ssl_local.h
@@ -258,6 +258,11 @@
 # define SSL_CONNECTION_IS_DTLS(s) \
     (SSL_CONNECTION_GET_SSL(s)->method->ssl3_enc->enc_flags & SSL_ENC_FLAG_DTLS)
 
+/* Check if an SSL structure is using DTLS */
+# define SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s) \
+    ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0 \
+     && !SSL_CONNECTION_IS_DTLS(s))
+
 /* Check if we are using DTLSv1.3 */
 # define SSL_CONNECTION_IS_DTLS13(s) (SSL_CONNECTION_IS_DTLS(s) \
     && DTLS_VERSION_GE(SSL_CONNECTION_GET_SSL(s)->method->version, DTLS1_3_VERSION) \
diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index 9d6cb5be08..c96ae8728b 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c
@@ -464,7 +464,7 @@ static WRITE_TRAN ossl_statem_client13_write_transition(SSL_CONNECTION *s)
         if (s->early_data_state == SSL_EARLY_DATA_WRITE_RETRY
                 || s->early_data_state == SSL_EARLY_DATA_FINISHED_WRITING)
             st->hand_state = TLS_ST_PENDING_EARLY_DATA_END;
-        else if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0
+        else if (SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s)
                  && s->hello_retry_request == SSL_HRR_NONE)
             st->hand_state = TLS_ST_CW_CHANGE;
         else if (s->s3.tmp.cert_req == 0)
@@ -565,7 +565,7 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s)
              * We are assuming this is a (D)TLSv1.3 connection, although we haven't
              * actually selected a version yet.
              */
-            if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0)
+            if (SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s))
                 st->hand_state = TLS_ST_CW_CHANGE;
             else
                 st->hand_state = TLS_ST_EARLY_DATA;
@@ -584,7 +584,7 @@ WRITE_TRAN ossl_statem_client_write_transition(SSL_CONNECTION *s)
          * CCS unless middlebox compat mode is off, or we already issued one
          * because we did early data.
          */
-        if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0
+        if (SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s)
                 && s->early_data_state != SSL_EARLY_DATA_FINISHED_WRITING)
             st->hand_state = TLS_ST_CW_CHANGE;
         else
@@ -799,7 +799,7 @@ WORK_STATE ossl_statem_client_post_work(SSL_CONNECTION *s, WORK_STATE wst)
              * cipher state function associated with the SSL_METHOD. Instead
              * we call tls13_change_cipher_state() directly.
              */
-            if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) == 0) {
+            if (!SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s)) {
                 if (!tls13_change_cipher_state(s,
                             SSL3_CC_EARLY | SSL3_CHANGE_CIPHER_CLIENT_WRITE)) {
                     /* SSLfatal() already called */
@@ -1251,7 +1251,7 @@ CON_FUNC_RETURN tls_construct_client_hello(SSL_CONNECTION *s, WPACKET *pkt)
             || s->session->ssl_version == TLS1_3_VERSION
             || s->session->ssl_version == DTLS1_3_VERSION) {
         if (s->version == TLS1_3_VERSION
-                && (s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0) {
+                && SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s)) {
             sess_id_len = sizeof(s->tmp_session_id);
             s->tmp_session_id_len = sess_id_len;
             session_id = s->tmp_session_id;
@@ -1791,7 +1791,7 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL_CONNECTION *s, PACKET *pkt)
          * compat this doesn't cause a problem.
          */
         if (s->early_data_state == SSL_EARLY_DATA_NONE
-                && (s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) == 0
+                && !SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s)
                 && !ssl->method->ssl3_enc->change_cipher_state(s,
                     SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_WRITE)) {
             /* SSLfatal() already called */
@@ -3790,7 +3790,7 @@ CON_FUNC_RETURN tls_construct_client_certificate(SSL_CONNECTION *s,
     if (SSL_CONNECTION_IS_VERSION13(s)
             && SSL_IS_FIRST_HANDSHAKE(s)
             && (s->early_data_state != SSL_EARLY_DATA_NONE
-                || (s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0)
+                || SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s))
             && (!ssl->method->ssl3_enc->change_cipher_state(s,
                     SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_WRITE))) {
         /*
diff --git a/ssl/statem/statem_lib.c b/ssl/statem/statem_lib.c
index 13b4a98c11..98b461016b 100644
--- a/ssl/statem/statem_lib.c
+++ b/ssl/statem/statem_lib.c
@@ -621,7 +621,7 @@ CON_FUNC_RETURN tls_construct_finished(SSL_CONNECTION *s, WPACKET *pkt)
     if (SSL_CONNECTION_IS_VERSION13(s)
             && !s->server
             && (s->early_data_state != SSL_EARLY_DATA_NONE
-                || (s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0)
+                || SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s))
             && s->s3.tmp.cert_req == 0
             && (!ssl->method->ssl3_enc->change_cipher_state(s,
                     SSL3_CC_HANDSHAKE | SSL3_CHANGE_CIPHER_CLIENT_WRITE))) {;
diff --git a/ssl/statem/statem_srvr.c b/ssl/statem/statem_srvr.c
index e217410f40..2d03f5e9dc 100644
--- a/ssl/statem/statem_srvr.c
+++ b/ssl/statem/statem_srvr.c
@@ -497,7 +497,7 @@ static WRITE_TRAN ossl_statem_server13_write_transition(SSL_CONNECTION *s)
         return WRITE_TRAN_CONTINUE;
 
     case TLS_ST_SW_SRVR_HELLO:
-        if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0
+        if (SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s)
                 && s->hello_retry_request != SSL_HRR_COMPLETE)
             st->hand_state = TLS_ST_SW_CHANGE;
         else if (s->hello_retry_request == SSL_HRR_PENDING)
@@ -907,7 +907,7 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst)
     case TLS_ST_SW_SRVR_HELLO:
         if (SSL_CONNECTION_IS_VERSION13(s)
             && s->hello_retry_request == SSL_HRR_PENDING) {
-            if ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) == 0
+            if (!SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s)
                     && statem_flush(s) != 1)
                 return WORK_MORE_A;
             break;
@@ -943,7 +943,7 @@ WORK_STATE ossl_statem_server_post_work(SSL_CONNECTION *s, WORK_STATE wst)
         }
 #endif
         if (!SSL_CONNECTION_IS_VERSION13(s)
-                || ((s->options & SSL_OP_ENABLE_MIDDLEBOX_COMPAT) != 0
+                || (SSL_CONNECTION_MIDDLEBOX_IS_ENABLED(s)
                     && s->hello_retry_request != SSL_HRR_COMPLETE))
             break;
         /* Fall through */