Always use uint8_t for TLS record type

Reviewed-by: Hugo Landau <hlandau@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/21823)

14 files changed, 54 insertions, 47 deletions

diff --git a/ssl/d1_msg.c b/ssl/d1_msg.c
index eb84ed6470..23cfa150fc 100644
--- a/ssl/d1_msg.c
+++ b/ssl/d1_msg.c
@@ -9,8 +9,8 @@
 
 #include "ssl_local.h"
 
-int dtls1_write_app_data_bytes(SSL *s, int type, const void *buf_, size_t len,
-                               size_t *written)
+int dtls1_write_app_data_bytes(SSL *s, uint8_t type, const void *buf_,
+                               size_t len, size_t *written)
 {
     int i;
     SSL_CONNECTION *sc = SSL_CONNECTION_FROM_SSL_ONLY(s);
diff --git a/ssl/quic/quic_tls.c b/ssl/quic/quic_tls.c
index 586042bcd7..8b9f812253 100644
--- a/ssl/quic/quic_tls.c
+++ b/ssl/quic/quic_tls.c
@@ -230,7 +230,8 @@ static int quic_processed_read_pending(OSSL_RECORD_LAYER *rl)
     return 0;
 }
 
-static size_t quic_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t len,
+static size_t quic_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type,
+                                   size_t len,
                                    size_t maxfrag, size_t *preffrag)
 {
     return 1;
diff --git a/ssl/record/methods/dtls_meth.c b/ssl/record/methods/dtls_meth.c
index 7b16f42c47..c5477b9b75 100644
--- a/ssl/record/methods/dtls_meth.c
+++ b/ssl/record/methods/dtls_meth.c
@@ -693,7 +693,7 @@ dtls_new_record_layer(OSSL_LIB_CTX *libctx, const char *propq, int vers,
 int dtls_prepare_record_header(OSSL_RECORD_LAYER *rl,
                                WPACKET *thispkt,
                                OSSL_RECORD_TEMPLATE *templ,
-                               unsigned int rectype,
+                               uint8_t rectype,
                                unsigned char **recdata)
 {
     size_t maxcomplen;
diff --git a/ssl/record/methods/ktls_meth.c b/ssl/record/methods/ktls_meth.c
index 5cd833d070..2d3c33bf7a 100644
--- a/ssl/record/methods/ktls_meth.c
+++ b/ssl/record/methods/ktls_meth.c
@@ -493,7 +493,7 @@ static int ktls_initialise_write_packets(OSSL_RECORD_LAYER *rl,
 static int ktls_prepare_record_header(OSSL_RECORD_LAYER *rl,
                                       WPACKET *thispkt,
                                       OSSL_RECORD_TEMPLATE *templ,
-                                      unsigned int rectype,
+                                      uint8_t rectype,
                                       unsigned char **recdata)
 {
     /* The kernel writes the record header, so nothing to do */
diff --git a/ssl/record/methods/recmethod_local.h b/ssl/record/methods/recmethod_local.h
index cf0af62f3a..86517f0062 100644
--- a/ssl/record/methods/recmethod_local.h
+++ b/ssl/record/methods/recmethod_local.h
@@ -138,7 +138,7 @@ struct record_functions_st
 
     /* Write related functions */
 
-    size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, int type, size_t len,
+    size_t (*get_max_records)(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len,
                               size_t maxfrag, size_t *preffrag);
 
     /* Return 1 for success or 0 for error */
@@ -167,13 +167,13 @@ struct record_functions_st
                                     size_t *wpinited);
 
     /* Get the actual record type to be used for a given template */
-    unsigned int (*get_record_type)(OSSL_RECORD_LAYER *rl,
-                                    OSSL_RECORD_TEMPLATE *template);
+    uint8_t (*get_record_type)(OSSL_RECORD_LAYER *rl,
+                               OSSL_RECORD_TEMPLATE *template);
 
     /* Write the record header data to the WPACKET */
     int (*prepare_record_header)(OSSL_RECORD_LAYER *rl, WPACKET *thispkt,
                                  OSSL_RECORD_TEMPLATE *templ,
-                                 unsigned int rectype,
+                                 uint8_t rectype,
                                  unsigned char **recdata);
 
     int (*add_record_padding)(OSSL_RECORD_LAYER *rl,
@@ -419,7 +419,7 @@ int dtls_get_more_records(OSSL_RECORD_LAYER *rl);
 int dtls_prepare_record_header(OSSL_RECORD_LAYER *rl,
                                WPACKET *thispkt,
                                OSSL_RECORD_TEMPLATE *templ,
-                               unsigned int rectype,
+                               uint8_t rectype,
                                unsigned char **recdata);
 int dtls_post_encryption_processing(OSSL_RECORD_LAYER *rl,
                                     size_t mac_size,
@@ -451,7 +451,7 @@ int tls_free(OSSL_RECORD_LAYER *rl);
 int tls_unprocessed_read_pending(OSSL_RECORD_LAYER *rl);
 int tls_processed_read_pending(OSSL_RECORD_LAYER *rl);
 size_t tls_app_data_pending(OSSL_RECORD_LAYER *rl);
-size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t len,
+size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len,
                            size_t maxfrag, size_t *preffrag);
 int tls_write_records(OSSL_RECORD_LAYER *rl, OSSL_RECORD_TEMPLATE *templates,
                       size_t numtempl);
@@ -480,9 +480,10 @@ int tls_write_records_multiblock(OSSL_RECORD_LAYER *rl,
                                  OSSL_RECORD_TEMPLATE *templates,
                                  size_t numtempl);
 
-size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, int type, size_t len,
+size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, uint8_t type,
+                                   size_t len,
                                    size_t maxfrag, size_t *preffrag);
-size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, int type,
+size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, uint8_t type,
                                       size_t len, size_t maxfrag,
                                       size_t *preffrag);
 int tls_allocate_write_buffers_default(OSSL_RECORD_LAYER *rl,
@@ -508,7 +509,7 @@ int tls1_initialise_write_packets(OSSL_RECORD_LAYER *rl,
 int tls_prepare_record_header_default(OSSL_RECORD_LAYER *rl,
                                       WPACKET *thispkt,
                                       OSSL_RECORD_TEMPLATE *templ,
-                                      unsigned int rectype,
+                                      uint8_t rectype,
                                       unsigned char **recdata);
 int tls_prepare_for_encryption_default(OSSL_RECORD_LAYER *rl,
                                        size_t mac_size,
diff --git a/ssl/record/methods/tls13_meth.c b/ssl/record/methods/tls13_meth.c
index 84db78a8f6..8704c72603 100644
--- a/ssl/record/methods/tls13_meth.c
+++ b/ssl/record/methods/tls13_meth.c
@@ -233,11 +233,11 @@ static int tls13_post_process_record(OSSL_RECORD_LAYER *rl, TLS_RL_RECORD *rec)
     return 1;
 }
 
-static unsigned int tls13_get_record_type(OSSL_RECORD_LAYER *rl,
-                                          OSSL_RECORD_TEMPLATE *template)
+static uint8_t tls13_get_record_type(OSSL_RECORD_LAYER *rl,
+                                     OSSL_RECORD_TEMPLATE *template)
 {
     if (rl->allow_plain_alerts && template->type == SSL3_RT_ALERT)
-        return  SSL3_RT_ALERT;
+        return SSL3_RT_ALERT;
 
     /*
      * Aside from the above case we always use the application data record type
diff --git a/ssl/record/methods/tls_common.c b/ssl/record/methods/tls_common.c
index 576b651aa7..6ea090df24 100644
--- a/ssl/record/methods/tls_common.c
+++ b/ssl/record/methods/tls_common.c
@@ -1487,7 +1487,8 @@ size_t tls_app_data_pending(OSSL_RECORD_LAYER *rl)
     return num;
 }
 
-size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, int type, size_t len,
+size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, uint8_t type,
+                                   size_t len,
                                    size_t maxfrag, size_t *preffrag)
 {
     /*
@@ -1511,7 +1512,7 @@ size_t tls_get_max_records_default(OSSL_RECORD_LAYER *rl, int type, size_t len,
     return 1;
 }
 
-size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, int type, size_t len,
+size_t tls_get_max_records(OSSL_RECORD_LAYER *rl, uint8_t type, size_t len,
                            size_t maxfrag, size_t *preffrag)
 {
     return rl->funcs->get_max_records(rl, type, len, maxfrag, preffrag);
@@ -1574,7 +1575,7 @@ int tls_initialise_write_packets_default(OSSL_RECORD_LAYER *rl,
 int tls_prepare_record_header_default(OSSL_RECORD_LAYER *rl,
                                       WPACKET *thispkt,
                                       OSSL_RECORD_TEMPLATE *templ,
-                                      unsigned int rectype,
+                                      uint8_t rectype,
                                       unsigned char **recdata)
 {
     size_t maxcomplen;
@@ -1755,7 +1756,7 @@ int tls_write_records_default(OSSL_RECORD_LAYER *rl,
     memset(wr, 0, sizeof(wr));
     for (j = 0; j < numtempl + prefix; j++) {
         unsigned char *compressdata = NULL;
-        unsigned int rectype;
+        uint8_t rectype;
 
         thispkt = &pkt[j];
         thiswr = &wr[j];
diff --git a/ssl/record/methods/tls_multib.c b/ssl/record/methods/tls_multib.c
index fb5a2b3e0e..8aa89f86b2 100644
--- a/ssl/record/methods/tls_multib.c
+++ b/ssl/record/methods/tls_multib.c
@@ -20,7 +20,7 @@
 # define EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK 0
 #endif
 
-static int tls_is_multiblock_capable(OSSL_RECORD_LAYER *rl, int type,
+static int tls_is_multiblock_capable(OSSL_RECORD_LAYER *rl, uint8_t type,
                                      size_t len, size_t fraglen)
 {
 #if !defined(OPENSSL_NO_MULTIBLOCK) && EVP_CIPH_FLAG_TLS1_1_MULTIBLOCK
@@ -38,7 +38,7 @@ static int tls_is_multiblock_capable(OSSL_RECORD_LAYER *rl, int type,
     return 0;
 }
 
-size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, int type,
+size_t tls_get_max_records_multiblock(OSSL_RECORD_LAYER *rl, uint8_t type,
                                       size_t len, size_t maxfrag,
                                       size_t *preffrag)
 {
diff --git a/ssl/record/rec_layer_d1.c b/ssl/record/rec_layer_d1.c
index fed57b65cd..75ed6775cb 100644
--- a/ssl/record/rec_layer_d1.c
+++ b/ssl/record/rec_layer_d1.c
@@ -195,8 +195,9 @@ static void dtls_unbuffer_record(SSL_CONNECTION *s)
  *     Application data protocol
  *             none of our business
  */
-int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
-                     size_t len, int peek, size_t *readbytes)
+int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type,
+                     unsigned char *buf, size_t len,
+                     int peek, size_t *readbytes)
 {
     int i, j, ret;
     size_t n;
@@ -603,7 +604,7 @@ int dtls1_read_bytes(SSL *s, int type, int *recvd_type, unsigned char *buf,
  * Call this to write data in records of type 'type' It will return <= 0 if
  * not all data has been sent or non-blocking IO.
  */
-int dtls1_write_bytes(SSL_CONNECTION *s, int type, const void *buf,
+int dtls1_write_bytes(SSL_CONNECTION *s, uint8_t type, const void *buf,
                       size_t len, size_t *written)
 {
     int i;
@@ -617,7 +618,7 @@ int dtls1_write_bytes(SSL_CONNECTION *s, int type, const void *buf,
     return i;
 }
 
-int do_dtls1_write(SSL_CONNECTION *sc, int type, const unsigned char *buf,
+int do_dtls1_write(SSL_CONNECTION *sc, uint8_t type, const unsigned char *buf,
                    size_t len, size_t *written)
 {
     int i;
diff --git a/ssl/record/rec_layer_s3.c b/ssl/record/rec_layer_s3.c
index 16726019f9..bf92a8e6ab 100644
--- a/ssl/record/rec_layer_s3.c
+++ b/ssl/record/rec_layer_s3.c
@@ -200,7 +200,7 @@ const char *SSL_rstate_string(const SSL *s)
     return shrt;
 }
 
-static int tls_write_check_pending(SSL_CONNECTION *s, int type,
+static int tls_write_check_pending(SSL_CONNECTION *s, uint8_t type,
                                    const unsigned char *buf, size_t len)
 {
     if (s->rlayer.wpend_tot == 0)
@@ -221,7 +221,7 @@ static int tls_write_check_pending(SSL_CONNECTION *s, int type,
  * Call this to write data in records of type 'type' It will return <= 0 if
  * not all data has been sent or non-blocking IO.
  */
-int ssl3_write_bytes(SSL *ssl, int type, const void *buf_, size_t len,
+int ssl3_write_bytes(SSL *ssl, uint8_t type, const void *buf_, size_t len,
                      size_t *written)
 {
     const unsigned char *buf = buf_;
@@ -558,8 +558,9 @@ int ssl_release_record(SSL_CONNECTION *s, TLS_RECORD *rr, size_t length)
  *     Application data protocol
  *             none of our business
  */
-int ssl3_read_bytes(SSL *ssl, int type, int *recvd_type, unsigned char *buf,
-                    size_t len, int peek, size_t *readbytes)
+int ssl3_read_bytes(SSL *ssl, uint8_t type, uint8_t *recvd_type,
+                    unsigned char *buf, size_t len,
+                    int peek, size_t *readbytes)
 {
     int i, j, ret;
     size_t n, curr_rec, totalbytes;
diff --git a/ssl/record/record.h b/ssl/record/record.h
index a8fa6833e9..e4d1e34036 100644
--- a/ssl/record/record.h
+++ b/ssl/record/record.h
@@ -107,7 +107,7 @@ typedef struct record_layer_st {
     /* partial write - check the numbers match */
     /* number bytes written */
     size_t wpend_tot;
-    int wpend_type;
+    uint8_t wpend_type;
     /* number of bytes submitted */
     size_t wpend_ret;
     const unsigned char *wpend_buf;
@@ -148,21 +148,21 @@ int RECORD_LAYER_processed_read_pending(const RECORD_LAYER *rl);
 int RECORD_LAYER_write_pending(const RECORD_LAYER *rl);
 int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl);
 __owur size_t ssl3_pending(const SSL *s);
-__owur int ssl3_write_bytes(SSL *s, int type, const void *buf, size_t len,
+__owur int ssl3_write_bytes(SSL *s, uint8_t type, const void *buf, size_t len,
                             size_t *written);
-__owur int ssl3_read_bytes(SSL *s, int type, int *recvd_type,
+__owur int ssl3_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type,
                            unsigned char *buf, size_t len, int peek,
                            size_t *readbytes);
 
 int DTLS_RECORD_LAYER_new(RECORD_LAYER *rl);
 void DTLS_RECORD_LAYER_free(RECORD_LAYER *rl);
 void DTLS_RECORD_LAYER_clear(RECORD_LAYER *rl);
-__owur int dtls1_read_bytes(SSL *s, int type, int *recvd_type,
+__owur int dtls1_read_bytes(SSL *s, uint8_t type, uint8_t *recvd_type,
                             unsigned char *buf, size_t len, int peek,
                             size_t *readbytes);
-__owur int dtls1_write_bytes(SSL_CONNECTION *s, int type, const void *buf,
+__owur int dtls1_write_bytes(SSL_CONNECTION *s, uint8_t type, const void *buf,
                              size_t len, size_t *written);
-int do_dtls1_write(SSL_CONNECTION *s, int type, const unsigned char *buf,
+int do_dtls1_write(SSL_CONNECTION *s, uint8_t type, const unsigned char *buf,
                    size_t len, size_t *written);
 void dtls1_increment_epoch(SSL_CONNECTION *s, int rw);
 int ssl_release_record(SSL_CONNECTION *s, TLS_RECORD *rr, size_t length);
diff --git a/ssl/ssl_local.h b/ssl/ssl_local.h
index d628f40262..4da83ab692 100644
--- a/ssl/ssl_local.h
+++ b/ssl/ssl_local.h
@@ -433,10 +433,10 @@ struct ssl_method_st {
     int (*ssl_shutdown) (SSL *s);
     int (*ssl_renegotiate) (SSL *s);
     int (*ssl_renegotiate_check) (SSL *s, int);
-    int (*ssl_read_bytes) (SSL *s, int type, int *recvd_type,
+    int (*ssl_read_bytes) (SSL *s, uint8_t type, uint8_t *recvd_type,
                            unsigned char *buf, size_t len, int peek,
                            size_t *readbytes);
-    int (*ssl_write_bytes) (SSL *s, int type, const void *buf_, size_t len,
+    int (*ssl_write_bytes) (SSL *s, uint8_t type, const void *buf_, size_t len,
                             size_t *written);
     int (*ssl_dispatch_alert) (SSL *s);
     long (*ssl_ctrl) (SSL *s, int cmd, long larg, void *parg);
@@ -2587,7 +2587,7 @@ int ssl3_init_finished_mac(SSL_CONNECTION *s);
 __owur int ssl3_setup_key_block(SSL_CONNECTION *s);
 __owur int ssl3_change_cipher_state(SSL_CONNECTION *s, int which);
 void ssl3_cleanup_key_block(SSL_CONNECTION *s);
-__owur int ssl3_do_write(SSL_CONNECTION *s, int type);
+__owur int ssl3_do_write(SSL_CONNECTION *s, uint8_t type);
 int ssl3_send_alert(SSL_CONNECTION *s, int level, int desc);
 __owur int ssl3_generate_master_secret(SSL_CONNECTION *s, unsigned char *out,
                                        unsigned char *p, size_t len,
@@ -2650,14 +2650,14 @@ __owur int ssl_get_min_max_version(const SSL_CONNECTION *s, int *min_version,
                                    int *max_version, int *real_max);
 
 __owur OSSL_TIME tls1_default_timeout(void);
-__owur int dtls1_do_write(SSL_CONNECTION *s, int type);
+__owur int dtls1_do_write(SSL_CONNECTION *s, uint8_t type);
 void dtls1_set_message_header(SSL_CONNECTION *s,
                               unsigned char mt,
                               size_t len,
                               size_t frag_off, size_t frag_len);
 
-int dtls1_write_app_data_bytes(SSL *s, int type, const void *buf_, size_t len,
-                               size_t *written);
+int dtls1_write_app_data_bytes(SSL *s, uint8_t type, const void *buf_,
+                               size_t len, size_t *written);
 
 __owur int dtls1_read_failed(SSL_CONNECTION *s, int code);
 __owur int dtls1_buffer_message(SSL_CONNECTION *s, int ccs);
diff --git a/ssl/statem/statem_dtls.c b/ssl/statem/statem_dtls.c
index 2e26a3f3df..31dc9f3511 100644
--- a/ssl/statem/statem_dtls.c
+++ b/ssl/statem/statem_dtls.c
@@ -111,7 +111,7 @@ void dtls1_hm_fragment_free(hm_fragment *frag)
  * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or
  * SSL3_RT_CHANGE_CIPHER_SPEC)
  */
-int dtls1_do_write(SSL_CONNECTION *s, int type)
+int dtls1_do_write(SSL_CONNECTION *s, uint8_t type)
 {
     int ret;
     size_t written;
@@ -809,7 +809,8 @@ static int dtls_get_reassembled_message(SSL_CONNECTION *s, int *errtype,
 {
     unsigned char wire[DTLS1_HM_HEADER_LENGTH];
     size_t mlen, frag_off, frag_len;
-    int i, ret, recvd_type;
+    int i, ret;
+    uint8_t recvd_type;
     struct hm_header_st msg_hdr;
     size_t readbytes;
     SSL *ssl = SSL_CONNECTION_GET_SSL(s);
diff --git a/ssl/statem/statem_lib.c b/ssl/statem/statem_lib.c
index f339b2694c..d979b6d5fc 100644
--- a/ssl/statem/statem_lib.c
+++ b/ssl/statem/statem_lib.c
@@ -58,7 +58,7 @@ int ossl_statem_set_mutator(SSL *s,
  * send s->init_buf in records of type 'type' (SSL3_RT_HANDSHAKE or
  * SSL3_RT_CHANGE_CIPHER_SPEC)
  */
-int ssl3_do_write(SSL_CONNECTION *s, int type)
+int ssl3_do_write(SSL_CONNECTION *s, uint8_t type)
 {
     int ret;
     size_t written = 0;
@@ -1530,7 +1530,8 @@ WORK_STATE tls_finish_handshake(SSL_CONNECTION *s, ossl_unused WORK_STATE wst,
 int tls_get_message_header(SSL_CONNECTION *s, int *mt)
 {
     /* s->init_num < SSL3_HM_HEADER_LENGTH */
-    int skip_message, i, recvd_type;
+    int skip_message, i;
+    uint8_t recvd_type;
     unsigned char *p;
     size_t l, readbytes;
     SSL *ssl = SSL_CONNECTION_GET_SSL(s);