From f50730d3615026d3d898e8c6821b23bdcf69cf96 Mon Sep 17 00:00:00 2001 From: Matt Caswell Date: Fri, 12 Dec 2014 15:32:24 +0000 Subject: DTLS fixes for signed/unsigned issues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Emilia Käsper (cherry picked from commit 1904d21123849a65dafde1705e6dd5b7c2f420eb) --- ssl/d1_both.c | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) (limited to 'ssl/d1_both.c') diff --git a/ssl/d1_both.c b/ssl/d1_both.c index 20f5628db0..f377ba0394 100644 --- a/ssl/d1_both.c +++ b/ssl/d1_both.c @@ -259,9 +259,9 @@ static int dtls1_query_mtu(SSL *s) int dtls1_do_write(SSL *s, int type) { int ret; - int curr_mtu; + unsigned int curr_mtu; int retry = 1; - unsigned int len, frag_off, mac_size, blocksize; + unsigned int len, frag_off, mac_size, blocksize, used_len; if(!dtls1_query_mtu(s)) return -1; @@ -284,10 +284,15 @@ int dtls1_do_write(SSL *s, int type) blocksize = 0; frag_off = 0; - while( s->init_num) + /* s->init_num shouldn't ever be < 0...but just in case */ + while( s->init_num > 0) { - curr_mtu = s->d1->mtu - BIO_wpending(SSL_get_wbio(s)) - - DTLS1_RT_HEADER_LENGTH - mac_size - blocksize; + used_len = BIO_wpending(SSL_get_wbio(s)) + DTLS1_RT_HEADER_LENGTH + + mac_size + blocksize; + if(s->d1->mtu > used_len) + curr_mtu = s->d1->mtu - used_len; + else + curr_mtu = 0; if ( curr_mtu <= DTLS1_HM_HEADER_LENGTH) { @@ -295,15 +300,23 @@ int dtls1_do_write(SSL *s, int type) ret = BIO_flush(SSL_get_wbio(s)); if ( ret <= 0) return ret; - curr_mtu = s->d1->mtu - DTLS1_RT_HEADER_LENGTH - - mac_size - blocksize; + used_len = DTLS1_RT_HEADER_LENGTH + mac_size + blocksize; + if(s->d1->mtu > used_len + DTLS1_HM_HEADER_LENGTH) + curr_mtu = s->d1->mtu - used_len; + else + /* Shouldn't happen */ + return -1; } - if ( s->init_num > curr_mtu) + /* We just checked that s->init_num > 0 so this cast should be safe */ + if (((unsigned int)s->init_num) > curr_mtu) len = curr_mtu; else len = s->init_num; + /* Shouldn't ever happen */ + if(len > INT_MAX) + len = INT_MAX; /* XDTLS: this function is too long. split out the CCS part */ if ( type == SSL3_RT_HANDSHAKE) @@ -314,12 +327,17 @@ int dtls1_do_write(SSL *s, int type) s->init_off -= DTLS1_HM_HEADER_LENGTH; s->init_num += DTLS1_HM_HEADER_LENGTH; - if ( s->init_num > curr_mtu) + /* We just checked that s->init_num > 0 so this cast should be safe */ + if (((unsigned int)s->init_num) > curr_mtu) len = curr_mtu; else len = s->init_num; } + /* Shouldn't ever happen */ + if(len > INT_MAX) + len = INT_MAX; + if ( len < DTLS1_HM_HEADER_LENGTH ) { /* -- cgit v1.2.3