Fix some missed size_t updates

Reviewed-by: Rich Salz <rsalz@openssl.org>

13 files changed, 38 insertions, 49 deletions

diff --git a/ssl/d1_lib.c b/ssl/d1_lib.c
index 72673863cb..aa6cdd2670 100644
--- a/ssl/d1_lib.c
+++ b/ssl/d1_lib.c
@@ -35,13 +35,11 @@ const SSL3_ENC_METHOD DTLSv1_enc_data = {
     tls1_generate_master_secret,
     tls1_change_cipher_state,
     tls1_final_finish_mac,
-    TLS1_FINISH_MAC_LENGTH,
     TLS_MD_CLIENT_FINISH_CONST, TLS_MD_CLIENT_FINISH_CONST_SIZE,
     TLS_MD_SERVER_FINISH_CONST, TLS_MD_SERVER_FINISH_CONST_SIZE,
     tls1_alert_code,
     tls1_export_keying_material,
     SSL_ENC_FLAG_DTLS | SSL_ENC_FLAG_EXPLICIT_IV,
-    DTLS1_HM_HEADER_LENGTH,
     dtls1_set_handshake_header,
     dtls1_close_construct_packet,
     dtls1_handshake_write
@@ -54,14 +52,12 @@ const SSL3_ENC_METHOD DTLSv1_2_enc_data = {
     tls1_generate_master_secret,
     tls1_change_cipher_state,
     tls1_final_finish_mac,
-    TLS1_FINISH_MAC_LENGTH,
     TLS_MD_CLIENT_FINISH_CONST, TLS_MD_CLIENT_FINISH_CONST_SIZE,
     TLS_MD_SERVER_FINISH_CONST, TLS_MD_SERVER_FINISH_CONST_SIZE,
     tls1_alert_code,
     tls1_export_keying_material,
     SSL_ENC_FLAG_DTLS | SSL_ENC_FLAG_EXPLICIT_IV | SSL_ENC_FLAG_SIGALGS
         | SSL_ENC_FLAG_SHA256_PRF | SSL_ENC_FLAG_TLS1_2_CIPHERS,
-    DTLS1_HM_HEADER_LENGTH,
     dtls1_set_handshake_header,
     dtls1_close_construct_packet,
     dtls1_handshake_write
@@ -435,7 +431,7 @@ int DTLSv1_listen(SSL *s, BIO_ADDR *client)
     unsigned char seq[SEQ_NUM_SIZE];
     const unsigned char *data;
     unsigned char *buf;
-    unsigned long fragoff, fraglen, msglen;
+    size_t fragoff, fraglen, msglen;
     unsigned int rectype, versmajor, msgseq, msgtype, clientvers, cookielen;
     BIO *rbio, *wbio;
     BUF_MEM *bufm;
@@ -663,8 +659,7 @@ int DTLSv1_listen(SSL *s, BIO_ADDR *client)
                 return -1;
             }
             if (s->ctx->app_verify_cookie_cb(s, PACKET_data(&cookiepkt),
-                                             PACKET_remaining(&cookiepkt)) ==
-                0) {
+                    (unsigned int)PACKET_remaining(&cookiepkt)) == 0) {
                 /*
                  * We treat invalid cookies in the same was as no cookie as
                  * per RFC6347
@@ -795,6 +790,7 @@ int DTLSv1_listen(SSL *s, BIO_ADDR *client)
             BIO_ADDR_free(tmpclient);
             tmpclient = NULL;
 
+            /* TODO(size_t): convert this call */
             if (BIO_write(wbio, buf, wreclen) < (int)wreclen) {
                 if (BIO_should_retry(wbio)) {
                     /*
@@ -1072,7 +1068,7 @@ int dtls1_query_mtu(SSL *s)
                 /* Set to min mtu */
                 s->d1->mtu = dtls1_min_mtu(s);
                 BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SET_MTU,
-                         s->d1->mtu, NULL);
+                         (long)s->d1->mtu, NULL);
             }
         } else
             return 0;
diff --git a/ssl/d1_srtp.c b/ssl/d1_srtp.c
index bcefb9ec3a..718f417d85 100644
--- a/ssl/d1_srtp.c
+++ b/ssl/d1_srtp.c
@@ -40,7 +40,7 @@ static SRTP_PROTECTION_PROFILE srtp_known_profiles[] = {
 };
 
 static int find_profile_by_name(char *profile_name,
-                                SRTP_PROTECTION_PROFILE **pptr, unsigned len)
+                                SRTP_PROTECTION_PROFILE **pptr, size_t len)
 {
     SRTP_PROTECTION_PROFILE *p;
 
@@ -76,7 +76,8 @@ static int ssl_ctx_make_profiles(const char *profiles_string,
     do {
         col = strchr(ptr, ':');
 
-        if (!find_profile_by_name(ptr, &p, col ? col - ptr : (int)strlen(ptr))) {
+        if (!find_profile_by_name(ptr, &p, col ? (size_t)(col - ptr)
+                                               : strlen(ptr))) {
             if (sk_SRTP_PROTECTION_PROFILE_find(profiles, p) >= 0) {
                 SSLerr(SSL_F_SSL_CTX_MAKE_PROFILES,
                        SSL_R_BAD_SRTP_PROTECTION_PROFILE_LIST);
diff --git a/ssl/pqueue.c b/ssl/pqueue.c
index b447e1dceb..9ff4132dcc 100644
--- a/ssl/pqueue.c
+++ b/ssl/pqueue.c
@@ -141,7 +141,7 @@ pitem *pqueue_next(pitem **item)
     return ret;
 }
 
-int pqueue_size(pqueue *pq)
+size_t pqueue_size(pqueue *pq)
 {
     pitem *item = pq->items;
     int count = 0;
diff --git a/ssl/record/rec_layer_s3.c b/ssl/record/rec_layer_s3.c
index c74799598d..b61c1f77e2 100644
--- a/ssl/record/rec_layer_s3.c
+++ b/ssl/record/rec_layer_s3.c
@@ -122,10 +122,9 @@ void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl)
     memset(rl->write_sequence, 0, sizeof(rl->write_sequence));
 }
 
-int ssl3_pending(const SSL *s)
+size_t ssl3_pending(const SSL *s)
 {
-    unsigned int i;
-    int num = 0;
+    size_t i, num = 0;
 
     if (s->rlayer.rstate == SSL_ST_READ_BODY)
         return 0;
@@ -429,7 +428,7 @@ int ssl3_write_bytes(SSL *s, int type, const void *buf_, size_t len,
 
             packlen = EVP_CIPHER_CTX_ctrl(s->enc_write_ctx,
                                           EVP_CTRL_TLS1_1_MULTIBLOCK_MAX_BUFSIZE,
-                                          max_send_fragment, NULL);
+                                          (int)max_send_fragment, NULL);
 
             if (len >= 8 * max_send_fragment)
                 packlen *= 8;
@@ -443,7 +442,8 @@ int ssl3_write_bytes(SSL *s, int type, const void *buf_, size_t len,
         } else if (tot == len) { /* done? */
             /* free jumbo buffer */
             ssl3_release_write_buffer(s);
-            return tot;
+            *written = tot;
+            return 1;
         }
 
         n = (len - tot);
diff --git a/ssl/record/record.h b/ssl/record/record.h
index 86b91a274e..bd0a0906a1 100644
--- a/ssl/record/record.h
+++ b/ssl/record/record.h
@@ -214,7 +214,7 @@ void RECORD_LAYER_reset_read_sequence(RECORD_LAYER *rl);
 void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl);
 int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl);
 size_t RECORD_LAYER_get_rrec_length(RECORD_LAYER *rl);
-__owur int ssl3_pending(const SSL *s);
+__owur size_t ssl3_pending(const SSL *s);
 __owur int ssl3_write_bytes(SSL *s, int type, const void *buf, size_t len,
                             size_t *written);
 int do_ssl3_write(SSL *s, int type, const unsigned char *buf,
diff --git a/ssl/record/ssl3_buffer.c b/ssl/record/ssl3_buffer.c
index 1252310394..df1f90092c 100644
--- a/ssl/record/ssl3_buffer.c
+++ b/ssl/record/ssl3_buffer.c
@@ -134,7 +134,7 @@ int ssl3_setup_buffers(SSL *s)
 int ssl3_release_write_buffer(SSL *s)
 {
     SSL3_BUFFER *wb;
-    unsigned int pipes;
+    size_t pipes;
 
     pipes = s->rlayer.numwpipes;
     while (pipes > 0) {
diff --git a/ssl/s3_lib.c b/ssl/s3_lib.c
index ca27e9908d..d6d7c46287 100644
--- a/ssl/s3_lib.c
+++ b/ssl/s3_lib.c
@@ -2756,7 +2756,6 @@ const SSL3_ENC_METHOD SSLv3_enc_data = {
     ssl3_generate_master_secret,
     ssl3_change_cipher_state,
     ssl3_final_finish_mac,
-    MD5_DIGEST_LENGTH + SHA_DIGEST_LENGTH,
     SSL3_MD_CLIENT_FINISHED_CONST, 4,
     SSL3_MD_SERVER_FINISHED_CONST, 4,
     ssl3_alert_code,
@@ -2764,7 +2763,6 @@ const SSL3_ENC_METHOD SSLv3_enc_data = {
              size_t, const unsigned char *, size_t,
              int use_context))ssl_undefined_function,
     0,
-    SSL3_HM_HEADER_LENGTH,
     ssl3_set_handshake_header,
     tls_close_construct_packet,
     ssl3_handshake_write
@@ -3037,6 +3035,8 @@ long ssl3_ctrl(SSL *s, int cmd, long larg, void *parg)
 
     case SSL_CTRL_GET_TLSEXT_STATUS_REQ_OCSP_RESP:
         *(unsigned char **)parg = s->tlsext_ocsp_resp;
+        if (s->tlsext_ocsp_resplen == 0)
+            return -1;
         return s->tlsext_ocsp_resplen;
 
     case SSL_CTRL_SET_TLSEXT_STATUS_REQ_OCSP_RESP:
diff --git a/ssl/s3_msg.c b/ssl/s3_msg.c
index c016429878..743a02b8d1 100644
--- a/ssl/s3_msg.c
+++ b/ssl/s3_msg.c
@@ -15,7 +15,7 @@ int ssl3_do_change_cipher_spec(SSL *s)
     int i;
     size_t finish_md_len;
     const char *sender;
-    int slen;
+    size_t slen;
 
     if (s->server)
         i = SSL3_CHANGE_CIPHER_SERVER_READ;
diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index 53cfcb7b47..3c0cb764bf 100644
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -67,7 +67,6 @@ SSL3_ENC_METHOD ssl3_undef_enc_method = {
     (int (*)(SSL *, int))ssl_undefined_function,
     (size_t (*)(SSL *, const char *, size_t, unsigned char *))
         ssl_undefined_function,
-    0,                          /* finish_mac_length */
     NULL,                       /* client_finished_label */
     0,                          /* client_finished_label_len */
     NULL,                       /* server_finished_label */
@@ -598,7 +597,7 @@ SSL *SSL_new(SSL_CTX *ctx)
     s->tlsext_ocsp_ids = NULL;
     s->tlsext_ocsp_exts = NULL;
     s->tlsext_ocsp_resp = NULL;
-    s->tlsext_ocsp_resplen = -1;
+    s->tlsext_ocsp_resplen = 0;
     SSL_CTX_up_ref(ctx);
     s->initial_ctx = ctx;
 #ifndef OPENSSL_NO_EC
@@ -1293,14 +1292,19 @@ int SSL_get_read_ahead(const SSL *s)
 
 int SSL_pending(const SSL *s)
 {
+    size_t pending = s->method->ssl_pending(s);
+
     /*
      * SSL_pending cannot work properly if read-ahead is enabled
      * (SSL_[CTX_]ctrl(..., SSL_CTRL_SET_READ_AHEAD, 1, NULL)), and it is
      * impossible to fix since SSL_pending cannot report errors that may be
      * observed while scanning the new data. (Note that SSL_pending() is
      * often used as a boolean value, so we'd better not return -1.)
+     *
+     * SSL_pending also cannot work properly if the value >INT_MAX. In that case
+     * we just return INT_MAX.
      */
-    return (s->method->ssl_pending(s));
+    return pending < INT_MAX ? pending : INT_MAX;
 }
 
 int SSL_has_pending(const SSL *s)
diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h
index eea9b86612..b540d890df 100644
--- a/ssl/ssl_locl.h
+++ b/ssl/ssl_locl.h
@@ -461,7 +461,7 @@ struct ssl_method_st {
     const SSL_CIPHER *(*get_cipher_by_char) (const unsigned char *ptr);
     int (*put_cipher_by_char) (const SSL_CIPHER *cipher, WPACKET *pkt,
                                size_t *len);
-    int (*ssl_pending) (const SSL *s);
+    size_t (*ssl_pending) (const SSL *s);
     int (*num_ciphers) (void);
     const SSL_CIPHER *(*get_cipher) (unsigned ncipher);
     long (*get_timeout) (void);
@@ -1061,7 +1061,7 @@ struct ssl_st {
     X509_EXTENSIONS *tlsext_ocsp_exts;
     /* OCSP response received or to be sent */
     unsigned char *tlsext_ocsp_resp;
-    int tlsext_ocsp_resplen;
+    size_t tlsext_ocsp_resplen;
     /* RFC4507 session ticket expected to be received or sent */
     int tlsext_ticket_expected;
 # ifndef OPENSSL_NO_EC
@@ -1374,7 +1374,7 @@ pitem *pqueue_pop(pqueue *pq);
 pitem *pqueue_find(pqueue *pq, unsigned char *prio64be);
 pitem *pqueue_iterator(pqueue *pq);
 pitem *pqueue_next(piterator *iter);
-int pqueue_size(pqueue *pq);
+size_t pqueue_size(pqueue *pq);
 
 typedef struct dtls1_state_st {
     unsigned char cookie[DTLS1_COOKIE_LENGTH];
@@ -1573,11 +1573,10 @@ typedef struct ssl3_enc_method {
                                    size_t, size_t *);
     int (*change_cipher_state) (SSL *, int);
     size_t (*final_finish_mac) (SSL *, const char *, size_t, unsigned char *);
-    int finish_mac_length;
     const char *client_finished_label;
-    int client_finished_label_len;
+    size_t client_finished_label_len;
     const char *server_finished_label;
-    int server_finished_label_len;
+    size_t server_finished_label_len;
     int (*alert_value) (int);
     int (*export_keying_material) (SSL *, unsigned char *, size_t,
                                    const char *, size_t,
@@ -1585,8 +1584,6 @@ typedef struct ssl3_enc_method {
                                    int use_context);
     /* Various flags indicating protocol version requirements */
     uint32_t enc_flags;
-    /* Handshake header length */
-    unsigned int hhlen;
     /* Set the handshake header */
     int (*set_handshake_header) (SSL *s, WPACKET *pkt, int type);
     /* Close construction of the handshake message */
@@ -1595,9 +1592,6 @@ typedef struct ssl3_enc_method {
     int (*do_write) (SSL *s);
 } SSL3_ENC_METHOD;
 
-# define SSL_HM_HEADER_LENGTH(s) s->method->ssl3_enc->hhlen
-# define ssl_handshake_start(s) \
-        (((unsigned char *)s->init_buf->data) + s->method->ssl3_enc->hhlen)
 # define ssl_set_handshake_header(s, pkt, htype) \
         s->method->ssl3_enc->set_handshake_header((s), (pkt), (htype))
 # define ssl_close_construct_packet(s, pkt, htype) \
diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index cdf8bb0d92..194243fc9d 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c
@@ -1968,7 +1968,8 @@ MSG_PROCESS_RETURN tls_process_new_session_ticket(SSL *s, PACKET *pkt)
 MSG_PROCESS_RETURN tls_process_cert_status(SSL *s, PACKET *pkt)
 {
     int al;
-    unsigned long resplen;
+    unsigned long resplenl;
+    size_t resplen;
     unsigned int type;
 
     if (!PACKET_get_1(pkt, &type)
@@ -1977,12 +1978,13 @@ MSG_PROCESS_RETURN tls_process_cert_status(SSL *s, PACKET *pkt)
         SSLerr(SSL_F_TLS_PROCESS_CERT_STATUS, SSL_R_UNSUPPORTED_STATUS_TYPE);
         goto f_err;
     }
-    if (!PACKET_get_net_3(pkt, &resplen)
-        || PACKET_remaining(pkt) != resplen) {
+    if (!PACKET_get_net_3(pkt, &resplenl)
+        || PACKET_remaining(pkt) != resplenl) {
         al = SSL_AD_DECODE_ERROR;
         SSLerr(SSL_F_TLS_PROCESS_CERT_STATUS, SSL_R_LENGTH_MISMATCH);
         goto f_err;
     }
+    resplen = resplenl;
     s->tlsext_ocsp_resp = OPENSSL_malloc(resplen);
     if (s->tlsext_ocsp_resp == NULL) {
         al = SSL_AD_INTERNAL_ERROR;
diff --git a/ssl/statem/statem_lib.c b/ssl/statem/statem_lib.c
index b622c5c837..30d674ef0f 100644
--- a/ssl/statem/statem_lib.c
+++ b/ssl/statem/statem_lib.c
@@ -76,7 +76,7 @@ int tls_construct_finished(SSL *s, WPACKET *pkt)
 {
     size_t finish_md_len;
     const char *sender;
-    int slen;
+    size_t slen;
 
     if (s->server) {
         sender = s->method->ssl3_enc->server_finished_label;
@@ -130,7 +130,7 @@ int tls_construct_finished(SSL *s, WPACKET *pkt)
 static void ssl3_take_mac(SSL *s)
 {
     const char *sender;
-    int slen;
+    size_t slen;
     /*
      * If no new cipher setup return immediately: other functions will set
      * the appropriate error.
diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index 3621fbb18d..6a1386cb36 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -33,13 +33,11 @@ SSL3_ENC_METHOD const TLSv1_enc_data = {
     tls1_generate_master_secret,
     tls1_change_cipher_state,
     tls1_final_finish_mac,
-    TLS1_FINISH_MAC_LENGTH,
     TLS_MD_CLIENT_FINISH_CONST, TLS_MD_CLIENT_FINISH_CONST_SIZE,
     TLS_MD_SERVER_FINISH_CONST, TLS_MD_SERVER_FINISH_CONST_SIZE,
     tls1_alert_code,
     tls1_export_keying_material,
     0,
-    SSL3_HM_HEADER_LENGTH,
     ssl3_set_handshake_header,
     tls_close_construct_packet,
     ssl3_handshake_write
@@ -52,13 +50,11 @@ SSL3_ENC_METHOD const TLSv1_1_enc_data = {
     tls1_generate_master_secret,
     tls1_change_cipher_state,
     tls1_final_finish_mac,
-    TLS1_FINISH_MAC_LENGTH,
     TLS_MD_CLIENT_FINISH_CONST, TLS_MD_CLIENT_FINISH_CONST_SIZE,
     TLS_MD_SERVER_FINISH_CONST, TLS_MD_SERVER_FINISH_CONST_SIZE,
     tls1_alert_code,
     tls1_export_keying_material,
     SSL_ENC_FLAG_EXPLICIT_IV,
-    SSL3_HM_HEADER_LENGTH,
     ssl3_set_handshake_header,
     tls_close_construct_packet,
     ssl3_handshake_write
@@ -71,14 +67,12 @@ SSL3_ENC_METHOD const TLSv1_2_enc_data = {
     tls1_generate_master_secret,
     tls1_change_cipher_state,
     tls1_final_finish_mac,
-    TLS1_FINISH_MAC_LENGTH,
     TLS_MD_CLIENT_FINISH_CONST, TLS_MD_CLIENT_FINISH_CONST_SIZE,
     TLS_MD_SERVER_FINISH_CONST, TLS_MD_SERVER_FINISH_CONST_SIZE,
     tls1_alert_code,
     tls1_export_keying_material,
     SSL_ENC_FLAG_EXPLICIT_IV | SSL_ENC_FLAG_SIGALGS | SSL_ENC_FLAG_SHA256_PRF
         | SSL_ENC_FLAG_TLS1_2_CIPHERS,
-    SSL3_HM_HEADER_LENGTH,
     ssl3_set_handshake_header,
     tls_close_construct_packet,
     ssl3_handshake_write
@@ -91,14 +85,12 @@ SSL3_ENC_METHOD const TLSv1_3_enc_data = {
     tls1_generate_master_secret,
     tls1_change_cipher_state,
     tls1_final_finish_mac,
-    TLS1_FINISH_MAC_LENGTH,
     TLS_MD_CLIENT_FINISH_CONST, TLS_MD_CLIENT_FINISH_CONST_SIZE,
     TLS_MD_SERVER_FINISH_CONST, TLS_MD_SERVER_FINISH_CONST_SIZE,
     tls1_alert_code,
     tls1_export_keying_material,
     SSL_ENC_FLAG_EXPLICIT_IV | SSL_ENC_FLAG_SIGALGS | SSL_ENC_FLAG_SHA256_PRF
         | SSL_ENC_FLAG_TLS1_2_CIPHERS,
-    SSL3_HM_HEADER_LENGTH,
     ssl3_set_handshake_header,
     tls_close_construct_packet,
     ssl3_handshake_write
@@ -2765,7 +2757,7 @@ int ssl_check_serverhello_tlsext(SSL *s)
      */
     OPENSSL_free(s->tlsext_ocsp_resp);
     s->tlsext_ocsp_resp = NULL;
-    s->tlsext_ocsp_resplen = -1;
+    s->tlsext_ocsp_resplen = 0;
 
     switch (ret) {
     case SSL_TLSEXT_ERR_ALERT_FATAL: