Convert master_secret_size code to size_t

Reviewed-by: Rich Salz <rsalz@openssl.org>

8 files changed, 93 insertions, 67 deletions

diff --git a/ssl/s3_enc.c b/ssl/s3_enc.c
index f32b68a0b3..58f63e24f0 100644
--- a/ssl/s3_enc.c
+++ b/ssl/s3_enc.c
@@ -113,7 +113,8 @@ int ssl3_change_cipher_state(SSL *s, int which)
     COMP_METHOD *comp;
 #endif
     const EVP_MD *m;
-    int n, i, j, k, cl;
+    int mdi;
+    size_t n, i, j, k, cl;
     int reuse_dd = 0;
 
     c = s->s3->tmp.new_sym_enc;
@@ -194,9 +195,10 @@ int ssl3_change_cipher_state(SSL *s, int which)
         EVP_CIPHER_CTX_reset(dd);
 
     p = s->s3->tmp.key_block;
-    i = EVP_MD_size(m);
-    if (i < 0)
+    mdi = EVP_MD_size(m);
+    if (mdi < 0)
         goto err2;
+    i = mdi;
     cl = EVP_CIPHER_key_length(c);
     j = cl;
     k = EVP_CIPHER_iv_length(c);
@@ -449,7 +451,7 @@ int ssl3_final_finish_mac(SSL *s, const char *sender, int len, unsigned char *p)
 }
 
 int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
-                                int len)
+                                size_t len, size_t *secret_size)
 {
     static const unsigned char *salt[3] = {
 #ifndef CHARSET_EBCDIC
@@ -464,11 +466,12 @@ int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
     };
     unsigned char buf[EVP_MAX_MD_SIZE];
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
-    int i, ret = 0;
+    int i, ret = 1;
     unsigned int n;
 #ifdef OPENSSL_SSL_TRACE_CRYPTO
     unsigned char *tmpout = out;
 #endif
+    size_t ret_secret_size = 0;
 
     if (ctx == NULL) {
         SSLerr(SSL_F_SSL3_GENERATE_MASTER_SECRET, ERR_R_MALLOC_FAILURE);
@@ -483,6 +486,7 @@ int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
                                 SSL3_RANDOM_SIZE) <= 0
             || EVP_DigestUpdate(ctx, &(s->s3->server_random[0]),
                                 SSL3_RANDOM_SIZE) <= 0
+               /* TODO(size_t) : convert me */
             || EVP_DigestFinal_ex(ctx, buf, &n) <= 0
             || EVP_DigestInit_ex(ctx, s->ctx->md5, NULL) <= 0
             || EVP_DigestUpdate(ctx, p, len) <= 0
@@ -493,12 +497,12 @@ int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
             break;
         }
         out += n;
-        ret += n;
+        ret_secret_size += n;
     }
     EVP_MD_CTX_free(ctx);
 
 #ifdef OPENSSL_SSL_TRACE_CRYPTO
-    if (ret > 0 && s->msg_callback) {
+    if (ret && s->msg_callback) {
         s->msg_callback(2, s->version, TLS1_RT_CRYPTO_PREMASTER,
                         p, len, s, s->msg_callback_arg);
         s->msg_callback(2, s->version, TLS1_RT_CRYPTO_CLIENT_RANDOM,
@@ -513,7 +517,9 @@ int ssl3_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
     }
 #endif
     OPENSSL_cleanse(buf, sizeof(buf));
-    return (ret);
+    if (ret)
+        *secret_size = ret_secret_size;
+    return ret;
 }
 
 int ssl3_alert_code(int code)
diff --git a/ssl/s3_lib.c b/ssl/s3_lib.c
index 52f9214c3d..44a6ab8f39 100644
--- a/ssl/s3_lib.c
+++ b/ssl/s3_lib.c
@@ -3964,6 +3964,8 @@ int ssl_generate_master_secret(SSL *s, unsigned char *pms, size_t pmslen,
                                int free_pms)
 {
     unsigned long alg_k = s->s3->tmp.new_cipher->algorithm_mkey;
+    int ret = 0;
+
     if (alg_k & SSL_PSK) {
 #ifndef OPENSSL_NO_PSK
         unsigned char *pskpms, *t;
@@ -3978,10 +3980,8 @@ int ssl_generate_master_secret(SSL *s, unsigned char *pms, size_t pmslen,
 
         pskpmslen = 4 + pmslen + psklen;
         pskpms = OPENSSL_malloc(pskpmslen);
-        if (pskpms == NULL) {
-            s->session->master_key_length = 0;
+        if (pskpms == NULL)
             goto err;
-        }
         t = pskpms;
         s2n(pmslen, t);
         if (alg_k & SSL_kPSK)
@@ -3994,23 +3994,23 @@ int ssl_generate_master_secret(SSL *s, unsigned char *pms, size_t pmslen,
 
         OPENSSL_clear_free(s->s3->tmp.psk, psklen);
         s->s3->tmp.psk = NULL;
-        s->session->master_key_length =
-            s->method->ssl3_enc->generate_master_secret(s,
-                                                        s->session->master_key,
-                                                        pskpms, pskpmslen);
+        if (!s->method->ssl3_enc->generate_master_secret(s,
+                    s->session->master_key,pskpms, pskpmslen,
+                    &s->session->master_key_length))
+            goto err;
         OPENSSL_clear_free(pskpms, pskpmslen);
 #else
         /* Should never happen */
-        s->session->master_key_length = 0;
         goto err;
 #endif
     } else {
-        s->session->master_key_length =
-            s->method->ssl3_enc->generate_master_secret(s,
-                                                        s->session->master_key,
-                                                        pms, pmslen);
+        if (!s->method->ssl3_enc->generate_master_secret(s,
+                s->session->master_key, pms, pmslen,
+                &s->session->master_key_length))
+            goto err;
     }
 
+    ret = 1;
  err:
     if (pms) {
         if (free_pms)
@@ -4020,7 +4020,7 @@ int ssl_generate_master_secret(SSL *s, unsigned char *pms, size_t pmslen,
     }
     if (s->server == 0)
         s->s3->tmp.pms = NULL;
-    return s->session->master_key_length >= 0;
+    return ret;
 }
 
 /* Generate a private key from parameters */
diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c
index acc1840ee1..478d9a9bd4 100644
--- a/ssl/ssl_ciph.c
+++ b/ssl/ssl_ciph.c
@@ -197,7 +197,7 @@ static int ssl_mac_pkey_id[SSL_MD_NUM_IDX] = {
     EVP_PKEY_HMAC,
 };
 
-static int ssl_mac_secret_size[SSL_MD_NUM_IDX];
+static size_t ssl_mac_secret_size[SSL_MD_NUM_IDX];
 
 #define CIPHER_ADD      1
 #define CIPHER_KILL     2
@@ -399,8 +399,9 @@ void ssl_load_ciphers(void)
         if (md == NULL) {
             disabled_mac_mask |= t->mask;
         } else {
-            ssl_mac_secret_size[i] = EVP_MD_size(md);
-            OPENSSL_assert(ssl_mac_secret_size[i] >= 0);
+            int tmpsize = EVP_MD_size(md);
+            OPENSSL_assert(tmpsize >= 0);
+            ssl_mac_secret_size[i] = tmpsize;
         }
     }
     /* Make sure we can access MD5 and SHA1 */
@@ -502,7 +503,7 @@ static int load_builtin_compressions(void)
 
 int ssl_cipher_get_evp(const SSL_SESSION *s, const EVP_CIPHER **enc,
                        const EVP_MD **md, int *mac_pkey_type,
-                       int *mac_secret_size, SSL_COMP **comp, int use_etm)
+                       size_t *mac_secret_size, SSL_COMP **comp, int use_etm)
 {
     int i;
     const SSL_CIPHER *c;
diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index d20f07cba1..5ca0334eff 100644
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -62,7 +62,7 @@ SSL3_ENC_METHOD ssl3_undef_enc_method = {
     (int (*)(SSL *, SSL3_RECORD *, size_t, int))ssl_undefined_function,
     (int (*)(SSL *, SSL3_RECORD *, unsigned char *, int))ssl_undefined_function,
     ssl_undefined_function,
-    (int (*)(SSL *, unsigned char *, unsigned char *, int))
+    (int (*)(SSL *, unsigned char *, unsigned char *, size_t, size_t *))
         ssl_undefined_function,
     (int (*)(SSL *, int))ssl_undefined_function,
     (int (*)(SSL *, const char *, int, unsigned char *))
@@ -3593,13 +3593,9 @@ size_t SSL_get_server_random(const SSL *ssl, unsigned char *out, size_t outlen)
 size_t SSL_SESSION_get_master_key(const SSL_SESSION *session,
                                   unsigned char *out, size_t outlen)
 {
-    if (session->master_key_length < 0) {
-        /* Should never happen */
-        return 0;
-    }
     if (outlen == 0)
         return session->master_key_length;
-    if (outlen > (size_t)session->master_key_length)
+    if (outlen > session->master_key_length)
         outlen = session->master_key_length;
     memcpy(out, session->master_key, outlen);
     return outlen;
@@ -3830,23 +3826,28 @@ void ssl_clear_hash_ctx(EVP_MD_CTX **hash)
 }
 
 /* Retrieve handshake hashes */
-int ssl_handshake_hash(SSL *s, unsigned char *out, int outlen)
+int ssl_handshake_hash(SSL *s, unsigned char *out, size_t outlen,
+                       size_t *hashlen)
 {
     EVP_MD_CTX *ctx = NULL;
     EVP_MD_CTX *hdgst = s->s3->handshake_dgst;
-    int ret = EVP_MD_CTX_size(hdgst);
-    if (ret < 0 || ret > outlen) {
-        ret = 0;
+    int hashleni = EVP_MD_CTX_size(hdgst);
+    int ret = 0;
+
+    if (hashleni < 0 || (size_t)hashleni > outlen)
         goto err;
-    }
+
     ctx = EVP_MD_CTX_new();
-    if (ctx == NULL) {
-        ret = 0;
+    if (ctx == NULL)
         goto err;
-    }
+
     if (!EVP_MD_CTX_copy_ex(ctx, hdgst)
         || EVP_DigestFinal_ex(ctx, out, NULL) <= 0)
-        ret = 0;
+        goto err;
+
+    *hashlen = hashleni;
+
+    ret = 1;
  err:
     EVP_MD_CTX_free(ctx);
     return ret;
diff --git a/ssl/ssl_locl.h b/ssl/ssl_locl.h
index 91ee34b102..6388d1087d 100644
--- a/ssl/ssl_locl.h
+++ b/ssl/ssl_locl.h
@@ -500,7 +500,7 @@ struct ssl_method_st {
 struct ssl_session_st {
     int ssl_version;            /* what ssl version session info is being kept
                                  * in here? */
-    int master_key_length;
+    size_t master_key_length;
     unsigned char master_key[SSL_MAX_MASTER_KEY_LENGTH];
     /* session_id - valid? */
     unsigned int session_id_length;
@@ -1200,7 +1200,7 @@ typedef struct ssl3_state_st {
         int ctype_num;
         char ctype[SSL3_CT_NUMBER];
         STACK_OF(X509_NAME) *ca_names;
-        int key_block_length;
+        size_t key_block_length;
         unsigned char *key_block;
         const EVP_CIPHER *new_sym_enc;
         const EVP_MD *new_hash;
@@ -1570,7 +1570,7 @@ typedef struct ssl3_enc_method {
     int (*mac) (SSL *, SSL3_RECORD *, unsigned char *, int);
     int (*setup_key_block) (SSL *);
     int (*generate_master_secret) (SSL *, unsigned char *, unsigned char *,
-                                   int);
+                                   size_t, size_t *);
     int (*change_cipher_state) (SSL *, int);
     int (*final_finish_mac) (SSL *, const char *, int, unsigned char *);
     int finish_mac_length;
@@ -1819,7 +1819,7 @@ __owur STACK_OF(SSL_CIPHER) *ssl_create_cipher_list(const SSL_METHOD *meth,
 void ssl_update_cache(SSL *s, int mode);
 __owur int ssl_cipher_get_evp(const SSL_SESSION *s, const EVP_CIPHER **enc,
                               const EVP_MD **md, int *mac_pkey_type,
-                              int *mac_secret_size, SSL_COMP **comp,
+                              size_t *mac_secret_size, SSL_COMP **comp,
                               int use_etm);
 __owur int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
                                    size_t *int_overhead, size_t *blocksize,
@@ -1879,7 +1879,8 @@ void ssl3_cleanup_key_block(SSL *s);
 __owur int ssl3_do_write(SSL *s, int type);
 int ssl3_send_alert(SSL *s, int level, int desc);
 __owur int ssl3_generate_master_secret(SSL *s, unsigned char *out,
-                                       unsigned char *p, int len);
+                                       unsigned char *p, size_t len,
+                                       size_t *secret_size);
 __owur int ssl3_get_req_cert_type(SSL *s, WPACKET *pkt);
 __owur int ssl3_num_ciphers(void);
 __owur const SSL_CIPHER *ssl3_get_cipher(unsigned int u);
@@ -1985,7 +1986,8 @@ __owur int tls1_setup_key_block(SSL *s);
 __owur int tls1_final_finish_mac(SSL *s,
                                  const char *str, int slen, unsigned char *p);
 __owur int tls1_generate_master_secret(SSL *s, unsigned char *out,
-                                       unsigned char *p, int len);
+                                       unsigned char *p, size_t len,
+                                       size_t *secret_size);
 __owur int tls1_export_keying_material(SSL *s, unsigned char *out, size_t olen,
                                        const char *label, size_t llen,
                                        const unsigned char *p, size_t plen,
@@ -2086,7 +2088,8 @@ __owur int ssl_cipher_disabled(SSL *s, const SSL_CIPHER *c, int op);
 __owur int ssl_parse_clienthello_use_srtp_ext(SSL *s, PACKET *pkt, int *al);
 __owur int ssl_parse_serverhello_use_srtp_ext(SSL *s, PACKET *pkt, int *al);
 
-__owur int ssl_handshake_hash(SSL *s, unsigned char *out, int outlen);
+__owur int ssl_handshake_hash(SSL *s, unsigned char *out, size_t outlen,
+                                 size_t *hashlen);
 __owur const EVP_MD *ssl_md(int idx);
 __owur const EVP_MD *ssl_handshake_md(SSL *s);
 __owur const EVP_MD *ssl_prf_md(SSL *s);
diff --git a/ssl/statem/statem_clnt.c b/ssl/statem/statem_clnt.c
index 5704654df1..c6b025cc51 100644
--- a/ssl/statem/statem_clnt.c
+++ b/ssl/statem/statem_clnt.c
@@ -973,11 +973,18 @@ MSG_PROCESS_RETURN tls_process_server_hello(SSL *s, PACKET *pkt)
     if (s->version >= TLS1_VERSION && s->tls_session_secret_cb &&
         s->session->tlsext_tick) {
         const SSL_CIPHER *pref_cipher = NULL;
-        s->session->master_key_length = sizeof(s->session->master_key);
+        /*
+         * s->session->master_key_length is a size_t, but this is an int for
+         * backwards compat reasons
+         */
+        int master_key_length;
+        master_key_length = sizeof(s->session->master_key);
         if (s->tls_session_secret_cb(s, s->session->master_key,
-                                     &s->session->master_key_length,
+                                     &master_key_length,
                                      NULL, &pref_cipher,
-                                     s->tls_session_secret_cb_arg)) {
+                                     s->tls_session_secret_cb_arg)
+                 && master_key_length > 0) {
+            s->session->master_key_length = master_key_length;
             s->session->cipher = pref_cipher ?
                 pref_cipher : ssl_get_cipher_by_char(s, cipherchars);
         } else {
diff --git a/ssl/statem/statem_srvr.c b/ssl/statem/statem_srvr.c
index a243161e3a..f936588f2f 100644
--- a/ssl/statem/statem_srvr.c
+++ b/ssl/statem/statem_srvr.c
@@ -1231,12 +1231,19 @@ MSG_PROCESS_RETURN tls_process_client_hello(SSL *s, PACKET *pkt)
 
     if (!s->hit && s->version >= TLS1_VERSION && s->tls_session_secret_cb) {
         const SSL_CIPHER *pref_cipher = NULL;
+        /*
+         * s->session->master_key_length is a size_t, but this is an int for
+         * backwards compat reasons
+         */
+        int master_key_length;
 
-        s->session->master_key_length = sizeof(s->session->master_key);
+        master_key_length = sizeof(s->session->master_key);
         if (s->tls_session_secret_cb(s, s->session->master_key,
-                                     &s->session->master_key_length, ciphers,
+                                     &master_key_length, ciphers,
                                      &pref_cipher,
-                                     s->tls_session_secret_cb_arg)) {
+                                     s->tls_session_secret_cb_arg)
+                && master_key_length > 0) {
+            s->session->master_key_length = master_key_length;
             s->hit = 1;
             s->session->ciphers = ciphers;
             s->session->verify_result = X509_V_OK;
diff --git a/ssl/t1_enc.c b/ssl/t1_enc.c
index 3a1e024de3..46895ef597 100644
--- a/ssl/t1_enc.c
+++ b/ssl/t1_enc.c
@@ -88,7 +88,8 @@ static int tls1_PRF(SSL *s,
     return ret;
 }
 
-static int tls1_generate_key_block(SSL *s, unsigned char *km, int num)
+/* TODO(size_t): convert me */
+static int tls1_generate_key_block(SSL *s, unsigned char *km, size_t num)
 {
     int ret;
     ret = tls1_PRF(s,
@@ -360,9 +361,9 @@ int tls1_setup_key_block(SSL *s)
     unsigned char *p;
     const EVP_CIPHER *c;
     const EVP_MD *hash;
-    int num;
     SSL_COMP *comp;
-    int mac_type = NID_undef, mac_secret_size = 0;
+    int mac_type = NID_undef;
+    size_t num, mac_secret_size = 0;
     int ret = 0;
 
     if (s->s3->tmp.key_block_length != 0)
@@ -409,7 +410,7 @@ int tls1_setup_key_block(SSL *s)
     }
     printf("master key\n");
     {
-        int z;
+        size_t z;
         for (z = 0; z < s->session->master_key_length; z++)
             printf("%02X%c", s->session->master_key[z],
                    ((z + 1) % 16) ? ' ' : '\n');
@@ -420,7 +421,7 @@ int tls1_setup_key_block(SSL *s)
 #ifdef SSL_DEBUG
     printf("\nkey block\n");
     {
-        int z;
+        size_t z;
         for (z = 0; z < num; z++)
             printf("%02X%c", p[z], ((z + 1) % 16) ? ' ' : '\n');
     }
@@ -452,15 +453,13 @@ int tls1_setup_key_block(SSL *s)
 
 int tls1_final_finish_mac(SSL *s, const char *str, int slen, unsigned char *out)
 {
-    int hashlen;
+    size_t hashlen;
     unsigned char hash[EVP_MAX_MD_SIZE];
 
     if (!ssl3_digest_cached_records(s, 0))
         return 0;
 
-    hashlen = ssl_handshake_hash(s, hash, sizeof(hash));
-
-    if (hashlen == 0)
+    if (!ssl_handshake_hash(s, hash, sizeof(hash), &hashlen))
         return 0;
 
     if (!tls1_PRF(s, str, slen, hash, hashlen, NULL, 0, NULL, 0, NULL, 0,
@@ -472,19 +471,20 @@ int tls1_final_finish_mac(SSL *s, const char *str, int slen, unsigned char *out)
 }
 
 int tls1_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
-                                int len)
+                                size_t len, size_t *secret_size)
 {
     if (s->session->flags & SSL_SESS_FLAG_EXTMS) {
         unsigned char hash[EVP_MAX_MD_SIZE * 2];
-        int hashlen;
+        size_t hashlen;
         /*
          * Digest cached records keeping record buffer (if present): this wont
          * affect client auth because we're freezing the buffer at the same
          * point (after client key exchange and before certificate verify)
          */
         if (!ssl3_digest_cached_records(s, 1))
-            return -1;
-        hashlen = ssl_handshake_hash(s, hash, sizeof(hash));
+            return 0;
+        if(!ssl_handshake_hash(s, hash, sizeof(hash), &hashlen))
+            return 0;
 #ifdef SSL_DEBUG
         fprintf(stderr, "Handshake hashes:\n");
         BIO_dump_fp(stderr, (char *)hash, hashlen);
@@ -536,7 +536,8 @@ int tls1_generate_master_secret(SSL *s, unsigned char *out, unsigned char *p,
     }
 #endif
 
-    return (SSL3_MASTER_SECRET_SIZE);
+    *secret_size = SSL3_MASTER_SECRET_SIZE;
+    return 1;
 }
 
 int tls1_export_keying_material(SSL *s, unsigned char *out, size_t olen,