ssl_cipher_get_overhead(): Replace size_t with int and add the checks

Replace the type of "mac", "out", and "blk" with int to avoid implicit
conversion when it is assigned by EVP_MD_get_size(),
EVP_CIPHER_get_iv_length(), and EVP_CIPHER_get_block_size().
Moreover, add the checks to avoid integer overflow.

Fixes: 045bd04706 ("Add DTLS_get_data_mtu() function")
Signed-off-by: Jiasheng Jiang <jiasheng@purdue.edu>

Reviewed-by: Tom Cosgrove <tom.cosgrove@arm.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/23935)

1 files changed, 10 insertions, 6 deletions

diff --git a/ssl/ssl_ciph.c b/ssl/ssl_ciph.c
index ddde21b968..a9cf6416b1 100644
--- a/ssl/ssl_ciph.c
+++ b/ssl/ssl_ciph.c
@@ -2184,7 +2184,7 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
                             size_t *int_overhead, size_t *blocksize,
                             size_t *ext_overhead)
 {
-    size_t mac = 0, in = 0, blk = 0, out = 0;
+    int mac = 0, in = 0, blk = 0, out = 0;
 
     /* Some hard-coded numbers for the CCM/Poly1305 MAC overhead
      * because there are no handy #defines for those. */
@@ -2208,6 +2208,8 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
             return 0;
 
         mac = EVP_MD_get_size(e_md);
+        if (mac <= 0)
+            return 0;
         if (c->algorithm_enc != SSL_eNULL) {
             int cipher_nid = SSL_CIPHER_get_cipher_nid(c);
             const EVP_CIPHER *e_ciph = EVP_get_cipherbynid(cipher_nid);
@@ -2220,16 +2222,18 @@ int ssl_cipher_get_overhead(const SSL_CIPHER *c, size_t *mac_overhead,
 
             in = 1; /* padding length byte */
             out = EVP_CIPHER_get_iv_length(e_ciph);
+            if (out < 0)
+                return 0;
             blk = EVP_CIPHER_get_block_size(e_ciph);
-            if (blk == 0)
+            if (blk <= 0)
                 return 0;
         }
     }
 
-    *mac_overhead = mac;
-    *int_overhead = in;
-    *blocksize = blk;
-    *ext_overhead = out;
+    *mac_overhead = (size_t)mac;
+    *int_overhead = (size_t)in;
+    *blocksize = (size_t)blk;
+    *ext_overhead = (size_t)out;
 
     return 1;
 }