Replace instances of OPENSSL_assert() with soft asserts in libssl

Reviewed-by: Tim Hudson <tjh@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/3496)

3 files changed, 47 insertions, 16 deletions

diff --git a/ssl/record/rec_layer_d1.c b/ssl/record/rec_layer_d1.c
index 487b096b24..879a9b039c 100644
--- a/ssl/record/rec_layer_d1.c
+++ b/ssl/record/rec_layer_d1.c
@@ -734,7 +734,8 @@ int dtls1_write_bytes(SSL *s, int type, const void *buf, size_t len,
 {
     int i;
 
-    OPENSSL_assert(len <= SSL3_RT_MAX_PLAIN_LENGTH);
+    if (!ossl_assert(len <= SSL3_RT_MAX_PLAIN_LENGTH))
+        return -1;
     s->rwstate = SSL_NOTHING;
     i = do_dtls1_write(s, type, buf, len, 0, written);
     return i;
@@ -757,9 +758,9 @@ int do_dtls1_write(SSL *s, int type, const unsigned char *buf,
      * first check if there is a SSL3_BUFFER still being written out.  This
      * will happen with non blocking IO
      */
-    if (SSL3_BUFFER_get_left(wb) != 0) {
-        OPENSSL_assert(0);      /* XDTLS: want to see if we ever get here */
-        return ssl3_write_pending(s, type, buf, len, written);
+    if (!ossl_assert(SSL3_BUFFER_get_left(wb) == 0)) {
+        SSLerr(SSL_F_DO_DTLS1_WRITE, ERR_R_INTERNAL_ERROR);
+        return 0;
     }
 
     /* If we have an alert to send, lets send it */
diff --git a/ssl/record/record_locl.h b/ssl/record/record_locl.h
index e249918ef4..2b55bece85 100644
--- a/ssl/record/record_locl.h
+++ b/ssl/record/record_locl.h
@@ -106,7 +106,7 @@ void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num);
 int ssl3_get_record(SSL *s);
 __owur int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr);
 __owur int ssl3_do_uncompress(SSL *ssl, SSL3_RECORD *rr);
-void ssl3_cbc_copy_mac(unsigned char *out,
+int ssl3_cbc_copy_mac(unsigned char *out,
                        const SSL3_RECORD *rec, size_t md_size);
 __owur int ssl3_cbc_remove_padding(SSL3_RECORD *rec,
                                    size_t block_size, size_t mac_size);
diff --git a/ssl/record/ssl3_record.c b/ssl/record/ssl3_record.c
index 8ebb1b92ee..bba0cc0f0b 100644
--- a/ssl/record/ssl3_record.c
+++ b/ssl/record/ssl3_record.c
@@ -515,7 +515,11 @@ int ssl3_get_record(SSL *s)
         unsigned char mac_tmp[EVP_MAX_MD_SIZE];
 
         mac_size = EVP_MD_CTX_size(s->read_hash);
-        OPENSSL_assert(mac_size <= EVP_MAX_MD_SIZE);
+        if (!ossl_assert(mac_size <= EVP_MAX_MD_SIZE)) {
+            al = SSL_AD_INTERNAL_ERROR;
+            SSLerr(SSL_F_SSL3_GET_RECORD, ERR_R_INTERNAL_ERROR);
+            goto f_err;
+        }
 
         for (j = 0; j < num_recs; j++) {
             thisrr = &rr[j];
@@ -542,7 +546,11 @@ int ssl3_get_record(SSL *s)
                  * contents of the padding bytes.
                  */
                 mac = mac_tmp;
-                ssl3_cbc_copy_mac(mac_tmp, thisrr, mac_size);
+                if (!ssl3_cbc_copy_mac(mac_tmp, thisrr, mac_size)) {
+                    al = SSL_AD_INTERNAL_ERROR;
+                    SSLerr(SSL_F_SSL3_GET_RECORD, ERR_R_INTERNAL_ERROR);
+                    goto f_err;
+                }
                 thisrr->length -= mac_size;
             } else {
                 /*
@@ -859,7 +867,10 @@ int tls1_enc(SSL *s, SSL3_RECORD *recs, size_t n_recs, int sending)
     if (sending) {
         if (EVP_MD_CTX_md(s->write_hash)) {
             int n = EVP_MD_CTX_size(s->write_hash);
-            OPENSSL_assert(n >= 0);
+            if (!ossl_assert(n >= 0)) {
+                SSLerr(SSL_F_TLS1_ENC, ERR_R_INTERNAL_ERROR);
+                return -1;
+            }
         }
         ds = s->enc_write_ctx;
         if (s->enc_write_ctx == NULL)
@@ -892,7 +903,10 @@ int tls1_enc(SSL *s, SSL3_RECORD *recs, size_t n_recs, int sending)
     } else {
         if (EVP_MD_CTX_md(s->read_hash)) {
             int n = EVP_MD_CTX_size(s->read_hash);
-            OPENSSL_assert(n >= 0);
+            if (!ossl_assert(n >= 0)) {
+                SSLerr(SSL_F_TLS1_ENC, ERR_R_INTERNAL_ERROR);
+                return -1;
+            }
         }
         ds = s->enc_read_ctx;
         if (s->enc_read_ctx == NULL)
@@ -1179,7 +1193,8 @@ int tls1_mac(SSL *ssl, SSL3_RECORD *rec, unsigned char *md, int sending)
     }
 
     t = EVP_MD_CTX_size(hash);
-    OPENSSL_assert(t >= 0);
+    if (!ossl_assert(t >= 0))
+        return 0;
     md_size = t;
 
     /* I should fix this up TLS TLS TLS TLS TLS XXXXXXXX */
@@ -1404,7 +1419,7 @@ int tls1_cbc_remove_padding(const SSL *s,
  */
 #define CBC_MAC_ROTATE_IN_PLACE
 
-void ssl3_cbc_copy_mac(unsigned char *out,
+int ssl3_cbc_copy_mac(unsigned char *out,
                        const SSL3_RECORD *rec, size_t md_size)
 {
 #if defined(CBC_MAC_ROTATE_IN_PLACE)
@@ -1428,8 +1443,9 @@ void ssl3_cbc_copy_mac(unsigned char *out,
     size_t i, j;
     size_t rotate_offset;
 
-    OPENSSL_assert(rec->orig_len >= md_size);
-    OPENSSL_assert(md_size <= EVP_MAX_MD_SIZE);
+    if (!ossl_assert(rec->orig_len >= md_size
+                     && md_size <= EVP_MAX_MD_SIZE))
+        return 0;
 
 #if defined(CBC_MAC_ROTATE_IN_PLACE)
     rotated_mac = rotated_mac_buf + ((0 - (size_t)rotated_mac_buf) & 63);
@@ -1474,6 +1490,8 @@ void ssl3_cbc_copy_mac(unsigned char *out,
         rotate_offset &= constant_time_lt_s(rotate_offset, md_size);
     }
 #endif
+
+    return 1;
 }
 
 int dtls1_process_record(SSL *s, DTLS1_BITMAP *bitmap)
@@ -1521,7 +1539,11 @@ int dtls1_process_record(SSL *s, DTLS1_BITMAP *bitmap)
     if (SSL_READ_ETM(s) && s->read_hash) {
         unsigned char *mac;
         mac_size = EVP_MD_CTX_size(s->read_hash);
-        OPENSSL_assert(mac_size <= EVP_MAX_MD_SIZE);
+        if (!ossl_assert(mac_size <= EVP_MAX_MD_SIZE)) {
+            al = SSL_AD_INTERNAL_ERROR;
+            SSLerr(SSL_F_DTLS1_PROCESS_RECORD, ERR_R_INTERNAL_ERROR);
+            goto f_err;
+        }
         if (rr->orig_len < mac_size) {
             al = SSL_AD_DECODE_ERROR;
             SSLerr(SSL_F_DTLS1_PROCESS_RECORD, SSL_R_LENGTH_TOO_SHORT);
@@ -1576,7 +1598,11 @@ int dtls1_process_record(SSL *s, DTLS1_BITMAP *bitmap)
             goto f_err;
         }
         mac_size = (size_t)imac_size;
-        OPENSSL_assert(mac_size <= EVP_MAX_MD_SIZE);
+        if (!ossl_assert(mac_size <= EVP_MAX_MD_SIZE)) {
+            al = SSL_AD_INTERNAL_ERROR;
+            SSLerr(SSL_F_DTLS1_PROCESS_RECORD, ERR_R_INTERNAL_ERROR);
+            goto f_err;
+        }
 
         /*
          * orig_len is the length of the record before any padding was
@@ -1601,7 +1627,11 @@ int dtls1_process_record(SSL *s, DTLS1_BITMAP *bitmap)
              * contents of the padding bytes.
              */
             mac = mac_tmp;
-            ssl3_cbc_copy_mac(mac_tmp, rr, mac_size);
+            if (!ssl3_cbc_copy_mac(mac_tmp, rr, mac_size)) {
+                al = SSL_AD_INTERNAL_ERROR;
+                SSLerr(SSL_F_DTLS1_PROCESS_RECORD, ERR_R_INTERNAL_ERROR);
+                goto f_err;
+            }
             rr->length -= mac_size;
         } else {
             /*