Lazily initialise the compression buffer

With read pipelining we use multiple SSL3_RECORD structures for reading.
There are SSL_MAX_PIPELINES (32) of them defined (typically not all of these
would be used). Each one has a 16k compression buffer allocated! This
results in a significant amount of memory being consumed which, most of the
time, is not needed.  This change swaps the allocation of the compression
buffer to be lazy so that it is only done immediately before it is actually
used.

Reviewed-by: Tim Hudson <tjh@openssl.org>

4 files changed, 9 insertions, 27 deletions

diff --git a/ssl/record/rec_layer_s3.c b/ssl/record/rec_layer_s3.c
index 1aceed3aa5..91b8205ee9 100644
--- a/ssl/record/rec_layer_s3.c
+++ b/ssl/record/rec_layer_s3.c
@@ -223,11 +223,6 @@ void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl)
     memset(rl->write_sequence, 0, sizeof(rl->write_sequence));
 }
 
-int RECORD_LAYER_setup_comp_buffer(RECORD_LAYER *rl)
-{
-    return SSL3_RECORD_setup((rl)->rrec, SSL_MAX_PIPELINES);
-}
-
 int ssl3_pending(const SSL *s)
 {
     unsigned int i;
diff --git a/ssl/record/record.h b/ssl/record/record.h
index bf3ffa3cfd..000fc85621 100644
--- a/ssl/record/record.h
+++ b/ssl/record/record.h
@@ -325,7 +325,6 @@ int RECORD_LAYER_write_pending(RECORD_LAYER *rl);
 int RECORD_LAYER_set_data(RECORD_LAYER *rl, const unsigned char *buf, int len);
 void RECORD_LAYER_reset_read_sequence(RECORD_LAYER *rl);
 void RECORD_LAYER_reset_write_sequence(RECORD_LAYER *rl);
-int RECORD_LAYER_setup_comp_buffer(RECORD_LAYER *rl);
 int RECORD_LAYER_is_sslv2_record(RECORD_LAYER *rl);
 unsigned int RECORD_LAYER_get_rrec_length(RECORD_LAYER *rl);
 __owur int ssl3_pending(const SSL *s);
diff --git a/ssl/record/record_locl.h b/ssl/record/record_locl.h
index 8239549beb..e5d2784f26 100644
--- a/ssl/record/record_locl.h
+++ b/ssl/record/record_locl.h
@@ -193,7 +193,6 @@ int ssl3_release_write_buffer(SSL *s);
 
 void SSL3_RECORD_clear(SSL3_RECORD *r, unsigned int num_recs);
 void SSL3_RECORD_release(SSL3_RECORD *r, unsigned int num_recs);
-int SSL3_RECORD_setup(SSL3_RECORD *r, unsigned int num_recs);
 void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num);
 int ssl3_get_record(SSL *s);
 __owur int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr);
diff --git a/ssl/record/ssl3_record.c b/ssl/record/ssl3_record.c
index 33122626ec..c910e690a2 100644
--- a/ssl/record/ssl3_record.c
+++ b/ssl/record/ssl3_record.c
@@ -157,24 +157,6 @@ void SSL3_RECORD_release(SSL3_RECORD *r, unsigned int num_recs)
     }
 }
 
-int SSL3_RECORD_setup(SSL3_RECORD *r, unsigned int num_recs)
-{
-    unsigned int i;
-
-    for (i = 0; i < num_recs; i++) {
-        if (r[i].comp == NULL)
-            r[i].comp = (unsigned char *)
-                OPENSSL_malloc(SSL3_RT_MAX_ENCRYPTED_LENGTH);
-        if (r[i].comp == NULL) {
-            if (i > 0)
-                SSL3_RECORD_release(r, i);
-            return 0;
-        }
-    }
-
-    return 1;
-}
-
 void SSL3_RECORD_set_seq_num(SSL3_RECORD *r, const unsigned char *seq_num)
 {
     memcpy(r->seq_num, seq_num, SEQ_NUM_SIZE);
@@ -626,16 +608,23 @@ int ssl3_do_uncompress(SSL *ssl, SSL3_RECORD *rr)
 #ifndef OPENSSL_NO_COMP
     int i;
 
+    if (rr->comp == NULL) {
+        rr->comp = (unsigned char *)
+            OPENSSL_malloc(SSL3_RT_MAX_ENCRYPTED_LENGTH);
+    }
+    if (rr->comp == NULL)
+        return 0;
+
     i = COMP_expand_block(ssl->expand, rr->comp,
                           SSL3_RT_MAX_PLAIN_LENGTH, rr->data,
                           (int)rr->length);
     if (i < 0)
-        return (0);
+        return 0;
     else
         rr->length = i;
     rr->data = rr->comp;
 #endif
-    return (1);
+    return 1;
 }
 
 int ssl3_do_compress(SSL *ssl, SSL3_RECORD *wr)