From 81c471292d3b15f48d236f8432aacf2b734f9d1e Mon Sep 17 00:00:00 2001 From: "Neal H. Walfield" Date: Fri, 6 Jan 2023 11:15:13 +0100 Subject: buffered-reader: Set the buffer size using an environment variable - If the environment variable `SEQUOIA_BUFFERED_READER_BUFFER` is set, and we are able to parse it as a usize, use it as the default buffer size. --- buffered-reader/src/decompress_deflate.rs | 6 ++--- buffered-reader/src/dup.rs | 4 +-- buffered-reader/src/generic.rs | 10 +++---- buffered-reader/src/lib.rs | 43 +++++++++++++++++++++++++++---- buffered-reader/src/limitor.rs | 6 ++--- buffered-reader/src/memory.rs | 4 +-- 6 files changed, 53 insertions(+), 20 deletions(-) (limited to 'buffered-reader/src') diff --git a/buffered-reader/src/decompress_deflate.rs b/buffered-reader/src/decompress_deflate.rs index 1bb0bc4f..790a5541 100644 --- a/buffered-reader/src/decompress_deflate.rs +++ b/buffered-reader/src/decompress_deflate.rs @@ -263,7 +263,7 @@ mod test { use std::io::prelude::*; // Test vector. - let size = 10 * DEFAULT_BUF_SIZE; + let size = 10 * default_buf_size(); let mut input_raw = Vec::with_capacity(size); let mut v = 0u8; for _ in 0..size { @@ -289,11 +289,11 @@ mod test { // Gather some stats to make it easier to figure out whether // this test is working. - let stats_count = 2 * DEFAULT_BUF_SIZE; + let stats_count = 2 * default_buf_size(); let mut stats = vec![0usize; stats_count]; for i in 0..input_raw.len() { - let data = reader.data(DEFAULT_BUF_SIZE + 1).unwrap().to_vec(); + let data = reader.data(default_buf_size() + 1).unwrap().to_vec(); assert!(!data.is_empty()); assert_eq!(data, reader.buffer()); // And, we may as well check to make sure we read the diff --git a/buffered-reader/src/dup.rs b/buffered-reader/src/dup.rs index c63ab9c8..cc9f4436 100644 --- a/buffered-reader/src/dup.rs +++ b/buffered-reader/src/dup.rs @@ -177,7 +177,7 @@ mod test { fn buffer_test() { // Test vector. A Dup returns all unconsumed // data. So, use a relatively small buffer size. - let size = DEFAULT_BUF_SIZE; + let size = default_buf_size(); let mut input = Vec::with_capacity(size); let mut v = 0u8; for _ in 0..size { @@ -193,7 +193,7 @@ mod test { let mut reader = Dup::new(reader); for i in 0..input.len() { - let data = reader.data(DEFAULT_BUF_SIZE + 1).unwrap().to_vec(); + let data = reader.data(default_buf_size() + 1).unwrap().to_vec(); assert!(!data.is_empty()); assert_eq!(data, reader.buffer()); // And, we may as well check to make sure we read the diff --git a/buffered-reader/src/generic.rs b/buffered-reader/src/generic.rs index 4a08e549..cff30b5e 100644 --- a/buffered-reader/src/generic.rs +++ b/buffered-reader/src/generic.rs @@ -81,7 +81,7 @@ impl Generic { unused_buffer: None, preferred_chunk_size: if let Some(s) = preferred_chunk_size { s } - else { DEFAULT_BUF_SIZE }, + else { default_buf_size() }, reader, error: None, eof: false, @@ -140,7 +140,7 @@ impl Generic { // available. Read some more. let capacity : usize = cmp::max(cmp::max( - DEFAULT_BUF_SIZE, + default_buf_size(), 2 * self.preferred_chunk_size), amount); let mut buffer_new = self.unused_buffer.take() @@ -361,7 +361,7 @@ mod test { #[test] fn buffer_test() { // Test vector. - let size = 10 * DEFAULT_BUF_SIZE; + let size = 10 * default_buf_size(); let mut input = Vec::with_capacity(size); let mut v = 0u8; for _ in 0..size { @@ -377,11 +377,11 @@ mod test { // Gather some stats to make it easier to figure out whether // this test is working. - let stats_count = 2 * DEFAULT_BUF_SIZE; + let stats_count = 2 * default_buf_size(); let mut stats = vec![0usize; stats_count]; for i in 0..input.len() { - let data = reader.data(DEFAULT_BUF_SIZE + 1).unwrap().to_vec(); + let data = reader.data(default_buf_size() + 1).unwrap().to_vec(); assert!(!data.is_empty()); assert_eq!(data, reader.buffer()); // And, we may as well check to make sure we read the diff --git a/buffered-reader/src/lib.rs b/buffered-reader/src/lib.rs index db9a5b46..57dc6251 100644 --- a/buffered-reader/src/lib.rs +++ b/buffered-reader/src/lib.rs @@ -280,7 +280,40 @@ pub use self::file_generic::File; pub use self::file_unix::File; // The default buffer size. -const DEFAULT_BUF_SIZE: usize = 8 * 1024; +// +// This is configurable by the SEQUOIA_BUFFERED_READER_BUFFER +// environment variable. +lazy_static::lazy_static! { + static ref DEFAULT_BUF_SIZE_: usize = { + use std::env::var_os; + use std::str::FromStr; + + let default = 8 * 1024; + + if let Some(size) = var_os("SEQUOIA_BUFFERED_READER_BUFFER") { + size.to_str() + .and_then(|s| { + match FromStr::from_str(s) { + Ok(s) => Some(s), + Err(err) => { + eprintln!("Unable to parse the value of \ + 'SEQUOIA_BUFFERED_READER_BUFFER'; \ + falling back to the default buffer \ + size ({}): {}", + err, default); + None + } + } + }) + .unwrap_or(default) + } else { + default + } + }; +} +fn default_buf_size() -> usize { + *DEFAULT_BUF_SIZE_ +} // On debug builds, Vec::truncate is very, very slow. For // instance, running the decrypt_test_stream test takes 51 seconds on @@ -450,7 +483,7 @@ pub trait BufferedReader : io::Read + fmt::Debug + fmt::Display + Send + Sync // implementation might try to actually allocate a buffer that // large! Instead, try with increasingly larger buffers until // the read is (strictly) shorter than the specified size. - let mut s = DEFAULT_BUF_SIZE; + let mut s = default_buf_size(); // We will break the loop eventually, because self.data(s) // must return a slice shorter than std::usize::MAX. loop { @@ -732,7 +765,7 @@ pub trait BufferedReader : io::Read + fmt::Debug + fmt::Display + Send + Sync // Try self.buffer. Only if it is empty, use // self.data. let buffer = if self.buffer().is_empty() { - self.data(DEFAULT_BUF_SIZE)? + self.data(default_buf_size())? } else { self.buffer() }; @@ -817,10 +850,10 @@ pub trait BufferedReader : io::Read + fmt::Debug + fmt::Display + Send + Sync fn drop_eof(&mut self) -> Result { let mut at_least_one_byte = false; loop { - let n = self.data(DEFAULT_BUF_SIZE)?.len(); + let n = self.data(default_buf_size())?.len(); at_least_one_byte |= n > 0; self.consume(n); - if n < DEFAULT_BUF_SIZE { + if n < default_buf_size() { // EOF. break; } diff --git a/buffered-reader/src/limitor.rs b/buffered-reader/src/limitor.rs index 54ac5a78..ebf8d6b7 100644 --- a/buffered-reader/src/limitor.rs +++ b/buffered-reader/src/limitor.rs @@ -242,7 +242,7 @@ mod test { #[test] fn buffer_test() { // Test vector. - let size = 10 * DEFAULT_BUF_SIZE; + let size = 10 * default_buf_size(); let mut input = Vec::with_capacity(size); let mut v = 0u8; for _ in 0..size { @@ -261,11 +261,11 @@ mod test { // Gather some stats to make it easier to figure out whether // this test is working. - let stats_count = 2 * DEFAULT_BUF_SIZE; + let stats_count = 2 * default_buf_size(); let mut stats = vec![0usize; stats_count]; for i in 0..input.len() { - let data = reader.data(DEFAULT_BUF_SIZE + 1).unwrap().to_vec(); + let data = reader.data(default_buf_size() + 1).unwrap().to_vec(); assert!(!data.is_empty()); assert_eq!(data, reader.buffer()); // And, we may as well check to make sure we read the diff --git a/buffered-reader/src/memory.rs b/buffered-reader/src/memory.rs index 6fb9e338..8c4a7261 100644 --- a/buffered-reader/src/memory.rs +++ b/buffered-reader/src/memory.rs @@ -151,7 +151,7 @@ mod test { fn buffer_test() { // Test vector. A Memory returns all unconsumed // data. So, use a relatively small buffer size. - let size = DEFAULT_BUF_SIZE; + let size = default_buf_size(); let mut input = Vec::with_capacity(size); let mut v = 0u8; for _ in 0..size { @@ -166,7 +166,7 @@ mod test { let mut reader = Memory::new(&input[..]); for i in 0..input.len() { - let data = reader.data(DEFAULT_BUF_SIZE + 1).unwrap().to_vec(); + let data = reader.data(default_buf_size() + 1).unwrap().to_vec(); assert!(!data.is_empty()); assert_eq!(data, reader.buffer()); // And, we may as well check to make sure we read the -- cgit v1.2.3