diff options
Diffstat (limited to 'buffered-reader')
-rw-r--r-- | buffered-reader/Cargo.toml | 8 | ||||
l--------- | buffered-reader/LICENSE.txt | 1 | ||||
-rw-r--r-- | buffered-reader/NEWS | 25 | ||||
-rw-r--r-- | buffered-reader/src/adapter.rs | 2 | ||||
-rw-r--r-- | buffered-reader/src/decompress_bzip2.rs | 22 | ||||
-rw-r--r-- | buffered-reader/src/decompress_deflate.rs | 46 | ||||
-rw-r--r-- | buffered-reader/src/dup.rs | 10 | ||||
-rw-r--r-- | buffered-reader/src/eof.rs | 17 | ||||
-rw-r--r-- | buffered-reader/src/file_error.rs | 1 | ||||
-rw-r--r-- | buffered-reader/src/file_unix.rs | 32 | ||||
-rw-r--r-- | buffered-reader/src/generic.rs | 80 | ||||
-rw-r--r-- | buffered-reader/src/lib.rs | 50 | ||||
-rw-r--r-- | buffered-reader/src/limitor.rs | 23 | ||||
-rw-r--r-- | buffered-reader/src/macros.rs | 113 | ||||
-rw-r--r-- | buffered-reader/src/memory.rs | 10 | ||||
-rw-r--r-- | buffered-reader/src/reserve.rs | 5 |
16 files changed, 308 insertions, 137 deletions
diff --git a/buffered-reader/Cargo.toml b/buffered-reader/Cargo.toml index 480170aa..4d3b2719 100644 --- a/buffered-reader/Cargo.toml +++ b/buffered-reader/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "buffered-reader" description = "A super-powered Reader" -version = "0.21.0" +version = "1.1.3" authors = [ "Justus Winter <justus@sequoia-pgp.org>", "Kai Michaelis <kai@sequoia-pgp.org>", @@ -11,8 +11,9 @@ documentation = "https://docs.rs/buffered-reader" homepage = "https://sequoia-pgp.org/" repository = "https://gitlab.com/sequoia-pgp/sequoia" readme = "README.md" -license = "GPL-2.0-or-later" +license = "LGPL-2.0-or-later" edition = "2018" +rust-version = "1.60" [badges] gitlab = { repository = "sequoia-pgp/sequoia" } @@ -23,6 +24,9 @@ bzip2 = { version = "0.4", optional = true } flate2 = { version = "1.0.1", optional = true } libc = "0.2.66" +[lib] +bench = false + [features] default = ["compression"] diff --git a/buffered-reader/LICENSE.txt b/buffered-reader/LICENSE.txt new file mode 120000 index 00000000..4ab43736 --- /dev/null +++ b/buffered-reader/LICENSE.txt @@ -0,0 +1 @@ +../LICENSE.txt
\ No newline at end of file diff --git a/buffered-reader/NEWS b/buffered-reader/NEWS new file mode 100644 index 00000000..327bf83a --- /dev/null +++ b/buffered-reader/NEWS @@ -0,0 +1,25 @@ + -*- org -*- +#+TITLE: buffered-reader NEWS – history of user-visible changes +#+STARTUP: content hidestars + +* Changes in 1.1.2 +** Notable changes + - The generic buffered reader now correctly handles end-of-file + situations. + +* Changes in 1.1.1 +** Notable changes + - The generic buffered reader now recycles buffers reducing + pressure on the heap allocator. + +* Changes in 1.1.0 +** Notable changes + - This crate is now licensed under the LGPL 2.0 or later. + +* Changes in 1.0.1 +** Notable fixes + - Fixed a crash in Dup::read. + +* Changes in 1.0.0 + +This is the initial stable release. diff --git a/buffered-reader/src/adapter.rs b/buffered-reader/src/adapter.rs index 3734d9bc..e319aa8b 100644 --- a/buffered-reader/src/adapter.rs +++ b/buffered-reader/src/adapter.rs @@ -11,9 +11,9 @@ use super::*; /// own. #[derive(Debug)] pub struct Adapter<T: BufferedReader<B>, B: fmt::Debug + Send + Sync, C: fmt::Debug + Sync + Send> { - reader: T, _ghostly_cookie: std::marker::PhantomData<B>, cookie: C, + reader: T, } assert_send_and_sync!(Adapter<T, B, C> diff --git a/buffered-reader/src/decompress_bzip2.rs b/buffered-reader/src/decompress_bzip2.rs index febe37fc..d292390f 100644 --- a/buffered-reader/src/decompress_bzip2.rs +++ b/buffered-reader/src/decompress_bzip2.rs @@ -53,48 +53,48 @@ impl<R: BufferedReader<C>, C: fmt::Debug + Sync + Send> fmt::Display for Bzip<R, impl<R: BufferedReader<C>, C: fmt::Debug + Send + Sync> BufferedReader<C> for Bzip<R, C> { fn buffer(&self) -> &[u8] { - return self.reader.buffer(); + self.reader.buffer() } fn data(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data(amount); + self.reader.data(amount) } fn data_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_hard(amount); + self.reader.data_hard(amount) } fn data_eof(&mut self) -> Result<&[u8], io::Error> { - return self.reader.data_eof(); + self.reader.data_eof() } fn consume(&mut self, amount: usize) -> &[u8] { - return self.reader.consume(amount); + self.reader.consume(amount) } fn data_consume(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_consume(amount); + self.reader.data_consume(amount) } fn data_consume_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_consume_hard(amount); + self.reader.data_consume_hard(amount) } fn read_be_u16(&mut self) -> Result<u16, io::Error> { - return self.reader.read_be_u16(); + self.reader.read_be_u16() } fn read_be_u32(&mut self) -> Result<u32, io::Error> { - return self.reader.read_be_u32(); + self.reader.read_be_u32() } fn steal(&mut self, amount: usize) -> Result<Vec<u8>, io::Error> { - return self.reader.steal(amount); + self.reader.steal(amount) } fn steal_eof(&mut self) -> Result<Vec<u8>, io::Error> { - return self.reader.steal_eof(); + self.reader.steal_eof() } fn get_mut(&mut self) -> Option<&mut dyn BufferedReader<C>> { diff --git a/buffered-reader/src/decompress_deflate.rs b/buffered-reader/src/decompress_deflate.rs index d1d1a16b..1bb0bc4f 100644 --- a/buffered-reader/src/decompress_deflate.rs +++ b/buffered-reader/src/decompress_deflate.rs @@ -54,48 +54,48 @@ impl<R: BufferedReader<C>, C: fmt::Debug + Sync + Send> fmt::Display for Deflate impl<R: BufferedReader<C>, C: fmt::Debug + Send + Sync> BufferedReader<C> for Deflate<R, C> { fn buffer(&self) -> &[u8] { - return self.reader.buffer(); + self.reader.buffer() } fn data(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data(amount); + self.reader.data(amount) } fn data_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_hard(amount); + self.reader.data_hard(amount) } fn data_eof(&mut self) -> Result<&[u8], io::Error> { - return self.reader.data_eof(); + self.reader.data_eof() } fn consume(&mut self, amount: usize) -> &[u8] { - return self.reader.consume(amount); + self.reader.consume(amount) } fn data_consume(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_consume(amount); + self.reader.data_consume(amount) } fn data_consume_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_consume_hard(amount); + self.reader.data_consume_hard(amount) } fn read_be_u16(&mut self) -> Result<u16, io::Error> { - return self.reader.read_be_u16(); + self.reader.read_be_u16() } fn read_be_u32(&mut self) -> Result<u32, io::Error> { - return self.reader.read_be_u32(); + self.reader.read_be_u32() } fn steal(&mut self, amount: usize) -> Result<Vec<u8>, io::Error> { - return self.reader.steal(amount); + self.reader.steal(amount) } fn steal_eof(&mut self) -> Result<Vec<u8>, io::Error> { - return self.reader.steal_eof(); + self.reader.steal_eof() } fn get_mut(&mut self) -> Option<&mut dyn BufferedReader<C>> { @@ -180,48 +180,48 @@ impl<R: BufferedReader<C>, C: fmt::Debug + Sync + Send> fmt::Debug for Zlib<R, C impl<R: BufferedReader<C>, C: fmt::Debug + Send + Sync> BufferedReader<C> for Zlib<R, C> { fn buffer(&self) -> &[u8] { - return self.reader.buffer(); + self.reader.buffer() } fn data(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data(amount); + self.reader.data(amount) } fn data_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_hard(amount); + self.reader.data_hard(amount) } fn data_eof(&mut self) -> Result<&[u8], io::Error> { - return self.reader.data_eof(); + self.reader.data_eof() } fn consume(&mut self, amount: usize) -> &[u8] { - return self.reader.consume(amount); + self.reader.consume(amount) } fn data_consume(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_consume(amount); + self.reader.data_consume(amount) } fn data_consume_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.reader.data_consume_hard(amount); + self.reader.data_consume_hard(amount) } fn read_be_u16(&mut self) -> Result<u16, io::Error> { - return self.reader.read_be_u16(); + self.reader.read_be_u16() } fn read_be_u32(&mut self) -> Result<u32, io::Error> { - return self.reader.read_be_u32(); + self.reader.read_be_u32() } fn steal(&mut self, amount: usize) -> Result<Vec<u8>, io::Error> { - return self.reader.steal(amount); + self.reader.steal(amount) } fn steal_eof(&mut self) -> Result<Vec<u8>, io::Error> { - return self.reader.steal_eof(); + self.reader.steal_eof() } fn get_mut(&mut self) -> Option<&mut dyn BufferedReader<C>> { @@ -294,7 +294,7 @@ mod test { for i in 0..input_raw.len() { let data = reader.data(DEFAULT_BUF_SIZE + 1).unwrap().to_vec(); - assert!(data.len() > 0); + assert!(!data.is_empty()); assert_eq!(data, reader.buffer()); // And, we may as well check to make sure we read the // right data. diff --git a/buffered-reader/src/dup.rs b/buffered-reader/src/dup.rs index d7d3f7e5..c63ab9c8 100644 --- a/buffered-reader/src/dup.rs +++ b/buffered-reader/src/dup.rs @@ -12,13 +12,13 @@ use super::*; /// at the underlying `BufferedReader`. #[derive(Debug)] pub struct Dup<T: BufferedReader<C>, C: fmt::Debug + Sync + Send> { - reader: T, - // The number of bytes that have been consumed. cursor: usize, // The user settable cookie. cookie: C, + + reader: T, } assert_send_and_sync!(Dup<T, C> @@ -57,7 +57,7 @@ impl<T: BufferedReader<C>, C: fmt::Debug + Sync + Send> Dup<T, C> { /// Returns the number of bytes that this reader has consumed. pub fn total_out(&self) -> usize { - return self.cursor; + self.cursor } /// Resets the cursor to the beginning of the stream. @@ -73,7 +73,7 @@ impl<T: BufferedReader<C>, C: fmt::Debug + Sync + Send> io::Read for Dup<T, C> { let data = &data[self.cursor..]; let amount = cmp::min(buf.len(), data.len()); - buf.copy_from_slice(&data[..amount]); + buf[..amount].copy_from_slice(&data[..amount]); self.cursor += amount; @@ -194,7 +194,7 @@ mod test { for i in 0..input.len() { let data = reader.data(DEFAULT_BUF_SIZE + 1).unwrap().to_vec(); - assert!(data.len() > 0); + assert!(!data.is_empty()); assert_eq!(data, reader.buffer()); // And, we may as well check to make sure we read the // right data. diff --git a/buffered-reader/src/eof.rs b/buffered-reader/src/eof.rs index b0312630..08b0a2b6 100644 --- a/buffered-reader/src/eof.rs +++ b/buffered-reader/src/eof.rs @@ -20,6 +20,7 @@ impl<C> fmt::Display for EOF<C> { } } +#[allow(clippy::new_without_default)] impl EOF<()> { /// Instantiates a new `EOF`. pub fn new() -> Self { @@ -40,26 +41,26 @@ impl<C> EOF<C> { impl<C> Read for EOF<C> { fn read(&mut self, _buf: &mut [u8]) -> Result<usize, io::Error> { - return Ok(0); + Ok(0) } } impl<C: fmt::Debug + Sync + Send> BufferedReader<C> for EOF<C> { fn buffer(&self) -> &[u8] { - return &b""[..]; + &b""[..] } fn data(&mut self, _amount: usize) -> Result<&[u8], io::Error> { - return Ok(&b""[..]); + Ok(&b""[..]) } fn consume(&mut self, amount: usize) -> &[u8] { assert_eq!(amount, 0); - return &b""[..]; + &b""[..] } fn data_consume(&mut self, _amount: usize) -> Result<&[u8], io::Error> { - return Ok(&b""[..]); + Ok(&b""[..]) } fn data_consume_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { @@ -73,17 +74,17 @@ impl<C: fmt::Debug + Sync + Send> BufferedReader<C> for EOF<C> { fn into_inner<'a>(self: Box<Self>) -> Option<Box<dyn BufferedReader<C> + 'a>> where Self: 'a { - return None; + None } fn get_mut(&mut self) -> Option<&mut dyn BufferedReader<C>> { - return None; + None } fn get_ref(&self) -> Option<&dyn BufferedReader<C>> { - return None; + None } diff --git a/buffered-reader/src/file_error.rs b/buffered-reader/src/file_error.rs index eba444cd..add00e6b 100644 --- a/buffered-reader/src/file_error.rs +++ b/buffered-reader/src/file_error.rs @@ -12,6 +12,7 @@ pub(crate) struct FileError { source: io::Error, } +#[allow(clippy::new_ret_no_self)] impl FileError { /// Returns a new `io::Error` backed by a `FileError`. pub fn new<P: AsRef<Path>>(path: P, source: io::Error) -> io::Error { diff --git a/buffered-reader/src/file_unix.rs b/buffered-reader/src/file_unix.rs index a1298737..89d0b808 100644 --- a/buffered-reader/src/file_unix.rs +++ b/buffered-reader/src/file_unix.rs @@ -48,7 +48,7 @@ impl<'a, C: fmt::Debug + Sync + Send> fmt::Debug for File<'a, C> { /// The implementation. enum Imp<'a, C: fmt::Debug + Sync + Send> { Generic(Generic<fs::File, C>), - MMAP { + Mmap { reader: Memory<'a, C>, } } @@ -57,7 +57,7 @@ impl<'a, C: fmt::Debug + Sync + Send> Drop for Imp<'a, C> { fn drop(&mut self) { match self { Imp::Generic(_) => (), - Imp::MMAP { reader, } => { + Imp::Mmap { reader, } => { let buf = reader.source_buffer(); unsafe { munmap(buf.as_ptr() as *mut _, buf.len()); @@ -72,7 +72,7 @@ impl<'a, C: fmt::Debug + Sync + Send> fmt::Display for Imp<'a, C> { write!(f, "File(")?; match self { Imp::Generic(_) => write!(f, "Generic")?, - Imp::MMAP { .. } => write!(f, "Memory")?, + Imp::Mmap { .. } => write!(f, "Memory")?, }; write!(f, ")") } @@ -85,8 +85,8 @@ impl<'a, C: fmt::Debug + Sync + Send> fmt::Debug for Imp<'a, C> { f.debug_tuple("Generic") .field(&g) .finish(), - Imp::MMAP { reader, } => - f.debug_struct("MMAP") + Imp::Mmap { reader, } => + f.debug_struct("Mmap") .field("addr", &reader.source_buffer().as_ptr()) .field("length", &reader.source_buffer().len()) .field("reader", reader) @@ -152,7 +152,7 @@ impl<'a, C: fmt::Debug + Sync + Send> File<'a, C> { }; Ok(File( - Imp::MMAP { + Imp::Mmap { reader: Memory::with_cookie(slice, cookie), }, path.into(), @@ -164,7 +164,7 @@ impl<'a, C: fmt::Debug + Sync + Send> io::Read for File<'a, C> { fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { match self.0 { Imp::Generic(ref mut reader) => reader.read(buf), - Imp::MMAP { ref mut reader, .. } => reader.read(buf), + Imp::Mmap { ref mut reader, .. } => reader.read(buf), }.map_err(|e| FileError::new(&self.1, e)) } } @@ -173,7 +173,7 @@ impl<'a, C: fmt::Debug + Sync + Send> BufferedReader<C> for File<'a, C> { fn buffer(&self) -> &[u8] { match self.0 { Imp::Generic(ref reader) => reader.buffer(), - Imp::MMAP { ref reader, .. } => reader.buffer(), + Imp::Mmap { ref reader, .. } => reader.buffer(), } } @@ -181,7 +181,7 @@ impl<'a, C: fmt::Debug + Sync + Send> BufferedReader<C> for File<'a, C> { let path = &self.1; match self.0 { Imp::Generic(ref mut reader) => reader.data(amount), - Imp::MMAP { ref mut reader, .. } => reader.data(amount), + Imp::Mmap { ref mut reader, .. } => reader.data(amount), }.map_err(|e| FileError::new(path, e)) } @@ -189,14 +189,14 @@ impl<'a, C: fmt::Debug + Sync + Send> BufferedReader<C> for File<'a, C> { let path = &self.1; match self.0 { Imp::Generic(ref mut reader) => reader.data_hard(amount), - Imp::MMAP { ref mut reader, .. } => reader.data_hard(amount), + Imp::Mmap { ref mut reader, .. } => reader.data_hard(amount), }.map_err(|e| FileError::new(path, e)) } fn consume(&mut self, amount: usize) -> &[u8] { match self.0 { Imp::Generic(ref mut reader) => reader.consume(amount), - Imp::MMAP { ref mut reader, .. } => reader.consume(amount), + Imp::Mmap { ref mut reader, .. } => reader.consume(amount), } } @@ -204,7 +204,7 @@ impl<'a, C: fmt::Debug + Sync + Send> BufferedReader<C> for File<'a, C> { let path = &self.1; match self.0 { Imp::Generic(ref mut reader) => reader.data_consume(amount), - Imp::MMAP { ref mut reader, .. } => reader.data_consume(amount), + Imp::Mmap { ref mut reader, .. } => reader.data_consume(amount), }.map_err(|e| FileError::new(path, e)) } @@ -212,7 +212,7 @@ impl<'a, C: fmt::Debug + Sync + Send> BufferedReader<C> for File<'a, C> { let path = &self.1; match self.0 { Imp::Generic(ref mut reader) => reader.data_consume_hard(amount), - Imp::MMAP { ref mut reader, .. } => reader.data_consume_hard(amount), + Imp::Mmap { ref mut reader, .. } => reader.data_consume_hard(amount), }.map_err(|e| FileError::new(path, e)) } @@ -232,21 +232,21 @@ impl<'a, C: fmt::Debug + Sync + Send> BufferedReader<C> for File<'a, C> { fn cookie_set(&mut self, cookie: C) -> C { match self.0 { Imp::Generic(ref mut reader) => reader.cookie_set(cookie), - Imp::MMAP { ref mut reader, .. } => reader.cookie_set(cookie), + Imp::Mmap { ref mut reader, .. } => reader.cookie_set(cookie), } } fn cookie_ref(&self) -> &C { match self.0 { Imp::Generic(ref reader) => reader.cookie_ref(), - Imp::MMAP { ref reader, .. } => reader.cookie_ref(), + Imp::Mmap { ref reader, .. } => reader.cookie_ref(), } } fn cookie_mut(&mut self) -> &mut C { match self.0 { Imp::Generic(ref mut reader) => reader.cookie_mut(), - Imp::MMAP { ref mut reader, .. } => reader.cookie_mut(), + Imp::Mmap { ref mut reader, .. } => reader.cookie_mut(), } } } diff --git a/buffered-reader/src/generic.rs b/buffered-reader/src/generic.rs index d2e9acfa..117a1301 100644 --- a/buffered-reader/src/generic.rs +++ b/buffered-reader/src/generic.rs @@ -5,24 +5,29 @@ use std::io::{Error, ErrorKind}; use super::*; +/// Controls tracing. +const TRACE: bool = false; + /// Wraps a `Read`er. /// /// This is useful when reading from a generic `std::io::Read`er. To /// read from a file, use [`File`]. To read from a buffer, use /// [`Memory`]. Both are more efficient than `Generic`. /// -/// [`File`]: struct.File.html -/// [`Memory`]: struct.Memory.html pub struct Generic<T: io::Read + Send + Sync, C: fmt::Debug + Sync + Send> { - buffer: Option<Box<[u8]>>, + buffer: Option<Vec<u8>>, // The next byte to read in the buffer. cursor: usize, + /// Currently unused buffer. + unused_buffer: Option<Vec<u8>>, // The preferred chunk size. This is just a hint. preferred_chunk_size: usize, // The wrapped reader. reader: T, // Stashed error, if any. error: Option<Error>, + /// Whether we hit EOF on the underlying reader. + eof: bool, // The user settable cookie. cookie: C, @@ -73,11 +78,13 @@ impl<T: io::Read + Send + Sync, C: fmt::Debug + Sync + Send> Generic<T, C> { Generic { buffer: None, cursor: 0, + unused_buffer: None, preferred_chunk_size: if let Some(s) = preferred_chunk_size { s } else { DEFAULT_BUF_SIZE }, reader, error: None, + eof: false, cookie, } } @@ -99,18 +106,22 @@ impl<T: io::Read + Send + Sync, C: fmt::Debug + Sync + Send> Generic<T, C> { /// Return the buffer. Ensure that it contains at least `amount` /// bytes. + // + // Note: + // + // If you find a bug in this function, consider whether + // sequoia_openpgp::armor::Reader::data_helper is also affected. fn data_helper(&mut self, amount: usize, hard: bool, and_consume: bool) -> Result<&[u8], io::Error> { - // println!("Generic.data_helper(\ - // amount: {}, hard: {}, and_consume: {} (cursor: {}, buffer: {:?})", - // amount, hard, and_consume, - // self.cursor, - // if let Some(ref buffer) = self.buffer { Some(buffer.len()) } - // else { None }); - + tracer!(TRACE, "Generic::data_helper"); + t!("amount: {}, hard: {}, and_consume: {} (cursor: {}, buffer: {:?})", + amount, hard, and_consume, + self.cursor, + self.buffer.as_ref().map(|buffer| buffer.len())); // See if there is an error from the last invocation. if let Some(e) = self.error.take() { + t!("Returning stashed error: {}", e); return Err(e); } @@ -132,14 +143,29 @@ impl<T: io::Read + Send + Sync, C: fmt::Debug + Sync + Send> Generic<T, C> { DEFAULT_BUF_SIZE, 2 * self.preferred_chunk_size), amount); - let mut buffer_new : Vec<u8> = vec![0u8; capacity]; + let mut buffer_new = self.unused_buffer.take() + .map(|mut v| { + vec_resize(&mut v, capacity); + v + }) + .unwrap_or_else(|| vec![0u8; capacity]); let mut amount_read = 0; while amount_buffered + amount_read < amount { + t!("Have {} bytes, need {} bytes", + amount_buffered + amount_read, amount); + + if self.eof { + t!("Hit EOF on the underlying reader, don't poll again."); + break; + } |