diff options
author | Neal H. Walfield <neal@pep.foundation> | 2017-12-12 10:28:13 +0100 |
---|---|---|
committer | Neal H. Walfield <neal@pep.foundation> | 2017-12-12 10:28:13 +0100 |
commit | 6f6a2e8fc95b7506f769f122b4b3e5982a52336c (patch) | |
tree | c9da1c265bec3ca1f25619347b1dd29bca5f97ea | |
parent | 8d0a293a38251e21ad25f72eeadc21759ace441f (diff) |
Split BufferedReader implementations into separate files.
- Break buffered_reader.rs into several files, one per
implementation.
- Rename the other implementations to remove the 'buffered_reader_'
prefix from the filename.
-rw-r--r-- | src/buffered_reader/buffered_reader.rs | 493 | ||||
-rw-r--r-- | src/buffered_reader/decompress.rs (renamed from src/buffered_reader/buffered_reader_decompress.rs) | 0 | ||||
-rw-r--r-- | src/buffered_reader/generic.rs | 250 | ||||
-rw-r--r-- | src/buffered_reader/limitor.rs | 167 | ||||
-rw-r--r-- | src/buffered_reader/memory.rs | 85 | ||||
-rw-r--r-- | src/buffered_reader/partial_body.rs (renamed from src/buffered_reader/buffered_reader_partial_body.rs) | 0 | ||||
-rw-r--r-- | src/openpgp/parse/parse.rs | 8 |
7 files changed, 514 insertions, 489 deletions
diff --git a/src/buffered_reader/buffered_reader.rs b/src/buffered_reader/buffered_reader.rs index 6118b3ce..5d5727b5 100644 --- a/src/buffered_reader/buffered_reader.rs +++ b/src/buffered_reader/buffered_reader.rs @@ -8,8 +8,14 @@ use std::fmt; // The default buffer size. const DEFAULT_BUF_SIZE: usize = 8 * 1024; -pub mod buffered_reader_decompress; -pub mod buffered_reader_partial_body; +pub mod decompress; +pub mod partial_body; +pub mod generic; +pub use self::generic::*; +pub mod memory; +pub use self::memory::*; +pub mod limitor; +pub use self::limitor::*; /// A `BufferedReader` is a type of `Read`er that has an internal /// buffer, and allows working directly from that buffer. Like a @@ -228,223 +234,6 @@ impl <'a> BufferedReader for Box<BufferedReader + 'a> { } } -/// A generic `BufferedReader` implementation that only requires a -/// source that implements the `Read` trait. This is sufficient when -/// reading from a file, and it even works with a `&[u8]` (but -/// `BufferedReaderMemory` is more efficient). -pub struct BufferedReaderGeneric<T: io::Read> { - buffer: Option<Box<[u8]>>, - // The next byte to read in the buffer. - cursor: usize, - // The preferred chunk size. This is just a hint. - preferred_chunk_size: usize, - // XXX: This is pub for the decompressors. It would be better to - // change this to some accessor method. - pub reader: Box<T>, - // Whether we saw an EOF. - saw_eof: bool, - // The last error that we encountered, but have not yet returned. - error: Option<io::Error>, -} - -impl<T: io::Read> std::fmt::Debug for BufferedReaderGeneric<T> { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let buffered_data = if let Some(ref buffer) = self.buffer { - buffer.len() - self.cursor - } else { - 0 - }; - - f.debug_struct("BufferedReaderGeneric") - .field("preferred_chunk_size", &self.preferred_chunk_size) - .field("buffer data", &buffered_data) - .field("saw eof", &self.saw_eof) - .field("error", &self.error) - .finish() - } -} - -impl<T: io::Read> BufferedReaderGeneric<T> { - /// Instantiate a new generic reader. `reader` is the source to - /// wrap. `preferred_chuck_size` is the preferred chuck size. If - /// None, then the default will be used, which is usually what you - /// want. - pub fn new(reader: T, preferred_chunk_size: Option<usize>) - -> BufferedReaderGeneric<T> { - BufferedReaderGeneric { - buffer: None, - cursor: 0, - preferred_chunk_size: - if let Some(s) = preferred_chunk_size { s } else { DEFAULT_BUF_SIZE }, - reader: Box::new(reader), - saw_eof: false, - error: None, - } - } - - /// Return the buffer. Ensure that it contains at least `amount` - /// bytes. - fn data_helper(&mut self, amount: usize, hard: bool, and_consume: bool) - -> Result<&[u8], io::Error> { - // println!("BufferedReaderGeneric.data_helper(\ - // amount: {}, hard: {}, and_consume: {} (cursor: {}, buffer: {:?})", - // amount, hard, and_consume, - // self.cursor, - // if let Some(ref buffer) = self.buffer { Some(buffer.len()) } - // else { None }); - - - if let Some(ref buffer) = self.buffer { - // We have a buffer. Make sure `cursor` is sane. - assert!(self.cursor <= buffer.len()); - } else { - // We don't have a buffer. Make sure cursor is 0. - assert_eq!(self.cursor, 0); - } - - let amount_buffered = - if let Some(ref buffer) = self.buffer { buffer.len() } else { 0 } - - self.cursor; - if !self.saw_eof && amount > amount_buffered { - // The caller wants more data than we have readily - // available. Read some more. - - let capacity : usize = cmp::max(cmp::max( - DEFAULT_BUF_SIZE, 2 * self.preferred_chunk_size), amount); - - let mut buffer_new : Vec<u8> = vec![0u8; capacity]; - - let mut amount_read = 0; - while amount_buffered + amount_read < amount { - match self.reader.read(&mut buffer_new - [amount_buffered + amount_read..]) { - Ok(read) => { - if read == 0 { - // XXX: Likely EOF. - self.saw_eof = true; - break; - } else { - amount_read += read; - continue; - } - }, - Err(err) => { - // Don't return yet, because we may have - // actually read something. - self.saw_eof = true; - self.error = Some(err); - break; - }, - } - } - - if amount_read > 0 { - // We read something. - - if let Some(ref buffer) = self.buffer { - // We need to copy in the old data. - buffer_new[0..amount_buffered] - .copy_from_slice( - &buffer[self.cursor..self.cursor + amount_buffered]); - } - - buffer_new.truncate(amount_buffered + amount_read); - buffer_new.shrink_to_fit(); - - self.buffer = Some(buffer_new.into_boxed_slice()); - self.cursor = 0; - } - } - - if self.error.is_some() { - // An error occured. If we have enough data to fulfill - // the caller's request, then delay returning the error. - if let Some(ref buffer) = self.buffer { - if amount > buffer.len() { - // We return an error at most once (Recall: take - // clears self.error). - return Err(self.error.take().unwrap()); - } - } - } - - match self.buffer { - Some(ref buffer) => { - let amount_buffered = buffer.len() - self.cursor; - if hard && amount_buffered < amount { - return Err(Error::new(ErrorKind::UnexpectedEof, "EOF")); - } - if and_consume { - let amount_consumed = cmp::min(amount_buffered, amount); - self.cursor += amount_consumed; - assert!(self.cursor <= buffer.len()); - return Ok(&buffer[self.cursor-amount_consumed..]); - } else { - return Ok(&buffer[self.cursor..]); - } - }, - None if self.saw_eof => { - return Ok(&b""[..]); - }, - None => { - unreachable!(); - } - } - } -} - -impl<T: io::Read> io::Read for BufferedReaderGeneric<T> { - fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> { - return buffered_reader_generic_read_impl(self, buf); - } -} - -impl<T: io::Read> BufferedReader for BufferedReaderGeneric<T> { - fn data(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.data_helper(amount, false, false); - } - - fn data_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.data_helper(amount, true, false); - } - - fn consume(&mut self, amount: usize) -> &[u8] { - // println!("BufferedReaderGeneric.consume({}) \ - // (cursor: {}, buffer: {:?})", - // amount, self.cursor, - // if let Some(ref buffer) = self.buffer { Some(buffer.len()) } - // else { None }); - - // The caller can't consume more than is buffered! - if let Some(ref buffer) = self.buffer { - assert!(self.cursor <= buffer.len()); - assert!(amount <= buffer.len() - self.cursor, - "buffer contains just {} bytes, but you are trying to - consume {} bytes. Did you forget to call data()?", - buffer.len() - self.cursor, amount); - - self.cursor += amount; - return &self.buffer.as_ref().unwrap()[self.cursor - amount..]; - } else { - assert_eq!(amount, 0); - return &b""[..]; - } - } - - fn data_consume(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.data_helper(amount, false, true); - } - - fn data_consume_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return self.data_helper(amount, true, true); - } - - fn into_inner<'b>(self: Box<Self>) -> Option<Box<BufferedReader + 'b>> - where Self: 'b { - None - } -} - // The file was created as follows: // // for i in $(seq 0 9999); do printf "%04d\n" $i; done > buffered-reader-test.txt @@ -467,272 +256,6 @@ fn buffered_reader_test_data_check<'a, T: BufferedReader + 'a>(bio: &mut T) { bio.consume(consumed); } } - -#[test] -fn buffered_reader_generic_test() { - // Test reading from a file. - { - use std::path::PathBuf; - use std::fs::File; - - let path : PathBuf = [env!("CARGO_MANIFEST_DIR"), - "src", "buffered_reader", "buffered-reader-test.txt"] - .iter().collect(); - let mut f = File::open(&path).expect(&path.to_string_lossy()); - let mut bio = BufferedReaderGeneric::new(&mut f, None); - - buffered_reader_test_data_check(&mut bio); - } - - // Same test, but as a slice. - { - let mut data : &[u8] = include_bytes!("buffered-reader-test.txt"); - let mut bio = BufferedReaderGeneric::new(&mut data, None); - - buffered_reader_test_data_check(&mut bio); - } -} - -/// A `BufferedReader` specialized for reading from memory buffers. -pub struct BufferedReaderMemory<'a> { - buffer: &'a [u8], - // The next byte to read in the buffer. - cursor: usize, -} - -impl <'a> std::fmt::Debug for BufferedReaderMemory<'a> { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.debug_struct("BufferedReaderMemory") - .field("buffer (bytes)", &&self.buffer.len()) - .field("cursor", &self.cursor) - .finish() - } -} - -impl<'a> BufferedReaderMemory<'a> { - pub fn new(buffer: &'a [u8]) -> BufferedReaderMemory<'a> { - BufferedReaderMemory { - buffer: buffer, - cursor: 0, - } - } -} - -impl<'a> io::Read for BufferedReaderMemory<'a> { - fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> { - let amount = cmp::min(buf.len(), self.buffer.len() - self.cursor); - buf[0..amount].copy_from_slice( - &self.buffer[self.cursor..self.cursor+amount]); - self.consume(amount); - return Ok(amount); - } -} - -impl<'a> BufferedReader for BufferedReaderMemory<'a> { - /// Return the buffer. Ensure that it contains at least `amount` - /// bytes. - fn data(&mut self, _amount: usize) -> Result<&[u8], io::Error> { - assert!(self.cursor <= self.buffer.len()); - return Ok(&self.buffer[self.cursor..]); - } - - fn consume(&mut self, amount: usize) -> &[u8] { - // The caller can't consume more than is buffered! - assert!(amount <= self.buffer.len() - self.cursor, - "Attempt to consume {} bytes, but buffer only has {} bytes!", - amount, self.buffer.len() - self.cursor); - self.cursor += amount; - assert!(self.cursor <= self.buffer.len()); - return &self.buffer[self.cursor - amount..]; - } - - fn data_consume(&mut self, amount: usize) -> Result<&[u8], io::Error> { - return Ok(self.consume(amount)); - } - - fn data_consume_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - if self.buffer.len() - self.cursor < amount { - return Err(Error::new(ErrorKind::UnexpectedEof, "EOF")); - } - return Ok(self.consume(amount)); - } - - fn into_inner<'b>(self: Box<Self>) -> Option<Box<BufferedReader + 'b>> - where Self: 'b { - None - } -} - -#[test] -fn buffered_reader_memory_test () { - let data : &[u8] = include_bytes!("buffered-reader-test.txt"); - let mut bio = BufferedReaderMemory::new(data); - - buffered_reader_test_data_check(&mut bio); -} - -/// A `BufferedReaderLimitor` limits the amount of data that can be -/// read from a `BufferedReader`. -#[derive(Debug)] -pub struct BufferedReaderLimitor<T: BufferedReader> { - reader: T, - limit: u64, -} - -impl<T: BufferedReader> BufferedReaderLimitor<T> { - pub fn new(reader: T, limit: u64) -> BufferedReaderLimitor<T> { - BufferedReaderLimitor { - reader: reader, - limit: limit, - } - } -} - -impl<T: BufferedReader> io::Read for BufferedReaderLimitor<T> { - fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> { - let len = cmp::min(self.limit, buf.len() as u64) as usize; - return self.reader.read(&mut buf[0..len]); - } -} - -impl<T: BufferedReader> BufferedReader for BufferedReaderLimitor<T> { - /// Return the buffer. Ensure that it contains at least `amount` - /// bytes. - fn data(&mut self, amount: usize) -> Result<&[u8], io::Error> { - let amount = cmp::min(amount as u64, self.limit) as usize; - let result = self.reader.data(amount); - match result { - Ok(ref buffer) => - if buffer.len() as u64 > self.limit { - return Ok(&buffer[0..self.limit as usize]); - } else { - return Ok(buffer); - }, - Err(err) => return Err(err), - } - } - - fn consume(&mut self, amount: usize) -> &[u8] { - assert!(amount as u64 <= self.limit); - self.limit -= amount as u64; - let data = self.reader.consume(amount); - return &data[..cmp::min(self.limit + amount as u64, data.len() as u64) as usize]; - } - - fn data_consume(&mut self, amount: usize) -> Result<&[u8], io::Error> { - let amount = cmp::min(amount as u64, self.limit) as usize; - let result = self.reader.data_consume(amount); - if let Ok(ref buffer) = result { - self.limit -= amount as u64; - return Ok(&buffer[ - ..cmp::min(buffer.len() as u64, self.limit + amount as u64) as usize]); - } - return result; - } - - fn data_consume_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { - if amount as u64 > self.limit { - return Err(Error::new(ErrorKind::UnexpectedEof, "EOF")); - } - let result = self.reader.data_consume_hard(amount); - if let Ok(ref buffer) = result { - self.limit -= amount as u64; - return Ok(&buffer[ - ..cmp::min(buffer.len() as u64, self.limit + amount as u64) as usize]); - } - return result; - } - - fn into_inner<'b>(self: Box<Self>) -> Option<Box<BufferedReader + 'b>> - where Self: 'b { - Some(Box::new(self.reader)) - } -} - -#[test] -fn buffered_reader_limitor_test() { - let data : &[u8] = b"01234567890123456789"; - - /* Add a single limitor. */ - { - let mut bio : Box<BufferedReader> - = Box::new(BufferedReaderMemory::new(data)); - - bio = { - let mut bio2 = Box::new(BufferedReaderLimitor::new(bio, 5)); - { - let result = bio2.data(5).unwrap(); - assert_eq!(result.len(), 5); - assert_eq!(result, &b"01234"[..]); - } - bio2.consume(5); - { - let result = bio2.data(1).unwrap(); - assert_eq!(result.len(), 0); - assert_eq!(result, &b""[..]); - } - - bio2.into_inner().unwrap() - }; - - { - { - let result = bio.data(15).unwrap(); - assert_eq!(result.len(), 15); - assert_eq!(result, &b"567890123456789"[..]); - } - bio.consume(15); - { - let result = bio.data(1).unwrap(); - assert_eq!(result.len(), 0); - assert_eq!(result, &b""[..]); - } - } - } - - /* Try with two limitors where the first one imposes the real - * limit. */ - { - let mut bio : Box<BufferedReader> - = Box::new(BufferedReaderMemory::new(data)); - - bio = { - let bio2 : Box<BufferedReader> - = Box::new(BufferedReaderLimitor::new(bio, 5)); - // We limit to 15 bytes, but bio2 will still limit us to 5 - // bytes. - let mut bio3 : Box<BufferedReader> - = Box::new(BufferedReaderLimitor::new(bio2, 15)); - { - let result = bio3.data(100).unwrap(); - assert_eq!(result.len(), 5); - assert_eq!(result, &b"01234"[..]); - } - bio3.consume(5); - { - let result = bio3.data(1).unwrap(); - assert_eq!(result.len(), 0); - assert_eq!(result, &b""[..]); - } - - bio3.into_inner().unwrap().into_inner().unwrap() - }; - - { - { - let result = bio.data(15).unwrap(); - assert_eq!(result.len(), 15); - assert_eq!(result, &b"567890123456789"[..]); - } - bio.consume(15); - { - let result = bio.data(1).unwrap(); - assert_eq!(result.len(), 0); - assert_eq!(result, &b""[..]); - } - } - } -} #[cfg(test)] mod test { diff --git a/src/buffered_reader/buffered_reader_decompress.rs b/src/buffered_reader/decompress.rs index 648037fc..648037fc 100644 --- a/src/buffered_reader/buffered_reader_decompress.rs +++ b/src/buffered_reader/decompress.rs diff --git a/src/buffered_reader/generic.rs b/src/buffered_reader/generic.rs new file mode 100644 index 00000000..7049a2fe --- /dev/null +++ b/src/buffered_reader/generic.rs @@ -0,0 +1,250 @@ +use std::io; +use std::fmt; +use std::cmp; +use std::io::{Error,ErrorKind}; + +use super::*; + +/// A generic `BufferedReader` implementation that only requires a +/// source that implements the `Read` trait. This is sufficient when +/// reading from a file, and it even works with a `&[u8]` (but +/// `BufferedReaderMemory` is more efficient). +pub struct BufferedReaderGeneric<T: io::Read> { + buffer: Option<Box<[u8]>>, + // The next byte to read in the buffer. + cursor: usize, + // The preferred chunk size. This is just a hint. + preferred_chunk_size: usize, + // XXX: This is pub for the decompressors. It would be better to + // change this to some accessor method. + pub reader: Box<T>, + // Whether we saw an EOF. + saw_eof: bool, + // The last error that we encountered, but have not yet returned. + error: Option<io::Error>, +} + +impl<T: io::Read> fmt::Debug for BufferedReaderGeneric<T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let buffered_data = if let Some(ref buffer) = self.buffer { + buffer.len() - self.cursor + } else { + 0 + }; + + f.debug_struct("BufferedReaderGeneric") + .field("preferred_chunk_size", &self.preferred_chunk_size) + .field("buffer data", &buffered_data) + .field("saw eof", &self.saw_eof) + .field("error", &self.error) + .finish() + } +} + +impl<T: io::Read> BufferedReaderGeneric<T> { + /// Instantiate a new generic reader. `reader` is the source to + /// wrap. `preferred_chuck_size` is the preferred chuck size. If + /// None, then the default will be used, which is usually what you + /// want. + pub fn new(reader: T, preferred_chunk_size: Option<usize>) + -> BufferedReaderGeneric<T> { + BufferedReaderGeneric { + buffer: None, + cursor: 0, + preferred_chunk_size: + if let Some(s) = preferred_chunk_size { s } + else { DEFAULT_BUF_SIZE }, + reader: Box::new(reader), + saw_eof: false, + error: None, + } + } + + /// Return the buffer. Ensure that it contains at least `amount` + /// bytes. + fn data_helper(&mut self, amount: usize, hard: bool, and_consume: bool) + -> Result<&[u8], io::Error> { + // println!("BufferedReaderGeneric.data_helper(\ + // amount: {}, hard: {}, and_consume: {} (cursor: {}, buffer: {:?})", + // amount, hard, and_consume, + // self.cursor, + // if let Some(ref buffer) = self.buffer { Some(buffer.len()) } + // else { None }); + + + if let Some(ref buffer) = self.buffer { + // We have a buffer. Make sure `cursor` is sane. + assert!(self.cursor <= buffer.len()); + } else { + // We don't have a buffer. Make sure cursor is 0. + assert_eq!(self.cursor, 0); + } + + let amount_buffered = + if let Some(ref buffer) = self.buffer { buffer.len() } else { 0 } + - self.cursor; + if !self.saw_eof && amount > amount_buffered { + // The caller wants more data than we have readily + // available. Read some more. + + let capacity : usize = cmp::max(cmp::max( + DEFAULT_BUF_SIZE, + 2 * self.preferred_chunk_size), amount); + + let mut buffer_new : Vec<u8> = vec![0u8; capacity]; + + let mut amount_read = 0; + while amount_buffered + amount_read < amount { + match self.reader.read(&mut buffer_new + [amount_buffered + amount_read..]) { + Ok(read) => { + if read == 0 { + // XXX: Likely EOF. + self.saw_eof = true; + break; + } else { + amount_read += read; + continue; + } + }, + Err(err) => { + // Don't return yet, because we may have + // actually read something. + self.saw_eof = true; + self.error = Some(err); + break; + }, + } + } + + if amount_read > 0 { + // We read something. + + if let Some(ref buffer) = self.buffer { + // We need to copy in the old data. + buffer_new[0..amount_buffered] + .copy_from_slice( + &buffer[self.cursor..self.cursor + amount_buffered]); + } + + buffer_new.truncate(amount_buffered + amount_read); + buffer_new.shrink_to_fit(); + + self.buffer = Some(buffer_new.into_boxed_slice()); + self.cursor = 0; + } + } + + if self.error.is_some() { + // An error occured. If we have enough data to fulfill + // the caller's request, then delay returning the error. + if let Some(ref buffer) = self.buffer { + if amount > buffer.len() { + // We return an error at most once (Recall: take + // clears self.error). + return Err(self.error.take().unwrap()); + } + } + } + + match self.buffer { + Some(ref buffer) => { + let amount_buffered = buffer.len() - self.cursor; + if hard && amount_buffered < amount { + return Err(Error::new(ErrorKind::UnexpectedEof, "EOF")); + } + if and_consume { + let amount_consumed = cmp::min(amount_buffered, amount); + self.cursor += amount_consumed; + assert!(self.cursor <= buffer.len()); + return Ok(&buffer[self.cursor-amount_consumed..]); + } else { + return Ok(&buffer[self.cursor..]); + } + }, + None if self.saw_eof => { + return Ok(&b""[..]); + }, + None => { + unreachable!(); + } + } + } +} + +impl<T: io::Read> io::Read for BufferedReaderGeneric<T> { + fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> { + return buffered_reader_generic_read_impl(self, buf); + } +} + +impl<T: io::Read> BufferedReader for BufferedReaderGeneric<T> { + fn data(&mut self, amount: usize) -> Result<&[u8], io::Error> { + return self.data_helper(amount, false, false); + } + + fn data_hard(&mut self, amount: usize) -> Result<&[u8], io::Error> { + return self.data_helper(amount, true, false); + } + + fn consume(&mut self, amount: usize) -> &[u8] { + // println!("BufferedReaderGeneric.consume({}) \ + // (cursor: {}, buffer: {:?})", + // amount, self.cursor, + // if let Some(ref buffer) = self.buffer { Some(buffer.len()) } + // else { None }); + + // The caller can't consume more than is buffered! + if let Some(ref buffer) = self.buffer { + assert!(self.cursor <= buffer.len()); + assert!(amount <= buffer.len() - self.cursor, + "buffer contains just {} bytes, but you are trying to + consume {} bytes. Did you forget to call data()?", + buffer.len() - self.cursor, amount); + + self.cursor += amount; + return &self.buffer.as_ref().unwrap()[self.cursor - amount..]; + } else { + assert_eq!(amount, 0); + re |