diff options
author | Neal H. Walfield <neal@pep.foundation> | 2019-08-02 21:58:24 +0200 |
---|---|---|
committer | Neal H. Walfield <neal@pep.foundation> | 2019-08-02 22:00:53 +0200 |
commit | adb30ff19a3ba162d3906da064092313d8740869 (patch) | |
tree | 1157f444f3ec08f0cdec46ce6f38fd59b57d538f | |
parent | 0bb2b150390491a8ef7dfc03c50c47c7c6437258 (diff) |
openpgp: Upgrade to base64 version 0.10.1.
- Version 0.10.1 removes support for the base64::MIME configuration.
That means, that we have to strip whitespace on our own.
- Our implementation tries hard to not double buffer.
- Closes #280.
-rw-r--r-- | Cargo.lock | 13 | ||||
-rw-r--r-- | openpgp/Cargo.toml | 2 | ||||
-rw-r--r-- | openpgp/src/armor.rs | 557 |
3 files changed, 308 insertions, 264 deletions
@@ -101,11 +101,10 @@ dependencies = [ [[package]] name = "base64" -version = "0.9.3" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1528,11 +1527,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "safemem" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] name = "schannel" version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1703,7 +1697,7 @@ dependencies = [ name = "sequoia-openpgp" version = "0.9.0" dependencies = [ - "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "buffered-reader 0.9.0", "bzip2 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2392,7 +2386,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum autocfg 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "0e49efa51329a5fd37e7c79db4621af617cd4e3e5bc224939808d076077077bf" "checksum backtrace 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)" = "18b50f5258d1a9ad8396d2d345827875de4261b158124d4c819d9b351454fae5" "checksum backtrace-sys 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "5b3a000b9c543553af61bc01cbfc403b04b5caa9e421033866f2e98061eb3e61" -"checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" +"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" "checksum bindgen 0.47.3 (registry+https://github.com/rust-lang/crates.io-index)" = "df683a55b54b41d5ea8ebfaebb5aa7e6b84e3f3006a78f010dadc9ca88469260" "checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80" "checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" @@ -2556,7 +2550,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum ryu 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c92464b447c0ee8c4fb3824ecc8383b81717b9f1e74ba2e72540aef7b9f82997" -"checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" "checksum schannel 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "f2f6abf258d99c3c1c5c2131d99d064e94b7b3dd5f416483057f308fea253339" "checksum scoped-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" "checksum scoped_threadpool 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" diff --git a/openpgp/Cargo.toml b/openpgp/Cargo.toml index fa78ce40..c93cef30 100644 --- a/openpgp/Cargo.toml +++ b/openpgp/Cargo.toml @@ -24,7 +24,7 @@ maintenance = { status = "actively-developed" } [dependencies] buffered-reader = { path = "../buffered-reader", version = "0.9", default-features = false } -base64 = "0.9.0" +base64 = "0.10.1" bzip2 = { version = "0.3.2", optional = true } failure = "0.1.2" flate2 = { version = "1.0.1", optional = true } diff --git a/openpgp/src/armor.rs b/openpgp/src/armor.rs index 0668055c..876b3500 100644 --- a/openpgp/src/armor.rs +++ b/openpgp/src/armor.rs @@ -32,6 +32,7 @@ use std::io::{Result, Error, ErrorKind}; use std::path::Path; use std::cmp; use std::str; +use std::borrow::Cow; use quickcheck::{Arbitrary, Gen}; use crate::packet::prelude::*; @@ -135,16 +136,6 @@ impl Kind { "-----BEGIN PGP -----".len() + self.blurb().len() } - - /// Returns the maximal size of the footer with CRC. - fn footer_max_len(&self) -> usize { - (5 // CRC - + 4 // CR NL CR NL - + 18 // "-----END PGP -----" - + self.blurb().len() - + 2 // CR NL - ) - } } /// A filter that applies ASCII Armor to the data written to it. @@ -579,12 +570,12 @@ impl<'a> Reader<'a> { let mut o = [ 0u8; 4 ]; CTBNew::new(tag).serialize_into(&mut ctb[..]).unwrap(); - base64::encode_config_slice(&ctb[..], base64::MIME, &mut o[..]); + base64::encode_config_slice(&ctb[..], base64::STANDARD, &mut o[..]); valid_start.push(o[0]); CTBOld::new(tag, BodyLength::Full(0)).unwrap() .serialize_into(&mut ctb[..]).unwrap(); - base64::encode_config_slice(&ctb[..], base64::MIME, &mut o[..]); + base64::encode_config_slice(&ctb[..], base64::STANDARD, &mut o[..]); valid_start.push(o[0]); } @@ -754,57 +745,139 @@ impl<'a> Reader<'a> { self.initialized = true; Ok(()) } +} - /// Parses the footer. - fn finalize(footer: &[u8], kind: Option<Kind>) -> Result<Option<u32>> { - let mut off = 0; +// Remove whitespace, etc. from the base64 data. +// +// This function returns the filtered base64 data (i.e., stripped of +// all skipable data like whitespace), and the amount of unfiltered +// data that corresponds to. Thus, if we have the following 7 bytes: +// +// ab cde +// 0123456 +// +// This function returns ("abcd", 6), because the 'd' is the last +// character in the last complete base64 chunk, and it is at offset 5. +// +// If 'd' is follow by whitespace, it is undefined whether that +// whitespace is included in the count. +// +// This function only returns full chunks of base64 data. As a +// consequence, if base64_data_max is less than 4, then this will not +// return any data. +// +// This function will stop after it sees base64 padding, and if it +// sees invalid base64 data. +fn base64_filter(mut bytes: Cow<[u8]>, base64_data_max: usize) + -> (Cow<[u8]>, usize) +{ + let mut leading_whitespace = 0; + + // Round down to the nearest chunk size. + let base64_data_max = base64_data_max / 4 * 4; + + // Number of bytes of base64 data. Since we update `bytes` in + // place, the base64 data is `&bytes[..base64_len]`. + let mut base64_len = 0; + + // Offset of the next byte of unfiltered data to process. + let mut unfiltered_offset = 0; + + // Offset of the last byte of the last ***complete*** base64 chunk + // in the unfiltered data. + let mut unfiltered_complete_len = 0; + + // Number of bytes of padding that we've seen so far. + let mut padding = 0; - /* Look for CRC. The CRC is optional. */ - let crc = if footer.len() >= 6 && footer[0] == '=' as u8 - && footer[1..5].iter().all(is_base64_char) - { - /* Found. */ - let crc = match base64::decode_config(&footer[1..5], base64::MIME) { - Ok(d) => d, - Err(e) => return Err(Error::new(ErrorKind::InvalidInput, e)), - }; + while unfiltered_offset < bytes.len() + && base64_len < base64_data_max + // A valid base64 chunk never starts with padding. + && ! (padding > 0 && base64_len % 4 == 0) + { + match bytes[unfiltered_offset] { + // White space. + c if c.is_ascii_whitespace() => { + if unfiltered_offset == 0 { + match bytes { + Cow::Borrowed(s) => { + // We're at the beginning. Avoid moving + // data by cutting off the start of the + // slice. + bytes = Cow::Borrowed(&s[1..]); + leading_whitespace += 1; + continue; + } + Cow::Owned(_) => (), + } + } + } - assert_eq!(crc.len(), 3); - let crc = - (crc[0] as u32) << 16 - | (crc[1] as u32) << 8 - | crc[2] as u32; + // Padding. + b'=' => { + if padding == 2 { + // There can never be more than two bytes of + // padding. + break; + } + if base64_len % 4 == 0 { + // Padding can never occur at the start of a + // base64 chunk. + break; + } - /* Update offset, skip whitespace. */ - off += 5; - while off < footer.len() && footer[off].is_ascii_whitespace() { - off += 1; + if unfiltered_offset != base64_len { + bytes.to_mut()[base64_len] = b'='; + } + base64_len += 1; + if base64_len % 4 == 0 { + unfiltered_complete_len = unfiltered_offset + 1; + } + padding += 1; } - Some(crc) - } else { - None - }; + // The only thing that can occur after padding is + // whitespace or padding. Those cases were covered above. + _ if padding > 0 => break, - if let Some(kind) = kind { - if ! footer[off..].starts_with(&kind.end().into_bytes()) { - return Err(Error::new(ErrorKind::InvalidInput, "Invalid ASCII Armor footer.")); + // Base64 data! + b if is_base64_char(&b) => { + if unfiltered_offset != base64_len { + bytes.to_mut()[base64_len] = b; + } + base64_len += 1; + if base64_len % 4 == 0 { + unfiltered_complete_len = unfiltered_offset + 1; + } } + + // Not base64 data. + _ => break, } - Ok(crc) + unfiltered_offset += 1; + } + + let base64_len = base64_len - (base64_len % 4); + unfiltered_complete_len += leading_whitespace; + match bytes { + Cow::Borrowed(s) => + (Cow::Borrowed(&s[..base64_len]), unfiltered_complete_len), + Cow::Owned(mut v) => { + v.truncate(base64_len); + (Cow::Owned(v), unfiltered_complete_len) + } } } /// Checks whether the given bytes contain armored OpenPGP data. fn is_armored_pgp_blob(bytes: &[u8]) -> bool { - let bytes = if let Some(msg) = get_base64_prefix(bytes) { - msg - } else { - return false; - }; + // Get up to 32 bytes of base64 data. That's 24 bytes of data + // (ignoring padding), which is more than enough to get the first + // packet's header. + let (bytes, _) = base64_filter(Cow::Borrowed(bytes), 32); - match base64::decode_config(bytes, base64::MIME) { + match base64::decode_config(&bytes, base64::STANDARD) { Ok(d) => { // Don't consider an empty message to be valid. if d.len() == 0 { @@ -823,244 +896,222 @@ fn is_armored_pgp_blob(bytes: &[u8]) -> bool { } } -/// Gets a slice containing the largest valid base64 prefix. -fn get_base64_prefix(bytes: &[u8]) -> Option<&[u8]> { - let mut padding = 0; - let mut base64_chars = 0; +/// Checks whether the given byte is in the base64 character set. +fn is_base64_char(b: &u8) -> bool { + b.is_ascii_alphanumeric() || *b == '+' as u8 || *b == '/' as u8 +} - let mut result = bytes; - for (i, c) in bytes.iter().enumerate() { - if c.is_ascii_whitespace() { - continue; - } +/// Returns the number of bytes of base64 data are needed to encode +/// `s` bytes of raw data. +fn base64_size(s: usize) -> usize { + (s + 3 - 1) / 3 * 4 +} - if padding > 0 && *c != '=' as u8 { - result = &bytes[..i]; - break; - } +#[test] +fn base64_size_test() { + assert_eq!(base64_size(0), 0); + assert_eq!(base64_size(1), 4); + assert_eq!(base64_size(2), 4); + assert_eq!(base64_size(3), 4); + assert_eq!(base64_size(4), 8); + assert_eq!(base64_size(5), 8); + assert_eq!(base64_size(6), 8); + assert_eq!(base64_size(7), 12); +} - if *c == '=' as u8 { - padding += 1; - base64_chars += 1; - if base64_chars % 4 == 0 || padding == 2 { - result = &bytes[..i]; - break; - } - } else if is_base64_char(c) { - base64_chars += 1; - } else { - if i == 0 { - return None; - } else { - result = &bytes[..i]; - break; - } +impl<'a> Read for Reader<'a> { + fn read(&mut self, buf: &mut [u8]) -> Result<usize> { + if ! self.initialized { + self.initialize()?; } - } - // If we have too much, just chop off a few characters. - while base64_chars % 4 != 0 { - let suffix = result[result.len() - 1]; - if is_base64_char(&suffix) || suffix == b'=' { - base64_chars -= 1; + if self.finalized { + assert_eq!(self.buffer.len(), 0); + return Ok(0); } - result = &result[..result.len() - 1]; - } - return Some(result); -} + let (consumed, decoded) = if self.buffer.len() > 0 { + // We have something buffered, use that. -/// Checks whether the given byte is in the base64 character set. -fn is_base64_char(b: &u8) -> bool { - b.is_ascii_alphanumeric() || *b == '+' as u8 || *b == '/' as u8 -} + let amount = cmp::min(buf.len(), self.buffer.len()); + buf[..amount].copy_from_slice(&self.buffer[..amount]); + self.buffer.drain(..amount); -/// Checks whether the given slice looks like an armor footer. If so, -/// returns the size of the footer. -fn is_footer(buf: &[u8], reference: &[u8]) -> Option<usize> { - if buf.len() < reference.len() { - return None; - } + (0, amount) + } else { + // We need to decode some data. We consider three cases, + // all a function of the size of `buf`: + // + // - Tiny: if `buf` can hold less than three bytes, then + // we almost certainly have to double buffer: except + // at the very end, a base64 chunk consists of 3 bytes + // of data. + // + // Note: this happens if the caller does `for c in + // Reader::new(...).bytes() ...`. Then it reads one + // byte of decoded data at a time. + // + // - Small: if the caller only requests a few bytes at a + // time, we may as well double buffer to reduce + // decoding overhead. + // + // - Large: if `buf` is large, we can decode directly + // into `buf` and avoid double buffering. But, + // because we ignore whitespace, it is hard to + // determine exactly how much data to read to + // maximally fill `buf`. + + // We use 64, because ASCII-armor text usually contains 64 + // characters of base64 data per line, and this prevents + // turning the borrow into an own. + const THRESHOLD : usize = 64; + + let to_read = + cmp::max( + // Tiny or small: + THRESHOLD + 2, + + // Large: a heuristic: + + base64_size(buf.len()) + // Assume about 2 bytes of whitespace (crlf) per + // 64 character line. + + 2 * ((buf.len() + 63) / 64)); + + let base64data = self.source.data(to_read)?; + let base64data = if base64data.len() > to_read { + &base64data[..to_read] + } else { + base64data + }; - let mut off = 0; + let (base64data, consumed) + = base64_filter(Cow::Borrowed(base64data), + // base64_size rounds up, but we want + // to round down as we have to double + // buffer partial chunks. + cmp::max(THRESHOLD, buf.len() / 3 * 4)); - // Look for CRC. The CRC is optional. - if buf.len() >= 6 && buf[0] == '=' as u8 - && buf[1..5].iter().all(is_base64_char) - { - // Found. Update offset, skip whitespace. - off += 5; - while off < buf.len() && buf[off].is_ascii_whitespace() { - off += 1; - } - } + // We shouldn't have any partial chunks. + assert_eq!(base64data.len() % 4, 0); - if buf[off..].starts_with(reference) { - Some(off + reference.len()) - } else { - None - } -} + let decoded = if base64data.len() / 4 * 3 > buf.len() { + // We need to double buffer. Decode into a vector. + // (Note: the computed size *might* be a slight + // overestimate, because the last base64 chunk may + // include padding.) + self.buffer = base64::decode_config( + &base64data, base64::STANDARD) + .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; -/// Looks for the footer, returning the footer's offset, and the end -/// of the footer. -fn find_footer(buf: &[u8], kind: Kind) -> Option<(usize, usize)> { - let reference = kind.end().into_bytes(); + self.crc.update(&self.buffer); - if buf.len() < reference.len() { - return None; - } + let copied = cmp::min(buf.len(), self.buffer.len()); + buf[..copied].copy_from_slice(&self.buffer[..copied]); + self.buffer.drain(..copied); - for i in 0..buf.len() - reference.len() { - if let Some(length) = is_footer(&buf[i..], &reference) { - // Found footer at offset i. - return Some((i, i + length)); - } - } - None -} + copied + } else { + // We can decode directly into the caller-supplied + // buffer. + let decoded = base64::decode_config_slice( + &base64data, base64::STANDARD, buf) + .map_err(|e| Error::new(ErrorKind::InvalidData, e))?; -impl<'a> Read for Reader<'a> { - fn read(&mut self, buf: &mut [u8]) -> Result<usize> { - self.initialize()?; + self.crc.update(&buf[..decoded]); - /* How much did we get? */ - let mut read = 0; + decoded + }; - // First, use what we have in the buffer. - let amount = cmp::min(buf.len(), self.buffer.len()); - &mut buf[..amount].copy_from_slice(&self.buffer[..amount]); - self.buffer.drain(..amount); - read += amount; + (consumed, decoded) + }; - // If we could satisfy the read from the buffer, we're done. - if read == buf.len() { - return Ok(read); - } - assert_eq!(self.buffer.len(), 0); + self.source.consume(consumed); - // Our buffer is drained. If we are finalized, nothing more - // can be read. - if self.finalized { - return Ok(read); - } + if decoded == 0 { + self.finalized = true; - let (consumed, decoded) = { - // Try to get enough bytes to fill buf, account for bytes - // filled using our buffer, round up, and add enough for - // the footer. - // - // Later, we may have to get some more until we have a - // multiple of four non-whitespace ASCII characters. - let mut want = (buf.len() - read + 2) / 3 * 4 - + self.kind.map(|k| k.footer_max_len()).unwrap_or(46); - - // Keep track of how much we got last time to detect - // hitting EOF. - let mut got = 0; - - 'readloop: loop { - let raw = self.source.data(want)?; - if raw.len() == got { - // EOF. Decide how to proceed. - - if self.mode != ReaderMode::VeryTolerant { - // If we are here, we should have seen a - // footer by now. - return Err(Error::new(ErrorKind::UnexpectedEof, - "Armor footer is missing")); - } else { - // Otherwise, we may have found only the blob, - // or the footer is damaged, or missing. Try - // to decode what we have got, then we are - // done. - - // We need to try to discard garbage at the end. - let mut end = cmp::min(raw.len(), want); - loop { - match base64::decode_config(&raw[..end], - base64::MIME) { - Ok(d) => break 'readloop (end, d), - Err(_) => - if end == 0 { - // No more valid data. - break 'readloop (raw.len(), vec![]); - } else { - end -= 1; - }, - } - } - } + /* Look for CRC. The CRC is optional. */ + let consumed = { + // Skip whitespace. + while self.source.data(1)?.len() > 0 + && self.source.buffer()[0].is_ascii_whitespace() + { + self.source.consume(1); + } + + let data = self.source.data(5)?; + let data = if data.len() > 5 { + &data[..5] + } else { + data + }; + + if data.len() == 5 + && data[0] == '=' as u8 + && data[1..5].iter().all(is_base64_char) + { + /* Found. */ + let crc = match base64::decode_config( + &data[1..5], base64::STANDARD) + { + Ok(d) => d, + Err(e) => return Err(Error::new(ErrorKind::InvalidInput, e)), + }; + + assert_eq!(crc.len(), 3); + let crc = + (crc[0] as u32) << 16 + | (crc[1] as u32) << 8 + | crc[2] as u32; + + self.expect_crc = Some(crc); + 5 } else { - got = raw.len(); + 0 + } + }; + self.source.consume(consumed); + + // Look for a footer. + let consumed = { + // Skip whitespace. + while self.source.data(1)?.len() > 0 + && self.source.buffer()[0].is_ascii_whitespace() + { + self.source.consume(1); } - // Check if we see the footer. If so, we're almost done. + // If we had a header, we require a footer. if let Some(kind) = self.kind { - if let Some((n, end)) = find_footer(&raw, kind) { - self.expect_crc = Reader::finalize(&raw[n..], self.kind)?; - self.finalized = true; - match base64::decode_config(&raw[..n], base64::MIME) { - Ok(d) => break (end, d), - Err(e) => - return Err(Error::new(ErrorKind::InvalidInput, e)), - } + let footer = kind.end(); + let got = self.source.data(footer.len())?; + let got = if got.len() > footer.len() { + &got[..footer.len()] + } else { + got + }; + if footer.as_bytes() != got { + return Err(Error::new(ErrorKind::InvalidInput, + "Invalid ASCII Armor footer.")); } - } - // See how many valid characters we got. - let n = &raw.iter().filter( - |c| ! (**c).is_ascii_whitespace()).count(); - if n % 4 == 0 { - // Enough! Try to decode them. - - // We need to try to discard garbage at the end. - let mut end = raw.len(); - loop { - match base64::decode_config(&raw[..end], - base64::MIME) { - Ok(d) => break 'readloop (end, d), - Err(_) => - if end == 0 { - // No more valid data. - break 'readloop (raw.len(), vec![]); - } else { - end -= 1; - }, - } - } + footer.len() + } else { + 0 } + }; + self.source.consume(consumed); - // Otherwise, get some more bytes. - want = got + 4 - n % 4; + if let Some(crc) = self.expect_crc { + if self.crc.finalize() != crc { + return Err(Error::new(ErrorKind::InvalidInput, + "Bad CRC sum.")); + } } - }; - self.source.consume(consumed); - self.crc.update(&decoded); - - /* Check how much we got vs how much was requested. */ - if decoded.len() <= (buf.len() - read) { - &mut buf[read..read + decoded.len()].copy_from_slice(&decoded); - read += decoded.len(); - } else { - // We got more than we wanted, spill the surplus into our - // buffer. - let spill = decoded.len() - (buf.len() - read); - - &mut buf[read..read + decoded.len() - spill].copy_from_slice( - &decoded[..decoded.len() - spill]); - read += decoded.len() - spill; - - self.buffer.extend_from_slice(&decoded[decoded.len() - spill..]); } - /* If we are finalized, we may have found a crc sum. */ - if let Some(crc) = self.expect_crc { - if self.crc.finalize() != crc { - return Err(Error::new(ErrorKind::InvalidInput, "Bad CRC sum.")); - } - } - Ok(read) + Ok(decoded) } } |