diff options
author | Neal H. Walfield <neal@pep.foundation> | 2022-12-23 15:26:40 +0100 |
---|---|---|
committer | Neal H. Walfield <neal@pep.foundation> | 2023-01-06 15:00:23 +0100 |
commit | cb0adbb4baa4a397d7763608cc08cd26782b3eb6 (patch) | |
tree | 520709ff2ed3e94e68071eb39c32f4621ca5dca5 /openpgp | |
parent | 36e0d368773017cafedb430042b29a633b081136 (diff) |
openpgp: Split certificates without parsing the packets.
- Add `RawCertParser`, which splits keyrings into individual
certificates, similar to `CertParser`, but without invoking the
heavy machinery of the `CertParser`.
- `RawCertParser` uses the OpenPGP framing information to identify
the packets, and it makes sure that the packets form a valid TPK or
TSK as per Sections 11.1 and 11.2 of RFC 4880, respectively.
Diffstat (limited to 'openpgp')
-rw-r--r-- | openpgp/NEWS | 4 | ||||
-rw-r--r-- | openpgp/src/cert.rs | 1 | ||||
-rw-r--r-- | openpgp/src/cert/parser/mod.rs | 9 | ||||
-rw-r--r-- | openpgp/src/cert/raw.rs | 1355 |
4 files changed, 1368 insertions, 1 deletions
diff --git a/openpgp/NEWS b/openpgp/NEWS index 7445feac..744d47c4 100644 --- a/openpgp/NEWS +++ b/openpgp/NEWS @@ -6,6 +6,10 @@ * Changes in 1.13.0 ** New cryptographic backends - We added a backend that uses OpenSSL. + * New functionality + - RawCertParser + - RawCert + - RawPacket * Changes in 1.12.0 - Bug fix release. * Changes in 1.11.0 diff --git a/openpgp/src/cert.rs b/openpgp/src/cert.rs index 135d7d90..70f5e3b5 100644 --- a/openpgp/src/cert.rs +++ b/openpgp/src/cert.rs @@ -183,6 +183,7 @@ mod builder; mod bindings; pub mod bundle; mod parser; +pub mod raw; mod revoke; pub use self::builder::{CertBuilder, CipherSuite, KeyBuilder, SubkeyBuilder}; diff --git a/openpgp/src/cert/parser/mod.rs b/openpgp/src/cert/parser/mod.rs index bf4e00f1..d376c0c1 100644 --- a/openpgp/src/cert/parser/mod.rs +++ b/openpgp/src/cert/parser/mod.rs @@ -342,7 +342,14 @@ impl CertValidator { /// this way, it is possible to propagate parse errors. /// /// A `CertParser` returns each [`TPK`] or [`TSK`] that it encounters. -/// Its behavior can be modeled using a simple state machine. +/// Note: if you don't actually need all of the certificates, it is +/// usually faster to use a [`RawCertParser`] and only fully parse and +/// canonicalize those certificates that are relevant. +/// +/// [`RawCertParser`]: crate::cert::raw::RawCertParser +/// +/// A `CertParser`'s behavior can be modeled using a simple state +/// machine. /// /// In the first and initial state, it looks for the start of a /// certificate, a [`Public Key`] packet or a [`Secret Key`] packet. diff --git a/openpgp/src/cert/raw.rs b/openpgp/src/cert/raw.rs new file mode 100644 index 00000000..91abc924 --- /dev/null +++ b/openpgp/src/cert/raw.rs @@ -0,0 +1,1355 @@ +//! Functionality for dealing with mostly unparsed certificates. +//! +//! Parsing a certificate is not cheap. When reading a keyring, most +//! certificates are discarded or never used as they are not relevant. +//! This module provides the [`RawCertParser`] and [`RawCert`] data +//! structures that can help reduce the amount of unnecessary +//! computation. +//! +//! [`RawCertParser`] splits a keyring into [`RawCert`]s by looking +//! primarily at the packet framing and the packet headers. This is +//! much faster than parsing the packets' contents, as the +//! [`CertParser`] does. +//! +//! [`CertParser`]: crate::cert::CertParser +//! +//! [`RawCert`] exposes just enough functionality to allow the user to +//! quickly check if a certificate is not relevant. Note: to check if +//! a certificate is really relevant, the check usually needs to be +//! repeated after canonicalizing it (by using, e.g., [`Cert::from`]) +//! and validating it (by using [`Cert::with_policy`]). +//! +//! [`Cert::from`]: From<RawCert> +//! +//! # Examples +//! +//! Search for a specific certificate in a keyring: +//! +//! ```rust +//! # use std::convert::TryFrom; +//! # +//! use sequoia_openpgp as openpgp; +//! +//! # use openpgp::Result; +//! use openpgp::cert::prelude::*; +//! use openpgp::cert::raw::RawCertParser; +//! use openpgp::parse::Parse; +//! # use openpgp::serialize::Serialize; +//! # +//! # fn main() -> Result<()> { +//! # fn doit() -> Result<Cert> { +//! # let (cert, _) = CertBuilder::new() +//! # .generate()?; +//! # let fpr = cert.fingerprint(); +//! # +//! # let mut bytes = Vec::new(); +//! # cert.serialize(&mut bytes); +//! for cert in RawCertParser::from_bytes(&bytes)? { +//! /// Ignore corrupt and invalid certificates. +//! let cert = if let Ok(cert) = cert { +//! cert +//! } else { +//! continue; +//! }; +//! +//! if cert.fingerprint() == fpr { +//! // Found it! Try to convert it to a Cert. +//! if let cert = Cert::try_from(cert) { +//! return cert; +//! } +//! } +//! } +//! +//! // Not found. +//! return Err(anyhow::anyhow!("Not found!").into()); +//! # } +//! # doit().expect("Found the certificate"); +//! # Ok(()) +//! # } +//! ``` +use std::borrow::Cow; +use std::convert::TryFrom; +use std::fmt; +use std::io::Read; +use std::path::Path; + +use buffered_reader; +use buffered_reader::BufferedReader; + +use crate::Error; +use crate::Fingerprint; +use crate::KeyID; +use crate::Result; +use crate::armor; +use crate::cert::Cert; +use crate::packet::Header; +use crate::packet::Key; +use crate::packet::Packet; +use crate::packet::Tag; +use crate::packet::header::BodyLength; +use crate::packet::header::CTB; +use crate::parse::PacketParser; +use crate::parse::Parse; +use crate::parse::RECOVERY_THRESHOLD; + +use super::TRACE; + +/// A mostly unparsed `Packet`. +/// +/// This is returned by [`RawCert::packets`]. +/// +/// The data includes the OpenPGP framing (i.e., the CTB, and length +/// information). [`RawPacket::body`] returns just the bytes +/// corresponding to the packet's body, i.e., without the OpenPGP +/// framing. +/// +/// You can convert it to a [`Packet`] using `TryFrom`. +/// +/// # Examples +/// +/// ```rust +/// use sequoia_openpgp as openpgp; +/// # use openpgp::Result; +/// # use openpgp::cert::prelude::*; +/// use openpgp::cert::raw::RawCertParser; +/// use openpgp::packet::Packet; +/// use openpgp::packet::Tag; +/// # use openpgp::parse::Parse; +/// # use openpgp::serialize::Serialize; +/// # +/// # fn main() -> Result<()> { +/// # let (cert, _) = CertBuilder::new() +/// # .add_signing_subkey() +/// # .add_certification_subkey() +/// # .add_transport_encryption_subkey() +/// # .add_storage_encryption_subkey() +/// # .add_authentication_subkey() +/// # .generate()?; +/// # +/// # let mut bytes = Vec::new(); +/// # cert.as_tsk().serialize(&mut bytes); +/// # let mut count = 0; +/// # +/// # let rawcert = RawCertParser::from_bytes(&bytes)? +/// # .next().expect("got a cert") +/// # .expect("valid cert"); +/// for p in rawcert.packets() { +/// if p.tag() == Tag::SecretSubkey { +/// if let Ok(packet) = Packet::try_from(p) { +/// // Do something with the packet. +/// # count += 1; +/// } +/// # else { panic!("Failed to parse packet"); } +/// } +/// } +/// # assert_eq!(count, 5); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, PartialEq, Eq)] +pub struct RawPacket<'a> { + tag: Tag, + header_len: usize, + data: &'a [u8], +} +assert_send_and_sync!(RawPacket<'_>); + +impl fmt::Debug for RawPacket<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RawPacket") + .field("tag", &self.tag) + .field("data (bytes)", &self.data.len()) + .finish() + } +} + +impl<'a> RawPacket<'a> { + fn new(tag: Tag, header_len: usize, bytes: &'a [u8]) -> Self { + Self { + tag, + header_len, + data: bytes, + } + } + + /// Returns the packet's tag. + pub fn tag(&self) -> Tag { + self.tag + } + + /// Returns the packet's bytes. + pub fn as_bytes(&self) -> &[u8] { + self.data + } + + /// Return the packet's body without the OpenPGP framing. + pub fn body(&self) -> &[u8] { + &self.data[self.header_len..] + } +} + +impl<'a> TryFrom<RawPacket<'a>> for Packet { + type Error = anyhow::Error; + + fn try_from(p: RawPacket<'a>) -> Result<Self> { + Packet::from_bytes(p.as_bytes()) + } +} + +impl<'a> crate::seal::Sealed for RawPacket<'a> {} +impl<'a> crate::serialize::Marshal for RawPacket<'a> { + fn serialize(&self, o: &mut dyn std::io::Write) -> Result<()> { + o.write_all(self.as_bytes())?; + Ok(()) + } +} + +/// A mostly unparsed `Cert`. +/// +/// This data structure contains the unparsed packets for a +/// certificate or key. The packet sequence is well formed in the +/// sense that the sequence of tags conforms to the [Transferable +/// Public Key grammar] or [Transferable Secret Key grammar], and that +/// it can extract the primary key's fingerprint. Beyond that, the +/// packets are not guaranteed to be valid. +/// +/// [Transferable Public Key grammar]: https://www.rfc-editor.org/rfc/rfc4880#section-11.1 +/// [Transferable Secret Key grammar]: https://www.rfc-editor.org/rfc/rfc4880#section-11.2 +/// +/// This data structure exists to quickly split a large keyring, and +/// only parse those certificates that appear to be relevant. +#[derive(Clone)] +pub struct RawCert<'a> { + data: Cow<'a, [u8]>, + + fingerprint: Fingerprint, + + // The packet's tag, the length of the header, and the offset of + // the start of the packet (including the header) into data. + packets: Vec<(Tag, usize, usize)>, +} +assert_send_and_sync!(RawCert<'_>); + +impl<'a> fmt::Debug for RawCert<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RawCert") + .field("fingerprint", &self.fingerprint) + .field("packets", + &self.packets + .iter() + .map(|p| format!("{} (offset: {})", p.0, p.1)) + .collect::<Vec<String>>() + .join(", ")) + .field("data (bytes)", &self.data.as_ref().len()) + .finish() + } +} + +impl<'a> PartialEq for RawCert<'a> { + fn eq(&self, other: &Self) -> bool { + self.data == other.data + } +} + +impl<'a> Eq for RawCert<'a> { +} + +impl<'a> RawCert<'a> { + /// Returns the certificate's bytes. + /// + /// If you want an individual packet's bytes, use + /// [`RawCert::packet`] or [`RawCert::packets`], and then call + /// [`RawPacket::as_bytes`]. + pub fn as_bytes(&'a self) -> &'a [u8] { + self.data.as_ref() + } + + /// Returns the certificate's fingerprint. + pub fn fingerprint(&self) -> Fingerprint { + self.fingerprint.clone() + } + + /// Returns the certificate's Key ID. + pub fn keyid(&self) -> KeyID { + KeyID::from(&self.fingerprint) + } + + /// Returns the ith packet. + pub fn packet(&self, i: usize) -> Option<RawPacket> { + let data: &[u8] = self.data.as_ref(); + + let &(tag, header_len, start) = self.packets.get(i)?; + let following = self.packets + .get(i + 1) + .map(|&(_, _, offset)| offset) + .unwrap_or(data.len()); + + Some(RawPacket::new(tag, header_len, &data[start..following])) + } + + /// Returns an iterator over each raw packet. + pub fn packets(&self) -> impl Iterator<Item=RawPacket> { + let data: &[u8] = self.data.as_ref(); + + let count = self.packets.len(); + (0..count) + .map(move |i| { + let (tag, header_len, start) = self.packets[i]; + let following = self.packets + .get(i + 1) + .map(|&(_, _, offset)| offset) + .unwrap_or(data.len()); + + RawPacket::new(tag, header_len, &data[start..following]) + }) + } + + /// Returns the number of packets. + pub fn count(&self) -> usize { + self.packets.len() + } +} + +impl<'a> TryFrom<&RawCert<'a>> for Cert { + type Error = anyhow::Error; + + fn try_from(c: &RawCert) -> Result<Self> { + Cert::from_bytes(c.as_bytes()) + } +} + +impl<'a> TryFrom<RawCert<'a>> for Cert { + type Error = anyhow::Error; + + fn try_from(c: RawCert) -> Result<Self> { + Cert::try_from(&c) + } +} + +impl<'a> crate::seal::Sealed for RawCert<'a> {} +impl<'a> crate::serialize::Marshal for RawCert<'a> { + fn serialize(&self, o: &mut dyn std::io::Write) -> Result<()> { + o.write_all(self.as_bytes())?; + Ok(()) + } +} + +/// An iterator over a sequence of unparsed certificates, i.e., an +/// OpenPGP keyring. +/// +/// A `RawCertParser` returns each certificate that it encounters. +/// +/// It implements the same state machine as [`CertParser`], however, a +/// `CertParser` is stricter. Specifically, a `CertParser` performs +/// some sanity checks on the content of the packets whereas a +/// `RawCertParser` doesn't do those checks, because it avoids parsing +/// the packets' contents; it primarily looks at the packets' framing, +/// and their headers. +/// +/// [`CertParser`]: crate::cert::CertParser +/// +/// `RawCertParser` checks that the packet sequence is well formed in +/// the sense that the sequence of tags conforms to the [Transferable +/// Public Key grammar] or [Transferable Secret Key grammar], and it +/// performs a few basic checks. See the documentation for +/// [`RawCert`] for details. +/// +/// [Transferable Public Key grammar]: https://www.rfc-editor.org/rfc/rfc4880#section-11.1 +/// [Transferable Secret Key grammar]: https://www.rfc-editor.org/rfc/rfc4880#section-11.2 +/// +/// Because a `RawCertParser` doesn't parse the contents of the +/// packets, it is significantly faster than a [`CertParser`] when +/// many of the certificates in a keyring are irrelevant. +/// +/// # Examples +/// +/// Search for a specific certificate in a keyring: +/// +/// ```rust +/// # use std::convert::TryFrom; +/// # +/// use sequoia_openpgp as openpgp; +/// +/// # use openpgp::Result; +/// use openpgp::cert::prelude::*; +/// use openpgp::cert::raw::RawCertParser; +/// use openpgp::parse::Parse; +/// # use openpgp::serialize::Serialize; +/// # +/// # fn main() -> Result<()> { +/// # fn doit() -> Result<Cert> { +/// # let (cert, _) = CertBuilder::new() +/// # .generate()?; +/// # let fpr = cert.fingerprint(); +/// # +/// # let mut bytes = Vec::new(); +/// # cert.serialize(&mut bytes); +/// for cert in RawCertParser::from_bytes(&bytes)? { +/// /// Ignore corrupt and invalid certificates. +/// let cert = if let Ok(cert) = cert { +/// cert +/// } else { +/// continue; +/// }; +/// +/// if cert.fingerprint() == fpr { +/// // Found it! Try to convert it to a Cert. +/// if let cert = Cert::try_from(cert) { +/// return cert; +/// } +/// } +/// } +/// +/// // Not found. +/// return Err(anyhow::anyhow!("Not found!").into()); +/// # } +/// # doit().expect("Found the certificate"); +/// # Ok(()) +/// # } +/// ``` +pub struct RawCertParser<'a> +{ + // If the data is being read from a slice, then the slice. This + // is used to avoid copying the data into the RawCert. + slice: Option<&'a [u8]>, + + // Where `RawCertParser` reads the data. When reading from a + // slice, this is a `buffered_reader::Memory`. Note: the slice + // field will not be set, if the input needs to be transferred + // (i.e., dearmored). + reader: Box<dyn BufferedReader<()> + 'a>, + + // Whether we are dearmoring the input. + dearmor: bool, + + // The total number of bytes read. + bytes_read: usize, + + // Any pending error. + pending_error: Option<anyhow::Error>, + + // Whether there was an unrecoverable error. + done: bool, +} +assert_send_and_sync!(RawCertParser<'_>); + +impl<'a> RawCertParser<'a> { + fn new<R>(reader: R) -> Result<Self> + where R: 'a + BufferedReader<()> + { + // Check that we can read the first header and that it is + // reasonable. Note: an empty keyring is not an error; we're + // just checking for bad data here. If not, try again after + // dearmoring the input. + let mut dearmor = false; + let mut dup = buffered_reader::Dup::new(reader); + if ! dup.eof() { + match Header::parse(&mut dup) { + Ok(header) => { + let tag = header.ctb().tag(); + if matches!(tag, Tag::Unknown(_) | Tag::Private(_)) { + return Err(Error::MalformedCert( + format!("A certificate must start with a \ + public key or a secret key packet, \ + got a {}", + tag)) + .into()); + } + } + Err(_err) => { + // We failed to read a header. Try to dearmor the + // input. + dearmor = true; + } + } + } + + // Strip the Dup reader. + let mut reader = dup.as_boxed().into_inner().expect("inner"); + + if dearmor { + // Unfortunately, armor::Reader is not generic over the + // Cookie type. Use a few adapters to get it to work + // anyway. + reader + = buffered_reader::Adapter::new( + armor::Reader::from_buffered_reader( + buffered_reader::Adapter::with_cookie( + reader, Default::default()).as_boxed(), + armor::ReaderMode::Tolerant(None), + Default::default()) + .as_boxed()) + .as_boxed(); + + let mut dup = buffered_reader::Dup::new(reader); + match Header::parse(&mut dup) { + Ok(header) => { + let tag = header.ctb().tag(); + if matches!(tag, Tag::Unknown(_) | Tag::Private(_)) { + return Err(Error::MalformedCert( + format!("A certificate must start with a \ + public key or a secret key packet, \ + got a {}", + tag)) + .into()); + } + } + Err(err) => { + return Err(err); + } + } + + reader = dup.as_boxed().into_inner().expect("inner"); + } + + Ok(RawCertParser { + slice: None, + reader, + dearmor, + bytes_read: 0, + pending_error: None, + done: false, + }) + } +} + +impl<'a> Parse<'a, RawCertParser<'a>> for RawCertParser<'a> +{ + /// Initializes a `RawCertParser` from a `Read`er. + fn from_reader<R: 'a + Read + Send + Sync>(reader: R) -> Result<Self> { + RawCertParser::new(buffered_reader::Generic::new(reader, None)) + } + + /// Initializes a `RawCertParser` from a `File`. + fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> { + RawCertParser::new(buffered_reader::File::open(path)?) + } + + /// Initializes a `RawCertParser` from a byte string. + fn from_bytes<D: AsRef<[u8]> + ?Sized + Send + Sync>(data: &'a D) -> Result<Self> { + let data = data.as_ref(); + let mut p = RawCertParser::new(buffered_reader::Memory::new(data))?; + + // If we are dearmoring the input, then the slice doesn't + // reflect the raw packets. + if ! p.dearmor { + p.slice = Some(data); + } + Ok(p) + } +} + +impl<'a> Iterator for RawCertParser<'a> +{ + type Item = Result<RawCert<'a>>; + + fn next(&mut self) -> Option<Self::Item> { + tracer!(TRACE, "RawCertParser::next", 0); + + // Return the pending error. + if let Some(err) = self.pending_error.take() { + t!("Returning the queued error: {}", err); + return Some(Err(err)); + } + + if self.done { + return None; + } + + if self.reader.eof() { + return None; + } + + let mut reader = buffered_reader::Dup::new( + std::mem::replace(&mut self.reader, + Box::new(buffered_reader::EOF::new()))); + + // The absolute start of this certificate in the stream. + let cert_start_absolute = self.bytes_read; + + // The number of bytes processed relative to the start of the + // dup'ed buffered reader. This may be less than the number + // of bytes read, e.g., when we encounter a new certificate, + // we read the header, but we don't necessarily want to + // consider it consumed. + let mut processed = 0; + + // The certificate's span relative to the start of the dup'ed + // buffered reader. The start will be larger than zero when + // we skip a marker packet. + let mut cert_start = 0; + let mut cert_end = 0; + + // (Tag, header length, offset from start of the certificate) + let mut packets: Vec<(Tag, usize, usize)> = Vec::new(); + let mut fingerprint = None; + + let mut pending_error = None; + 'packet_parser: loop { + if reader.eof() { + break; + } + + let packet_start = reader.total_out(); + processed = packet_start; + + let mut skip = 0; + let mut header_len = 0; + let header = loop { + match Header::parse(&mut reader) { + Err(err) => { + if skip == 0 { + t!("Reading the next packet's header: {}", err); + } + + if skip >= RECOVERY_THRESHOLD { + pending_error = Some(err.context( + format!("Splitting keyring at offset {}", + self.bytes_read + packet_start))); + processed = reader.total_out(); + + // We tried to recover and failed. Once + // we return the above error, we're done. + self.done = true; + + break 'packet_parser; + } else if reader.eof() { + t!("EOF while trying to recover"); + skip += 1; + break Header::new(CTB::new(Tag::Reserved), + BodyLength::Full(skip as u32)); + } else { + skip += 1; + reader.rewind(); + reader.consume(packet_start + skip); + } + } + Ok(header) if skip > 0 => { + if PacketParser::plausible_cert(&mut reader, &header) + .is_ok() + { + // We recovered. First return an error. The + // next time this function is called, we'll + // resume here. + t!("Found a valid header after {} bytes \ + of junk: {:?}", + skip, header); + + break Header::new(CTB::new(Tag::Reserved), + BodyLength::Full(skip as u32)); + } else { + skip += 1; + reader.rewind(); + reader.consume(packet_start + skip); + } + } + Ok(header) => { + header_len = reader.total_out() - packet_start; + break header; + } + } + }; + + if skip > 0 { + // Fabricate a header. + t!("Recovered after {} bytes of junk", skip); + + pending_error = Some(Error::MalformedPacket( + format!("Encountered {} bytes of junk at offset {}", + skip, self.bytes_read)).into()); + + // Be careful: if we recovered, then we + // reader.total_out() includes the good header. + processed += skip; + + break; + } + + let tag = header.ctb().tag(); + t!("Found a {:?}, length: {:?}", + tag, header.length()); + + if packet_start > cert_start + && (tag == Tag::PublicKey || tag == Tag::SecretKey) + { + // Start of new cert. Note: we don't advanced + // processed! That would consume the header that + // we want to read the next time this function is + // called. + t!("Stopping: found the start of a new cert ({})", tag); + break; + } + + match header.length() { + BodyLength::Full(l) => { + let l = *l as usize; + + match reader.data_consume_hard(l) { + Err(err) => { + t!("Stopping: reading {}'s body: {}", tag, err); + pending_error = Some(err.into()); + break; + } + Ok(data) => { + if tag == Tag::PublicKey + || tag == Tag::SecretKey + { + let data = &data[..l]; + match Key::from_bytes(data) { + Err(err) => { + t!("Stopping: parsing public key: {}", + err); + pending_error = Some(err); + break; + } + Ok(key) => { + fingerprint = Some(key.fingerprint()); + } + } + } + } + } + } + BodyLength::Partial(_) => { + t!("Stopping: Partial body length not allowed \ + for {} packets", + tag); + pending_error = Some( + Error::MalformedPacket( + format!("Packet {} uses partial body length \ + encoding, which is not allowed in \ + certificates", + tag)) + .into()); + self.done = true; + break; + } + |