From 645833f747db227a28cc7bab2ad16d6cafa19843 Mon Sep 17 00:00:00 2001
From: Matthias Beyer <matthias.beyer@atos.net>
Date: Mon, 25 Jan 2021 13:49:42 +0100
Subject: Reimplement hash verification using streaming

This patch re-implements hashing using streams and buffered readers instead of
reading a full file to RAM before hashing it.

Signed-off-by: Matthias Beyer <matthias.beyer@atos.net>
---
 src/commands/source.rs |  2 +-
 src/package/source.rs  | 34 ++++++++++++++++++++++++++++------
 src/source/mod.rs      | 29 +++++++++++++++--------------
 3 files changed, 44 insertions(+), 21 deletions(-)

(limited to 'src')

diff --git a/src/commands/source.rs b/src/commands/source.rs
index b806939..00a3ca0 100644
--- a/src/commands/source.rs
+++ b/src/commands/source.rs
@@ -96,7 +96,7 @@ where
             let bar = multi.add(progressbars.bar());
             if source.path().exists() {
                 trace!("Exists: {}", source.path().display());
-                source.verify_hash().with_context(|| {
+                source.verify_hash().await.with_context(|| {
                     anyhow!("Hash verification failed for: {}", source.path().display())
                 })?;
 
diff --git a/src/package/source.rs b/src/package/source.rs
index c99b701..d60444a 100644
--- a/src/package/source.rs
+++ b/src/package/source.rs
@@ -8,6 +8,8 @@
 // SPDX-License-Identifier: EPL-2.0
 //
 
+use std::io::Read;
+
 use anyhow::anyhow;
 use anyhow::Result;
 use getset::Getters;
@@ -50,9 +52,9 @@ pub struct SourceHash {
 }
 
 impl SourceHash {
-    pub fn matches_hash_of(&self, buf: &[u8]) -> Result<()> {
+    pub fn matches_hash_of<R: Read>(&self, reader: R) -> Result<()> {
         trace!("Hashing buffer with: {:?}", self.hashtype);
-        let h = self.hashtype.hash_buffer(&buf)?;
+        let h = self.hashtype.hash_from_reader(reader)?;
         trace!("Hashing buffer with: {} finished", self.hashtype);
 
         if h == self.value {
@@ -90,24 +92,44 @@ pub enum HashType {
 }
 
 impl HashType {
-    fn hash_buffer(&self, buffer: &[u8]) -> Result<HashValue> {
+    fn hash_from_reader<R: Read>(&self, mut reader: R) -> Result<HashValue> {
+        let mut buffer = [0; 1024];
+
         match self {
             HashType::Sha1 => {
                 trace!("SHA1 hashing buffer");
                 let mut m = sha1::Sha1::new();
-                m.update(buffer);
+                loop {
+                    let count = reader.read(&mut buffer)?;
+                    if count == 0 {
+                        break;
+                    }
+                    m.update(&buffer[..count]);
+                }
                 Ok(HashValue(m.digest().to_string()))
             }
             HashType::Sha256 => {
                 trace!("SHA256 hashing buffer");
                 let mut m = sha2::Sha256::new();
-                m.update(buffer);
+                loop {
+                    let count = reader.read(&mut buffer)?;
+                    if count == 0 {
+                        break;
+                    }
+                    m.update(&buffer[..count]);
+                }
                 Ok(HashValue(String::from_utf8(m.finalize()[..].to_vec())?))
             }
             HashType::Sha512 => {
                 trace!("SHA512 hashing buffer");
                 let mut m = sha2::Sha512::new();
-                m.update(buffer);
+                loop {
+                    let count = reader.read(&mut buffer)?;
+                    if count == 0 {
+                        break;
+                    }
+                    m.update(&buffer[..count]);
+                }
                 Ok(HashValue(String::from_utf8(m.finalize()[..].to_vec())?))
             }
         }
diff --git a/src/source/mod.rs b/src/source/mod.rs
index b356c31..1c97a87 100644
--- a/src/source/mod.rs
+++ b/src/source/mod.rs
@@ -97,23 +97,24 @@ impl SourceEntry {
         Ok(())
     }
 
-    pub fn verify_hash(&self) -> Result<()> {
-        use std::io::Read;
-
+    pub async fn verify_hash(&self) -> Result<()> {
         let p = self.source_file_path();
-        trace!("Reading to buffer: {}", p.display());
+        trace!("Verifying : {}", p.display());
 
         let path = p.clone();
-        let mut buf = vec![];
-        std::fs::OpenOptions::new()
-            .create(false)
-            .create_new(false)
-            .read(true)
-            .open(path)?
-            .read_to_end(&mut buf)?;
-
-        trace!("Reading to buffer finished: {}", p.display());
-        self.package_source.hash().matches_hash_of(&buf)
+        let reader = tokio::task::spawn_blocking(move || {
+            std::fs::OpenOptions::new()
+                .create(false)
+                .create_new(false)
+                .read(true)
+                .open(path)
+                .map(std::io::BufReader::new)
+        })
+        .await??;
+
+        self.package_source
+            .hash()
+            .matches_hash_of(reader)
     }
 
     pub async fn create(&self) -> Result<tokio::fs::File> {
-- 
cgit v1.2.3