Allow multiple sources per package

This patch implements multiple (unnamed) sources per package.
This means that a package can have an array of sources.

What was adapted to allow multiple sources per package:

* Downloads are made in parallel now
* The cache structure was changed to /<package>-<version>/<hash>.source
* The UI was changed to contain the full `Package` struct (as JSON
  object) in a UI format string

Tests were adapted.

Signed-off-by: Matthias Beyer <mail@beyermatthias.de>

16 files changed, 152 insertions, 160 deletions

diff --git a/config.toml b/config.toml
index 458bf13..5804fa0 100644
--- a/config.toml
+++ b/config.toml
@@ -29,15 +29,7 @@ script_highlight_theme = "Solarized (dark)"
 #
 # Possible tokens are:
 #     i                         - Incrementing number of package that is printed
-#     name                      - Name of the package
-#     version                   - Version of the package
-#     source_url                - URL from where the source was retrieved
-#     source_hash_type          - Type of hash noted in the package
-#     source_hash               - Hash of the sources
-#     system_deps               - System dependencies, as list
-#     system_runtime_deps       - System runtime dependencies, as list
-#     build_deps                - Build dependencies, as list
-#     runtime_deps              - Runtime dependencies, as list
+#     p                         - The package data
 #     print_system_deps         - boolean flag to use in format string, true if user wants to see system deps
 #     print_system_runtime_deps - boolean flag to use in format string, true if user wants to see system runtime deps
 #     print_build_deps          - boolean flag to use in format string, true if user wants to see build deps
diff --git a/examples/packages/example_1/pkgA/pkg.toml b/examples/packages/example_1/pkgA/pkg.toml
index cb9a556..d82df02 100644
--- a/examples/packages/example_1/pkgA/pkg.toml
+++ b/examples/packages/example_1/pkgA/pkg.toml
@@ -4,7 +4,7 @@ version           = "1.0.0"
 [dependencies]
 runtime = [ "pkgB =1.0.0" ]
 
-[source]
+[[sources]]
 url  = "http://somerandomthi.ng/foo/bar.tar.gz"
 hash.type = "sha1"
 hash.hash = "23465"
diff --git a/examples/packages/example_1/pkgB/pkg.toml b/examples/packages/example_1/pkgB/pkg.toml
index 14a062a..8685b5d 100644
--- a/examples/packages/example_1/pkgB/pkg.toml
+++ b/examples/packages/example_1/pkgB/pkg.toml
@@ -1,7 +1,7 @@
 name              = "pkgB"
 version           = "1.0.0"
 
-[source]
+[[sources]]
 url  = "http://somerandomthi.ng/foo/bar.tar.gz"
 hash.type = "sha1"
 hash.hash = "23465"
diff --git a/examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml b/examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml
index bcb9678..9788036 100644
--- a/examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml
+++ b/examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml
@@ -1,6 +1,6 @@
 version = "1.0.0"
 
-[source]
+[[sources]]
 url  = "http://somerandomthi.ng/foo/bar-1.0.0.tar.gz"
 hash.type = "sha1"
 hash.hash = "23465"
diff --git a/examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml b/examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml
index 2eee1df..a46727f 100644
--- a/examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml
+++ b/examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml
@@ -1,6 +1,6 @@
 version = "2.0.0"
 
-[source]
+[[sources]]
 url  = "http://somerandomthi.ng/foo/bar-2.0.0.tar.gz"
 hash.type = "sha1"
 hash.hash = "23465"
diff --git a/examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml b/examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml
index c167d09..9e9d3a8 100644
--- a/examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml
+++ b/examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml
@@ -1,6 +1,6 @@
 version = "1.0.0"
 
-[source]
+[[sources]]
 url  = "http://somerandomthi.ng/foo/baz-1.0.0.tar.gz"
 hash.type = "sha1"
 hash.hash = "23465"
diff --git a/examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml b/examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml
index 225aebd..2a3990a 100644
--- a/examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml
+++ b/examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml
@@ -1,6 +1,6 @@
 version = "2.0.0"
 
-[source]
+[[sources]]
 url  = "http://somerandomthi.ng/foo/baz-2.0.0.tar.gz"
 hash.type = "sha1"
 hash.hash = "23465"
diff --git a/examples/packages/example_3/pkg.toml b/examples/packages/example_3/pkg.toml
index f844921..d399047 100644
--- a/examples/packages/example_3/pkg.toml
+++ b/examples/packages/example_3/pkg.toml
@@ -4,7 +4,7 @@ version_is_semver = false
 # no patches by default
 patches = []
 
-[source]
+[[sources]]
 url       = "https://github.com/user/repo/archive/sources.tar.gz"
 hash.type = "sha1"
 hash.hash = "5e8bcaa3c758f84f01935a914e2bbf01309462ae"
diff --git a/examples/packages/example_tmux/pkg.toml b/examples/packages/example_tmux/pkg.toml
index 7d4119e..beac147 100644
--- a/examples/packages/example_tmux/pkg.toml
+++ b/examples/packages/example_tmux/pkg.toml
@@ -5,7 +5,7 @@ version_is_semver = false
 # no patches by default
 patches = []
 
-[source]
+[[sources]]
 url       = "https://github.com/tmux/tmux/archive/3.1c.tar.gz"
 hash.type = "sha1"
 hash.hash = "5e8bcaa3c758f84f01935a914e2bbf01309462ae"
diff --git a/src/commands/source.rs b/src/commands/source.rs
index 46c548d..8892a79 100644
--- a/src/commands/source.rs
+++ b/src/commands/source.rs
@@ -36,22 +36,24 @@ pub async fn verify<'a>(matches: &ArgMatches, config: &Configuration<'a>, repo:
         .filter(|p| pname.as_ref().map(|n| p.name() == n).unwrap_or(true))
         .filter(|p| pvers.as_ref().map(|v| v.matches(p.version())).unwrap_or(true))
         .map(|p| {
-            let source = sc.source_for(p);
-            async move {
-                let out = std::io::stdout();
-                if source.exists() {
-                    if source.verify_hash().await? {
-                        writeln!(out.lock(), "Ok: {}", source.path().display())?;
+            sc.sources_for(p)
+                .into_iter()
+                .map(|source| async move {
+                    let path = source.path();
+                    let mut out = std::io::stdout();
+                    if path.exists() {
+                        if source.verify_hash().await? {
+                            writeln!(out, "Ok: {}", source.path().display())
+                        } else {
+                            writeln!(out, "Hash Mismatch: {}", source.path().display())
+                        }
                     } else {
-                        writeln!(out.lock(), "Hash Mismatch: {}", source.path().display())?;
+                        writeln!(out, "Source missing: {}", source.path().display())
                     }
-                } else {
-                    writeln!(out.lock(), "Source missing: {}", source.path().display())?;
-                }
-
-                Ok(())
-            }
+                    .map_err(Error::from)
+                })
         })
+        .flatten()
         .collect::<futures::stream::FuturesUnordered<_>>()
         .collect::<Result<()>>()
         .await
@@ -64,9 +66,10 @@ pub async fn list_missing<'a>(_: &ArgMatches, config: &Configuration<'a>, repo:
 
     repo.packages()
         .map(|p| {
-            let s = sc.source_for(p);
-            if !s.exists() {
-                writeln!(outlock, "{} {} -> {}", p.name(), p.version(), s.path().display())?;
+            for source in sc.sources_for(p) {
+                if !source.exists() {
+                    writeln!(outlock, "{} {} -> {}", p.name(), p.version(), source.path().display())?;
+                }
             }
 
             Ok(())
@@ -84,7 +87,12 @@ pub async fn url<'a>(matches: &ArgMatches, config: &Configuration<'a>, repo: Rep
     repo.packages()
         .filter(|p| pname.as_ref().map(|n| p.name() == n).unwrap_or(true))
         .filter(|p| pvers.as_ref().map(|v| v.matches(p.version())).unwrap_or(true))
-        .map(|p| writeln!(outlock, "{} {} -> {}", p.name(), p.version(), p.source().url()).map_err(Error::from))
+        .map(|p| {
+            p.sources()
+                .iter()
+                .map(|source| writeln!(outlock, "{} {} -> {}", p.name(), p.version(), source.url()).map_err(Error::from))
+                .collect()
+        })
         .collect()
 }
 
@@ -100,36 +108,40 @@ pub async fn download<'a>(matches: &ArgMatches, config: &Configuration<'a>, repo
         .filter(|p| pname.as_ref().map(|n| p.name() == n).unwrap_or(true))
         .filter(|p| pvers.as_ref().map(|v| v.matches(p.version())).unwrap_or(true))
         .map(|p| {
-            let source = sc.source_for(p);
-            let bar = multi.add(progressbars.download_bar(source.url()));
-            async move {
-                if source.exists() && !force {
-                    Err(anyhow!("Source exists: {}", source.path().display()))
-                } else {
-                    if source.exists() {
-                        let _ = source.remove_file().await?;
+            sc.sources_for(p)
+                .into_iter()
+                .map(|source| {
+                    let bar = multi.add(progressbars.download_bar(source.url()));
+                    async move {
+                        if source.exists() && !force {
+                            Err(anyhow!("Source exists: {}", source.path().display()))
+                        } else {
+                            if source.exists() {
+                                let _ = source.remove_file().await?;
+                            }
+
+                            trace!("Starting download...");
+                            let file = source.create().await?;
+                            let mut file = tokio::io::BufWriter::new(file);
+                            let response = reqwest::get(source.url().as_ref()).await?;
+                            if let Some(len) = response.content_length() {
+                                bar.set_length(len);
+                            }
+                            let mut stream = reqwest::get(source.url().as_ref()).await?.bytes_stream();
+                            while let Some(bytes) = stream.next().await {
+                                let bytes = bytes?;
+                                file.write_all(bytes.as_ref()).await?;
+                                bar.inc(bytes.len() as u64);
+                            }
+
+                            file.flush().await?;
+                            bar.finish_with_message("Finished download");
+                            Ok(())
+                        }
                     }
-
-                    trace!("Starting download...");
-                    let file = source.create().await?;
-                    let mut file = tokio::io::BufWriter::new(file);
-                    let response = reqwest::get(source.url().as_ref()).await?;
-                    if let Some(len) = response.content_length() {
-                        bar.set_length(len);
-                    }
-                    let mut stream = reqwest::get(source.url().as_ref()).await?.bytes_stream();
-                    while let Some(bytes) = stream.next().await {
-                        let bytes = bytes?;
-                        file.write_all(bytes.as_ref()).await?;
-                        bar.inc(bytes.len() as u64);
-                    }
-
-                    file.flush().await?;
-                    bar.finish_with_message("Finished download");
-                    Ok(())
-                }
-            }
+                })
         })
+        .flatten()
         .collect::<futures::stream::FuturesUnordered<_>>()
         .collect::<Result<()>>()
         .await;
diff --git a/src/config/util.rs b/src/config/util.rs
index 17aeadf..ecd94c8 100644
--- a/src/config/util.rs
+++ b/src/config/util.rs
@@ -4,13 +4,12 @@ pub fn default_progress_format() -> String {
 
 pub fn default_package_print_format() -> String {
     String::from(indoc::indoc!(r#"
-            {{i}} - {{name}} : {{version}}
-            Source: {{source_url}}
-            Hash ({{source_hash_type}}): {{source_hash}}
-            {{#if print_system_deps}}System Deps: {{ system_deps }} {{/if}}
-            {{#if print_system_runtime_deps}}System runtime Deps: {{ system_runtime_deps }} {{/if}}
-            {{#if print_build_deps}}Build Deps: {{ build_deps }} {{/if}}
-            {{#if print_runtime_deps}}Runtime Deps: {{ runtime_deps }} {{/if}}
+            {{i}} - {{p.name}} : {{p.version}}
+            {{#each p.sources}}Source: {{this.url}} - {{this.hash.hash}} ({{this.hash.type}}){{/each}}
+            {{#if print_system_deps}}System Deps: {{ p.dependencies.system }} {{/if}}
+            {{#if print_system_runtime_deps}}System runtime Deps: {{ p.dependencies.system_runtime }} {{/if}}
+            {{#if print_build_deps}}Build Deps: {{ p.dependencies.build }} {{/if}}
+            {{#if print_runtime_deps}}Runtime Deps: {{ p.dependencies.runtime }} {{/if}}
 
     "#))
 }
diff --git a/src/endpoint/configured.rs b/src/endpoint/configured.rs
index d09bd37..27e1823 100644
--- a/src/endpoint/configured.rs
+++ b/src/endpoint/configured.rs
@@ -9,6 +9,7 @@ use anyhow::Error;
 use anyhow::Result;
 use anyhow::anyhow;
 use futures::FutureExt;
+use futures::future::TryFutureExt;
 use getset::{Getters, CopyGetters};
 use shiplift::Docker;
 use shiplift::ExecContainerOptions;
@@ -221,33 +222,39 @@ impl Endpoint {
         { // copy source to container
             use tokio::io::AsyncReadExt;
 
-            let pkgsource   = job.package_source();
-            let source_path = pkgsource.path();
-            let destination = PathBuf::from("/inputs").join({
-                source_path.file_name()
-                    .ok_or_else(|| anyhow!("Not a file: {}", source_path.display()))
-                    .with_context(|| anyhow!("Copying package source from {} to container {}", source_path.display(), self.name))?
-            });
-            trace!("Package source = {:?}", pkgsource);
-            trace!("Source path    = {:?}", source_path);
-            trace!("Source dest    = {:?}", destination);
-            let mut buf = vec![];
-            tokio::fs::OpenOptions::new()
-                .create(false)
-                .create_new(false)
-                .append(false)
-                .write(false)
-                .read(true)
-                .open(source_path)
-                .await
-                .with_context(|| anyhow!("Getting source file: {}", source_path.display()))?
-                .read_to_end(&mut buf)
-                .await
-                .with_context(|| anyhow!("Reading file {}", source_path.display()))?;
+            job.package_sources()
+                .into_iter()
+                .map(|entry| async {
+                    let source_path = entry.path();
+                    let destination = PathBuf::from("/inputs").join({
+                        source_path.file_name()
+                            .ok_or_else(|| anyhow!("Not a file: {}", source_path.display()))
+                            .with_context(|| anyhow!("Copying package source from {} to container {}", source_path.display(), self.name))?
+                    });
+                    trace!("Source path    = {:?}", source_path);
+                    trace!("Source dest    = {:?}", destination);
+                    let mut buf = vec![];
+                    tokio::fs::OpenOptions::new()
+                        .create(false)
+                        .create_new(false)
+                        .append(false)
+                        .write(false)
+                        .read(true)
+                        .open(&source_path)
+                        .await
+                        .with_context(|| anyhow!("Getting source file: {}", source_path.display()))?
+                        .read_to_end(&mut buf)
+                        .await
+                        .with_context(|| anyhow!("Reading file {}", source_path.display()))?;
 
-            let _ = container.copy_file_into(destination, &buf)
+                    drop(entry);
+                    let _ = container.copy_file_into(destination, &buf).await?;
+                    Ok(())
+                })
+                .collect::<futures::stream::FuturesUnordered<_>>()
+                .collect::<Result<()>>()
                 .await
-                .with_context(|| anyhow!("Copying {} to container {}", source_path.display(), container_id))?;
+                .with_context(|| anyhow!("Copying sources to container {}", container_id))?;
         }
         { // Copy all Path artifacts to the container
             job.resources()
diff --git a/src/job/runnable.rs b/src/job/runnable.rs
index 67219fb..ab2d6ca 100644
--- a/src/job/runnable.rs
+++ b/src/job/runnable.rs
@@ -80,8 +80,8 @@ impl RunnableJob {
 
     }
 
-    pub fn package_source(&self) -> SourceEntry {
-        self.source_cache.source_for(&self.package())
+    pub fn package_sources(&self) -> Vec<SourceEntry> {
+        self.source_cache.sources_for(&self.package())
     }
 
     pub fn environment(&self) -> Vec<(String, String)> {
diff --git a/src/package/package.rs b/src/package/package.rs
index c9ad4f0..698b8b1 100644
--- a/src/package/package.rs
+++ b/src/package/package.rs
@@ -26,7 +26,7 @@ pub struct Package {
     version_is_semver: bool,
 
     #[getset(get = "pub")]
-    source: Source,
+    sources: Vec<Source>,
 
     #[getset(get = "pub")]
     dependencies: Dependencies,
@@ -53,12 +53,12 @@ pub struct Package {
 impl Package {
 
     #[cfg(test)]
-    pub fn new(name: PackageName, version: PackageVersion, version_is_semver: bool, source: Source, dependencies: Dependencies) -> Self {
+    pub fn new(name: PackageName, version: PackageVersion, version_is_semver: bool, sources: Vec<Source>, dependencies: Dependencies) -> Self {
         Package {
             name,
             version,
             version_is_semver,
-            source,
+            sources,
             dependencies,
             patches: vec![],
             environment: None,
@@ -216,13 +216,13 @@ pub mod tests {
         let name    = pname(name);
         let version = pversion(vers);
         let version_is_semver = false;
-        let source = {
+        let sources = {
             let url       = Url::parse(srcurl).unwrap();
             let hashvalue = HashValue::from(String::from(hash));
-            Source::new(url, SourceHash::new(HashType::Sha1, hashvalue))
+            vec![Source::new(url, SourceHash::new(HashType::Sha1, hashvalue))]
         };
         let dependencies = Dependencies::empty();
-        Package::new(name, version, version_is_semver, source, dependencies)
+        Package::new(name, version, version_is_semver, sources, dependencies)
     }
 
 }
diff --git a/src/source/mod.rs b/src/source/mod.rs
index 455785c..c89021e 100644
--- a/src/source/mod.rs
+++ b/src/source/mod.rs
@@ -19,7 +19,7 @@ impl SourceCache {
         SourceCache { root }
     }
 
-    pub fn source_for(&self, p: &Package) -> SourceEntry {
+    pub fn sources_for(&self, p: &Package) -> Vec<SourceEntry> {
         SourceEntry::for_package(self.root.clone(), p)
     }
 }
@@ -30,29 +30,35 @@ pub struct SourceEntry {
     package_name: PackageName,
     package_version: PackageVersion,
     package_source: Source,
-    package_source_path: PathBuf,
 }
 
 impl SourceEntry {
 
-    fn for_package(cache_root: PathBuf, package: &Package) -> Self {
-        let package_source_path = cache_root.join(format!("{}-{}.source", package.name(), package.version()));
+    fn source_file_path(&self) -> PathBuf {
+        self.cache_root.join(format!("{}-{}/{}.source", self.package_name, self.package_version, self.package_source.hash().value()))
+    }
 
-        SourceEntry {
-            cache_root,
-            package_name: package.name().clone(),
-            package_version: package.version().clone(),
-            package_source: package.source().clone(),
-            package_source_path
-        }
+    fn for_package(cache_root: PathBuf, package: &Package) -> Vec<Self> {
+        package.sources()
+            .clone()
+            .into_iter()
+            .map(|source| {
+                SourceEntry {
+                    cache_root: cache_root.clone(),
+                    package_name: package.name().clone(),
+                    package_version: package.version().clone(),
+                    package_source: source,
+                }
+            })
+            .collect()
     }
 
     pub fn exists(&self) -> bool {
-        self.package_source_path.exists()
+        self.source_file_path().exists()
     }
 
-    pub fn path(&self) -> &PathBuf {
-        &self.package_source_path
+    pub fn path(&self) -> PathBuf {
+        self.source_file_path()
     }
 
     pub fn url(&self) -> &Url {
@@ -60,41 +66,50 @@ impl SourceEntry {
     }
 
     pub async fn remove_file(&self) -> Result<()> {
-        tokio::fs::remove_file(&self.package_source_path).await.map_err(Error::from)
+        let p = self.source_file_path();
+        tokio::fs::remove_file(&p).await?;
+        Ok(())
     }
 
     pub async fn verify_hash(&self) -> Result<bool> {
         use tokio::io::AsyncReadExt;
 
+        let p = self.source_file_path();
         let mut buf = vec![];
         tokio::fs::OpenOptions::new()
             .create(false)
             .create_new(false)
             .read(true)
-            .open(&self.package_source_path)
+            .open(&p)
             .await?
             .read_to_end(&mut buf)
             .await?;
 
-        self.package_source.hash().matches_hash_of(&buf)
+        self.package_source
+            .hash()
+            .matches_hash_of(&buf)
     }
 
     pub async fn open(&self) -> Result<tokio::fs::File> {
+        let p = self.source_file_path();
+
         tokio::fs::OpenOptions::new()
             .create(false)
             .create_new(false)
             .read(true)
-            .open(&self.package_source_path)
+            .open(&p)
             .await
             .map_err(Error::from)
     }
 
     pub async fn create(&self) -> Result<tokio::fs::File> {
+        let p = self.source_file_path();
+
         tokio::fs::OpenOptions::new()
             .create(true)
             .create_new(true)
             .write(true)
-            .open(&self.package_source_path)
+            .open(&p)
             .await
             .map_err(Error::from)
     }
diff --git a/src/ui.rs b/src/ui.rs
index fe7bec7..d05014c 100644
--- a/src/ui.rs
+++ b/src/ui.rs
@@ -61,57 +61,24 @@ fn print_package(out: &mut dyn Write,
     -> Result<()>
 {
     let mut data = BTreeMap::new();
-    data.insert("i", i.to_string());
-    data.insert("name",             format!("{}", package.name()));
-    data.insert("version",          format!("{}", package.version()));
-    data.insert("source_url",       format!("{}", package.source().url()));
-    data.insert("source_hash_type", format!("{}", package.source().hash().hashtype()));
-    data.insert("source_hash",      format!("{}", package.source().hash().value()));
+    data.insert("i", serde_json::Value::Number(serde_json::Number::from(i)));
+    data.insert("p", serde_json::to_value(package)?);
 
     // This is an ugly hack. Because the `data` is a <String, String>, we do only insert the flag
     // if it is set, because handlebars renders a non-present value as false
     if print_runtime_deps {
-        data.insert("print_runtime_deps",     format!("{}", print_runtime_deps));
+        data.insert("print_runtime_deps", serde_json::Value::Bool(print_runtime_deps));
     }
     if print_build_deps {
-        data.insert("print_build_deps",       format!("{}", print_build_deps));
+        data.insert("print_build_deps", serde_json::Value::Bool(print_build_deps));
     }
     if print_sys_deps {
-        data.insert("print_system_deps",         format!("{}", print_sys_deps));
+        data.insert("print_system_deps", serde_json::Value::Bool(print_sys_deps));
     }
     if print_sys_runtime_deps {
-        data.insert("print_system_runtime_deps", format!("{}", print_sys_runtime_deps));
+        data.insert("print_system_runtime_deps", serde_json::Value::Bool(print_sys_runtime_deps));
     }
 
-    data.insert("runtime_deps",     {
-        format!("[{}]", package.dependencies()
-                .runtime()
-                .iter()
-                .map(|p| p.as_ref())
-                .join(", "))
-    });
-    data.insert("build_deps",     {
-        format!("[{}]", package.dependencies()
-                .build()
-                .iter()
-                .map(|p| p.as_ref())
-                .join(", "))
-    });
-    data.insert("system_deps",     {
-        format!("[{}]", package.dependencies()
-                .system()
-                .iter()
-                .map(|p| p.as_ref())
-                .join(", "))
-    });
-    data.insert("system_runtime_deps",     {
-        format!("[{}]", package.dependencies()
-                .system_runtime()
-                .iter()
-                .map(|p| p.as_ref())
-                .join(", "))
-    });
-
     hb.render("package", &data)
         .map_err(Error::from)
         .and_then(|r| writeln!(out, "{}", r).map_err(Error::from))