diff options
-rw-r--r-- | config.toml | 10 | ||||
-rw-r--r-- | examples/packages/example_1/pkgA/pkg.toml | 2 | ||||
-rw-r--r-- | examples/packages/example_1/pkgB/pkg.toml | 2 | ||||
-rw-r--r-- | examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml | 2 | ||||
-rw-r--r-- | examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml | 2 | ||||
-rw-r--r-- | examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml | 2 | ||||
-rw-r--r-- | examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml | 2 | ||||
-rw-r--r-- | examples/packages/example_3/pkg.toml | 2 | ||||
-rw-r--r-- | examples/packages/example_tmux/pkg.toml | 2 | ||||
-rw-r--r-- | src/commands/source.rs | 102 | ||||
-rw-r--r-- | src/config/util.rs | 13 | ||||
-rw-r--r-- | src/endpoint/configured.rs | 57 | ||||
-rw-r--r-- | src/job/runnable.rs | 4 | ||||
-rw-r--r-- | src/package/package.rs | 12 | ||||
-rw-r--r-- | src/source/mod.rs | 53 | ||||
-rw-r--r-- | src/ui.rs | 45 |
16 files changed, 152 insertions, 160 deletions
diff --git a/config.toml b/config.toml index 458bf13..5804fa0 100644 --- a/config.toml +++ b/config.toml @@ -29,15 +29,7 @@ script_highlight_theme = "Solarized (dark)" # # Possible tokens are: # i - Incrementing number of package that is printed -# name - Name of the package -# version - Version of the package -# source_url - URL from where the source was retrieved -# source_hash_type - Type of hash noted in the package -# source_hash - Hash of the sources -# system_deps - System dependencies, as list -# system_runtime_deps - System runtime dependencies, as list -# build_deps - Build dependencies, as list -# runtime_deps - Runtime dependencies, as list +# p - The package data # print_system_deps - boolean flag to use in format string, true if user wants to see system deps # print_system_runtime_deps - boolean flag to use in format string, true if user wants to see system runtime deps # print_build_deps - boolean flag to use in format string, true if user wants to see build deps diff --git a/examples/packages/example_1/pkgA/pkg.toml b/examples/packages/example_1/pkgA/pkg.toml index cb9a556..d82df02 100644 --- a/examples/packages/example_1/pkgA/pkg.toml +++ b/examples/packages/example_1/pkgA/pkg.toml @@ -4,7 +4,7 @@ version = "1.0.0" [dependencies] runtime = [ "pkgB =1.0.0" ] -[source] +[[sources]] url = "http://somerandomthi.ng/foo/bar.tar.gz" hash.type = "sha1" hash.hash = "23465" diff --git a/examples/packages/example_1/pkgB/pkg.toml b/examples/packages/example_1/pkgB/pkg.toml index 14a062a..8685b5d 100644 --- a/examples/packages/example_1/pkgB/pkg.toml +++ b/examples/packages/example_1/pkgB/pkg.toml @@ -1,7 +1,7 @@ name = "pkgB" version = "1.0.0" -[source] +[[sources]] url = "http://somerandomthi.ng/foo/bar.tar.gz" hash.type = "sha1" hash.hash = "23465" diff --git a/examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml b/examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml index bcb9678..9788036 100644 --- a/examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml +++ b/examples/packages/example_2/sub/pkg1/1.0.0/pkg.toml @@ -1,6 +1,6 @@ version = "1.0.0" -[source] +[[sources]] url = "http://somerandomthi.ng/foo/bar-1.0.0.tar.gz" hash.type = "sha1" hash.hash = "23465" diff --git a/examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml b/examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml index 2eee1df..a46727f 100644 --- a/examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml +++ b/examples/packages/example_2/sub/pkg1/2.0.0/pkg.toml @@ -1,6 +1,6 @@ version = "2.0.0" -[source] +[[sources]] url = "http://somerandomthi.ng/foo/bar-2.0.0.tar.gz" hash.type = "sha1" hash.hash = "23465" diff --git a/examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml b/examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml index c167d09..9e9d3a8 100644 --- a/examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml +++ b/examples/packages/example_2/sub/pkg2/1.0.0/pkg.toml @@ -1,6 +1,6 @@ version = "1.0.0" -[source] +[[sources]] url = "http://somerandomthi.ng/foo/baz-1.0.0.tar.gz" hash.type = "sha1" hash.hash = "23465" diff --git a/examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml b/examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml index 225aebd..2a3990a 100644 --- a/examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml +++ b/examples/packages/example_2/sub/pkg2/2.0.0/pkg.toml @@ -1,6 +1,6 @@ version = "2.0.0" -[source] +[[sources]] url = "http://somerandomthi.ng/foo/baz-2.0.0.tar.gz" hash.type = "sha1" hash.hash = "23465" diff --git a/examples/packages/example_3/pkg.toml b/examples/packages/example_3/pkg.toml index f844921..d399047 100644 --- a/examples/packages/example_3/pkg.toml +++ b/examples/packages/example_3/pkg.toml @@ -4,7 +4,7 @@ version_is_semver = false # no patches by default patches = [] -[source] +[[sources]] url = "https://github.com/user/repo/archive/sources.tar.gz" hash.type = "sha1" hash.hash = "5e8bcaa3c758f84f01935a914e2bbf01309462ae" diff --git a/examples/packages/example_tmux/pkg.toml b/examples/packages/example_tmux/pkg.toml index 7d4119e..beac147 100644 --- a/examples/packages/example_tmux/pkg.toml +++ b/examples/packages/example_tmux/pkg.toml @@ -5,7 +5,7 @@ version_is_semver = false # no patches by default patches = [] -[source] +[[sources]] url = "https://github.com/tmux/tmux/archive/3.1c.tar.gz" hash.type = "sha1" hash.hash = "5e8bcaa3c758f84f01935a914e2bbf01309462ae" diff --git a/src/commands/source.rs b/src/commands/source.rs index 46c548d..8892a79 100644 --- a/src/commands/source.rs +++ b/src/commands/source.rs @@ -36,22 +36,24 @@ pub async fn verify<'a>(matches: &ArgMatches, config: &Configuration<'a>, repo: .filter(|p| pname.as_ref().map(|n| p.name() == n).unwrap_or(true)) .filter(|p| pvers.as_ref().map(|v| v.matches(p.version())).unwrap_or(true)) .map(|p| { - let source = sc.source_for(p); - async move { - let out = std::io::stdout(); - if source.exists() { - if source.verify_hash().await? { - writeln!(out.lock(), "Ok: {}", source.path().display())?; + sc.sources_for(p) + .into_iter() + .map(|source| async move { + let path = source.path(); + let mut out = std::io::stdout(); + if path.exists() { + if source.verify_hash().await? { + writeln!(out, "Ok: {}", source.path().display()) + } else { + writeln!(out, "Hash Mismatch: {}", source.path().display()) + } } else { - writeln!(out.lock(), "Hash Mismatch: {}", source.path().display())?; + writeln!(out, "Source missing: {}", source.path().display()) } - } else { - writeln!(out.lock(), "Source missing: {}", source.path().display())?; - } - - Ok(()) - } + .map_err(Error::from) + }) }) + .flatten() .collect::<futures::stream::FuturesUnordered<_>>() .collect::<Result<()>>() .await @@ -64,9 +66,10 @@ pub async fn list_missing<'a>(_: &ArgMatches, config: &Configuration<'a>, repo: repo.packages() .map(|p| { - let s = sc.source_for(p); - if !s.exists() { - writeln!(outlock, "{} {} -> {}", p.name(), p.version(), s.path().display())?; + for source in sc.sources_for(p) { + if !source.exists() { + writeln!(outlock, "{} {} -> {}", p.name(), p.version(), source.path().display())?; + } } Ok(()) @@ -84,7 +87,12 @@ pub async fn url<'a>(matches: &ArgMatches, config: &Configuration<'a>, repo: Rep repo.packages() .filter(|p| pname.as_ref().map(|n| p.name() == n).unwrap_or(true)) .filter(|p| pvers.as_ref().map(|v| v.matches(p.version())).unwrap_or(true)) - .map(|p| writeln!(outlock, "{} {} -> {}", p.name(), p.version(), p.source().url()).map_err(Error::from)) + .map(|p| { + p.sources() + .iter() + .map(|source| writeln!(outlock, "{} {} -> {}", p.name(), p.version(), source.url()).map_err(Error::from)) + .collect() + }) .collect() } @@ -100,36 +108,40 @@ pub async fn download<'a>(matches: &ArgMatches, config: &Configuration<'a>, repo .filter(|p| pname.as_ref().map(|n| p.name() == n).unwrap_or(true)) .filter(|p| pvers.as_ref().map(|v| v.matches(p.version())).unwrap_or(true)) .map(|p| { - let source = sc.source_for(p); - let bar = multi.add(progressbars.download_bar(source.url())); - async move { - if source.exists() && !force { - Err(anyhow!("Source exists: {}", source.path().display())) - } else { - if source.exists() { - let _ = source.remove_file().await?; + sc.sources_for(p) + .into_iter() + .map(|source| { + let bar = multi.add(progressbars.download_bar(source.url())); + async move { + if source.exists() && !force { + Err(anyhow!("Source exists: {}", source.path().display())) + } else { + if source.exists() { + let _ = source.remove_file().await?; + } + + trace!("Starting download..."); + let file = source.create().await?; + let mut file = tokio::io::BufWriter::new(file); + let response = reqwest::get(source.url().as_ref()).await?; + if let Some(len) = response.content_length() { + bar.set_length(len); + } + let mut stream = reqwest::get(source.url().as_ref()).await?.bytes_stream(); + while let Some(bytes) = stream.next().await { + let bytes = bytes?; + file.write_all(bytes.as_ref()).await?; + bar.inc(bytes.len() as u64); + } + + file.flush().await?; + bar.finish_with_message("Finished download"); + Ok(()) + } } - - trace!("Starting download..."); - let file = source.create().await?; - let mut file = tokio::io::BufWriter::new(file); - let response = reqwest::get(source.url().as_ref()).await?; - if let Some(len) = response.content_length() { - bar.set_length(len); - } - let mut stream = reqwest::get(source.url().as_ref()).await?.bytes_stream(); - while let Some(bytes) = stream.next().await { - let bytes = bytes?; - file.write_all(bytes.as_ref()).await?; - bar.inc(bytes.len() as u64); - } - - file.flush().await?; - bar.finish_with_message("Finished download"); - Ok(()) - } - } + }) }) + .flatten() .collect::<futures::stream::FuturesUnordered<_>>() .collect::<Result<()>>() .await; diff --git a/src/config/util.rs b/src/config/util.rs index 17aeadf..ecd94c8 100644 --- a/src/config/util.rs +++ b/src/config/util.rs @@ -4,13 +4,12 @@ pub fn default_progress_format() -> String { pub fn default_package_print_format() -> String { String::from(indoc::indoc!(r#" - {{i}} - {{name}} : {{version}} - Source: {{source_url}} - Hash ({{source_hash_type}}): {{source_hash}} - {{#if print_system_deps}}System Deps: {{ system_deps }} {{/if}} - {{#if print_system_runtime_deps}}System runtime Deps: {{ system_runtime_deps }} {{/if}} - {{#if print_build_deps}}Build Deps: {{ build_deps }} {{/if}} - {{#if print_runtime_deps}}Runtime Deps: {{ runtime_deps }} {{/if}} + {{i}} - {{p.name}} : {{p.version}} + {{#each p.sources}}Source: {{this.url}} - {{this.hash.hash}} ({{this.hash.type}}){{/each}} + {{#if print_system_deps}}System Deps: {{ p.dependencies.system }} {{/if}} + {{#if print_system_runtime_deps}}System runtime Deps: {{ p.dependencies.system_runtime }} {{/if}} + {{#if print_build_deps}}Build Deps: {{ p.dependencies.build }} {{/if}} + {{#if print_runtime_deps}}Runtime Deps: {{ p.dependencies.runtime }} {{/if}} "#)) } diff --git a/src/endpoint/configured.rs b/src/endpoint/configured.rs index d09bd37..27e1823 100644 --- a/src/endpoint/configured.rs +++ b/src/endpoint/configured.rs @@ -9,6 +9,7 @@ use anyhow::Error; use anyhow::Result; use anyhow::anyhow; use futures::FutureExt; +use futures::future::TryFutureExt; use getset::{Getters, CopyGetters}; use shiplift::Docker; use shiplift::ExecContainerOptions; @@ -221,33 +222,39 @@ impl Endpoint { { // copy source to container use tokio::io::AsyncReadExt; - let pkgsource = job.package_source(); - let source_path = pkgsource.path(); - let destination = PathBuf::from("/inputs").join({ - source_path.file_name() - .ok_or_else(|| anyhow!("Not a file: {}", source_path.display())) - .with_context(|| anyhow!("Copying package source from {} to container {}", source_path.display(), self.name))? - }); - trace!("Package source = {:?}", pkgsource); - trace!("Source path = {:?}", source_path); - trace!("Source dest = {:?}", destination); - let mut buf = vec![]; - tokio::fs::OpenOptions::new() - .create(false) - .create_new(false) - .append(false) - .write(false) - .read(true) - .open(source_path) - .await - .with_context(|| anyhow!("Getting source file: {}", source_path.display()))? - .read_to_end(&mut buf) - .await - .with_context(|| anyhow!("Reading file {}", source_path.display()))?; + job.package_sources() + .into_iter() + .map(|entry| async { + let source_path = entry.path(); + let destination = PathBuf::from("/inputs").join({ + source_path.file_name() + .ok_or_else(|| anyhow!("Not a file: {}", source_path.display())) + .with_context(|| anyhow!("Copying package source from {} to container {}", source_path.display(), self.name))? + }); + trace!("Source path = {:?}", source_path); + trace!("Source dest = {:?}", destination); + let mut buf = vec![]; + tokio::fs::OpenOptions::new() + .create(false) + .create_new(false) + .append(false) + .write(false) + .read(true) + .open(&source_path) + .await + .with_context(|| anyhow!("Getting source file: {}", source_path.display()))? + .read_to_end(&mut buf) + .await + .with_context(|| anyhow!("Reading file {}", source_path.display()))?; - let _ = container.copy_file_into(destination, &buf) + drop(entry); + let _ = container.copy_file_into(destination, &buf).await?; + Ok(()) + }) + .collect::<futures::stream::FuturesUnordered<_>>() + .collect::<Result<()>>() .await - .with_context(|| anyhow!("Copying {} to container {}", source_path.display(), container_id))?; + .with_context(|| anyhow!("Copying sources to container {}", container_id))?; } { // Copy all Path artifacts to the container job.resources() diff --git a/src/job/runnable.rs b/src/job/runnable.rs index 67219fb..ab2d6ca 100644 --- a/src/job/runnable.rs +++ b/src/job/runnable.rs @@ -80,8 +80,8 @@ impl RunnableJob { } - pub fn package_source(&self) -> SourceEntry { - self.source_cache.source_for(&self.package()) + pub fn package_sources(&self) -> Vec<SourceEntry> { + self.source_cache.sources_for(&self.package()) } pub fn environment(&self) -> Vec<(String, String)> { diff --git a/src/package/package.rs b/src/package/package.rs index c9ad4f0..698b8b1 100644 --- a/src/package/package.rs +++ b/src/package/package.rs @@ -26,7 +26,7 @@ pub struct Package { version_is_semver: bool, #[getset(get = "pub")] - source: Source, + sources: Vec<Source>, #[getset(get = "pub")] dependencies: Dependencies, @@ -53,12 +53,12 @@ pub struct Package { impl Package { #[cfg(test)] - pub fn new(name: PackageName, version: PackageVersion, version_is_semver: bool, source: Source, dependencies: Dependencies) -> Self { + pub fn new(name: PackageName, version: PackageVersion, version_is_semver: bool, sources: Vec<Source>, dependencies: Dependencies) -> Self { Package { name, version, version_is_semver, - source, + sources, dependencies, patches: vec![], environment: None, @@ -216,13 +216,13 @@ pub mod tests { let name = pname(name); let version = pversion(vers); let version_is_semver = false; - let source = { + let sources = { let url = Url::parse(srcurl).unwrap(); let hashvalue = HashValue::from(String::from(hash)); - Source::new(url, SourceHash::new(HashType::Sha1, hashvalue)) + vec![Source::new(url, SourceHash::new(HashType::Sha1, hashvalue))] }; let dependencies = Dependencies::empty(); - Package::new(name, version, version_is_semver, source, dependencies) + Package::new(name, version, version_is_semver, sources, dependencies) } } diff --git a/src/source/mod.rs b/src/source/mod.rs index 455785c..c89021e 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -19,7 +19,7 @@ impl SourceCache { SourceCache { root } } - pub fn source_for(&self, p: &Package) -> SourceEntry { + pub fn sources_for(&self, p: &Package) -> Vec<SourceEntry> { SourceEntry::for_package(self.root.clone(), p) } } @@ -30,29 +30,35 @@ pub struct SourceEntry { package_name: PackageName, package_version: PackageVersion, package_source: Source, - package_source_path: PathBuf, } impl SourceEntry { - fn for_package(cache_root: PathBuf, package: &Package) -> Self { - let package_source_path = cache_root.join(format!("{}-{}.source", package.name(), package.version())); + fn source_file_path(&self) -> PathBuf { + self.cache_root.join(format!("{}-{}/{}.source", self.package_name, self.package_version, self.package_source.hash().value())) + } - SourceEntry { - cache_root, - package_name: package.name().clone(), - package_version: package.version().clone(), - package_source: package.source().clone(), - package_source_path - } + fn for_package(cache_root: PathBuf, package: &Package) -> Vec<Self> { + package.sources() + .clone() + .into_iter() + .map(|source| { + SourceEntry { + cache_root: cache_root.clone(), + package_name: package.name().clone(), + package_version: package.version().clone(), + package_source: source, + } + }) + .collect() } pub fn exists(&self) -> bool { - self.package_source_path.exists() + self.source_file_path().exists() } - pub fn path(&self) -> &PathBuf { - &self.package_source_path + pub fn path(&self) -> PathBuf { + self.source_file_path() } pub fn url(&self) -> &Url { @@ -60,41 +66,50 @@ impl SourceEntry { } pub async fn remove_file(&self) -> Result<()> { - tokio::fs::remove_file(&self.package_source_path).await.map_err(Error::from) + let p = self.source_file_path(); + tokio::fs::remove_file(&p).await?; + Ok(()) } pub async fn verify_hash(&self) -> Result<bool> { use tokio::io::AsyncReadExt; + let p = self.source_file_path(); let mut buf = vec![]; tokio::fs::OpenOptions::new() .create(false) .create_new(false) .read(true) - .open(&self.package_source_path) + .open(&p) .await? .read_to_end(&mut buf) .await?; - self.package_source.hash().matches_hash_of(&buf) + self.package_source + .hash() + .matches_hash_of(&buf) } pub async fn open(&self) -> Result<tokio::fs::File> { + let p = self.source_file_path(); + tokio::fs::OpenOptions::new() .create(false) .create_new(false) .read(true) - .open(&self.package_source_path) + .open(&p) .await .map_err(Error::from) } pub async fn create(&self) -> Result<tokio::fs::File> { + let p = self.source_file_path(); + tokio::fs::OpenOptions::new() .create(true) .create_new(true) .write(true) - .open(&self.package_source_path) + .open(&p) .await .map_err(Error::from) } @@ -61,57 +61,24 @@ fn print_package(out: &mut dyn Write, -> Result<()> { let mut data = BTreeMap::new(); - data.insert("i", i.to_string()); - data.insert("name", format!("{}", package.name())); - data.insert("version", format!("{}", package.version())); - data.insert("source_url", format!("{}", package.source().url())); - data.insert("source_hash_type", format!("{}", package.source().hash().hashtype())); - data.insert("source_hash", format!("{}", package.source().hash().value())); + data.insert("i", serde_json::Value::Number(serde_json::Number::from(i))); + data.insert("p", serde_json::to_value(package)?); // This is an ugly hack. Because the `data` is a <String, String>, we do only insert the flag // if it is set, because handlebars renders a non-present value as false if print_runtime_deps { - data.insert("print_runtime_deps", format!("{}", print_runtime_deps)); + data.insert("print_runtime_deps", serde_json::Value::Bool(print_runtime_deps)); } if print_build_deps { - data.insert("print_build_deps", format!("{}", print_build_deps)); + data.insert("print_build_deps", serde_json::Value::Bool(print_build_deps)); } if print_sys_deps { - data.insert("print_system_deps", format!("{}", print_sys_deps)); + data.insert("print_system_deps", serde_json::Value::Bool(print_sys_deps)); } if print_sys_runtime_deps { - data.insert("print_system_runtime_deps", format!("{}", print_sys_runtime_deps)); + data.insert("print_system_runtime_deps", serde_json::Value::Bool(print_sys_runtime_deps)); } - data.insert("runtime_deps", { - format!("[{}]", package.dependencies() - .runtime() - .iter() - .map(|p| p.as_ref()) - .join(", ")) - }); - data.insert("build_deps", { - format!("[{}]", package.dependencies() - .build() - .iter() - .map(|p| p.as_ref()) - .join(", ")) - }); - data.insert("system_deps", { - format!("[{}]", package.dependencies() - .system() - .iter() - .map(|p| p.as_ref()) - .join(", ")) - }); - data.insert("system_runtime_deps", { - format!("[{}]", package.dependencies() - .system_runtime() - .iter() - .map(|p| p.as_ref()) - .join(", ")) - }); - hb.render("package", &data) .map_err(Error::from) .and_then(|r| writeln!(out, "{}", r).map_err(Error::from)) |