Merge branch 'parallelize-package-print-preparation'

6 files changed, 323 insertions, 248 deletions

diff --git a/src/commands/dependencies_of.rs b/src/commands/dependencies_of.rs
index 3e4ff21..827da92 100644
--- a/src/commands/dependencies_of.rs
+++ b/src/commands/dependencies_of.rs
@@ -10,14 +10,19 @@
 
 //! Implementation of the 'dependencies-of' subcommand
 
+use std::io::Write;
+
 use anyhow::Result;
 use clap::ArgMatches;
+use futures::stream::StreamExt;
+use futures::stream::TryStreamExt;
 use log::trace;
 
 use crate::commands::util::getbool;
 use crate::config::*;
 use crate::package::PackageName;
 use crate::repository::Repository;
+use crate::ui::*;
 
 /// Implementation of the "dependencies_of" subcommand
 pub async fn dependencies_of(
@@ -39,11 +44,9 @@ pub async fn dependencies_of(
     };
 
     let format = config.package_print_format();
-    let mut stdout = std::io::stdout();
-    let iter = repo
-        .packages()
-        .filter(|package| package_filter.filter(package))
-        .inspect(|pkg| trace!("Found package: {:?}", pkg));
+    let hb = crate::ui::handlebars_for_package_printing(format)?;
+    let stdout = std::io::stdout();
+    let mut outlock = stdout.lock();
 
     let print_runtime_deps = getbool(
         matches,
@@ -80,5 +83,18 @@ pub async fn dependencies_of(
         script_highlighting: false,
     };
 
-    crate::ui::print_packages(&mut stdout, format, iter, config, &flags)
+    let iter = repo
+        .packages()
+        .filter(|package| package_filter.filter(package))
+        .inspect(|pkg| trace!("Found package: {:?}", pkg))
+        .enumerate()
+        .map(|(i, p)| p.prepare_print(config, &flags, &hb, i));
+
+    tokio_stream::iter(iter)
+        .map(|pp| pp.into_displayable())
+        .try_for_each(|p| {
+            let r = writeln!(&mut outlock, "{}", p).map_err(anyhow::Error::from);
+            futures::future::ready(r)
+        })
+        .await
 }
diff --git a/src/commands/find_pkg.rs b/src/commands/find_pkg.rs
index d45c946..8982911 100644
--- a/src/commands/find_pkg.rs
+++ b/src/commands/find_pkg.rs
@@ -16,10 +16,13 @@ use anyhow::Context;
 use anyhow::Result;
 use clap::ArgMatches;
 use log::trace;
+use futures::stream::StreamExt;
+use futures::stream::TryStreamExt;
 
 use crate::config::Configuration;
 use crate::package::PackageVersionConstraint;
 use crate::repository::Repository;
+use crate::ui::*;
 
 /// Implementation of the "find_pkg" subcommand
 pub async fn find_pkg(
@@ -85,6 +88,17 @@ pub async fn find_pkg(
         };
 
         let format = config.package_print_format();
-        crate::ui::print_packages(&mut outlock, format, iter, config, &flags)
+        let hb = crate::ui::handlebars_for_package_printing(format)?;
+
+        tokio_stream::iter({
+            iter.enumerate()
+                .map(|(i, p)| p.prepare_print(config, &flags, &hb, i))
+        })
+        .map(|pp| pp.into_displayable())
+        .try_for_each(|p| {
+            let r = writeln!(&mut outlock, "{}", p).map_err(anyhow::Error::from);
+            futures::future::ready(r)
+        })
+        .await
     }
 }
diff --git a/src/commands/what_depends.rs b/src/commands/what_depends.rs
index 10c9edc..1b28620 100644
--- a/src/commands/what_depends.rs
+++ b/src/commands/what_depends.rs
@@ -10,8 +10,12 @@
 
 //! Implementation of the 'what_depends' subcommand
 
+use std::io::Write;
+
 use anyhow::Result;
 use clap::ArgMatches;
+use futures::stream::StreamExt;
+use futures::stream::TryStreamExt;
 use log::trace;
 use resiter::Filter;
 use resiter::Map;
@@ -20,6 +24,7 @@ use crate::commands::util::getbool;
 use crate::config::*;
 use crate::package::PackageName;
 use crate::repository::Repository;
+use crate::ui::*;
 
 /// Implementation of the "what_depends" subcommand
 pub async fn what_depends(
@@ -54,16 +59,9 @@ pub async fn what_depends(
         )
     };
 
-    let format = config.package_print_format();
-    let mut stdout = std::io::stdout();
-
-    let packages = repo
-        .packages()
-        .map(|package| package_filter.filter(package).map(|b| (b, package)))
-        .filter_ok(|(b, _)| *b)
-        .map_ok(|tpl| tpl.1)
-        .inspect(|pkg| trace!("Found package: {:?}", pkg))
-        .collect::<Result<Vec<_>>>()?;
+    let hb = crate::ui::handlebars_for_package_printing(config.package_print_format())?;
+    let stdout = std::io::stdout();
+    let mut outlock = stdout.lock();
 
     let flags = crate::ui::PackagePrintFlags {
         print_all: false,
@@ -82,5 +80,23 @@ pub async fn what_depends(
         script_highlighting: false,
     };
 
-    crate::ui::print_packages(&mut stdout, format, packages.into_iter(), config, &flags)
+    let mut i = 0;
+    let iter = repo
+        .packages()
+        .map(|package| package_filter.filter(package).map(|b| (b, package)))
+        .filter_ok(|(b, _)| *b)
+        .map_ok(|tpl| tpl.1)
+        .inspect(|pkg| trace!("Found package: {:?}", pkg))
+        .map_ok(|p| { // poor mans enumerate_ok()
+            i += 1;
+            p.prepare_print(config, &flags, &hb, i)
+        });
+
+    tokio_stream::iter(iter)
+        .map(|pp| pp.and_then(|p| p.into_displayable()))
+        .try_for_each(|p| {
+            let r = writeln!(&mut outlock, "{}", p).map_err(anyhow::Error::from);
+            futures::future::ready(r)
+        })
+        .await
 }
diff --git a/src/ui.rs b/src/ui.rs
deleted file mode 100644
index 526db8c..0000000
--- a/src/ui.rs
+++ /dev/null
@@ -1,230 +0,0 @@
-//
-// Copyright (c) 2020-2021 science+computing ag and other contributors
-//
-// This program and the accompanying materials are made
-// available under the terms of the Eclipse Public License 2.0
-// which is available at https://www.eclipse.org/legal/epl-2.0/
-//
-// SPDX-License-Identifier: EPL-2.0
-//
-
-//! Utility functions for the UI
-
-use std::collections::BTreeMap;
-use std::io::Write;
-use std::path::Path;
-use std::path::PathBuf;
-
-use anyhow::anyhow;
-use anyhow::Context;
-use anyhow::Error;
-use anyhow::Result;
-use handlebars::Handlebars;
-use itertools::Itertools;
-use log::error;
-
-use crate::config::Configuration;
-use crate::package::Package;
-use crate::package::Script;
-use crate::package::ScriptBuilder;
-use crate::package::Shebang;
-
-pub fn package_repo_cleanness_check(repo: &git2::Repository) -> Result<()> {
-    if !crate::util::git::repo_is_clean(&repo)? {
-        error!(
-            "Repository not clean, refusing to go on: {}",
-            repo.path().display()
-        );
-        Err(anyhow!(
-            "Repository not clean, refusing to go on: {}",
-            repo.path().display()
-        ))
-    } else {
-        Ok(())
-    }
-}
-
-pub struct PackagePrintFlags {
-    pub print_all: bool,
-    pub print_runtime_deps: bool,
-    pub print_build_deps: bool,
-    pub print_sources: bool,
-    pub print_dependencies: bool,
-    pub print_patches: bool,
-    pub print_env: bool,
-    pub print_flags: bool,
-    pub print_allowed_images: bool,
-    pub print_denied_images: bool,
-    pub print_phases: bool,
-    pub print_script: bool,
-    pub script_line_numbers: bool,
-    pub script_highlighting: bool,
-}
-
-impl PackagePrintFlags {
-    // Helper to check whether any of the CLI args requested one of these flags.
-    //
-    // The print_build_deps and print_runtime_deps as well as the script_highlighting and
-    // script_line_numbers is not included, because these only modify what to print and not whether
-    // to print.
-    fn print_any(&self) -> bool {
-        self.print_all || {
-            self.print_sources
-                || self.print_dependencies
-                || self.print_patches
-                || self.print_env
-                || self.print_flags
-                || self.print_allowed_images
-                || self.print_denied_images
-                || self.print_phases
-                || self.print_script
-        }
-    }
-}
-
-pub fn print_packages<'a, I>(
-    out: &mut dyn Write,
-    format: &str,
-    iter: I,
-    config: &Configuration,
-    flags: &PackagePrintFlags,
-) -> Result<()>
-where
-    I: Iterator<Item = &'a Package>,
-{
-    let mut hb = Handlebars::new();
-    hb.register_escape_fn(handlebars::no_escape);
-    hb.register_template_string("package", format)?;
-
-    iter.enumerate()
-        .try_for_each(|(i, package)| print_package(out, &hb, i, package, config, flags))
-}
-
-fn print_package(
-    out: &mut dyn Write,
-    hb: &Handlebars,
-    i: usize,
-    package: &Package,
-    config: &Configuration,
-    flags: &PackagePrintFlags,
-) -> Result<()> {
-    let script = ScriptBuilder::new(&Shebang::from(config.shebang().clone())).build(
-        package,
-        config.available_phases(),
-        *config.strict_script_interpolation(),
-    ).context("Rendering script for printing it failed")?;
-
-    let script = crate::ui::script_to_printable(
-        &script,
-        flags.script_highlighting,
-        config
-            .script_highlight_theme()
-            .as_ref()
-            .ok_or_else(|| anyhow!("Highlighting for script enabled, but no theme configured"))?,
-        flags.script_line_numbers,
-    )?;
-
-    let mut data = BTreeMap::new();
-    data.insert("i", serde_json::Value::Number(serde_json::Number::from(i)));
-    data.insert("p", serde_json::to_value(package)?);
-    data.insert("script", serde_json::Value::String(script));
-    data.insert("print_any", serde_json::Value::Bool(flags.print_any()));
-    data.insert(
-        "print_runtime_deps",
-        serde_json::Value::Bool(flags.print_runtime_deps),
-    );
-    data.insert(
-        "print_build_deps",
-        serde_json::Value::Bool(flags.print_build_deps),
-    );
-
-    data.insert(
-        "print_sources",
-        serde_json::Value::Bool(flags.print_all || flags.print_sources),
-    );
-    data.insert(
-        "print_dependencies",
-        serde_json::Value::Bool(flags.print_all || flags.print_dependencies),
-    );
-    data.insert(
-        "print_patches",
-        serde_json::Value::Bool(flags.print_all || flags.print_patches),
-    );
-    data.insert(
-        "print_env",
-        serde_json::Value::Bool(flags.print_all || flags.print_env),
-    );
-    data.insert(
-        "print_flags",
-        serde_json::Value::Bool(flags.print_all || flags.print_flags),
-    );
-    data.insert(
-        "print_allowed_images",
-        serde_json::Value::Bool(flags.print_all || flags.print_allowed_images),
-    );
-    data.insert(
-        "print_denied_images",
-        serde_json::Value::Bool(flags.print_all || flags.print_denied_images),
-    );
-    data.insert(
-        "print_phases",
-        serde_json::Value::Bool(flags.print_all || flags.print_phases),
-    );
-    data.insert(
-        "print_script",
-        serde_json::Value::Bool(flags.print_all || flags.print_script),
-    );
-
-    hb.render("package", &data)
-        .map_err(Error::from)
-        .and_then(|r| writeln!(out, "{}", r).map_err(Error::from))
-}
-
-pub fn script_to_printable(
-    script: &Script,
-    highlight: bool,
-    highlight_theme: &str,
-    line_numbers: bool,
-) -> Result<String> {
-    let script = if highlight {
-        let script = script.highlighted(highlight_theme);
-        if line_numbers {
-            script
-                .lines_numbered()?
-                .map(|(i, s)| format!("{:>4} | {}", i, s))
-                .join("")
-        } else {
-            script.lines()?.join("")
-        }
-    } else if line_numbers {
-        script
-            .lines_numbered()
-            .map(|(i, s)| format!("{:>4} | {}", i, s))
-            .join("")
-    } else {
-        script.to_string()
-    };
-
-    Ok(script)
-}
-
-pub fn find_linter_command(repo_path: &Path, config: &Configuration) -> Result<Option<PathBuf>> {
-    match config.script_linter().as_ref() {
-        None => Ok(None),
-        Some(linter) => {
-            if linter.is_absolute() {
-                Ok(Some(linter.to_path_buf()))
-            } else {
-                let linter = repo_path.join(linter);
-                if !linter.is_file() {
-                    Err(anyhow!(
-                        "Cannot find linter command, searched in: {}",
-                        linter.display()
-                    ))
-                } else {
-                    Ok(Some(linter))
-                }
-            }
-        }
-    }
-}
diff --git a/src/ui/mod.rs b/src/ui/mod.rs
new file mode 100644
index 0000000..b0d2167
--- /dev/null
+++ b/src/ui/mod.rs
@@ -0,0 +1,89 @@
+//
+// Copyright (c) 2020-2021 science+computing ag and other contributors
+//
+// This program and the accompanying materials are made
+// available under the terms of the Eclipse Public License 2.0
+// which is available at https://www.eclipse.org/legal/epl-2.0/
+//
+// SPDX-License-Identifier: EPL-2.0
+//
+
+//! Utility functions for the UI
+
+use std::path::Path;
+use std::path::PathBuf;
+
+use anyhow::Result;
+use anyhow::anyhow;
+use itertools::Itertools;
+use log::error;
+
+use crate::config::Configuration;
+use crate::package::Script;
+
+mod package;
+pub use crate::ui::package::*;
+
+pub fn package_repo_cleanness_check(repo: &git2::Repository) -> Result<()> {
+    if !crate::util::git::repo_is_clean(&repo)? {
+        error!(
+            "Repository not clean, refusing to go on: {}",
+            repo.path().display()
+        );
+        Err(anyhow!(
+            "Repository not clean, refusing to go on: {}",
+            repo.path().display()
+        ))
+    } else {
+        Ok(())
+    }
+}
+
+pub fn script_to_printable(
+    script: &Script,
+    highlight: bool,
+    highlight_theme: &str,
+    line_numbers: bool,
+) -> Result<String> {
+    let script = if highlight {
+        let script = script.highlighted(highlight_theme);
+        if line_numbers {
+            script
+                .lines_numbered()?
+                .map(|(i, s)| format!("{:>4} | {}", i, s))
+                .join("")
+        } else {
+            script.lines()?.join("")
+        }
+    } else if line_numbers {
+        script
+            .lines_numbered()
+            .map(|(i, s)| format!("{:>4} | {}", i, s))
+            .join("")
+    } else {
+        script.to_string()
+    };
+
+    Ok(script)
+}
+
+pub fn find_linter_command(repo_path: &Path, config: &Configuration) -> Result<Option<PathBuf>> {
+    match config.script_linter().as_ref() {
+        None => Ok(None),
+        Some(linter) => {
+            if linter.is_absolute() {
+                Ok(Some(linter.to_path_buf()))
+            } else {
+                let linter = repo_path.join(linter);
+                if !linter.is_file() {
+                    Err(anyhow!(
+                        "Cannot find linter command, searched in: {}",
+                        linter.display()
+                    ))
+                } else {
+                    Ok(Some(linter))
+                }
+            }
+        }
+    }
+}
diff --git a/src/ui/package.rs b/src/ui/package.rs
new file mode 100644
index 0000000..5ac27b0
--- /dev/null
+++ b/src/ui/package.rs
@@ -0,0 +1,170 @@
+use std::borrow::Borrow;
+use std::collections::BTreeMap;
+
+use anyhow::anyhow;
+use anyhow::Context;
+use anyhow::Result;
+use handlebars::Handlebars;
+
+use crate::config::Configuration;
+use crate::package::Package;
+use crate::package::ScriptBuilder;
+use crate::package::Shebang;
+
+pub struct PackagePrintFlags {
+    pub print_all: bool,
+    pub print_runtime_deps: bool,
+    pub print_build_deps: bool,
+    pub print_sources: bool,
+    pub print_dependencies: bool,
+    pub print_patches: bool,
+    pub print_env: bool,
+    pub print_flags: bool,
+    pub print_allowed_images: bool,
+    pub print_denied_images: bool,
+    pub print_phases: bool,
+    pub print_script: bool,
+    pub script_line_numbers: bool,
+    pub script_highlighting: bool,
+}
+
+impl PackagePrintFlags {
+    // Helper to check whether any of the CLI args requested one of these flags.
+    //
+    // The print_build_deps and print_runtime_deps as well as the script_highlighting and
+    // script_line_numbers is not included, because these only modify what to print and not whether
+    // to print.
+    fn print_any(&self) -> bool {
+        self.print_all || {
+            self.print_sources
+                || self.print_dependencies
+                || self.print_patches
+                || self.print_env
+                || self.print_flags
+                || self.print_allowed_images
+                || self.print_denied_images
+                || self.print_phases
+                || self.print_script
+        }
+    }
+}
+
+
+pub trait PreparePrintable<'a>
+    where Self: Borrow<Package> + Sized
+{
+    fn prepare_print(self, config: &'a Configuration, flags: &'a PackagePrintFlags, handlebars: &'a Handlebars<'a>, i: usize) -> PreparePrintPackage<'a, Self>;
+}
+
+impl<'a, P> PreparePrintable<'a> for P
+    where P: Borrow<Package>
+{
+    fn prepare_print(self, config: &'a Configuration, flags: &'a PackagePrintFlags, handlebars: &'a Handlebars<'a>, i: usize) -> PreparePrintPackage<'a, P> {
+        PreparePrintPackage {
+            package: self,
+            config,
+            flags,
+            handlebars,
+            i,
+        }
+    }
+}
+
+pub struct PreparePrintPackage<'a, P: Borrow<Package>> {
+    package: P,
+    config: &'a Configuration,
+    flags: &'a PackagePrintFlags,
+    handlebars: &'a Handlebars<'a>,
+    i: usize,
+}
+
+
+pub fn handlebars_for_package_printing(format: &str) -> Result<Handlebars> {
+    let mut hb = Handlebars::new();
+    hb.register_escape_fn(handlebars::no_escape);
+    hb.register_template_string("package", format)?;
+    Ok(hb)
+}
+
+impl<'a, P: Borrow<Package>> PreparePrintPackage<'a, P> {
+    pub fn into_displayable(self) -> Result<PrintablePackage> {
+        let script = ScriptBuilder::new(&Shebang::from(self.config.shebang().clone())).build(
+            self.package.borrow(),
+            self.config.available_phases(),
+            *self.config.strict_script_interpolation(),
+        ).context("Rendering script for printing it failed")?;
+
+        let script = crate::ui::script_to_printable(
+            &script,
+            self.flags.script_highlighting,
+            self.config
+                .script_highlight_theme()
+                .as_ref()
+                .ok_or_else(|| anyhow!("Highlighting for script enabled, but no theme configured"))?,
+            self.flags.script_line_numbers,
+        )?;
+
+        let mut data = BTreeMap::new();
+        data.insert("i", serde_json::Value::Number(serde_json::Number::from(self.i)));
+        data.insert("p", serde_json::to_value(self.package.borrow())?);
+        data.insert("script", serde_json::Value::String(script));
+        data.insert("print_any", serde_json::Value::Bool(self.flags.print_any()));
+        data.insert(
+            "print_runtime_deps",
+            serde_json::Value::Bool(self.flags.print_runtime_deps),
+        );
+        data.insert(
+            "print_build_deps",
+            serde_json::Value::Bool(self.flags.print_build_deps),
+        );
+
+        data.insert(
+            "print_sources",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_sources),
+        );
+        data.insert(
+            "print_dependencies",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_dependencies),
+        );
+        data.insert(
+            "print_patches",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_patches),
+        );
+        data.insert(
+            "print_env",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_env),
+        );
+        data.insert(
+            "print_flags",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_flags),
+        );
+        data.insert(
+            "print_allowed_images",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_allowed_images),
+        );
+        data.insert(
+            "print_denied_images",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_denied_images),
+        );
+        data.insert(
+            "print_phases",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_phases),
+        );
+        data.insert(
+            "print_script",
+            serde_json::Value::Bool(self.flags.print_all || self.flags.print_script),
+        );
+
+        let string = self.handlebars.render("package", &data)?;
+        Ok(PrintablePackage { string })
+    }
+}
+
+pub struct PrintablePackage { string: String }
+
+impl std::fmt::Display for PrintablePackage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{}", self.string)
+    }
+}
+