Merge branch 'imag-mail/scan' into master

Signed-off-by: Matthias Beyer <mail@beyermatthias.de>

3 files changed, 94 insertions, 0 deletions

diff --git a/bin/domain/imag-mail/Cargo.toml b/bin/domain/imag-mail/Cargo.toml
index ef543231..fd871089 100644
--- a/bin/domain/imag-mail/Cargo.toml
+++ b/bin/domain/imag-mail/Cargo.toml
@@ -24,6 +24,8 @@ log = "0.4.6"
 failure = "0.1.5"
 resiter = "0.4"
 handlebars = "2"
+walkdir = "2"
+rayon = "1"
 
 libimagrt       = { version = "0.10.0", path = "../../../lib/core/libimagrt" }
 libimagstore    = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
diff --git a/bin/domain/imag-mail/src/lib.rs b/bin/domain/imag-mail/src/lib.rs
index f428bf3f..4bb217ff 100644
--- a/bin/domain/imag-mail/src/lib.rs
+++ b/bin/domain/imag-mail/src/lib.rs
@@ -40,6 +40,8 @@ extern crate clap;
 extern crate toml_query;
 extern crate resiter;
 extern crate handlebars;
+extern crate walkdir;
+extern crate rayon;
 
 extern crate libimagrt;
 extern crate libimagmail;
@@ -58,10 +60,14 @@ use failure::Error;
 use toml_query::read::TomlValueReadTypeExt;
 use clap::App;
 use resiter::AndThen;
+use resiter::Filter;
 use resiter::IterInnerOkOrElse;
+use resiter::Map;
+use rayon::prelude::*;
 
 use libimagmail::store::MailStore;
 use libimagentryref::util::get_ref_config;
+use libimagentryref::reference::Config as RefConfig;
 use libimagrt::runtime::Runtime;
 use libimagrt::application::ImagApplication;
 use libimagutil::info_result::*;
@@ -79,6 +85,7 @@ pub enum ImagMail {}
 impl ImagApplication for ImagMail {
     fn run(rt: Runtime) -> Result<()> {
         match rt.cli().subcommand_name().ok_or_else(|| err_msg("No subcommand called"))? {
+            "scan"        => scan(&rt),
             "import-mail" => import_mail(&rt),
             "list"        => list(&rt),
             "mail-store"  => mail_store(&rt),
@@ -110,6 +117,64 @@ impl ImagApplication for ImagMail {
     }
 }
 
+fn scan(rt: &Runtime) -> Result<()> {
+    let collection_name = get_ref_collection_name(rt)?;
+    let refconfig       = get_ref_config(rt, "imag-mail")?;
+    let scmd            = rt.cli().subcommand_matches("scan").unwrap();
+
+    /// Helper function to get an Iterator for all files from one PathBuf
+    fn walk_pathes(path: PathBuf) -> impl Iterator<Item = Result<PathBuf>> {
+        walkdir::WalkDir::new(path)
+            .follow_links(false)
+            .into_iter()
+            .filter_ok(|entry| entry.file_type().is_file())
+            .map_err(Error::from)
+            .map_ok(|entry| entry.into_path())
+            .inspect(|e| trace!("Processing = {:?}", e))
+    }
+
+    /// Helper function to process an iterator over Result<PathBuf> and create store entries for
+    /// each path in the iterator
+    fn process_iter(i: &mut dyn Iterator<Item = Result<PathBuf>>, rt: &Runtime, collection_name: &str, refconfig: &RefConfig) -> Result<()> {
+        let scmd = rt.cli().subcommand_matches("scan").unwrap();
+        i.and_then_ok(|path| {
+                if scmd.is_present("ignore-existing-ids") {
+                    rt.store().retrieve_mail_from_path(path, collection_name, refconfig, true)
+                } else {
+                    rt.store().create_mail_from_path(path, collection_name, refconfig)
+                }
+            })
+            .and_then_ok(|e| rt.report_touched(e.get_location()).map_err(Error::from))
+            .collect::<Result<Vec<_>>>()
+            .map(|_| ())
+    }
+
+    let pathes = scmd.values_of("path")
+        .unwrap() // enforced by clap
+        .map(PathBuf::from)
+        .collect::<Vec<_>>();
+
+    if scmd.is_present("scan-parallel") {
+        debug!("Fetching pathes in parallel");
+        let mut i = pathes.into_par_iter()
+            .map(|path| walk_pathes(path).collect::<Result<_>>())
+            .collect::<Result<Vec<PathBuf>>>()?
+            .into_iter()
+            .map(Ok);
+
+        debug!("Processing pathes");
+        process_iter(&mut i, rt, &collection_name, &refconfig)
+    } else {
+        debug!("Fetching pathes not in parallel");
+        let mut i = pathes
+            .into_iter()
+            .map(walk_pathes)
+            .flatten();
+
+        debug!("Processing pathes");
+        process_iter(&mut i, rt, &collection_name, &refconfig)
+    }
+}
 
 fn import_mail(rt: &Runtime) -> Result<()> {
     let collection_name = get_ref_collection_name(rt)?;
diff --git a/bin/domain/imag-mail/src/ui.rs b/bin/domain/imag-mail/src/ui.rs
index 51142867..2462e15e 100644
--- a/bin/domain/imag-mail/src/ui.rs
+++ b/bin/domain/imag-mail/src/ui.rs
@@ -47,6 +47,33 @@ pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> {
                          .value_name("PATH"))
                     )
 
+        .subcommand(SubCommand::with_name("scan")
+                    .about("Scan a directory for mails")
+                    .version("0.1")
+                    .arg(Arg::with_name("ignore-existing-ids")
+                         .long("ignore-existing")
+                         .short("I")
+                         .takes_value(false)
+                         .required(false)
+                         .help("Ignore errors that might occur when store entries exist already"))
+
+                    .arg(Arg::with_name("scan-parallel")
+                         .long("parallel")
+                         .takes_value(false)
+                         .required(false)
+                         .multiple(false)
+                         .help("Scan with multiple threads. Might be faster, but might slow down other tasks"))
+
+                    .arg(Arg::with_name("path")
+                         .index(1)
+                         .takes_value(true)
+                         .multiple(true)
+                         .required(true)
+                         .validator(libimagutil::cli_validators::is_directory)
+                         .value_name("DIR")
+                         .help("Path to the directory containing mails"))
+                    )
+
         .subcommand(SubCommand::with_name("list")
                     .about("List all stored references to mails")
                     .version("0.1")