imag-mail: Reimplement

Signed-off-by: Matthias Beyer <mail@beyermatthias.de>

6 files changed, 294 insertions, 603 deletions

diff --git a/bin/domain/imag-mail/Cargo.toml b/bin/domain/imag-mail/Cargo.toml
index 32cae649..91eb1e40 100644
--- a/bin/domain/imag-mail/Cargo.toml
+++ b/bin/domain/imag-mail/Cargo.toml
@@ -23,9 +23,7 @@ maintenance                       = { status     = "actively-developed" }
 log = "0.4.6"
 anyhow = "1"
 resiter = "0.4"
-handlebars = "3"
-walkdir = "2"
-rayon = "1"
+handlebars = "2"
 
 libimagrt       = { version = "0.10.0", path = "../../../lib/core/libimagrt" }
 libimagstore    = { version = "0.10.0", path = "../../../lib/core/libimagstore" }
diff --git a/bin/domain/imag-mail/src/lib.rs b/bin/domain/imag-mail/src/lib.rs
index ceee854d..30df9d0c 100644
--- a/bin/domain/imag-mail/src/lib.rs
+++ b/bin/domain/imag-mail/src/lib.rs
@@ -40,8 +40,6 @@ extern crate clap;
 extern crate toml_query;
 extern crate resiter;
 extern crate handlebars;
-extern crate walkdir;
-extern crate rayon;
 
 extern crate libimagrt;
 extern crate libimagmail;
@@ -51,28 +49,27 @@ extern crate libimagutil;
 extern crate libimagentryref;
 extern crate libimaginteraction;
 
-use std::path::PathBuf;
+use std::io::Write;
 
-use anyhow::Result;
-
-use anyhow::Error;
-use toml_query::read::TomlValueReadTypeExt;
+use failure::Fallible as Result;
+use failure::err_msg;
 use clap::App;
 use resiter::AndThen;
-use resiter::Filter;
 use resiter::IterInnerOkOrElse;
+use resiter::Filter;
 use resiter::Map;
-use rayon::prelude::*;
+use handlebars::Handlebars;
 
 use libimagmail::store::MailStore;
+use libimagmail::store::Sorting;
 use libimagmail::mail::Mail;
-use libimagentryref::util::get_ref_config;
-use libimagentryref::reference::Config as RefConfig;
+use libimagmail::store::MailStoreWithConnection;
+use libimagmail::mailtree::Mailtree;
+use libimagmail::notmuch::connection::NotmuchConnection;
 use libimagrt::runtime::Runtime;
 use libimagrt::application::ImagApplication;
-use libimagutil::info_result::*;
-use libimagstore::storeid::StoreIdIterator;
 use libimagstore::iter::get::StoreIdGetIteratorExtension;
+use libimagstore::store::FileLockEntry;
 
 mod ui;
 mod util;
@@ -84,13 +81,11 @@ mod util;
 pub enum ImagMail {}
 impl ImagApplication for ImagMail {
     fn run(rt: Runtime) -> Result<()> {
-        match rt.cli().subcommand_name().ok_or_else(|| anyhow!("No subcommand called"))? {
-            "scan"        => scan(&rt),
-            "import-mail" => import_mail(&rt),
-            "list"        => list(&rt),
-            "unread"      => unread(&rt),
-            "mail-store"  => mail_store(&rt),
-            other               => {
+        match rt.cli().subcommand_name().ok_or_else(|| err_msg("No subcommand called"))? {
+            "import"   => import(&rt),
+            "list"     => list(&rt),
+            "print-id" => print_id(&rt),
+            other      => {
                 debug!("Unknown command");
                 if rt.handle_unknown_subcommand("imag-mail", other, rt.cli())?.success() {
                     Ok(())
@@ -118,172 +113,217 @@ impl ImagApplication for ImagMail {
     }
 }
 
-fn scan(rt: &Runtime) -> Result<()> {
-    let collection_name = get_ref_collection_name(rt)?;
-    let refconfig       = get_ref_config(rt, "imag-mail")?;
-    let scmd            = rt.cli().subcommand_matches("scan").unwrap();
-
-    /// Helper function to get an Iterator for all files from one PathBuf
-    fn walk_pathes(path: PathBuf) -> impl Iterator<Item = Result<PathBuf>> {
-        walkdir::WalkDir::new(path)
-            .follow_links(false)
-            .into_iter()
-            .filter_ok(|entry| entry.file_type().is_file())
-            .map_err(Error::from)
-            .map_ok(|entry| entry.into_path())
-            .inspect(|e| trace!("Processing = {:?}", e))
-    }
+fn import(rt: &Runtime) -> Result<()> {
+    let scmd               = rt.cli().subcommand_matches("import").unwrap();
+    let query              = scmd.value_of("query").unwrap();
+    let store              = rt.store();
+    let notmuch_path       = crate::util::get_notmuch_database_path(rt)?;
+    let notmuch_connection = NotmuchConnection::open(notmuch_path)?;
 
-    /// Helper function to process an iterator over Result<PathBuf> and create store entries for
-    /// each path in the iterator
-    fn process_iter(i: &mut dyn Iterator<Item = Result<PathBuf>>, rt: &Runtime, collection_name: &str, refconfig: &RefConfig) -> Result<()> {
-        let scmd = rt.cli().subcommand_matches("scan").unwrap();
-        i.and_then_ok(|path| {
-                if scmd.is_present("ignore-existing-ids") {
-                    rt.store().retrieve_mail_from_path(path, collection_name, refconfig, true)
-                } else {
-                    rt.store().create_mail_from_path(path, collection_name, refconfig)
-                }
-            })
-            .and_then_ok(|e| rt.report_touched(e.get_location()).map_err(Error::from))
-            .collect::<Result<Vec<_>>>()
-            .map(|_| ())
-    }
+    debug!("Importing mail with query = '{}'", query);
 
-    let pathes = scmd.values_of("path")
-        .unwrap() // enforced by clap
-        .map(PathBuf::from)
-        .collect::<Vec<_>>();
+    let r = store
+        .with_connection(&notmuch_connection)
+        .import_with_query(query)?
+        .into_iter()
+        .map(|fle| rt.report_touched(fle.get_location()))
+        .collect::<Result<Vec<_>>>()
+        .map(|_| ());
 
-    if scmd.is_present("scan-parallel") {
-        debug!("Fetching pathes in parallel");
-        let mut i = pathes.into_par_iter()
-            .map(|path| walk_pathes(path).collect::<Result<_>>())
-            .collect::<Result<Vec<PathBuf>>>()?
-            .into_iter()
-            .map(Ok);
+    r
+}
 
-        debug!("Processing pathes");
-        process_iter(&mut i, rt, &collection_name, &refconfig)
-    } else {
-        debug!("Fetching pathes not in parallel");
-        let mut i = pathes
-            .into_iter()
-            .map(walk_pathes)
-            .flatten();
+fn list(rt: &Runtime) -> Result<()> {
+    debug!("Listing mail");
+    let scmd               = rt.cli().subcommand_matches("list").unwrap();
+    let store              = rt.store();
+    let notmuch_path       = crate::util::get_notmuch_database_path(rt)?;
+    debug!("notmuch path: {:?}", notmuch_path);
 
-        debug!("Processing pathes");
-        process_iter(&mut i, rt, &collection_name, &refconfig)
-    }
-}
+    let tree               = scmd.is_present("tree");
+    debug!("tree:         {}", tree);
 
-fn import_mail(rt: &Runtime) -> Result<()> {
-    let collection_name = get_ref_collection_name(rt)?;
-    let refconfig       = get_ref_config(rt, "imag-mail")?;
-    let scmd            = rt.cli().subcommand_matches("import-mail").unwrap();
-    let store           = rt.store();
-
-    debug!(r#"Importing mail with
-    collection_name = {}
-    refconfig = {:?}
-    "#, collection_name, refconfig);
-
-    scmd.values_of("path")
-        .unwrap() // enforced by clap
-        .map(PathBuf::from)
-        .map(|path| {
-            if scmd.is_present("ignore-existing-ids") {
-                store.retrieve_mail_from_path(path, &collection_name, &refconfig, true)
-            } else {
-                store.create_mail_from_path(path, &collection_name, &refconfig)
-            }
-            .map_info_str("Ok")
+    let list_format        = if tree {
+        util::get_mail_print_format("mail.list_tree_format", rt, &scmd)
+    } else {
+        util::get_mail_print_format("mail.list_format", rt, &scmd)
+    }?;
+    debug!("List-format:  {:?}", list_format);
+
+    let notmuch_connection = NotmuchConnection::open(notmuch_path)?;
+
+    let find_root          = scmd.is_present("find-root");
+    debug!("find_root:    {}", find_root);
+
+    let mut out            = rt.stdout();
+    let mut i              = 0;
+
+    /// Helper function for processing a tree
+    ///
+    /// This function implements the normal-style printing functionality (without "--tree").
+    fn process_nontree<'a, I>(iter: I,
+                              i: &mut usize,
+                              conn: &NotmuchConnection,
+                              list_format: &Handlebars,
+                              rt: &Runtime,
+                              out: &mut dyn Write)
+        -> Result<()>
+        where I: Iterator<Item = Result<FileLockEntry<'a>>>
+    {
+        debug!("Processing non-tree");
+        iter.and_then_ok(|fle| {
+            trace!("Loading: {}", fle.get_location());
+            let loaded = fle.load(&conn)?
+                .ok_or_else(|| format_err!("Mail not found: {}", fle.get_location()))?;
+            trace!("Loaded: {}", fle.get_location());
+
+            let parsed = loaded.parsed()?;
+            trace!("Parsed: {}", fle.get_location());
+
+            let r = crate::util::list_mail(&parsed, *i, 0, list_format, out); // indentation hard coded to zero
+            trace!("Listed: {}", fle.get_location());
+            *i += 1; // poor mans enumerate()
+            r.map(|_| fle)
         })
-        .and_then_ok(|e| rt.report_touched(e.get_location()).map_err(Error::from))
-        .collect::<Result<Vec<()>>>()
+        .and_then_ok(|fle| rt.report_touched(fle.get_location()))
+        .collect::<Result<Vec<_>>>()
         .map(|_| ())
-}
+    };
+
+    /// Helper function for processing a tree
+    ///
+    /// This function implements the tree-style printing functionality ("--tree").
+    fn process_tree<'a, I>(iter: I,
+                           mailstore: &'a MailStoreWithConnection<'a>,
+                           i: &mut usize,
+                           conn: &NotmuchConnection,
+                           list_format: &Handlebars,
+                           rt: &Runtime,
+                           out: &mut dyn Write,
+                           find_root: bool)
+        -> Result<()>
+        where I: Iterator<Item = Result<FileLockEntry<'a>>>
+    {
+        debug!("Processing tree");
+        let iter = if find_root {
+            let iter = iter.and_then_ok(|fle| {
+                trace!("Loading: {}", fle.get_location());
+                let id = fle
+                    .load(conn)?
+                    .ok_or_else(|| format_err!("Cannot load mail: {}", fle.get_location()))?
+                    .parsed()?
+                    .root_parent(&mailstore)?
+                    .ok_or_else(|| format_err!("Failed to find root parent: {}", fle.get_location()))
+                    .map(|p| p.get_id().clone());
+
+                drop(fle);
+                trace!("Loaded and parsed {:?} successfully", id);
+                id
+            });
+
+            Box::new(iter) as Box<dyn Iterator<Item = Result<String>>>
+        } else {
+            let iter = iter.and_then_ok(|fle| fle.get_cached_id());
+            Box::new(iter) as Box<dyn Iterator<Item = Result<String>>>
+        };
+
+        trace!("Printing mailtrees now!");
+
+        // we have to collect here, so that all FLEs are drop()ed
+        let ids = iter.collect::<Result<Vec<String>>>()?;
+        trace!("ids = {:?}", ids);
+
+        let mss = ids.into_iter()
+            .map(|id: String| mailstore.get_mailtree(&id))
+            .collect::<Result<Vec<_>>>()?;
+        trace!("mss = {:?}", mss);
+
+        mss.into_iter()
+            .map(|mt| print_traverse(&mailstore, mt, i, conn, list_format, rt, out))
+            .collect::<Result<Vec<_>>>()
+            .map(|_| ())
+    }
 
-fn list(rt: &Runtime) -> Result<()> {
-    let refconfig   = get_ref_config(rt, "imag-mail")?;
-    let scmd        = rt.cli().subcommand_matches("list").unwrap(); // safe via clap
-    let list_format = util::get_mail_print_format("mail.list_format", rt, &scmd)?;
-    let mut out     = rt.stdout();
-    let mut i = 0;
-
-    if rt.ids_from_stdin() {
-        let iter = rt
-            .ids::<crate::ui::PathProvider>()?
-            .ok_or_else(|| anyhow!("No ids supplied"))?
-            .into_iter()
-            .map(Ok);
+    if let Some(query) = scmd.value_of("query") {
+        // Use notmuch to find mails
 
-        StoreIdIterator::new(Box::new(iter))
-    } else {
-        rt.store()
-            .all_mails()?
-            .into_storeid_iter()
-    }
-    .inspect(|id| debug!("Found: {:?}", id))
-    .into_get_iter(rt.store())
-    .map_inner_ok_or_else(|| anyhow!("Did not find one entry"))
-    .and_then_ok(|m| {
-        crate::util::list_mail(&m, i, &refconfig, &list_format, &mut out)?;
-        rt.report_touched(m.get_location())?;
-        i += 1;
-        Ok(())
-    })
-    .collect::<Result<Vec<_>>>()
-    .map(|_| ())
-}
+        debug!("Importing mail with query = '{}'", query);
 
-fn unread(rt: &Runtime) -> Result<()> {
-    let refconfig   = get_ref_config(rt, "imag-mail")?;
-    let scmd        = rt.cli().subcommand_matches("unread").unwrap(); // safe via clap
-    let list_format = util::get_mail_print_format("mail.list_format", rt, &scmd)?;
-    let mut out     = rt.stdout();
-    let mut i       = 0;
-
-    if rt.ids_from_stdin() {
-        let iter = rt
-            .ids::<crate::ui::PathProvider>()?
-            .ok_or_else(|| anyhow!("No ids supplied"))?
+        let mailstore = store.with_connection(&notmuch_connection);
+        let iter = mailstore
+            .build_query(query)
+            .sorted(Sorting::OldestFirst)
+            .execute()?
             .into_iter()
             .map(Ok);
 
-        StoreIdIterator::new(Box::new(iter))
+        if tree {
+            process_tree(iter, &mailstore, &mut i, &notmuch_connection, &list_format, rt, &mut out, find_root)
+        } else {
+            process_nontree(iter, &mut i, &notmuch_connection, &list_format, rt, &mut out)
+        }
     } else {
-        rt.store().all_mails()?.into_storeid_iter()
-    }
-    .inspect(|id| debug!("Found: {:?}", id))
-    .into_get_iter(rt.store())
-    .map_inner_ok_or_else(|| anyhow!("Did not find one entry"))
-    .and_then_ok(|m| {
-        if !m.is_seen(&refconfig)? {
-            crate::util::list_mail(&m, i, &refconfig, &list_format, &mut out)?;
-            rt.report_touched(m.get_location())?;
-            i += 1;
+        // use StoreIds to find mails
+        let iter = rt.ids::<crate::ui::PathProvider>()?
+            .ok_or_else(|| err_msg("No ids supplied"))?
+            .into_iter()
+            .map(Ok)
+            .into_get_iter(rt.store())
+            .map_inner_ok_or_else(|| err_msg("Did not find one entry"))
+            .and_then_ok(|m| m.is_mail().map(|b| (b, m)))
+            .filter_ok(|tpl| tpl.0)
+            .map_ok(|tpl| tpl.1);
+
+        if tree {
+            let mailstore = store.with_connection(&notmuch_connection);
+            process_tree(iter, &mailstore, &mut i, &notmuch_connection, &list_format, rt, &mut out, find_root)
+        } else {
+            process_nontree(iter, &mut i, &notmuch_connection, &list_format, rt, &mut out)
         }
-
-        Ok(())
-    })
-    .collect::<Result<Vec<_>>>()
-    .map(|_| ())
+    }
 }
 
-fn mail_store(rt: &Runtime) -> Result<()> {
-    let _ = rt.cli().subcommand_matches("mail-store").unwrap();
-    Err(anyhow!("This feature is currently not implemented."))
+fn print_id(rt: &Runtime) -> Result<()> {
+    rt.ids::<crate::ui::PathProvider>()?
+        .ok_or_else(|| err_msg("No ids supplied"))?
+        .into_iter()
+        .map(Ok)
+        .into_get_iter(rt.store())
+        .map_inner_ok_or_else(|| err_msg("Did not find one entry"))
+        .and_then_ok(|fle| {
+            let id = fle.get_cached_id()?;
+            writeln!(rt.stdout(), "{}", id)?;
+            Ok(fle)
+        })
+        .and_then_ok(|fle| rt.report_touched(fle.get_location()))
+        .collect::<Result<Vec<_>>>()
+        .map(|_| ())
 }
 
-fn get_ref_collection_name(rt: &Runtime) -> Result<String> {
-    let setting_name = "mail.ref_collection_name";
-
-    debug!("Getting configuration: {}", setting_name);
+fn print_traverse<'a>(store: &'a MailStoreWithConnection<'a>,
+                      tree: Mailtree,
+                      i: &mut usize,
+                      conn: &NotmuchConnection,
+                      list_format: &Handlebars,
+                      rt: &Runtime,
+                      out: &mut dyn Write) -> Result<()> {
+    trace!("Print-Traversing starting with: {:?}", tree.root());
+    for (indent, mid) in tree.traverse() {
+        let fle    = store.get_mail_by_id(&mid)?.ok_or_else(|| format_err!("Cannot find mail with id = {}", mid))?;
+        trace!("Printing: {}", fle.get_location());
+
+        let loaded = fle.load(&conn)?
+            .ok_or_else(|| format_err!("Mail not found: {}", fle.get_location()))?;
+        trace!("Loaded: {}", fle.get_location());
+
+        let parsed = loaded.parsed()?;
+        trace!("Parsed: {}", fle.get_location());
+
+        crate::util::list_mail(&parsed, *i, indent, list_format, out)?;
+        trace!("Listed: {}", fle.get_location());
+        rt.report_touched(fle.get_location())?;
+        *i += 1; // poor mans enumerate()
+    }
 
-    rt.config()
-        .ok_or_else(|| anyhow!("No configuration, cannot find collection name for mail collection"))?
-        .read_string(setting_name)?
-        .ok_or_else(|| anyhow!("Setting missing: {}", setting_name))
+    Ok(())
 }
-
diff --git a/bin/domain/imag-mail/src/ui.rs b/bin/domain/imag-mail/src/ui.rs
index 319073d5..22562275 100644
--- a/bin/domain/imag-mail/src/ui.rs
+++ b/bin/domain/imag-mail/src/ui.rs
@@ -24,102 +24,93 @@ use libimagstore::storeid::StoreId;
 use libimagrt::runtime::IdPathProvider;
 use libimagstore::storeid::IntoStoreId;
 
-use clap::{Arg, ArgMatches, App, SubCommand};
+use clap::{Arg, ArgMatches, ArgGroup, App, SubCommand};
 
 pub fn build_ui<'a>(app: App<'a, 'a>) -> App<'a, 'a> {
     app
-        .subcommand(SubCommand::with_name("import-mail")
-                    .about("Import a mail (create a reference to it) (Maildir)")
-                    .version("0.1")
-                    .arg(Arg::with_name("ignore-existing-ids")
-                         .long("ignore-existing")
-                         .short("I")
-                         .takes_value(false)
-                         .required(false)
-                         .help("Ignore errors that might occur when store entries exist already"))
 
-                    .arg(Arg::with_name("path")
-                         .index(1)
-                         .takes_value(true)
-                         .multiple(true)
-                         .required(true)
-                         .help("Path to the mail file(s) to import")
-                         .value_name("PATH"))
-                    )
+        .arg(Arg::with_name("database_path")
+             .long("db")
+             .takes_value(true)
+             .required(false)
+             .value_name("PATH")
+             .help("The path to the notmuch database to use (defaults to value in config)"))
 
-        .subcommand(SubCommand::with_name("scan")
-                    .about("Scan a directory for mails")
+        .subcommand(SubCommand::with_name("import")
+                    .about("Import mails from notmuch")
                     .version("0.1")
-                    .arg(Arg::with_name("ignore-existing-ids")
-                         .long("ignore-existing")
-                         .short("I")
-                         .takes_value(false)
-                         .required(false)
-                         .help("Ignore errors that might occur when store entries exist already"))
-
-                    .arg(Arg::with_name("scan-parallel")
-                         .long("parallel")
-                         .takes_value(false)
-                         .required(false)
-                         .multiple(false)
-                         .help("Scan with multiple threads. Might be faster, but might slow down other tasks"))
-
-                    .arg(Arg::with_name("path")
+                    .arg(Arg::with_name("query")
                          .index(1)
                          .takes_value(true)
-                         .multiple(true)
                          .required(true)
-                         .validator(libimagutil::cli_validators::is_directory)
-                         .value_name("DIR")
-                         .help("Path to the directory containing mails"))
+                         .value_name("QUERY")
+                         .help("Imports mails from notmuch to imag, using a query to fetch mails to import"))
                     )
 
         .subcommand(SubCommand::with_name("list")
-                    .about("List all stored references to mails")
+                    .about("List mails in imag")
                     .version("0.1")
+                    .arg(Arg::with_name("query")
+                         .long("query")
+                         .short("q")
+                         .takes_value(true)
+                         .required(false)
+                         .value_name("QUERY")
+                         .help("List mails in imag by using this notmuch query"))
 
-                    .arg(Arg::with_name("list-id")
+                    .arg(Arg::with_name("ID")
                          .index(1)
                          .takes_value(true)
                          .required(false)
                          .multiple(true)
-                         .help("The ids of the mails to list information for"))
+                         .value_name("ID")
+                         .help("List mails by Store ID (non-mails will be ignored)"))
 
                     .arg(Arg::with_name("format")
                          .long("format")
+                         .short("f")
                          .takes_value(true)
                          .required(false)
-                         .multiple(false)
-                         .help("The format to list the mails with"))
-                    )
+                         .value_name("FORMAT")
+                         .help("Format to list entries with (default in config)"))
 
-        .subcommand(SubCommand::with_name("unread")
-                    .about("Show unread mail")
-                    .version("0.1")
+                    .arg(Arg::with_name("tree")
+                         .long("tree")
+                         .short("t")
+                         .takes_value(false)
+                         .required(false)
+                         .help("Build a tree and print that. When this argument is passed, the {{indent}} variable is available in the format, otherwise it is set always to zero"))
 
-                    .arg(Arg::with_name("unread-read")
-                         .long("read")
-                         .short("r")
+                    .arg(Arg::with_name("find-root")
+                         .long("find-root")
+                         .short("R")
                          .takes_value(false)
                          .required(false)
-                         .multiple(false)
-                         .help("Print the textual content of the mail itself as well"))
+                         .help("If --tree is passed, this can be used to tell imag to find the root message and starting the tree there instead of the passed mail"))
+
+                    .group(ArgGroup::with_name("tree-root")
+                        .arg("find-root")
+                        .requires("tree"))
                     )
 
-        .subcommand(SubCommand::with_name("mail-store")
-                    .about("Operations on (subsets of) all mails")
+        .subcommand(SubCommand::with_name("print-id")
+                    .about("Print id of mail(s)")
                     .version("0.1")
-                    .subcommand(SubCommand::with_name("update-refs")
-                                .about("Create references based on Message-IDs for all loaded mails")
-                                .version("0.1"))
-                    // TODO: We really should be able to filter here.
+                    .arg(Arg::with_name("ID")
+                         .index(1)
+                         .takes_value(true)
+                         .multiple(true)
+                         .required(false)
+                         .value_name("ID")
+                         .help("Print ID of mail(s) identified by Store ID(s)"))
                     )
+
 }
 
 pub struct PathProvider;
 impl IdPathProvider for PathProvider {
     fn get_ids(matches: &ArgMatches) -> Result<Option<Vec<StoreId>>> {
-        matches.values_of("list-id")
+        matches.values_of("ID")
             .map(|v| v
                  .map(PathBuf::from)
                  .map(|pb| pb.into_storeid())
diff --git a/bin/domain/imag-mail/src/util.rs b/bin/domain/imag-mail/src/util.rs
index 151c82fb..469bf362 100644
--- a/bin/domain/imag-mail/src/util.rs
+++ b/bin/domain/imag-mail/src/util.rs
@@ -19,6 +19,7 @@
 
 use std::collections::BTreeMap;
 use std::io::Write;
+use std::path::PathBuf;
 
 use clap::ArgMatches;
 use anyhow::Error;
@@ -27,10 +28,9 @@ use anyhow::Result;
 use handlebars::Handlebars;
 use toml_query::read::TomlValueReadTypeExt;
 
-use libimagmail::mail::Mail;
+use libimaginteraction::format::HandlebarsData;
+use libimagmail::mail::ParsedMail;
 use libimagrt::runtime::Runtime;
-use libimagstore::store::FileLockEntry;
-use libimagentryref::reference::Config as RefConfig;
 
 pub fn get_mail_print_format<'rc>(config_value_path: &'static str, rt: &Runtime, scmd: &ArgMatches) -> Result<Handlebars<'rc>> {
     let fmt = match scmd.value_of("format").map(String::from) {
@@ -53,43 +53,60 @@ pub fn get_mail_print_format<'rc>(config_value_path: &'static str, rt: &Runtime,
     Ok(hb)
 }
 
-pub fn build_data_object_for_handlebars(i: usize, m: &FileLockEntry, refconfig: &RefConfig) -> Result<BTreeMap<&'static str, String>> {
+pub fn build_data_object_for_handlebars(i: usize, m: &ParsedMail) -> Result<BTreeMap<&'static str, HandlebarsData>> {
     let mut data = BTreeMap::new();
 
-    data.insert("i", format!("{}", i));
+    data.insert("i", HandlebarsData::Int(i));
 
-    let id = match m.get_message_id(refconfig)? {
-        Some(f) => f.into(),
+    let id = match m.message_id()? {
+        Some(f) => f,
         None => "<no id>".to_owned(),
     };
 
-    let from = match m.get_from(refconfig)? {
+    let from = match m.from()? {
         Some(f) => f,
         None => "<no from>".to_owned(),
     };
 
-    let to = match m.get_to(refconfig)? {
+    let to = match m.to()? {
         Some(f) => f,
         None => "<no to>".to_owned(),
     };
 
-    let subject = match m.get_subject(refconfig)? {
+    let subject = match m.subject()? {
         Some(f) => f,
         None => "<no subject>".to_owned(),
     };
 
-    data.insert("id"      , id);
-    data.insert("from"    , from);
-    data.insert("to"      , to);
-    data.insert("subject" , subject);
+    data.insert("id"      , HandlebarsData::Str(id));
+    data.insert("from"    , HandlebarsData::Str(from));
+    data.insert("to"      , HandlebarsData::Str(to));
+    data.insert("subject" , HandlebarsData::Str(subject));
 
     Ok(data)
 }
 
-pub fn list_mail(m: &FileLockEntry, i: usize, refconfig: &RefConfig, list_format: &Handlebars, out: &mut dyn Write) -> Result<()> {
-    let data = build_data_object_for_handlebars(i, m, refconfig)?;
+pub fn list_mail(m: &ParsedMail, i: usize, indent: usize, list_format: &Handlebars, out: &mut dyn Write) -> Result<()> {
+    let mut data = build_data_object_for_handlebars(i, m)?;
+    data.insert("indent", HandlebarsData::Int(indent));
     let s    = list_format.render("format", &data)?;
     writeln!(out, "{}", s)?;
     Ok(())
 }
 
+pub fn get_notmuch_database_path(rt: &Runtime) -> Result<PathBuf> {
+    if let Some(pb) = rt.cli()
+        .value_of("database_path")
+        .map(String::from)
+        .map(PathBuf::from)
+    {
+        return Ok(pb)
+    } else {
+        rt.config()
+            .ok_or_else(|| err_msg("No configuration file"))?
+            .read_string("mail.notmuch_database")
+            .map_err(Error::from)?
+            .map(PathBuf::from)
+            .ok_or_else(|| format_err!("Configuration 'mail.notmuch_database' does not exist"))
+    }
+}
diff --git a/doc/src/04020-module-mails.md b/doc/src/04020-module-mails.md
index 1786dfe9..75d70ccc 100644
--- a/doc/src/04020-module-mails.md
+++ b/doc/src/04020-module-mails.md
@@ -1,359 +1,4 @@
 ## Mails {#sec:modules:mails}
 
----
-
-**NOTE:** This is mostly a todo-list for the `imag-mail` command. Nothing shown
-here is implemented. This "documentation-to-be" should be moved to
-`imag-mail --help` eventually.
-This list might be incomplete, details might be not possible to implement in the
-way described or other dragons.
-
-**Target audience:** People who want to implement `imag-mail`.
-
----
-
-The Mails module implements a commandline email client. Emails can be written
-(via `$EDITOR`) and viewed, also in threads. Emails can be crawled for creating
-new contacts.
-
-A Text User Interface is not planned, but might be there at some point.
-
-The mail module implements a minimal Email client. It does not handle IMAP
-syncing or SMTP things, it is just a _viewer_ for emails (a MUA).
-
-The goal of the initial implementation is only a CLI, not a TUI like mutt
-offers, for example (but that might be implemented later). As this is an imag
-module, it also creates references to mails inside the imag store which can be
-used by other tools then (for example `imag-link` to link an entry with a mail -
-or the imag entry representing that mail).
-
-So this module offers functionality to read (Maildir) mailboxes, search for and
-list mails and mail-threads and reply to mails (by spawning the `$EDITOR`).
-
-Outgoing mails are pushed to a special directory and can later on be send via
-`imag-mail` which calls a MTA (for example msmtp) and also creates store entries
-for the outgoing mails.
-
-
-### Configuration
-
-The following configuration variables are available for the imag-mail command:
-
-* `mail.defaultaccount`: The name of the default account to use if the
-  commandline parameters do not specify which account to use. The name must be
-  in the `mail.accounts` array.
-* `mail.accounts`:
-  An array of account configuration. Each element in the array is a table of the
-  following key-value pairs:
-  * `name`: the name of the account. Names must be unique. Required.
-  * `outgoingbox`: Path to mailbox to use for outgoing email. Required.
-  * `draftbox`: Path to mailbox to use for outgoing email. Required.
-  * `sentbox`: Path to mailbox to use for sent email. Required.
-  ` `maildirroot`: Path to folder where all mailboxes for this account are
-  located. Required.
-  * `fetchcommand`: What commandline to invoke for fetching mails for this
-    account. Optional - if not used, the global `mail.fetchcommand` will be
-    used.
-  * `postfetchcommand`: What commandline to invoke after fetching mails for this
-    account. Optional - if not used, the global `mail.postfetchcommand` will be
-    used.
-  * `sendcommand`: What commandline to invoke for sending mails for this
-    account. Optional - if not used, the global `mail.sendcommand` will be used.
-  * `postsendcommand`: What commandline to invoke after sending mails for this
-    account. Optional - if not used, the global `mail.postsendcommand` will be
-    used.
-* `mail.fetchcommand`: Command to use for fetching mail if no account-specific
-  command was specified
-  Available variables:
-    * `{{accountname}}` - name of the account to fetch mail for.
-    * `{{boxes}}` - a list of maildir paths to the boxes to fetch email for.
-      imag provides primitives to transform this array.
-      An example configuration for fetching with `offlineimap` might look like
-      this: `offlineimap -a {{accountname}} -f {{concatsep "," (replace "/home/user/mails/" "" boxes)}}`
-      to concatenate all boxes with a comma after removing a prefix.
-      For a complete list of transformation functions, the `--help` flag shall
-      be consulted.
-      For more complicated transformations a bash/ruby/python script might be
-      appropriate.
-* `mail.postfetchcommand`: Command to use after fetching mail if no
-  account-specific command was specified
-  Available variables: Same as `mail.fetchcommand`.
-* `mail.postsendcommand`: Command to use after sending mail if no
-  account-specific command was specified
-  Available variables: Same as `mail.sendcommand`.
-* `mail.sendcommand`: Command to use for sending mail if no account-specific
-  command was specified
-    * `{{accountname}}` - name of the account to fetch mail for.
-    * `{{mailfile}}` - The path of the mail to send
-
-
-### CLI
-
-The CLI of the imag-mail module is planned as follows:
-
-* imag mail
-
-    -A, --account   - Specify the "account" to use for the opperation by name.
-                      If none is specified, the configuration is searched for a
-                      default command.
-
-* imag mail track <path> [opts...]
-  Track a new mail, mail file passed as path
-
-* imag mail scan <path> [opts...]
-  Scan a maildir and track all untracked mails
-
-    --refind        - re-find messages. Loads all messages which are known to imag
-                      and compares identifiers, to update the imag-internal cache if
-                      a mail got moved.
-                      Without this flag, a modified email file might be added to
-                      the imag store again, even if there's another entry in the
-                      imag store refering to the same file.
-
-* imag mail list <args...>
-  List mails in a given mailbox for a given account or the default account
-
-    -S, --style     - print messages in a certain style
-                      Available:
-                        - 'linewise'
-                        - 'thread'
-
-    -g, --grep      - Filter by grepping for a pattern in body and subject
-
-    -d, --daterange - Filter by date(range)
-
-    -F, --filter    - Filter by passed filter
-
-        --thread    - Print only messages from the same thread as the found ones
-
-    --format=<fmt>  - Format mails for showing.
-                      --format always colorizes output (specify color in config)
-                      except when using --no-pager or piping output.
-
-                      When --tree is passed, the format is applied to the
-                      fragment _after_ the tree graphic.
-
-                      Default mode is 'default'.
-
-                      Modes:
-                        - 'subject': <Subject>
-                        - 'simple': <From>: <Subject>
-                        - 'default': <Date> - <From>: <Subject>
-                        - 'fmt:<fmt>' format with passed format
-
-                      Additional formats can be specified via the configuration
-                      file. If a format has the same name as a predefined one,
-                      the config overrides the predefined formats.
-
-    --color         - Colorize output (default).
-    --no-color      - Do never colorize output.
-
-* imag mail show <args...>
-  Show mail(s) - either in pager or by printing them to stdout.
-
-    Mails are specified by message id or imag entry
-
-    --refind        - If a imag entry is passed but the mail file is not there,