Refactor fetcher packages (#477)

8 files changed, 169 insertions, 10 deletions

diff --git a/src/cheatsh.rs b/src/cheatsh.rs
index e0684cd..3007031 100644
--- a/src/cheatsh.rs
+++ b/src/cheatsh.rs
@@ -1,7 +1,7 @@
 use crate::display::Writer;
-use crate::fetcher;
 use crate::parser;
 use crate::structures::cheat::VariableMap;
+use crate::structures::fetcher;
 use anyhow::Context;
 use anyhow::Error;
 use std::collections::HashSet;
diff --git a/src/cmds/alfred.rs b/src/cmds/alfred.rs
index 489393e..52410c5 100644
--- a/src/cmds/alfred.rs
+++ b/src/cmds/alfred.rs
@@ -1,9 +1,9 @@
 use crate::display;
-use crate::fetcher::Fetcher;
 use crate::filesystem;
 use crate::shell::BashSpawnError;
 use crate::structures::cheat::Suggestion;
 use crate::structures::config::Config;
+use crate::structures::fetcher::Fetcher;
 use anyhow::Context;
 use anyhow::Error;
 use std::env;
diff --git a/src/cmds/core.rs b/src/cmds/core.rs
index cda972b..c3c0c13 100644
--- a/src/cmds/core.rs
+++ b/src/cmds/core.rs
@@ -4,10 +4,10 @@ use crate::cheatsh;
 use crate::display;
 
 use crate::extractor;
-use crate::fetcher::Fetcher;
 use crate::filesystem;
 use crate::finder::structures::{Opts as FinderOpts, SuggestionType};
 use crate::finder::Finder;
+use crate::structures::fetcher::Fetcher;
 
 use crate::structures::cheat::VariableMap;
 
diff --git a/src/filesystem.rs b/src/filesystem.rs
index 50ee7a6..957d57f 100644
--- a/src/filesystem.rs
+++ b/src/filesystem.rs
@@ -1,9 +1,170 @@
 use crate::display::Writer;
-use crate::fetcher;
-pub use crate::fetcher::filesystem::{all_cheat_files, default_cheat_pathbuf, read_all};
-pub use crate::fs::{create_dir, exe_string, pathbuf_to_string, remove_dir, InvalidPath, UnreadableDir};
+pub use crate::fs::{
+    create_dir, exe_string, pathbuf_to_string, read_lines, remove_dir, InvalidPath, UnreadableDir,
+};
+use crate::parser;
 use crate::structures::cheat::VariableMap;
+use crate::structures::fetcher;
 use anyhow::Error;
+use directories_next::BaseDirs;
+use std::collections::HashSet;
+use std::path::PathBuf;
+use walkdir::WalkDir;
+
+pub fn all_cheat_files(path_str: &str) -> Vec<String> {
+    let path_str_with_trailing_slash = if path_str.ends_with('/') {
+        path_str.to_string()
+    } else {
+        format!("{}/", &path_str)
+    };
+
+    WalkDir::new(&path_str)
+        .follow_links(true)
+        .into_iter()
+        .filter_map(|e| e.ok())
+        .map(|e| e.path().to_str().unwrap_or("").to_string())
+        .filter(|e| e.ends_with(".cheat"))
+        .map(|e| e.replace(&path_str_with_trailing_slash, ""))
+        .collect::<Vec<String>>()
+}
+
+fn paths_from_path_param(env_var: &str) -> impl Iterator<Item = &str> {
+    env_var.split(':').filter(|folder| folder != &"")
+}
+
+// TODO: move
+fn read_file(
+    path: &str,
+    file_index: usize,
+    variables: &mut VariableMap,
+    visited_lines: &mut HashSet<u64>,
+    writer: &mut dyn Writer,
+    stdin: &mut std::process::ChildStdin,
+) -> Result<(), Error> {
+    let lines = read_lines(path)?;
+    parser::read_lines(lines, path, file_index, variables, visited_lines, writer, stdin)
+}
+
+pub fn default_cheat_pathbuf() -> Result<PathBuf, Error> {
+    let base_dirs = BaseDirs::new().ok_or_else(|| anyhow!("Unable to get base dirs"))?;
+
+    let mut pathbuf = PathBuf::from(base_dirs.data_dir());
+    pathbuf.push("navi");
+    pathbuf.push("cheats");
+    Ok(pathbuf)
+}
+
+pub fn cheat_paths(path: Option<String>) -> Result<String, Error> {
+    if let Some(p) = path {
+        Ok(p)
+    } else {
+        pathbuf_to_string(default_cheat_pathbuf()?)
+    }
+}
+
+pub fn read_all(
+    path: Option<String>,
+    files: &mut Vec<String>,
+    stdin: &mut std::process::ChildStdin,
+    writer: &mut dyn Writer,
+) -> Result<Option<VariableMap>, Error> {
+    let mut variables = VariableMap::new();
+    let mut found_something = false;
+    let mut visited_lines = HashSet::new();
+    let paths = cheat_paths(path);
+
+    if paths.is_err() {
+        return Ok(None);
+    };
+
+    let paths = paths.expect("Unable to get paths");
+    let folders = paths_from_path_param(&paths);
+
+    for folder in folders {
+        for file in all_cheat_files(folder) {
+            let full_filename = format!("{}/{}", &folder, &file);
+            files.push(full_filename.clone());
+            let index = files.len() - 1;
+            if read_file(
+                &full_filename,
+                index,
+                &mut variables,
+                &mut visited_lines,
+                writer,
+                stdin,
+            )
+            .is_ok()
+                && !found_something
+            {
+                found_something = true
+            }
+        }
+    }
+
+    if !found_something {
+        return Ok(None);
+    }
+
+    Ok(Some(variables))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::display;
+    use crate::finder::structures::{Opts as FinderOpts, SuggestionType};
+    use std::process::{Command, Stdio};
+
+    #[test]
+    fn test_read_file() {
+        let path = "tests/cheats/ssh.cheat";
+        let mut variables = VariableMap::new();
+        let mut child = Command::new("cat")
+            .stdin(Stdio::piped())
+            .stdout(Stdio::null())
+            .spawn()
+            .unwrap();
+        let child_stdin = child.stdin.as_mut().unwrap();
+        let mut visited_lines: HashSet<u64> = HashSet::new();
+        let mut writer: Box<dyn Writer> = Box::new(display::terminal::Writer::new());
+        read_file(
+            path,
+            0,
+            &mut variables,
+            &mut visited_lines,
+            &mut *writer,
+            child_stdin,
+        )
+        .unwrap();
+        let expected_suggestion = (
+            r#" echo -e "$(whoami)\nroot" "#.to_string(),
+            Some(FinderOpts {
+                header_lines: 0,
+                column: None,
+                delimiter: None,
+                suggestion_type: SuggestionType::SingleSelection,
+                ..Default::default()
+            }),
+        );
+        let actual_suggestion = variables.get_suggestion("ssh", "user");
+        assert_eq!(Some(&expected_suggestion), actual_suggestion);
+    }
+
+    #[test]
+    fn splitting_of_dirs_param_may_not_contain_empty_items() {
+        // Trailing colon indicates potential extra path. Split returns an empty item for it. This empty item should be filtered away, which is what this test checks.
+        let given_path_config = "SOME_PATH:ANOTHER_PATH:";
+
+        let found_paths = paths_from_path_param(given_path_config);
+
+        let mut expected_paths = vec!["SOME_PATH", "ANOTHER_PATH"].into_iter();
+
+        for found in found_paths {
+            let expected = expected_paths.next().unwrap();
+            assert_eq!(found, expected)
+        }
+    }
+}
 
 pub fn tmp_path_str() -> Result<String, Error> {
     let cheat_path_str = pathbuf_to_string(default_cheat_pathbuf()?)?;
diff --git a/src/lib.rs b/src/lib.rs
index d1955a4..4c5bcb5 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,7 +10,6 @@ mod cmds;
 mod display;
 mod env_vars;
 mod extractor;
-mod fetcher;
 mod filesystem;
 mod finder;
 mod fs;
diff --git a/src/fetcher/mod.rs b/src/structures/fetcher.rs
index 3225df6..02e7b47 100644
--- a/src/fetcher/mod.rs
+++ b/src/structures/fetcher.rs
@@ -1,5 +1,3 @@
-pub mod filesystem;
-
 use crate::display::Writer;
 use crate::structures::cheat::VariableMap;
 use anyhow::Error;
diff --git a/src/structures/mod.rs b/src/structures/mod.rs
index e3def95..543dac8 100644
--- a/src/structures/mod.rs
+++ b/src/structures/mod.rs
@@ -1,3 +1,4 @@
 pub mod cheat;
 pub mod config;
+pub mod fetcher;
 pub mod item;
diff --git a/src/tldr.rs b/src/tldr.rs
index 2c30a99..1096bdd 100644
--- a/src/tldr.rs
+++ b/src/tldr.rs
@@ -1,7 +1,7 @@
 use crate::display::Writer;
-use crate::fetcher;
 use crate::parser;
 use crate::structures::cheat::VariableMap;
+use crate::structures::fetcher;
 use anyhow::{Context, Error};
 use regex::Regex;
 use std::collections::HashSet;