Implement --list-sites and --update-sites

and validation on supplied site argument.

7 files changed, 163 insertions, 59 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 5914994..0689438 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -467,6 +467,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49"
 
 [[package]]
+name = "linked-hash-map"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
+
+[[package]]
 name = "log"
 version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -885,6 +891,18 @@ dependencies = [
 ]
 
 [[package]]
+name = "serde_yaml"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16c7a592a1ec97c9c1c68d75b6e537dcbf60c7618e038e7841e00af1d9ccf0c4"
+dependencies = [
+ "dtoa",
+ "linked-hash-map",
+ "serde",
+ "yaml-rust",
+]
+
+[[package]]
 name = "slab"
 version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -907,6 +925,7 @@ dependencies = [
  "reqwest",
  "serde",
  "serde_json",
+ "serde_yaml",
 ]
 
 [[package]]
@@ -1239,3 +1258,12 @@ dependencies = [
  "winapi 0.2.8",
  "winapi-build",
 ]
+
+[[package]]
+name = "yaml-rust"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d"
+dependencies = [
+ "linked-hash-map",
+]
diff --git a/Cargo.toml b/Cargo.toml
index 9e33491..44cd9c5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,3 +14,4 @@ flate2 = "1.0"
 reqwest = { version = "0.10", features = ["blocking"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+serde_yaml = "0.8"
diff --git a/TODO.md b/TODO.md
index 14bb052..380dea0 100644
--- a/TODO.md
+++ b/TODO.md
@@ -5,15 +5,18 @@
 0. Implement --update-sites command
 3. Parse markdown (`pulldown_cmark`)
 4. Maybe default --validate-sites off (parsing 30k file a big hit)
+5. Print to stderr in style
 
 ### resources for later
 0. [Intro to async rust](http://jamesmcm.github.io/blog/2020/05/06/a-practical-introduction-to-async-programming-in-rust/)
 1. Async API calls [tokio](https://stackoverflow.com/a/57770687)
 2. Parallel calls against multiple sites [vid](https://www.youtube.com/watch?v=O-LagKc0MPA)
+0. OR JUST THREADS [see here](https://rust-lang.github.io/async-book/01_getting_started/02_why_async.html)
 3. [config mgmt](https://github.com/rust-cli/confy) or just use directories
 4. Test if pre-made filter can be used for various api keys
 5. Add sort option, e.g. relevance|votes|date
 6. Google stuff [scraping with reqwest](https://rust-lang-nursery.github.io/rust-cookbook/web/scraping.html))
-7. App Distribution [cross-platform binaries](https://github.com/rustwasm/wasm-pack/blob/51e6351c28fbd40745719e6d4a7bf26dadd30c85/.travis.yml#L74-L91)
-
 8. Keep track of quota in a data file, inform user when getting close?
+7. App Distribution
+   [cross-platform binaries via travis](https://github.com/rustwasm/wasm-pack/blob/51e6351c28fbd40745719e6d4a7bf26dadd30c85/.travis.yml#L74-L91)
+   also see lobster script in this [repo](https://git.sr.ht/~wezm/lobsters).
diff --git a/src/cli.rs b/src/cli.rs
index bc522f0..b7c7de7 100644
--- a/src/cli.rs
+++ b/src/cli.rs
@@ -1,16 +1,26 @@
-use clap::{App, AppSettings, Arg, ArgMatches};
+use clap::{App, AppSettings, Arg};
+
+use crate::config;
+use crate::config::Config;
 
 // TODO maybe consts for these keywords?
 
-// TODO pull defaults from config file
 // TODO --set-api-key KEY
 // TODO --update-sites
 // TODO --install-filter-key --force
 // TODO --sites plural
 // TODO --add-site (in addition to defaults)
-//?TODO --set-default-opt opt val # e.g. --set-default-opt sites site1;site2;site3
-pub fn mk_app<'a, 'b>() -> App<'a, 'b> {
-    App::new("so")
+pub struct Opts {
+    pub list_sites: bool,
+    pub update_sites: bool,
+    pub query: Option<String>,
+    pub config: Config,
+}
+
+pub fn get_opts() -> Opts {
+    let config = config::user_config();
+    let limit = &config.limit.to_string();
+    let matches = App::new("so")
         .setting(AppSettings::ColoredHelp)
         .version(clap::crate_version!())
         .author(clap::crate_authors!())
@@ -21,13 +31,18 @@ pub fn mk_app<'a, 'b>() -> App<'a, 'b> {
                 .help("Print available StackExchange sites"),
         )
         .arg(
+            Arg::with_name("update-sites")
+                .long("update-sites")
+                .help("Update cache of StackExchange sites"),
+        )
+        .arg(
             Arg::with_name("site")
                 .long("site")
                 .short("s")
                 .multiple(true)
                 .number_of_values(1)
                 .takes_value(true)
-                .default_value("stackoverflow")
+                .default_value(&config.site)
                 .help("StackExchange site code to search"), // TODO sites plural
         )
         .arg(
@@ -36,7 +51,7 @@ pub fn mk_app<'a, 'b>() -> App<'a, 'b> {
                 .short("l")
                 .number_of_values(1)
                 .takes_value(true)
-                .default_value("1")
+                .default_value(limit)
                 .validator(|s| s.parse::<u32>().map_err(|e| e.to_string()).map(|_| ()))
                 .help("Question limit per site query")
                 .hidden(true), // TODO unhide once more than just --lucky
@@ -51,30 +66,26 @@ pub fn mk_app<'a, 'b>() -> App<'a, 'b> {
             Arg::with_name("query")
                 .multiple(true)
                 .index(1)
-                .required(true)
-                .required_unless("list-sites"),
+                .required_unless_one(&["list-sites", "update-sites"]),
         )
-}
-
-pub fn get_query(matches: ArgMatches) -> Option<String> {
-    let q = matches
-        .values_of("query")?
-        .into_iter()
-        .collect::<Vec<_>>()
-        .join(" ");
-    Some(q)
+        .get_matches();
+    Opts {
+        list_sites: matches.is_present("list-sites"),
+        update_sites: matches.is_present("update-sites"),
+        query: matches
+            .values_of("query")
+            .map(|q| q.into_iter().collect::<Vec<_>>().join(" ")),
+        config: Config {
+            // these unwraps are safe b.c. default value
+            limit: matches.value_of("limit").unwrap().parse::<u16>().unwrap(),
+            site: matches.value_of("site").unwrap().to_string(),
+            // TODO if set_api_key passed, pass it here too
+            ..config
+        },
+    }
 }
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-
-    #[test]
-    fn test_cli() {
-        let m = mk_app().get_matches_from(vec![
-            "so", "--site", "meta", "how", "do", "I", "exit", "Vim",
-        ]);
-        assert_eq!(m.value_of("site"), Some("meta"));
-        assert_eq!(get_query(m).unwrap(), "how do I exit Vim");
-    }
+    // TODO how can I test this now that it depends on user dir?
 }
diff --git a/src/config.rs b/src/config.rs
index 02d03bc..957a46d 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -1,9 +1,46 @@
+use directories::ProjectDirs;
+use serde::{Deserialize, Serialize};
+use serde_yaml;
+use std::fs;
+use std::fs::File;
+
+#[derive(Deserialize, Serialize, Debug)]
 pub struct Config {
     pub api_key: Option<String>,
     pub limit: u16,
     pub site: String,
 }
 
+impl Default for Config {
+    fn default() -> Self {
+        Config {
+            api_key: None,
+            limit: 20,
+            site: String::from("stackoverflow"),
+        }
+    }
+}
+
+/// Get user config (writes default if none found)
+pub fn user_config() -> Config {
+    let project = project_dir();
+    let dir = project.config_dir();
+    fs::create_dir_all(&dir).unwrap(); // TODO bubble to main
+    let filename = dir.join("config.yml");
+    match File::open(&filename) {
+        Err(_) => {
+            let file = File::create(&filename).unwrap();
+            let def = Config::default();
+            serde_yaml::to_writer(file, &def).unwrap();
+            def
+        }
+        Ok(file) => serde_yaml::from_reader(file).expect(&format!(
+            "Local config corrupted; try removing it `rm {}`",
+            filename.display()
+        )),
+    }
+}
+
 /// Get project directory; might panic on unexpected OS
 pub fn project_dir() -> ProjectDirs {
     ProjectDirs::from("io", "Sam Tay", "so").expect(
diff --git a/src/main.rs b/src/main.rs
index f6f4399..d5568cf 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -6,22 +6,44 @@ use config::Config;
 use stackexchange::{LocalStorage, StackExchange};
 
 fn main() {
-    let config = config::user_config();
-    let matches = cli::mk_app(&config).get_matches();
+    // TODO wrap inner function with Result<(), ErrorMessage>, propagate, print to stderr at the top level.
+    let opts = cli::get_opts();
+    let config = opts.config;
+    let site = &config.site;
+    let mut ls = LocalStorage::new();
 
-    if matches.is_present("update-sites") {
-        LocalStorage::new().update_sites();
+    if opts.update_sites {
+        ls.update_sites();
+    }
+
+    if opts.list_sites {
+        for s in ls.sites() {
+            println!("{}: {}", s.api_site_parameter, s.site_url);
+        }
+        return;
+    }
+
+    // TODO make this validation optional
+    if !ls.validate_site(site) {
+        // TODO tooling for printing to stderr with color, etc.
+        println!(
+            "{} is not a valid StackExchange site. If you think this
+            is in error, try running `so --update-sites` to update
+            the cached site listing.  Run `so --list-sites` for all
+            available sites.",
+            site
+        );
+        return;
     }
 
-    // TODO merge config from ArgMatch
     let se = StackExchange::new(Config {
-        api_key: Some(String::from("8o9g7WcfwnwbB*Qp4VsGsw((")),
-        limit: 1,
-        site: String::from("stackoverflow"),
+        api_key: Some(String::from("8o9g7WcfwnwbB*Qp4VsGsw((")), // TODO stash this
+        ..config
     });
 
+    let query = opts.query;
     (|| -> Option<_> {
-        let q = cli::get_query(matches)?;
+        let q = query?;
         let que = se.search(&q).unwrap(); // TODO eventually be graceful
         let ans = que.first()?.answers.first()?;
         println!("{}", ans.body);
diff --git a/src/stackexchange.rs b/src/stackexchange.rs
index d98643d..1aed7d8 100644
--- a/src/stackexchange.rs
+++ b/src/stackexchange.rs
@@ -36,8 +36,8 @@ pub struct LocalStorage {
 
 #[derive(Deserialize, Serialize, Debug)]
 pub struct Site {
-    api_site_parameter: String,
-    site_url: String,
+    pub api_site_parameter: String,
+    pub site_url: String,
 }
 
 /// Represents a StackExchange answer with a custom selection of fields from
@@ -124,36 +124,38 @@ impl LocalStorage {
     pub fn new() -> Self {
         let project = project_dir();
         let dir = project.cache_dir();
-        fs::create_dir_all(&dir);
+        fs::create_dir_all(&dir).unwrap(); // TODO bubble to main
         LocalStorage {
             sites: None,
             filename: dir.join("sites.json"),
         }
     }
 
-    // TODO this function is disgusting; how do in idiomatic rust?
     // TODO make this async, inform user if we are downloading
     pub fn sites(&mut self) -> &Vec<Site> {
-        if let Some(ref sites) = self.sites {
-            return sites;
-        }
-        self.fetch_local_sites();
-        if let Some(ref sites) = self.sites {
-            return sites;
-        }
-        self.fetch_remote_sites();
-        self.sites.as_ref().unwrap()
+        self.sites
+            .as_ref()
+            .map(|_| ()) // stop if we already have sites
+            .or_else(|| self.fetch_local_sites()) // otherwise try local cache
+            .unwrap_or_else(|| self.fetch_remote_sites()); // otherwise remote fetch
+        self.sites.as_ref().unwrap() // we will have paniced earlier on failure
     }
 
     pub fn update_sites(&mut self) {
-        self.fetch_remote_sites()
+        self.fetch_remote_sites();
     }
 
-    fn fetch_local_sites(&mut self) {
-        if let Ok(file) = File::open(&self.filename) {
-            self.sites = serde_json::from_reader(file)
-                .expect("Local cache corrupted; try running `so --update-sites`")
-        }
+    pub fn validate_site(&mut self, site_code: &String) -> bool {
+        self.sites()
+            .iter()
+            .any(|site| site.api_site_parameter == *site_code)
+    }
+
+    fn fetch_local_sites(&mut self) -> Option<()> {
+        let file = File::open(&self.filename).ok()?;
+        self.sites = serde_json::from_reader(file)
+            .expect("Local cache corrupted; try running `so --update-sites`");
+        Some(())
     }
 
     // TODO decide whether or not I should give LocalStorage an api key..
@@ -169,7 +171,7 @@ impl LocalStorage {
             .send()
             .unwrap(); // TODO inspect response for errors e.g. throttle
         let gz = GzDecoder::new(resp_body);
-        let wrapper: ResponseWrapper<Site> = serde_json::from_reader(gz).unwrap(); // TODO
+        let wrapper: ResponseWrapper<Site> = serde_json::from_reader(gz).unwrap();
         self.sites = Some(wrapper.items);
         self.store_local_sites();
     }