From 5f88657a75c4443ba93936e0f14bb3be0435fd41 Mon Sep 17 00:00:00 2001 From: Sam Tay Date: Thu, 18 Jun 2020 12:43:19 -0700 Subject: Fetch q/a in the background while viewing --lucky --- README.md | 11 +++++++++++ TODO.md | 26 ++++++++++++++++---------- roadmap.md | 10 +++++----- src/main.rs | 15 +++++++++------ src/stackexchange.rs | 32 ++++++++++++++++++-------------- 5 files changed, 59 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 6986419..52e57c1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,17 @@ **Note:** under development, not ready for prime time. +### async considerations +Implemented async with tokio in ec92f93, but unclear if this is necessary. For +< 10 simultaneous network requests, it might be better and simpler to just use +rayon (i.e. OS threads). + +### TUI considerations +Going with cursive because it is way more flexible than tui-rs. +**Important note** Tables are not currently allowed in stackexchange... so the +benefit of incorporating termimad features will not be felt. But, this is +changing [soon](https://meta.stackexchange.com/q/348746). + ### to troubleshoot ``` export RUST_BACKTRACE=full diff --git a/TODO.md b/TODO.md index d232f52..672e993 100644 --- a/TODO.md +++ b/TODO.md @@ -1,11 +1,5 @@ # TODO -## TUI considerations -Going with cursive because it is way more flexible than tui-rs. -**Important note** Tables are not currently allowed in stackexchange... so the -benefit of incorporating termimad features will not be felt. But, this is -changing [soon](https://meta.stackexchange.com/q/348746). - ### v0.2.2 1. Site can be multiple 2. do tokio async on SE api @@ -40,16 +34,28 @@ etc. 7. Small tray at the bottom with "notifications", e.g. "GitHub Theme loaded!" ### resources for later + +#### async +1. start with [this](http://patshaughnessy.net/2020/1/20/downloading-100000-files-using-async-rust) but also see the following gist and thread through the below links to make sure its actually async.. +0. breakdown of futures+reqwest [here](https://stackoverflow.com/questions/51044467/how-can-i-perform-parallel-asynchronous-http-get-requests-with-reqwest) +0. general concurrency in rust [info](https://blog.yoshuawuyts.com/streams-concurrency/) 0. [Intro to async rust](http://jamesmcm.github.io/blog/2020/05/06/a-practical-introduction-to-async-programming-in-rust/) 1. Async API calls [tokio](https://stackoverflow.com/a/57770687) 2. Parallel calls against multiple sites [vid](https://www.youtube.com/watch?v=O-LagKc0MPA) 0. OR JUST THREADS [see here](https://rust-lang.github.io/async-book/01_getting_started/02_why_async.html) -3. [config mgmt](https://github.com/rust-cli/confy) or just use directories -5. Add sort option, e.g. relevance|votes|date + +#### scraping 6. Google stuff [scraping with reqwest](https://rust-lang-nursery.github.io/rust-cookbook/web/scraping.html)) -8. Keep track of quota in a data file, inform user when getting close? + +#### distribution 7. App Distribution [cross-platform binaries via travis](https://github.com/rustwasm/wasm-pack/blob/51e6351c28fbd40745719e6d4a7bf26dadd30c85/.travis.yml#L74-L91) also see lobster script in this [repo](https://git.sr.ht/~wezm/lobsters). 9. Great tui-rs [example app](https://github.com/SoptikHa2/desed/blob/master/src/ui/tui.rs) -10 nah look at [termimad example](https://github.com/Canop/whalespotter) +11. general CI & deploy [info](https://rust-cli.github.io/book/tutorial/packaging.html) +12. window binaries deployed via [github actions](https://github.com/rust-av/av-metrics) +13. oh game over [dawg](https://github.com/japaric/trust) + +#### ideas +5. Add sort option, e.g. relevance|votes|date +8. Keep track of quota in a data file, inform user when getting close? diff --git a/roadmap.md b/roadmap.md index 1a1bc4c..47afdf9 100644 --- a/roadmap.md +++ b/roadmap.md @@ -14,8 +14,8 @@ [x] Termimad interface for viewing multiple results ### v0.2.1 -[ ] Add --no-lucky option -[ ] For --lucky, async parsing first q/a, then the rest +[x] Add --no-lucky option +[x] For --lucky, async parsing first q/a, then the rest ### v0.2.2 [ ] Support multiple --site args & searches @@ -24,12 +24,12 @@ [ ] Add duckduckgo scraper ### at some point -[ ] ask SE forums if I should bundle my api-key? +[ ] use trust to distrubute app binaries +[ ] ask SE forums if I should bundle my api-key? (if so use an env var macro) [ ] allow new queries from TUI, e.g. hit `/` for a prompt [ ] or `/` searches current q/a [ ] clean up error.rs and term.rs ; only keep whats actually ergonomic [ ] ask legal@stackoverflow.com for permission to logo stackoverflow/stackexchange in readme [ ] add duckduckgo logo to readme -[ ] cross-platform release binaries [ ] per platform package mgmt -[ ] testing +[ ] more testing diff --git a/src/main.rs b/src/main.rs index 27c7109..e7c0414 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,12 +7,13 @@ mod tui; mod utils; use crossterm::style::Color; -use error::Error; +use error::{Error, Result}; use lazy_static::lazy_static; use minimad::mad_inline; use stackexchange::{LocalStorage, StackExchange}; use term::mk_print_error; use termimad::{CompoundStyle, MadSkin}; +use tokio::task; #[tokio::main] async fn main() -> Result<(), Error> { @@ -77,19 +78,21 @@ async fn run(skin: &mut MadSkin) -> Result<(), Error> { } if let Some(q) = opts.query { - let se = StackExchange::new(config); - // TODO get the rest of the results in the background + let se = StackExchange::new(config, q); if lucky { // TODO this needs preprocessing; all the more reason to do it at SE level - let md = se.search_lucky(&q).await?; + let md = se.search_lucky().await?; skin.print_text(&md); skin.print_text("\nPress **[SPACE]** to see more results, or any other key to exit"); + // Kick off the rest of the search in the background + let qs = task::spawn(async move { se.search().await }); if !utils::wait_for_char(' ')? { return Ok(()); } + tui::run(qs.await.unwrap()?)?; + } else { + tui::run(se.search().await?)?; } - let qs = se.search(&q).await?; - tui::run(qs)?; } Ok(()) } diff --git a/src/stackexchange.rs b/src/stackexchange.rs index 2b4da67..f86c8c0 100644 --- a/src/stackexchange.rs +++ b/src/stackexchange.rs @@ -27,11 +27,12 @@ const SE_SITES_PAGESIZE: u16 = 10000; pub struct StackExchange { client: Client, config: Config, + query: String, } /// This structure allows interacting with locally cached StackExchange metadata. pub struct LocalStorage { - sites: Option>, + sites: Option>, // TODO this should be a hashmap! filename: PathBuf, } @@ -75,16 +76,20 @@ struct ResponseWrapper { } impl StackExchange { - pub fn new(config: Config) -> Self { + pub fn new(config: Config, query: String) -> Self { let client = Client::new(); - StackExchange { client, config } + StackExchange { + client, + config, + query, + } } // TODO also return a future with the rest of the questions /// Search query at stack exchange and get the top answer body - pub async fn search_lucky(&self, q: &str) -> Result { - let ans = self - .search_advanced(q, 1) + pub async fn search_lucky(&self) -> Result { + Ok(self + .search_advanced(1) .await? .into_iter() .next() @@ -92,28 +97,27 @@ impl StackExchange { .answers .into_iter() .next() - .ok_or_else(|| { - Error::StackExchange(String::from("Received question with no answers")) - })?; - Ok(ans.body) + .ok_or_else(|| Error::StackExchange(String::from("Received question with no answers")))? + .body) } /// Search query at stack exchange and get a list of relevant questions - pub async fn search(&self, q: &str) -> Result> { - self.search_advanced(q, self.config.limit).await + pub async fn search(&self) -> Result> { + self.search_advanced(self.config.limit).await } + /// Search against the search/advanced endpoint with a given query. /// Only fetches questions that have at least one answer. /// TODO async /// TODO parallel requests over multiple sites - async fn search_advanced(&self, q: &str, limit: u16) -> Result> { + async fn search_advanced(&self, limit: u16) -> Result> { Ok(self .client .get(stackexchange_url("search/advanced")) .header("Accepts", "application/json") .query(&self.get_default_opts()) .query(&[ - ("q", q), + ("q", self.query.as_str()), ("pagesize", &limit.to_string()), ("page", "1"), ("answers", "1"), -- cgit v1.2.3