diff options
-rw-r--r-- | README.md | 17 | ||||
-rw-r--r-- | TODO.md | 5 | ||||
-rw-r--r-- | roadmap.md | 2 | ||||
-rw-r--r-- | src/stackexchange.rs | 13 |
4 files changed, 26 insertions, 11 deletions
@@ -13,6 +13,12 @@ Going with cursive because it is way more flexible than tui-rs. benefit of incorporating termimad features will not be felt. But, this is changing [soon](https://meta.stackexchange.com/q/348746). +### to stress test +Produces a long answer with noticeable pause on markdown view: +``` +cargo run -- --site stackoverflow --site serverfault how do I exit Vim +``` + ### to troubleshoot ``` export RUST_BACKTRACE=full @@ -29,5 +35,14 @@ run into throttling issues, get a key so --set-api-key <KEY> ``` -Recall my api key is: `8o9g7WcfwnwbB*Qp4VsGsw((` +### multi-site searching +As stated in the docs linked above, + +> If a single IP is making more than 30 requests a second, new requests will be dropped. + +So, don't go crazy with the multi-site search, since it is all done in parallel. +In particular, if you specify more than 30 sites, SE will likely ban you for a short time. + + +**Remove this** Recall my api key is: `8o9g7WcfwnwbB*Qp4VsGsw((` @@ -1,10 +1,5 @@ # TODO -### v0.2.2 -1. Site can be multiple -2. do tokio async on SE api -3. add warning to README about throttling on excessive requests - ### v0.3.0 1. Duckduck go search ftw, e.g. ``` @@ -18,7 +18,7 @@ [x] For --lucky, async parsing first q/a, then the rest ### v0.2.2 -[ ] Support multiple --site args & searches +[x] Support multiple --site args & searches ### v0.3.0 [ ] Add duckduckgo scraper diff --git a/src/stackexchange.rs b/src/stackexchange.rs index b6c6a16..ddc3c48 100644 --- a/src/stackexchange.rs +++ b/src/stackexchange.rs @@ -116,7 +116,7 @@ impl StackExchange { /// Parallel searches against the search/advanced endpoint across all configured sites async fn search_advanced(&self, limit: u16) -> Result<Vec<Question>> { - let results = futures::stream::iter(self.config.sites.clone()) + futures::stream::iter(self.config.sites.clone()) .map(|site| { let clone = self.clone(); tokio::spawn(async move { @@ -126,12 +126,17 @@ impl StackExchange { }) .buffer_unordered(CONCURRENT_REQUESTS_LIMIT) .collect::<Vec<_>>() - .await; - results + .await .into_iter() .map(|r| r.map_err(Error::from).and_then(|x| x)) .collect::<Result<Vec<Vec<_>>>>() - .map(|v| v.into_iter().flatten().collect()) + .map(|v| { + let mut all_qs: Vec<Question> = v.into_iter().flatten().collect(); + if self.config.sites.len() > 1 { + all_qs.sort_unstable_by_key(|q| -q.score); + } + all_qs + }) } /// Search against the site's search/advanced endpoint with a given query. |