path: root/src/stackexchange.rs

use futures::stream::StreamExt;
use percent_encoding::percent_decode_str;
use rayon::prelude::*;
use reqwest::header;
use reqwest::Client;
use reqwest::Url;
use scraper::html::Html;
use scraper::selector::Selector;
use serde::{Deserialize, Serialize};
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::fs;
use std::path::PathBuf;

use crate::config::{project_dir, Config};
use crate::error::{Error, Result};
use crate::tui::markdown;
use crate::tui::markdown::Markdown;
use crate::utils;

/// DuckDuckGo URL
const DUCKDUCKGO_URL: &str = "https://duckduckgo.com";

/// StackExchange API v2.2 URL
// TODO why not https?
const SE_API_URL: &str = "http://api.stackexchange.com";
const SE_API_VERSION: &str = "2.2";

/// Filter generated to include only the fields needed to populate
/// the structs below. Go here to make new filters:
/// [create filter](https://api.stackexchange.com/docs/create-filter).
const SE_FILTER: &str = ".DND5X2VHHUH8HyJzpjo)5NvdHI3w6auG";

/// Pagesize when fetching all SE sites. Should be good for many years...
const SE_SITES_PAGESIZE: u16 = 10000;

/// Limit on concurrent requests (gets passed to `buffer_unordered`)
const CONCURRENT_REQUESTS_LIMIT: usize = 8;

/// Mock user agent to get real DuckDuckGo results
// TODO copy other user agents and use random one each time
const USER_AGENT: &str =
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0";

/// This structure allows interacting with parts of the StackExchange
/// API, using the `Config` struct to determine certain API settings and options.
#[derive(Clone)]
pub struct StackExchange {
    client: Client,
    config: Config,
    sites: HashMap<String, String>,
    query: String,
}

/// This structure allows interacting with locally cached StackExchange metadata.
pub struct LocalStorage {
    pub sites: Vec<Site>,
}

#[derive(Deserialize, Serialize, Debug)]
pub struct Site {
    pub api_site_parameter: String,
    pub site_url: String,
}

/// Represents a StackExchange answer with a custom selection of fields from
/// the [StackExchange docs](https://api.stackexchange.com/docs/types/answer)
#[derive(Clone, Deserialize, Debug)]
pub struct Answer<S> {
    #[serde(rename = "answer_id")]
    pub id: u32,
    pub score: i32,
    #[serde(rename = "body_markdown")]
    pub body: S,
    pub is_accepted: bool,
}

/// Represents a StackExchange question with a custom selection of fields from
/// the [StackExchange docs](https://api.stackexchange.com/docs/types/question)
// TODO container over answers should be generic iterator
#[derive(Clone, Deserialize, Debug)]
pub struct Question<S> {
    #[serde(rename = "question_id")]
    pub id: u32,
    pub score: i32,
    pub answers: Vec<Answer<S>>,
    pub title: String,
    #[serde(rename = "body_markdown")]
    pub body: S,
}

/// Internal struct that represents the boilerplate response wrapper from SE API.
#[derive(Deserialize, Debug)]
struct ResponseWrapper<T> {
    items: Vec<T>,
}

// Iss question_id unique across all sites? If not, then this edge case is
// unaccounted for when sorting.
//
// If this is ever an issue, it wouldn't be too hard to account for this; just
// keep track of site in the `ordering` field and also return site from the
// spawned per-site tasks.
#[derive(Debug, PartialEq)]
struct ScrapedData {
    /// Mapping of site code to question ids
    question_ids: HashMap<String, Vec<String>>,
    /// Mapping of question_id to its ordinal place in search results
    ordering: HashMap<String, usize>,
}

impl StackExchange {
    pub fn new(config: Config, local_storage: LocalStorage, query: String) -> Self {
        let client = Client::new();
        StackExchange {
            client,
            sites: local_storage.get_urls(&config.sites),
            config,
            query,
        }
    }

    /// Search query and get the top answer body
    ///
    /// For StackExchange engine, use only the first configured site,
    /// since, parodoxically, sites with the worst results will finish
    /// executing first, because there's less data to retrieve.
    ///
    /// Needs mut because it temporarily changes self.config
    pub async fn search_lucky(&mut self) -> Result<String> {
        let original_config = self.config.clone();
        // Temp set lucky config
        self.config.limit = 1;
        if !self.config.duckduckgo {
            self.config.sites.truncate(1);
        }
        // Run search with temp config
        let result = self.search().await;
        // Reset config
        self.config = original_config;

        Ok(result?
            .into_iter()
            .next()
            .ok_or(Error::NoResults)?
            .answers
            .into_iter()
            .next()
            .ok_or_else(|| Error::StackExchange(String::from("Received question with no answers")))?
            .body)
    }

    /// Search and parse to Markdown for TUI
    pub async fn search_md(&self) -> Result<Vec<Question<Markdown>>> {
        Ok(parse_markdown(self.search().await?))
    }

    /// Search query and get a list of relevant questions
    pub async fn search(&self) -> Result<Vec<Question<String>>> {
        if self.config.duckduckgo {
            self.search_duckduck_go().await
        } else {
            // TODO after duckduck go finished, refactor to _not_ thread this limit, its unnecessary
            self.se_search_advanced(self.config.limit).await
        }
    }

    /// Search query at duckduckgo and then fetch the resulting questions from SE.
    async fn search_duckduck_go(&self) -> Result<Vec<Question<String>>> {
        let url = duckduckgo_url(&self.query, self.sites.values());
        let html = self
            .client
            .get(url)
            .header(header::USER_AGENT, USER_AGENT)
            .send()
            .await?
            .text()
            .await?;
        let data = parse_questions_from_ddg_html(&html, &self.sites, self.config.limit)?;
        self.se_questions(data).await
    }

    /// Parallel searches against the SE question endpoint across the sites in `ids`.
    // TODO I'm sure there is a way to DRY the se_question & se_search_advanced functions
    async fn se_questions(&self, data: ScrapedData) -> Result<Vec<Question<String>>> {
        let ScrapedData {
            question_ids,
            ordering,
        } = data;
        futures::stream::iter(question_ids)
            .map(|(site, ids)| {
                let clone = self.clone();
                tokio::spawn(async move {
                    let clone = &clone;
                    clone.se_questions_site(&site, ids).await
                })
            })
            .buffer_unordered(CONCURRENT_REQUESTS_LIMIT)
            .collect::<Vec<_>>()
            .await
            .into_iter()
            .map(|r| r.map_err(Error::from).and_then(|x| x))
            .collect::<Result<Vec<Vec<_>>>>()
            .map(|v| {
                let mut qs: Vec<Question<String>> = v.into_iter().flatten().collect();
                qs.sort_unstable_by_key(|q| ordering.get(&q.id.to_string()).unwrap());
                qs
            })
    }

    /// Parallel searches against the SE search/advanced endpoint across all configured sites
    async fn se_search_advanced(&self, limit: u16) -> Result<Vec<Question<String>>> {
        futures::stream::iter(self.config.sites.clone())
            .map(|site| {
                let clone