use repo_url::SimpleRepo; use simple_cache::TempCache; use std::future::Future; use std::path::Path; use github_v3::StatusCode; use serde::{Deserialize, Serialize}; use urlencoding::encode; mod model; pub use crate::model::*; pub type CResult = Result; use quick_error::quick_error; quick_error! { #[derive(Debug)] pub enum Error { NoBody { display("Reponse with no body") } TryAgainLater { display("Accepted, but no data available yet") } Cache(err: Box) { display("GH can't decode cache: {}", err) from(e: simple_cache::Error) -> (Box::new(e)) cause(err) } GitHub(err: String) { display("{}", err) from(e: github_v3::GHError) -> (e.to_string()) // non-Sync } Json(err: Box, call: Option<&'static str>) { display("JSON decode error {} in {}", err, call.unwrap_or("github_info")) from(e: serde_json::Error) -> (Box::new(e), None) cause(err) } Time(err: std::time::SystemTimeError) { display("{}", err) from() cause(err) } } } impl Error { pub fn context(self, ctx: &'static str) -> Self { match self { Error::Json(e, _) => Error::Json(e, Some(ctx)), as_is => as_is, } } } pub struct GitHub { client: github_v3::Client, user_orgs: TempCache<(String, Option>)>, orgs: TempCache<(String, Option)>, users: TempCache<(String, Option)>, commits: TempCache<(String, Option>)>, releases: TempCache<(String, Option>)>, contribs: TempCache<(String, Option>)>, repos: TempCache<(String, Option)>, emails: TempCache<(String, Option>)>, } impl GitHub { pub fn new(cache_path: impl AsRef, token: &str) -> CResult { Ok(Self { client: github_v3::Client::new(Some(token)), user_orgs: TempCache::new(&cache_path.as_ref().with_file_name("github_user_orgs.bin"))?, orgs: TempCache::new(&cache_path.as_ref().with_file_name("github_orgs2.bin"))?, users: TempCache::new(&cache_path.as_ref().with_file_name("github_users3.bin"))?, commits: TempCache::new(&cache_path.as_ref().with_file_name("github_commits2.bin"))?, releases: TempCache::new(&cache_path.as_ref().with_file_name("github_releases2.bin"))?, contribs: TempCache::new(&cache_path.as_ref().with_file_name("github_contribs.bin"))?, repos: TempCache::new(&cache_path.as_ref().with_file_name("github_repos2.bin"))?, emails: TempCache::new(&cache_path.as_ref().with_file_name("github_emails.bin"))?, }) } pub async fn user_by_email(&self, email: &str) -> CResult>> { let std_suffix = "@users.noreply.github.com"; if email.ends_with(std_suffix) { let login = email[0..email.len() - std_suffix.len()].split('+').last().unwrap(); if let Some(user) = self.user_by_login(login).await? { return Ok(Some(vec![user])); } } let enc_email = encode(email); self.get_cached(&self.emails, (email, ""), |client| client.get() .path(&format!("search/users?q=in:email%20{}", enc_email)) .send(), |res: SearchResults| { println!("Found {} = {:#?}", email, res.items); res.items }).await } pub async fn user_by_login(&self, login: &str) -> CResult> { let key = login.to_ascii_lowercase(); self.get_cached(&self.users, (&key, ""), |client| client.get() .path("users").arg(login) .send(), id).await.map_err(|e| e.context("user_by_login")) } pub async fn user_orgs(&self, login: &str) -> CResult>> { let key = login.to_ascii_lowercase(); self.get_cached(&self.user_orgs, (&key, ""), |client| client.get() .path("users").arg(login).path("orgs") .send(), id).await.map_err(|e| e.context("user_orgs")) } pub async fn org(&self, login: &str) -> CResult> { let key = login.to_ascii_lowercase(); self.get_cached(&self.orgs, (&key, ""), |client| client.get() .path("orgs").arg(login) .send(), id).await.map_err(|e| e.context("user_orgs")) } pub async fn commits(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult>> { let key = format!("commits/{}/{}", repo.owner, repo.repo); self.get_cached(&self.commits, (&key, as_of_version), |client| client.get() .path("repos").arg(&repo.owner).arg(&repo.repo) .path("commits") .send(), id).await.map_err(|e| e.context("commits")) } pub async fn releases(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult>> { let key = format!("release/{}/{}", repo.owner, repo.repo); let path = format!("repos/{}/{}/releases", repo.owner, repo.repo); self.get_cached(&self.releases, (&key, as_of_version), |client| client.get() .path(&path) .send(), id).await.map_err(|e| e.context("releases")) } pub async fn topics(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult>> { let repo = self.repo(repo, as_of_version).await?; Ok(repo.map(|r| r.topics)) } pub async fn repo(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult> { let key = format!("{}/{}", repo.owner, repo.repo); self.get_cached(&self.repos, (&key, as_of_version), |client| client.get() .path("repos").arg(&repo.owner).arg(&repo.repo) .send(), |mut ghdata: GitHubRepo| { // Keep GH-specific logic in here if ghdata.has_pages { // Name is case-sensitive ghdata.github_page_url = Some(format!("https://{}.github.io/{}/", repo.owner, ghdata.name)); } // Some homepages are empty strings if ghdata.homepage.as_ref().map_or(false, |h| !h.starts_with("http")) { ghdata.homepage = None; } if !ghdata.has_issues { ghdata.open_issues_count = None; } ghdata }).await .map_err(|e| e.context("repo")) } pub async fn contributors(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult>> { let path = format!("repos/{}/{}/stats/contributors", repo.owner, repo.repo); let key = (path.as_str(), as_of_version); let callback = |client: &github_v3::Client| { client.get().path(&path).send() }; self.get_cached(&self.contribs, key, callback, id).await } async fn get_cached(&self, cache: &TempCache<(String, Option)>, key: (&str, &str), cb: F, postproc: P) -> CResult> where P: FnOnce(B) -> R, F: FnOnce(&github_v3::Client) -> A, A: Future>, B: for<'de> serde::Deserialize<'de> + serde::Serialize + Clone + Send + 'static, R: for<'de> serde::Deserialize<'de> + serde::Serialize + Clone + Send + 'static, { if let Some((ver, payload)) = cache.get(key.0)? { if ver == key.1 { return Ok(payload); } eprintln!("Cache near miss {}@{} vs {}", key.0, ver, key.1); } let (status, res) = match Box::pin(cb(&self.client)).await { Ok(res) => { let status = res.status(); let headers = res.headers(); eprintln!("Recvd {}@{} {:?} {:?}", key.0, key.1, status, headers); (status, Some(res)) }, Err(github_v3::GHError::Response { status, message }) => { eprintln!("GH Error {} {}", status, message.as_deref().unwrap_or("??")); (status, None) }, Err(e) => return Err(e.into()), }; let non_parsable_body = match status { StatusCode::ACCEPTED | StatusCode::CREATED => return Err(Error::TryAgainLater), StatusCode::NO_CONTENT | StatusCode::NOT_FOUND | StatusCode::GONE | StatusCode::MOVED_PERMANENTLY => true, _ => false, }; let keep_cached = match status { StatusCode::NOT_FOUND | StatusCode::GONE | StatusCode::MOVED_PERMANENTLY => true, _ => status.is_success(), }; let body = match res { Some(res) if !non_parsable_body => Some(Box::pin(res.obj()).await?), _ => None, }; match body.ok_or(Error::NoBody).and_then(|stats| { let dbg = format!("{:?}", stats); Ok(postproc(serde_json::from_value(stats).map_err(|e| { eprintln!("Error matching JSON: {}\n data: {}", e, dbg); e })?)) }) { Ok(val) => { let res = (key.1.to_string(), Some(val)); if keep_cached { cache.set(key.0, &res)?; } Ok(res.1) }, Err(_) if non_parsable_body => { if keep_cached { cache.set(key.0, (key.1.to_string(), None))?; } Ok(None) }, Err(err) => Err(err)?, } } } fn id(v: T) -> T { v } #[derive(Serialize, Deserialize, Debug, Clone)] enum Payload { Meta(Vec), Contrib(Vec), Res(SearchResults), User(User), Topics(Topics), GitHubRepo(GitHubRepo), Dud, } impl Payloadable for Vec { fn to(&self) -> Payload { Payload::Meta(self.clone()) } fn from(p: Payload) -> Option { match p { Payload::Meta(d) => Some(d), _ => None, } } } impl Payloadable for Vec { fn to(&self) -> Payload { Payload::Contrib(self.clone()) } fn from(p: Payload) -> Option { match p { Payload::Contrib(d) => Some(d), _ => None, } } } impl Payloadable for SearchResults { fn to(&self) -> Payload { Payload::Res(self.clone()) } fn from(p: Payload) -> Option { match p { Payload::Res(d) => Some(d), _ => None, } } } impl Payloadable for User { fn to(&self) -> Payload { Payload::User(self.clone()) } fn from(p: Payload) -> Option { match p { Payload::User(d) => Some(d), _ => None, } } } pub(crate) trait Payloadable: Sized { fn to(&self) -> Payload; fn from(val: Payload) -> Option; } #[cfg(test)] #[tokio::test] async fn github_contrib() { let gh = GitHub::new( "../data/github.db", &std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap(); let repo = SimpleRepo{ owner:"visionmedia".into(), repo:"superagent".into(), }; gh.contributors(&repo, "").await.unwrap(); gh.commits(&repo, "").await.unwrap(); } #[cfg(test)] #[tokio::test] async fn github_releases() { let gh = GitHub::new( "../data/github.db", &std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap(); let repo = SimpleRepo{ owner:"kornelski".into(), repo:"pngquant".into(), }; assert!(gh.releases(&repo, "").await.unwrap().unwrap().len() > 2); } #[cfg(test)] #[tokio::test] async fn test_user_by_email() { let gh = GitHub::new( "../data/github.db", &std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap(); let user = gh.user_by_email("github@pornel.net").await.unwrap().unwrap(); assert_eq!("kornelski", user[0].login); }