Async github client

9 files changed, 608 insertions, 160 deletions

diff --git a/github_info/Cargo.toml b/github_info/Cargo.toml
index 2b544c1..ecddb8e 100644
--- a/github_info/Cargo.toml
+++ b/github_info/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 edition = "2018"
 name = "github_info"
-version = "0.8.0"
+version = "0.9.0"
 authors = ["Kornel <kornel@geekhood.net>"]
 
 [lib]
@@ -10,10 +10,12 @@ path = "src/lib_github.rs"
 
 [dependencies]
 repo_url = { path = "../repo_url" }
+github_v3 = { path = "../github_v3" }
 simple_cache = { path = "../simple_cache", version = "0.7.0" }
 serde = { version = "1.0.104", features = ["derive"] }
 serde_json = "1.0.44"
 urlencoding = "1.0.0"
 quick-error = "1.2.2"
-hyper = "0.12.0"
-github-rs = {git = "https://github.com/kornelski/github-rs"}
+
+[dev-dependencies]
+tokio = { version = "0.2.12", features = ["macros", "rt-threaded"] }
diff --git a/github_info/src/lib_github.rs b/github_info/src/lib_github.rs
index 606d16a..cb3e9cb 100644
--- a/github_info/src/lib_github.rs
+++ b/github_info/src/lib_github.rs
@@ -1,15 +1,12 @@
-use github_rs::client::Executor;
-use github_rs::client;
-use github_rs::headers::{rate_limit_remaining, rate_limit_reset};
-use github_rs::{HeaderMap, StatusCode};
+use std::future::Future;
 use repo_url::SimpleRepo;
 use simple_cache::TempCache;
 use std::path::Path;
-use std::thread;
-use std::time::Duration;
-use std::time::{SystemTime, UNIX_EPOCH};
+
+
 use urlencoding::encode;
 use serde::{Serialize, Deserialize};
+use github_v3::StatusCode;
 
 mod model;
 pub use crate::model::*;
@@ -33,7 +30,7 @@ quick_error! {
         }
         GitHub(err: String) {
             display("{}", err)
-            from(e: github_rs::errors::Error) -> (e.to_string()) // non-Sync
+            from(e: github_v3::GHError) -> (e.to_string()) // non-Sync
         }
         Json(err: Box<serde_json::Error>, call: Option<&'static str>) {
             display("JSON decode error {} in {}", err, call.unwrap_or("github_info"))
@@ -58,7 +55,7 @@ impl Error {
 }
 
 pub struct GitHub {
-    token: String,
+    client: github_v3::Client,
     orgs: TempCache<(String, Option<Vec<UserOrg>>)>,
     users: TempCache<(String, Option<User>)>,
     commits: TempCache<(String, Option<Vec<CommitMeta>>)>,
@@ -69,9 +66,9 @@ pub struct GitHub {
 }
 
 impl GitHub {
-    pub fn new(cache_path: impl AsRef<Path>, token: impl Into<String>) -> CResult<Self> {
+    pub fn new(cache_path: impl AsRef<Path>, token: &str) -> CResult<Self> {
         Ok(Self {
-            token: token.into(),
+            client: github_v3::Client::new(Some(token)),
             orgs: TempCache::new(&cache_path.as_ref().with_file_name("github_orgs.bin"))?,
             users: TempCache::new(&cache_path.as_ref().with_file_name("github_users.bin"))?,
             commits: TempCache::new(&cache_path.as_ref().with_file_name("github_commits.bin"))?,
@@ -82,74 +79,70 @@ impl GitHub {
         })
     }
 
-    fn client(&self) -> CResult<client::Github> {
-        Ok(client::Github::new(&self.token)?)
-    }
-
-    pub fn user_by_email(&self, email: &str) -> CResult<Option<Vec<User>>> {
+    pub async fn user_by_email(&self, email: &str) -> CResult<Option<Vec<User>>> {
         let std_suffix = "@users.noreply.github.com";
         if email.ends_with(std_suffix) {
             let login = email[0..email.len() - std_suffix.len()].split('+').last().unwrap();
-            if let Some(user) = self.user_by_login(login)? {
+            if let Some(user) = self.user_by_login(login).await? {
                 return Ok(Some(vec![user]));
             }
         }
         let enc_email = encode(email);
         self.get_cached(&self.emails, (email, ""), |client| client.get()
-                       .custom_endpoint(&format!("search/users?q=in:email%20{}", enc_email))
-                       .execute(), |res: SearchResults<User>| {
+                       .path(&format!("search/users?q=in:email%20{}", enc_email))
+                       .send(), |res: SearchResults<User>| {
                         println!("Found {} = {:#?}", email, res.items);
                         res.items
-                    })
+                    }).await
     }
 
-    pub fn user_by_login(&self, login: &str) -> CResult<Option<User>> {
+    pub async fn user_by_login(&self, login: &str) -> CResult<Option<User>> {
         let key = login.to_ascii_lowercase();
         self.get_cached(&self.users, (&key, ""), |client| client.get()
-                       .users().username(login)
-                       .execute(), id).map_err(|e| e.context("user_by_login"))
+                       .path("users").arg(login)
+                       .send(), id).await.map_err(|e| e.context("user_by_login"))
     }
 
-    pub fn user_by_id(&self, user_id: u32) -> CResult<Option<User>> {
+    pub async fn user_by_id(&self, user_id: u32) -> CResult<Option<User>> {
         let user_id = user_id.to_string();
         self.get_cached(&self.users, (&user_id, ""), |client| client.get()
-                       .users().username(&user_id)
-                       .execute(), id).map_err(|e| e.context("user_by_id"))
+                       .path("users").arg(&user_id)
+                       .send(), id).await.map_err(|e| e.context("user_by_id"))
     }
 
-    pub fn user_orgs(&self, login: &str) -> CResult<Option<Vec<UserOrg>>> {
+    pub async fn user_orgs(&self, login: &str) -> CResult<Option<Vec<UserOrg>>> {
         let key = login.to_ascii_lowercase();
         self.get_cached(&self.orgs, (&key, ""), |client| client.get()
-                       .users().username(login).orgs()
-                       .execute(), id).map_err(|e| e.context("user_orgs"))
+                       .path("users").arg(login).path("orgs")
+                       .send(), id).await.map_err(|e| e.context("user_orgs"))
     }
 
-    pub fn commits(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<Vec<CommitMeta>>> {
+    pub async fn commits(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<Vec<CommitMeta>>> {
         let key = format!("commits/{}/{}", repo.owner, repo.repo);
         self.get_cached(&self.commits, (&key, as_of_version), |client| client.get()
-                           .repos().owner(&repo.owner).repo(&repo.repo)
-                           .commits()
-                           .execute(), id).map_err(|e| e.context("commits"))
+                           .path("repos").arg(&repo.owner).arg(&repo.repo)
+                           .path("commits")
+                           .send(), id).await.map_err(|e| e.context("commits"))
     }
 
-    pub fn releases(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<Vec<GitHubRelease>>> {
+    pub async fn releases(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<Vec<GitHubRelease>>> {
         let key = format!("release/{}/{}", repo.owner, repo.repo);
         let path = format!("repos/{}/{}/releases", repo.owner, repo.repo);
         self.get_cached(&self.releases, (&key, as_of_version), |client| client.get()
-                           .custom_endpoint(&path)
-                           .execute(), id).map_err(|e| e.context("releases"))
+                           .path(&path)
+                           .send(), id).await.map_err(|e| e.context("releases"))
     }
 
-    pub fn topics(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<Vec<String>>> {
-        let repo = self.repo(repo, as_of_version)?;
+    pub async fn topics(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<Vec<String>>> {
+        let repo = self.repo(repo, as_of_version).await?;
         Ok(repo.map(|r| r.topics))
     }
 
-    pub fn repo(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<GitHubRepo>> {
+    pub async fn repo(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<GitHubRepo>> {
         let key = format!("{}/{}", repo.owner, repo.repo);
         self.get_cached(&self.repos, (&key, as_of_version), |client| client.get()
-                .repos().owner(&repo.owner).repo(&repo.repo)
-                .execute(), |mut ghdata: GitHubRepo| {
+                .path("repos").arg(&repo.owner).arg(&repo.repo)
+                .send(), |mut ghdata: GitHubRepo| {
                     // Keep GH-specific logic in here
                     if ghdata.has_pages {
                         // Name is case-sensitive
@@ -163,35 +156,24 @@ impl GitHub {
                         ghdata.open_issues_count = None;
                     }
                     ghdata
-                })
+                }).await
                 .map_err(|e| e.context("repo"))
     }
 
-    pub fn contributors(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<Vec<UserContrib>>> {
+    pub async fn contributors(&self, repo: &SimpleRepo, as_of_version: &str) -> CResult<Option<Vec<UserContrib>>> {
         let path = format!("repos/{}/{}/stats/contributors", repo.owner, repo.repo);
         let key = (path.as_str(), as_of_version);
-        let callback = |client: &client::Github| {
-            client.get().custom_endpoint(&path).execute()
+        let callback = |client: &github_v3::Client| {
+            client.get().path(&path).send()
         };
-        let mut retries = 5;
-        let mut delay = 1;
-        loop {
-            match self.get_cached(&self.contribs, key, callback, id) {
-                Err(Error::TryAgainLater) if retries > 0 => {
-                    thread::sleep(Duration::from_secs(delay));
-                    retries -= 1;
-                    delay *= 2;
-                },
-                Err(e) => return Err(e.context("contributors")),
-                res => return res,
-            }
-        }
+        self.get_cached(&self.contribs, key, callback, id).await
     }
 
-    fn get_cached<F, P, B, R>(&self, cache: &TempCache<(String, Option<R>)>, key: (&str, &str), cb: F, postproc: P) -> CResult<Option<R>>
+    async fn get_cached<F, P, B, R, A>(&self, cache: &TempCache<(String, Option<R>)>, key: (&str, &str), cb: F, postproc: P) -> CResult<Option<R>>
     where
         P: FnOnce(B) -> R,
-        F: FnOnce(&client::Github) -> Result<(HeaderMap, StatusCode, Option<serde_json::Value>), github_rs::errors::Error>,
+        F: FnOnce(&github_v3::Client) -> A,
+        A: Future<Output=Result<github_v3::Response, github_v3::GHError>>,
         B: for<'de> serde::Deserialize<'de> + serde::Serialize + Clone + Send + 'static,
         R: for<'de> serde::Deserialize<'de> + serde::Serialize + Clone + Send + 'static,
     {
@@ -202,22 +184,10 @@ impl GitHub {
             eprintln!("Cache near miss {}@{} vs {}", key.0, ver, key.1);
         }
 
-        let client = &self.client()?;
-        // eprintln!("Cache miss {}@{}", key.0, key.1);
-        let (headers, status, body) = cb(&*client)?;
+        let res = cb(&self.client).await?;
+        let status = res.status();
+        let headers = res.headers();
         eprintln!("Recvd {}@{} {:?} {:?}", key.0, key.1, status, headers);
-        if let (Some(rl), Some(rs)) = (rate_limit_remaining(&headers), rate_limit_reset(&headers)) {
-            let end_timestamp = Duration::from_secs(rs.into());
-            let now = SystemTime::now().duration_since(UNIX_EPOCH)?;
-            let wait = (end_timestamp.checked_sub(now)).and_then(|d| d.checked_div(rl + 2));
-            if let Some(wait) = wait {
-                if wait.as_secs() > 2 && (rl < 8 || wait.as_secs() < 15) {
-                    eprintln!("need to wait! {:?}", wait);
-                    thread::sleep(wait);
-                }
-            }
-        }
-
         let non_parsable_body = match status {
             StatusCode::ACCEPTED |
             StatusCode::CREATED => return Err(Error::TryAgainLater),
@@ -234,8 +204,8 @@ impl GitHub {
             StatusCode::MOVED_PERMANENTLY => true,
             _ => status.is_success(),
         };
-
-        match body.ok_or(Error::NoBody).and_then(|stats| {
+        let body = res.obj().await;
+        match body.map_err(|_| Error::NoBody).and_then(|stats| {
             let dbg = format!("{:?}", stats);
             Ok(postproc(serde_json::from_value(stats).map_err(|e| {
                 eprintln!("Error matching JSON: {}\n data: {}", e, dbg); e
@@ -328,37 +298,40 @@ pub(crate) trait Payloadable: Sized {
 }
 
 
-#[test]
-fn github_contrib() {
+#[cfg(test)]
+#[tokio::test]
+async fn github_contrib() {
     let gh = GitHub::new(
         "../data/github.db",
-        std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap();
+        &std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap();
     let repo = SimpleRepo{
         owner:"visionmedia".into(),
         repo:"superagent".into(),
     };
-    gh.contributors(&repo, "").unwrap();
-    gh.commits(&repo, "").unwrap();
+    gh.contributors(&repo, "").await.unwrap();
+    gh.commits(&repo, "").await.unwrap();
 }
 
-#[test]
-fn github_releases() {
+#[cfg(test)]
+#[tokio::test]
+async fn github_releases() {
     let gh = GitHub::new(
         "../data/github.db",
-        std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap();
+        &std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap();
     let repo = SimpleRepo{
         owner:"kornelski".into(),
         repo:"pngquant".into(),
     };
-    assert!(gh.releases(&repo, "").unwrap().unwrap().len() > 2);
+    assert!(gh.releases(&repo, "").await.unwrap().unwrap().len() > 2);
 }
 
-#[test]
-fn test_user_by_email() {
+#[cfg(test)]
+#[tokio::test]
+async fn test_user_by_email() {
     let gh = GitHub::new(
         "../data/github.db",
-        std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap();
-    let user = gh.user_by_email("github@pornel.net").unwrap().unwrap();
+        &std::env::var("GITHUB_TOKEN").expect("GITHUB_TOKEN env var")).unwrap();
+    let user = gh.user_by_email("github@pornel.net").await.unwrap().unwrap();
     assert_eq!("kornelski", user[0].login);
 }
 
diff --git a/github_v3/Cargo.toml b/github_v3/Cargo.toml
new file mode 100644
index 0000000..617bb0b
--- /dev/null
+++ b/github_v3/Cargo.toml
@@ -0,0 +1,25 @@
+[package]
+name = "github_v3"
+description = "Async GitHub API v3 client"
+version = "0.3.0"
+authors = ["Kornel <kornel@geekhood.net>"]
+keywords = ["github", "rest-api", "async"]
+categories = ["web-programming", "web-programming::http-client"]
+edition = "2018"
+
+[dependencies]
+reqwest = { version = "0.10.1", features = ["json"] }
+serde = "1.0.104"
+serde_json = "1.0.47"
+thiserror = "1.0.10"
+serde_derive = "1.0.104"
+futures = "0.3.4"
+async-stream = "0.2.1"
+tokio = { version = "0.2.11", features = ["time"] }
+
+[dev-dependencies]
+tokio = { version = "0.2.11", features = ["rt-threaded", "macros", "time"] }
+
+[features]
+default = ["gzip"]
+gzip = ["reqwest/gzip"]
diff --git a/github_v3/examples/users.rs b/github_v3/examples/users.rs
new file mode 100644
index 0000000..14d0f66
--- /dev/null
+++ b/github_v3/examples/users.rs
@@ -0,0 +1,13 @@
+use futures::StreamExt;
+use github_v3::*;
+
+#[tokio::main]
+async fn main() -> Result<(), GHError> {
+    let gh = Client::new_from_env();
+    let mut users = gh.get().path("users").send().await?.array::<model::User>();
+
+    while let Some(Ok(user)) = users.next().await {
+        println!("User! {:#?}", user);
+    }
+    Ok(())
+}
diff --git a/github_v3/src/lib.rs b/github_v3/src/lib.rs
new file mode 100644
index 0000000..2f03e27
--- /dev/null
+++ b/github_v3/src/lib.rs
@@ -0,0 +1,260 @@
+use std::sync::atomic::Ordering::SeqCst;
+use std::sync::atomic::AtomicU32;
+use std::sync::Arc;
+pub use reqwest::StatusCode;
+pub use reqwest::header::HeaderMap;
+pub use reqwest::header::HeaderValue;
+pub use futures::StreamExt;
+use futures::Stream;
+use serde::de::DeserializeOwned;
+use std::time::{Duration, SystemTime};
+
+pub mod model;
+
+pub struct Response {
+    res: reqwest::Response,
+    client: Arc<ClientInner>
+}
+
+impl Response {
+    pub async fn obj<T: DeserializeOwned>(self) -> Result<T, GHError> {
+        Ok(self.res.json().await?)
+    }
+
+    pub fn array<T: DeserializeOwned + std::marker::Unpin + 'static>(self) -> impl Stream<Item=Result<T, GHError>> {
+        let mut res = self.res;
+        let client = self.client;
+
+        // Pin is required for easy iteration, otherwise the caller would have to pin it
+        Box::pin(async_stream::try_stream! {
+            loop {
+                let next_link = res.headers().get("link")
+                    .and_then(|h| h.to_str().ok())
+                    .and_then(parse_next_link);
+                let items = res.json::<Vec<T>>().await?;
+                for item in items {
+                    yield item;
+                }
+                match next_link {
+                    Some(url) => res = client.raw_get(&url).await?,
+                    None => break,
+                }
+            }
+        })
+    }
+
+    pub fn headers(&self) -> &HeaderMap {
+        self.res.headers()
+    }
+
+    pub fn status(&self) -> StatusCode {
+        self.res.status()
+    }
+}
+
+pub struct Builder {
+    client: Arc<ClientInner>,
+    url: String,
+}
+
+impl Builder {
+    pub fn path(mut self, url_part: &str) -> Self {
+        debug_assert_eq!(url_part, url_part.trim_matches('/'));
+
+        self.url.push('/');
+        self.url.push_str(url_part);
+        self
+    }
+
+    pub fn arg(mut self, arg: &str) -> Self {
+        self.url.push('/');
+        self.url.push_str(arg);
+        self
+    }
+
+    pub async fn send(self) -> Result<Response, GHError> {
+        let res = self.client.raw_get(&self.url).await?;
+        Ok(Response {
+            client: self.client,
+            res
+        })
+    }
+}
+
+struct ClientInner {
+    client: reqwest::Client,
+    // FIXME: this should be per endpoint, because search and others have different throttling
+    wait_sec: AtomicU32,
+}
+
+pub struct Client {
+    inner: Arc<ClientInner>,
+}
+
+impl Client {
+    pub fn new_from_env() -> Self {
+        Self::new(std::env::var("GITHUB_TOKEN").ok().as_deref())
+    }
+
+    pub fn new(token: Option<&str>) -> Self {
+        let mut default_headers = HeaderMap::with_capacity(2);
+        default_headers.insert("Accept", HeaderValue::from_static("application/vnd.github.v3+json"));
+        if let Some(token) = token {
+            default_headers.insert("Authorization", HeaderValue::from_str(&format!("token {}", token)).unwrap());
+        }
+
+        Self {
+            inner: Arc::new(ClientInner {
+                client: reqwest::Client::builder()
+                    .user_agent(concat!("rust-github-v3/{}", env!("CARGO_PKG_VERSION")))
+                    .default_headers(default_headers)
+                    .connect_timeout(Duration::from_secs(7))
+                    .timeout(Duration::from_secs(30))
+                    .build()
+                    .unwrap(),
+                wait_sec: AtomicU32::new(0),
+            }),
+        }
+    }
+
+    pub fn get(&self) -> Builder {
+        let mut url = String::with_capacity(60);
+        url.push_str("https://api.github.com");
+        Builder {
+            client: self.inner.clone(),
+            url,
+        }
+    }
+}
+
+impl ClientInner {
+    // Get a single response
+    async fn raw_get(&self, url: &str) -> Result<reqwest::Response, GHError> {
+        debug_assert!(url.starts_with("https://api.github.com/"));
+
+        let mut retries = 5u8;
+        let mut retry_delay = 1;
+        loop {
+            let wait_sec = self.wait_sec.load(SeqCst);
+            if wait_sec > 0 {
+                // This has poor behavior with concurrency. It should be pacing all requests.
+                tokio::time::delay_for(Duration::from_secs(wait_sec.into())).await;
+            }
+
+            let res = self.client.get(url).send().await?;
+
+            let headers = res.headers();
+            let status = res.status();
+
+            let wait_sec = match (Self::rate_limit_remaining(headers), Self::rate_limit_reset(headers)) {
+                (Some(rl), Some(rs)) => {
+                    rs.duration_since(SystemTime::now()).ok()
+                        .and_then(|d| d.checked_div(rl + 2))
+                        .map(|d| d.as_secs() as u32)
+                        .unwrap_or(0)
+                }
+                _ => if status == StatusCode::TOO_MANY_REQUESTS {3} else {0},
+            };
+            self.wait_sec.store(wait_sec, SeqCst);
+
+            if status == StatusCode::ACCEPTED && retries > 0 {
+                tokio::time::delay_for(Duration::from_secs(retry_delay)).await;
+                retry_delay *= 2;
+                retries -= 1;
+                continue;
+            }
+
+            return if status.is_success() && status != StatusCode::ACCEPTED {
+                Ok(res)
+            } else {
+                Err(error_for_response(res).await)
+            };
+        }
+    }
+
+    pub fn rate_limit_remaining(headers: &HeaderMap) -> Option<u32> {
+        headers.get("x-ratelimit-remaining")
+            .and_then(|s| s.to_str().ok())
+            .and_then(|s| s.parse().ok())
+    }
+
+    pub fn rate_limit_reset(headers: &HeaderMap) -> Option<SystemTime> {
+        headers.get("x-ratelimit-reset")
+            .and_then(|s| s.to_str().ok())
+            .and_then(|s| s.parse().ok())
+            .map(|s| SystemTime::UNIX_EPOCH + Duration::from_secs(s))
+    }
+}
+
+async fn error_for_response(res: reqwest::Response) -> GHError {
+    let status = res.status().as_u16();
+    let mime = res.headers().get("content-type").and_then(|h| h.to_str().ok()).unwrap_or("");
+    GHError::Response {
+        status,
+        message: if mime.starts_with("application/json") {
+            res.json::<GitHubErrorResponse>().await.ok().map(|res| res.message)
+        } else {
+            None
+        },
+    }
+}
+
+fn parse_next_link(link: &str) -> Option<String> {
+    for part in link.split(',') {
+        if part.contains(r#"; rel="next""#) {
+            if let Some(start) = link.find('<') {
+                let link = &link[start+1..];
+                if let Some(end) = link.find('>') {
+                    return Some(link[..end].to_owned());
+                }
+            }
+        }
+    }
+    None
+}
+
+#[derive(serde_derive::Deserialize)]
+struct GitHubErrorResponse {
+    message: String,
+}
+
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum GHError {
+    #[error("Request timed out")]
+    Timeout,
+    #[error("Request error: {}", _0)]
+    Request(String),
+    #[error("{} ({})", message.as_deref().unwrap_or("HTTP error"), status)]
+    Response { status: u16, message: Option<String> },
+    #[error("Internal error")]
+    Internal,
+}
+
+impl From<reqwest::Error> for GHError {
+    fn from(e: reqwest::Error) -> Self {
+        if e.is_timeout() {
+            return Self::Timeout;
+        }
+        if let Some(s) = e.status() {
+            Self::Response {
+                status: s.as_u16(),
+                message: Some(e.to_string()),
+            }
+        } else {
+            Self::Request(e.to_string())
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[tokio::test]
+    async fn req_test() {
+        let gh = Client::new_from_env();
+        gh.get().path("users/octocat/orgs").send().await.unwrap();
+    }
+}
diff --git a/github_v3/src/model.rs b/github_v3/src/model.rs
new file mode 100644
index 0000000..9c26acf
--- /dev/null
+++ b/github_v3/src/model.rs
@@ -0,0 +1,186 @@
+#[derive(Debug, Copy, Eq, PartialEq, Clone)]
+pub enum UserType {
+    Org,
+    User,
+    Bot,
+}
+
+use serde::de;
+use serde::de::{Deserializer, Visitor};
+use serde_derive::Deserialize;
+use serde_derive::Serialize;
+use serde::Serializer;
+use std::fmt;
+
+/// Case-insensitive enum
+impl<'de> serde::Deserialize<'de> for UserType {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+        where D: Deserializer<'de>,
+    {
+        struct UserTypeVisitor;
+
+        impl<'a> Visitor<'a> for UserTypeVisitor {
+            type Value = UserType;
+
+            fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+                formatter.write_str("user/org/bot")
+            }
+
+            fn visit_str<E: de::Error>(self, v: &str) -> Result<Self::Value, E> {
+                match v.to_ascii_lowercase().as_str() {
+                    "org" | "organization" => Ok(UserType::Org),
+                    "user" => Ok(UserType::User),
+                    "bot" => Ok(UserType::Bot),
+                    x => Err(de::Error::unknown_variant(x, &["user", "org", "bot"])),
+                }
+            }
+
+            fn visit_string<E: de::Error>(self, v: String) -> Result<Self::Value, E> {
+                self.visit_str(&v)
+            }
+        }
+
+        deserializer.deserialize_string(UserTypeVisitor)
+    }
+}
+
+impl serde::Serialize for UserType {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+        where S: Serializer,
+    {
+        serializer.serialize_str(match *self {
+            UserType::User => "user",
+            UserType::Org => "org",
+            UserType::Bot => "bot",
+        })
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct User {
+    pub id: u32,
+    pub login: String,
+    pub name: Option<String>,
+    pub avatar_url: Option<String>,  // "https://avatars0.githubusercontent.com/u/1111?v=4",
+    pub gravatar_id: Option<String>, // "",
+    pub html_url: String,            // "https://github.com/zzzz",
+    pub blog: Option<String>,        // "https://example.com
+    #[serde(rename = "type")]
+    pub user_type: UserType,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ContribWeek {
+    #[serde(rename = "w")]
+    pub week_timestamp: u32,
+    #[serde(rename = "a")]
+    pub added: u32,
+    #[serde(rename = "d")]
+    pub deleted: u32,
+    #[serde(rename = "c")]
+    pub commits: u32,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SearchResults<T> {
+    pub items: Vec<T>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct UserContrib {
+    pub total: u32,
+    pub weeks: Vec<ContribWeek>,
+    pub author: Option<User>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GitCommitAuthor {
+    pub date: String, // "2018-04-30T16:24:52Z",
+    pub email: String,
+    pub name: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GitCommit {
+    pub author: GitCommitAuthor,
+    pub committer: GitCommitAuthor,
+    pub message: String,
+    pub comment_count: u32,
+    // tree.sha
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CommitMeta {
+    pub sha: String, // TODO: deserialize to bin
+    pub author: Option<User>,
+    pub committer: Option<User>,
+    pub commit: GitCommit,
+    // parents: [{sha}]
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GitHubRepo {
+    pub name: String,
+    pub description: Option<String>,
+    pub fork: bool,
+    pub created_at: String,
+    pub updated_at: Option<String>,
+    pub pushed_at: Option<String>,
+    pub homepage: Option<String>,
+    pub stargazers_count: u32,  // Stars
+    pub forks_count: u32,       // Real number of forks
+    pub subscribers_count: u32, // Real number of watches
+    pub has_issues: bool,
+    pub open_issues_count: Option<u32>,
+    // language: JavaScript,
+    pub has_downloads: bool,
+    // has_wiki: true,
+    pub has_pages: bool,
+    pub archived: bool,
+    pub default_branch: Option<String>,
+    pub owner: Option<User>,
+    #[serde(default)]
+    pub topics: Vec<String>,
+
+    #[serde(default)]
+    pub is_template: Vec<String>,
+
+    /// My custom addition!
+    pub github_page_url: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GitHubRelease {
+    // url: Option<String>, // "https://api.github.com/repos/octocat/Hello-World/releases/1",
+    // html_url: Option<String>, // "https://github.com/octocat/Hello-World/releases/v1.0.0",
+    // assets_url: Option<String>, // "https://api.github.com/repos/octocat/Hello-World/releases/1/assets",
+    // upload_url: Option<String>, // "https://uploads.github.com/repos/octocat/Hello-World/releases/1/assets{?name,label}",
+    // tarball_url: Option<String>, // "https://api.github.com/repos/octocat/Hello-World/tarball/v1.0.0",
+    // zipball_url: Option<String>, // "https://api.github.com/repos/octocat/Hello-World/zipball/v1.0.0",
+    // id: Option<String>, // 1,
+    // node_id: Option<String>, // "MDc6UmVsZWFzZTE=",
+    pub tag_name: Option<String>, // "v1.0.0",
+    // target_commitish: Option<String>, // "master",
+    // name: Option<String>, // "v1.0.0",
+    pub body: Option<String>,         // "Description of the release",
+    pub draft: Option<bool>,          // false,
+    pub prerelease: Option<bool>,     // false,
+    pub created_at: Option<String>,   // "2013-02-27T19:35:32Z",
+    pub published_at: Option<String>, // "2013-02-27T19:35:32Z",
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Topics {
+    pub names: Vec<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct UserOrg {
+    pub login: String, // "github",
+    //id: String, // 1,
+    // node_id: String, // "MDEyOk9yZ2FuaXphdGlvbjE=",
+    pub url: String, // "https://api.github.com/orgs/github",
+    // public_members_url: String, // "https://api.github.com/orgs/github/public_members{/member}",
+    // avatar_url: String, // "https://github.com/images/error/octocat_happy.gif",
+    pub description: Option<String>, // "A great organization"
+}
diff --git a/kitchen_sink/Cargo.toml b/kitchen_sink/Cargo.toml
index 4d01659..430ec44 100644
--- a/kitchen_sink/Cargo.toml
+++ b/kitchen_sink/Cargo.toml
@@ -13,7 +13,7 @@ path = "src/lib_kitchen_sink.rs"
 crates_io_client = { path = "../crates_io_client" }
 deps_index = { path = "../deps_index" }
 docs_rs_client = { git = "https://gitlab.com/crates.rs/docs_rs_client.git", version = "0.4.0" }
-github_info = { path = "../github_info", version = "0.8.0" }
+github_info = { path = "../github_info", version = "0.9" }
 crate_git_checkout = { git = "https://gitlab.com/crates.rs/crate_git_checkout.git", version = "0.4.3" }
 user_db = { path = "../user_db", version = "0.3" }
 crate_db = { path = "../crate_db", version = "0.4.0" }
diff --git a/kitchen_sink/src/lib_kitchen_sink.rs b/kitchen_sink/src/lib_kitchen_sink.rs
index ecaed64..e8073a6 100644
--- a/kitchen_sink/src/lib_kitchen_sink.rs
+++ b/kitchen_sink/src/lib_kitchen_sink.rs
@@ -520,7 +520,7 @@ impl KitchenSink {
             Origin::GitHub { repo, package } => {
                 let host = RepoHost::GitHub(repo.clone()).try_into().map_err(|_| KitchenSinkErr::CrateNotFound(origin.clone())).context("ghrepo host bad")?;
                 let cachebust = self.cachebust_string_for_repo(&host).await.context("ghrepo")?;
-                let gh = self.gh.repo(repo, &cachebust)?
+                let gh = self.gh.repo(repo, &cachebust).await?
                     .ok_or_else(|| KitchenSinkErr::CrateNotFound(origin.clone()))
                     .context(format!("ghrepo {:?} not found", repo))?;
                 let versions = self.get_repo_versions(origin, &host, &cachebust).await?;
@@ -554,7 +554,7 @@ impl KitchenSink {
         let package = match origin {
             Origin::GitLab { package, .. } => package,
             Origin::GitHub { repo, package } => {
-                let releases = self.gh.releases(repo, cachebust)?.ok_or_else(|| KitchenSinkErr::CrateNotFound(origin.clone())).context("releases not found")?;
+                let releases = self.gh.releases(repo, cachebust).await?.ok_or_else(|| KitchenSinkErr::CrateNotFound(origin.clone())).context("releases not found")?;
                 let versions: Vec<_> = releases.into_iter().filter_map(|r| {
                     let date = r.published_at.or(r.created_at)?;
                     let num_full = r.tag_name?;
@@ -710,7 +710,7 @@ impl KitchenSink {
     pub async fn changelog_url(&self, k: &RichCrateVersion) -> Option<String> {
         let repo = k.repository()?;
         if let RepoHost::GitHub(ref gh) = repo.host() {
-            let releases = self.gh.releases(gh, &self.cachebust_string_for_repo(repo).await.ok()?).ok()??;
+            let releases = self.gh.releases(gh, &self.cachebust_string_for_repo(repo).await.ok()?).await.ok()??;
             if releases.iter().any(|rel| rel.body.as_ref().map_or(false, |b| b.len() > 15)) {
                 return Some(format!("https://github.com/{}/{}/releases", gh.owner, gh.repo));
             }
@@ -857,7 +857,7 @@ impl KitchenSink {
         // TODO: also ignore useless keywords that are unique db-wide
         let gh = match maybe_repo.as_ref() {
             Some(repo) => if let RepoHost::GitHub(ref gh) = repo.host() {
-                self.gh.topics(gh, &self.cachebust_string_for_repo(repo).await.context("fetch topics")?)?
+                self.gh.topics(gh, &self.cachebust_string_for_repo(repo).await.context("fetch topics")?).await?
             } else {None},
             _ => None,
         };
@@ -1015,7 +1015,7 @@ impl KitchenSink {
         Ok(match crate_repo.host() {
             RepoHost::GitHub(ref repo) => {
                 let cachebust = self.cachebust_string_for_repo(crate_repo).await.context("ghrepo")?;
-                self.gh.repo(repo, &cachebust)?
+                self.gh.repo(repo, &cachebust).await?
             },
             _ => None,
         })
@@ -1259,10 +1259,10 @@ impl KitchenSink {
     }
 
     /// Maintenance: add user to local db index
-    pub fn index_email(&self, email: &str, name: Option<&str>) -> CResult<()> {
+    pub async fn index_email(&self, email: &str, name: Option<&str>) -> CResult<()> {
         if stopped() {Err(KitchenSinkErr::Stopped)?;}
         if !self.user_db.email_has_github(&email)? {
-            match self.gh.user_by_email(&email) {
+            match self.gh.user_by_ema