From 9e72f373a1042bfe3a8510d004b83a2af394772a Mon Sep 17 00:00:00 2001 From: Kornel Date: Mon, 2 Mar 2020 19:00:49 +0000 Subject: author v0 --- crate_db/src/lib_crate_db.rs | 4 +- datadump/src/main.rs | 21 +++-- front_end/src/author_page.rs | 147 +++++++++++++++++++++++++++++++++++ front_end/src/front_end.rs | 15 ++++ front_end/templates/author.rs.html | 39 ++++++++++ kitchen_sink/src/lib_kitchen_sink.rs | 27 ++++++- server/src/main.rs | 27 ++++++- 7 files changed, 267 insertions(+), 13 deletions(-) create mode 100644 front_end/src/author_page.rs create mode 100644 front_end/templates/author.rs.html diff --git a/crate_db/src/lib_crate_db.rs b/crate_db/src/lib_crate_db.rs index 6d7e4d3..4ddad18 100644 --- a/crate_db/src/lib_crate_db.rs +++ b/crate_db/src/lib_crate_db.rs @@ -643,8 +643,8 @@ impl CrateDb { }).await } - pub async fn crates_by_author(&self, github_id: u32) -> FResult> { - self.with_read("crates_by_author", |conn| { + pub async fn crates_of_author(&self, github_id: u32) -> FResult> { + self.with_read("crates_of_author", |conn| { let mut query = conn.prepare_cached(r#"SELECT ac.crate_id, ac.invited_by_github_id, ac.invited_at, max(cv.created) FROM author_crates ac JOIN crate_versions cv USING(crate_id) WHERE ac.github_id = ?1 diff --git a/datadump/src/main.rs b/datadump/src/main.rs index 5f21e59..c1da0e3 100644 --- a/datadump/src/main.rs +++ b/datadump/src/main.rs @@ -3,6 +3,7 @@ use chrono::prelude::*; use kitchen_sink::CrateOwner; use kitchen_sink::KitchenSink; +use kitchen_sink::Origin; use kitchen_sink::OwnerKind; use libflate::gzip::Decoder; use serde_derive::Deserialize; @@ -18,8 +19,8 @@ type BoxErr = Box; #[tokio::main] async fn main() -> Result<(), BoxErr> { - tokio::runtime::Handle::current() - .spawn(async move { + tokio::runtime::Handle::current().spawn(async move { + let handle = tokio::runtime::Handle::current(); let mut a = Archive::new(Decoder::new(BufReader::new(File::open("db-dump.tar.gz")?))?); let ksink = KitchenSink::new_default().await?; @@ -76,12 +77,15 @@ async fn main() -> Result<(), BoxErr> { index_downloads(crates, versions, &downloads, &ksink)?; } } - if let (Some(crates), Some(teams), Some(users)) = (&crates, &teams, &users) { + } + } + + if let (Some(crates), Some(teams), Some(users)) = (crates, teams, users) { if let Some(crate_owners) = crate_owners.take() { eprintln!("Indexing {} owners", crate_owners.len()); - index_owners(crates, crate_owners, teams, users, &ksink)?; - } - } + handle.spawn(async move { + index_owners(&crates, crate_owners, &teams, &users, &ksink).await.unwrap(); + }); } } Ok(()) @@ -113,7 +117,7 @@ fn index_downloads(crates: &CratesMap, versions: &VersionsMap, downloads: &Versi } #[inline(never)] -fn index_owners(crates: &CratesMap, owners: CrateOwners, teams: &Teams, users: &Users, ksink: &KitchenSink) -> Result<(), BoxErr> { +async fn index_owners(crates: &CratesMap, owners: CrateOwners, teams: &Teams, users: &Users, ksink: &KitchenSink) -> Result<(), BoxErr> { for (crate_id, owners) in owners { if let Some(k) = crates.get(&crate_id) { let owners: Vec<_> = owners @@ -157,7 +161,8 @@ fn index_owners(crates: &CratesMap, owners: CrateOwners, teams: &Teams, users: & }) }) .collect(); - ksink.set_crates_io_crate_owners(&k.to_ascii_lowercase(), owners).map_err(|_| "ugh")?; + let origin = Origin::from_crates_io_name(k); + ksink.index_crates_io_crate_owners(&origin, owners).await?; } } Ok(()) diff --git a/front_end/src/author_page.rs b/front_end/src/author_page.rs new file mode 100644 index 0000000..bf3a548 --- /dev/null +++ b/front_end/src/author_page.rs @@ -0,0 +1,147 @@ +use crate::templates; +use crate::Page; +use kitchen_sink::CResult; +use kitchen_sink::KitchenSink; +use kitchen_sink::RichAuthor; +use kitchen_sink::UserType; +use render_readme::Renderer; + +// pub struct User { +// pub id: u32, +// pub login: String, +// pub name: Option, +// pub avatar_url: Option, // "https://avatars0.githubusercontent.com/u/1111?v=4", +// pub gravatar_id: Option, // "", +// pub html_url: String, // "https://github.com/zzzz", +// pub blog: Option, // "https://example.com +// #[serde(rename = "type")] +// pub user_type: UserType, +// } + +/// Data sources used in `author.rs.html` +pub struct AuthorPage<'a> { + pub aut: &'a RichAuthor, + pub kitchen_sink: &'a KitchenSink, + pub markup: &'a Renderer, +} + +impl<'a> AuthorPage<'a> { + pub async fn new(aut: &'a RichAuthor, kitchen_sink: &'a KitchenSink, markup: &'a Renderer) -> CResult> { + dbg!(&aut); + let crates = kitchen_sink.crates_of_author(aut).await?; + + Ok(Self { + aut, + kitchen_sink, + markup, + }) + } + + pub fn is_org(&self) -> bool { + self.aut.github.user_type == UserType::Org + } + + pub fn login(&self) -> &str { + &self.aut.github.login + } + + pub fn github_url(&self) -> String { + format!("https://github.com/{}", self.aut.github.login) + } + + pub fn page(&self) -> Page { + Page { + title: format!("Rust crates by @{}", self.login()), + ..Default::default() + } + } + + pub fn render_markdown_str(&self, s: &str) -> templates::Html { + templates::Html(self.markup.markdown_str(s, true, None)) + } + + // fn block(&self, f: impl Future) -> O { + // self.handle.enter(|| futures::executor::block_on(f)) + // } + + // pub fn format_number(&self, num: impl Display) -> String { + // Numeric::english().format_int(num) + // } + + // pub fn format_knumber(&self, num: usize) -> (String, &'static str) { + // let (num, unit) = match num { + // 0..=899 => (num, ""), + // 0..=8000 => return (format!("{}", ((num + 250) / 500) as f64 * 0.5), "K"), // 3.5K + // 0..=899_999 => ((num + 500) / 1000, "K"), + // 0..=9_999_999 => return (format!("{}", ((num + 250_000) / 500_000) as f64 * 0.5), "M"), // 3.5M + // _ => ((num + 500_000) / 1_000_000, "M"), // 10M + // }; + // (Numeric::english().format_int(num), unit) + // } + + // pub fn format_kbytes(&self, bytes: usize) -> String { + // let (num, unit) = match bytes { + // 0..=100_000 => ((bytes + 999) / 1000, "KB"), + // 0..=800_000 => ((bytes + 3999) / 5000 * 5, "KB"), + // 0..=9_999_999 => return format!("{}MB", ((bytes + 250_000) / 500_000) as f64 * 0.5), + // _ => ((bytes + 500_000) / 1_000_000, "MB"), + // }; + // format!("{}{}", Numeric::english().format_int(num), unit) + // } + + // fn format_number_frac(num: f64) -> String { + // if num > 0.05 && num < 10. && num.fract() > 0.09 && num.fract() < 0.9 { + // if num < 3. { + // format!("{:.1}", num) + // } else { + // format!("{}", (num * 2.).round() / 2.) + // } + // } else { + // Numeric::english().format_int(if num > 500. { + // (num / 10.).round() * 10. + // } else if num > 100. { + // (num / 5.).round() * 5. + // } else { + // num.round() + // }) + // } + // } + + // pub fn format_kbytes_range(&self, a: usize, b: usize) -> String { + // let min_bytes = a.min(b); + // let max_bytes = a.max(b); + + // // if the range is small, just display the upper number + // if min_bytes * 4 > max_bytes * 3 || max_bytes < 250_000 { + // return self.format_kbytes(max_bytes); + // } + + // let (denom, unit) = match max_bytes { + // 0..=800_000 => (1000., "KB"), + // _ => (1_000_000., "MB"), + // }; + // let mut low_val = min_bytes as f64 / denom; + // let high_val = max_bytes as f64 / denom; + // if low_val > 1. && high_val > 10. { + // low_val = low_val.round(); // spread is so high that precision of low end isn't relevant + // } + // format!("{}–{}{}", Self::format_number_frac(low_val), Self::format_number_frac(high_val), unit) + // } + + // /// Display number 0..1 as percent + // pub fn format_fraction(&self, num: f64) -> String { + // if num < 1.9 { + // format!("{:0.1}%", num) + // } else { + // format!("{}%", Numeric::english().format_int(num.round() as usize)) + // } + // } + + // pub fn format(date: &DateTime) -> String { + // date.format("%b %e, %Y").to_string() + // } + + // pub fn format_month(date: &DateTime) -> String { + // date.format("%b %Y").to_string() + // } +} diff --git a/front_end/src/front_end.rs b/front_end/src/front_end.rs index 31c32c8..0cea8f3 100644 --- a/front_end/src/front_end.rs +++ b/front_end/src/front_end.rs @@ -5,6 +5,7 @@ //! because the template engine Ructe doesn't support //! complex expressions in the templates. +mod author_page; mod cat_page; mod crate_page; mod download_graph; @@ -19,6 +20,7 @@ pub use crate::not_found_page::*; pub use crate::search_page::*; use futures::future::try_join_all; +use crate::author_page::*; use crate::crate_page::*; use crate::urler::Urler; use categories::Category; @@ -26,6 +28,7 @@ use chrono::prelude::*; use failure; use failure::ResultExt; use kitchen_sink::Compat; +use kitchen_sink::RichAuthor; use kitchen_sink::KitchenSink; use kitchen_sink::{stopped, KitchenSinkErr}; use render_readme::Links; @@ -134,6 +137,18 @@ pub async fn render_sitemap(sitemap: &mut impl Write, crates: &KitchenSink) -> R Ok(()) } +/// See `author.rs.html` +pub async fn render_author_page(out: &mut W, aut: &RichAuthor, kitchen_sink: &KitchenSink, renderer: &Renderer) -> Result<(), failure::Error> { + if stopped() { + Err(KitchenSinkErr::Stopped)?; + } + + let urler = Urler::new(None); + let c = AuthorPage::new(aut, kitchen_sink, renderer).await.context("New crate page")?; + templates::author(out, &urler, &c).context("author page io")?; + Ok(()) +} + /// See `crate_page.rs.html` pub async fn render_crate_page(out: &mut W, all: &RichCrate, ver: &RichCrateVersion, kitchen_sink: &KitchenSink, renderer: &Renderer) -> Result>, failure::Error> { if stopped() { diff --git a/front_end/templates/author.rs.html b/front_end/templates/author.rs.html new file mode 100644 index 0000000..244ddf2 --- /dev/null +++ b/front_end/templates/author.rs.html @@ -0,0 +1,39 @@ +@use crate::templates::base; +@use crate::Urler; +@use crate::AuthorPage; + +@(url: &Urler, c: &AuthorPage) + +@:base(&c.page(), { +
+
+
+

Lib.rs

› @if c.is_org() { + Orgs + } else { + Users + } +
+ +

+ @"@"@c.login() +

+ + +
+
+
+
+ Hi +
+
+ + +
+
+ Bye +
+
+}) diff --git a/kitchen_sink/src/lib_kitchen_sink.rs b/kitchen_sink/src/lib_kitchen_sink.rs index 78cf4dd..0498091 100644 --- a/kitchen_sink/src/lib_kitchen_sink.rs +++ b/kitchen_sink/src/lib_kitchen_sink.rs @@ -13,6 +13,7 @@ mod ctrlcbreak; mod tarball; pub use crate::ctrlcbreak::*; +pub use crate_db::CrateOwnerRow; pub use crate_db::builddb::Compat; pub use crate_db::builddb::CompatibilityInfo; pub use crates_io_client::CrateDepKind; @@ -107,6 +108,8 @@ pub enum KitchenSinkErr { CategoryQueryFailed, #[fail(display = "crate not found: {:?}", _0)] CrateNotFound(Origin), + #[fail(display = "author not found: {}", _0)] + AuthorNotFound(String), #[fail(display = "crate {} not found in repo {}", _0, _1)] CrateNotFoundInRepo(String, String), #[fail(display = "crate is not a package: {:?}", _0)] @@ -1865,6 +1868,10 @@ impl KitchenSink { Err(KitchenSinkErr::OwnerWithoutLogin)? } + pub async fn crates_of_author(&self, aut: &RichAuthor) -> CResult> { + self.crate_db.crates_of_author(aut.github.id).await + } + async fn crate_owners(&self, origin: &Origin) -> CResult> { match origin { Origin::CratesIo(name) => { @@ -1893,8 +1900,12 @@ impl KitchenSink { } } - pub fn set_crates_io_crate_owners(&self, crate_name: &str, owners: Vec) -> Result<(), ()> { - self.crates_io_owners_cache.set(crate_name, owners).map_err(drop) + pub async fn index_crates_io_crate_owners(&self, origin: &Origin, owners: Vec) -> CResult<()> { + self.crate_db.index_crate_owners(origin, &owners).await?; + if let Origin::CratesIo(name) = origin { + self.crates_io_owners_cache.set(&**name, owners)?; + } + Ok(()) } // Sorted from the top, returns origins @@ -2069,6 +2080,18 @@ impl KitchenSink { let _ = self.url_check_cache.save(); self.crates_io.cleanup(); } + + pub async fn author_by_login(&self, login: &str) -> CResult { + let github = self.gh.user_by_login(login).await?.ok_or_else(|| KitchenSinkErr::AuthorNotFound(login.to_owned()))?; + Ok(RichAuthor { + github + }) + } +} + +#[derive(Debug, Clone)] +pub struct RichAuthor { + pub github: User, } /// This is used to uniquely identify authors based on as little information as is available diff --git a/server/src/main.rs b/server/src/main.rs index cfe6c44..cd38a34 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -146,9 +146,9 @@ async fn run_server() -> Result<(), failure::Error> { eprintln!("Refresh failed: {}", e); std::process::exit(1); }, + } } } - } }}); // watchdog @@ -179,6 +179,7 @@ async fn run_server() -> Result<(), failure::Error> { .route("/keywords/{keyword}", web::get().to(handle_keyword)) .route("/crates/{crate}", web::get().to(handle_crate)) .route("/crates/{crate}/rev", web::get().to(handle_crate_reverse_dependencies)) + .route("/~{author}", web::get().to(handle_author)) .route("/install/{crate:.*}", web::get().to(handle_install)) .route("/debug/{crate:.*}", web::get().to(handle_debug)) .route("/gh/{owner}/{repo}/{crate}", web::get().to(handle_github_crate)) @@ -440,6 +441,30 @@ async fn handle_install(req: HttpRequest) -> Result { Ok(serve_cached((page, 7200, false, last_mod))) } +async fn handle_author(req: HttpRequest) -> Result { + let login = req.match_info().query("author"); + println!("author page for {:?}", login); + let state: &AServerState = req.app_data().expect("appdata"); + if !is_alnum(login) { + return render_404_page(state, login); + } + let cache_file = state.page_cache_dir.join(format!("@{}.html", login)); + Ok(serve_cached( + with_file_cache(state, cache_file, 3600, { + let login = login.to_owned(); + let state = state.clone(); + run_timeout(60, async move { + let crates = state.crates.load(); + let mut page: Vec = Vec::with_capacity(32000); + let aut = crates.author_by_login(&login).await?; + front_end::render_author_page(&mut page, &aut, &crates, &state.markup).await?; + Ok::<_, failure::Error>((page, None)) + }) + }) + .await?, + )) +} + async fn handle_crate(req: HttpRequest) -> Result { let crate_name = req.match_info().query("crate"); println!("crate page for {:?}", crate_name); -- cgit v1.2.3