Move deps stats

15 files changed, 123 insertions, 65 deletions

diff --git a/Cargo.toml b/Cargo.toml
index eb2dcdd..418c68b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,6 +24,7 @@ members = [
 "search_index",
 "ranking",
 "server",
+"deps_index",
 "render_readme/dump",
 ]
 
diff --git a/builder/Cargo.toml b/builder/Cargo.toml
index f862de2..83805ec 100644
--- a/builder/Cargo.toml
+++ b/builder/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2018"
 default-run = "builder"
 
 [dependencies]
-kitchen_sink = { path = "../kitchen_sink", version = "0.8" }
+kitchen_sink = { path = "../kitchen_sink", version = "0.9" }
 rusqlite = "0.21.0"
 parking_lot = "0.10.0"
 chrono = "0.4.10"
@@ -18,5 +18,5 @@ scopeguard = "1.0.0"
 rand = "0.7.2"
 crate_db = { path = "../crate_db", version = "0.4.5" }
 lts = "0.1.5"
-crates-index = "0.13.4"
+crates-index = "0.14"
 tokio = { version = "0.2.11", features = ["rt-threaded", "macros"] }
diff --git a/crate_git_checkout b/crate_git_checkout
-Subproject 1279387d5a3acd7e474848f14a6d9ffb91cf4ac
+Subproject 1b6c0252ac14872e3c3003e1205196b36e75a02
diff --git a/datadump/Cargo.toml b/datadump/Cargo.toml
index b16750d..1ec6d75 100644
--- a/datadump/Cargo.toml
+++ b/datadump/Cargo.toml
@@ -11,5 +11,5 @@ libflate = "0.1.27"
 serde = "1.0.104"
 serde_derive = "1.0.104"
 chrono = "0.4.10"
-kitchen_sink = { path = "../kitchen_sink", version = "0.8" }
+kitchen_sink = { path = "../kitchen_sink", version = "0.9" }
 tokio = { version = "0.2", features = ["rt-threaded", "macros"] }
diff --git a/deps_index/Cargo.toml b/deps_index/Cargo.toml
new file mode 100644
index 0000000..1c6a479
--- /dev/null
+++ b/deps_index/Cargo.toml
@@ -0,0 +1,31 @@
+[package]
+name = "deps_index"
+version = "0.1.0"
+authors = ["Kornel <kornel@geekhood.net>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+crates-index = "0.14"
+rich_crate = { path = "../rich_crate", version = "0.5" }
+repo_url = { path = "../repo_url", version = "0.3.0" }
+serde = "1.0.104"
+serde_derive = "1.0.104"
+serde_json = "1.0.44"
+toml = "0.5.5"
+lazy_static = "1.4.0"
+regex = "1.3.1"
+url = "2.1.0"
+failure = "0.1.6"
+tokio = { version = "0.2", features = ["rt-threaded", "macros", "time", "sync", "blocking"] }
+rayon = "1.3.0"
+reqwest = "0.10"
+semver-parser = "0.9.0"
+semver = { version = "0.9.0", features = ["serde"] }
+string-interner = "0.7.1"
+parking_lot = "0.10.0"
+ahash = "0.3"
+futures = "0.3.4"
+double-checked-cell-async = "2.0.2"
+
diff --git a/kitchen_sink/src/deps_stats.rs b/deps_index/src/deps_stats.rs
index 437bbed..4c641f5 100644
--- a/kitchen_sink/src/deps_stats.rs
+++ b/deps_index/src/deps_stats.rs
@@ -1,5 +1,5 @@
 use crate::index::*;
-use crate::KitchenSinkErr;
+use crate::DepsErr;
 use parking_lot::Mutex;
 use rayon::prelude::*;
 use string_interner::Sym;
@@ -105,7 +105,7 @@ impl DepVisitor {
 }
 
 impl Index {
-    pub(crate) fn all_dependencies_flattened(&self, c: &impl ICrate) -> Result<DepInfMap, KitchenSinkErr> {
+    pub fn all_dependencies_flattened(&self, c: &impl ICrate) -> Result<DepInfMap, DepsErr> {
         let mut collected = FxHashMap::with_capacity_and_hasher(120, Default::default());
         let mut visitor = DepVisitor::new();
 
@@ -154,7 +154,7 @@ impl Index {
         Ok(converted)
     }
 
-    pub(crate) async fn get_deps_stats(&self) -> DepsStats {
+    pub async fn get_deps_stats(&self) -> DepsStats {
         let crates = self.crates_io_crates();
         let crates: Vec<(Box<str>, FxHashMap<_,_>)> = crates
         .par_iter()
diff --git a/kitchen_sink/src/git_crates_index.rs b/deps_index/src/git_crates_index.rs
index 814e538..51b3adb 100644
--- a/kitchen_sink/src/git_crates_index.rs
+++ b/deps_index/src/git_crates_index.rs
@@ -1,4 +1,4 @@
-use crate::KitchenSinkErr;
+use crate::DepsErr;
 use crate::Origin;
 use std::fs;
 use std::path::Path;
@@ -10,12 +10,12 @@ pub struct GitIndex {
 }
 
 impl GitIndex {
-    pub fn new(dir: &Path) -> Result<Self, KitchenSinkErr> {
+    pub fn new(dir: &Path) -> Result<Self, DepsErr> {
         let path = dir.join("git_crates.txt");
         let index = if path.exists() {
             match fs::read_to_string(&path) {
                 Ok(file) => file.split('\n').map(|s| s.trim()).filter(|s| !s.is_empty()).map(Origin::from_str).collect(),
-                Err(e) => return Err(KitchenSinkErr::GitIndexFile(path, e.to_string())),
+                Err(e) => return Err(DepsErr::GitIndexFile(path, e.to_string())),
             }
         } else {
             Default::default()
diff --git a/kitchen_sink/src/index.rs b/deps_index/src/index.rs
index 4ec3c88..c53ab8e 100644
--- a/kitchen_sink/src/index.rs
+++ b/deps_index/src/index.rs
@@ -1,7 +1,6 @@
 use crate::deps_stats::DepsStats;
 use crate::git_crates_index::*;
-use crate::KitchenSink;
-use crate::KitchenSinkErr;
+use crate::DepsErr;
 use crates_index;
 use crates_index::Crate;
 use crates_index::Dependency;
@@ -21,6 +20,7 @@ use string_interner::StringInterner;
 use string_interner::Sym;
 use std::time::Duration;
 use rayon::prelude::*;
+use serde_derive::*;
 
 type FxHashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;
 type FxHashSet<V> = std::collections::HashSet<V, ahash::RandomState>;
@@ -46,7 +46,7 @@ impl MiniVer {
     }
 }
 
-pub(crate) trait FeatureGetter {
+pub trait FeatureGetter {
     fn get(&self, key: &str) -> Option<&Vec<String>>;
 }
 impl FeatureGetter for std::collections::HashMap<String, Vec<String>> {
@@ -60,7 +60,7 @@ impl FeatureGetter for std::collections::BTreeMap<String, Vec<String>> {
     }
 }
 
-pub(crate) trait IVersion {
+pub trait IVersion {
     type Features: FeatureGetter;
     fn name(&self) -> &str;
     fn version(&self) -> &str;
@@ -78,7 +78,7 @@ impl IVersion for Version {
     fn is_yanked(&self) -> bool {self.is_yanked()}
 }
 
-pub(crate) trait ICrate {
+pub trait ICrate {
     type Ver: IVersion;
     fn latest_version_with_features(&self, all_optional: bool) -> (&Self::Ver, Box<[Box<str>]>);
 }
@@ -100,7 +100,7 @@ impl ICrate for Crate {
     }
 }
 
-pub(crate) enum Fudge<'a> {
+pub enum Fudge<'a> {
     CratesIo(&'a [Dependency]),
     Manifest((Vec<RichDep>, Vec<RichDep>, Vec<RichDep>)),
 }
@@ -128,20 +128,16 @@ impl ICrate for RichCrateVersion {
 
 pub struct Index {
     indexed_crates: FxHashMap<Box<str>, Crate>,
-    pub(crate) crates_io_index: crates_index::Index,
+    pub crates_io_index: crates_index::Index,
     git_index: GitIndex,
 
-    pub(crate) inter: RwLock<StringInterner<Sym>>,
-    pub(crate) cache: RwLock<FxHashMap<(Box<str>, Features), ArcDepSet>>,
+    pub inter: RwLock<StringInterner<Sym>>,
+    pub cache: RwLock<FxHashMap<(Box<str>, Features), ArcDepSet>>,
     deps_stats: DoubleCheckedCell<DepsStats>,
 }
 
 impl Index {
-    pub fn new_default() -> Result<Self, KitchenSinkErr> {
-        Self::new(&KitchenSink::data_path()?)
-    }
-
-    pub fn new(data_dir: &Path) -> Result<Self, KitchenSinkErr> {
+    pub fn new(data_dir: &Path) -> Result<Self, DepsErr> {
         let crates_io_index = crates_index::Index::new(data_dir.join("index"));
         let indexed_crates = crates_io_index.crate_index_paths().par_bridge()
                 .filter_map(|path| {
@@ -184,20 +180,20 @@ impl Index {
         self.git_index.crates().cloned().chain(self.crates_io_crates().keys().map(|n| Origin::from_crates_io_name(&n)))
     }
 
-    pub async fn deps_stats(&self) -> Result<&DepsStats, KitchenSinkErr> {
+    pub async fn deps_stats(&self) -> Result<&DepsStats, DepsErr> {
         Ok(tokio::time::timeout(Duration::from_secs(30), self.deps_stats.get_or_init(self.get_deps_stats())).await
-            .map_err(|_| KitchenSinkErr::DepsNotAvailable)?)
+            .map_err(|_| DepsErr::DepsNotAvailable)?)
     }
 
     #[inline]
-    pub fn crates_io_crate_by_lowercase_name(&self, name: &str) -> Result<&Crate, KitchenSinkErr> {
+    pub fn crates_io_crate_by_lowercase_name(&self, name: &str) -> Result<&Crate, DepsErr> {
         debug_assert_eq!(name, name.to_ascii_lowercase());
         self.crates_io_crates()
         .get(name)
-        .ok_or_else(|| KitchenSinkErr::CrateNotFound(Origin::from_crates_io_name(name)))
+        .ok_or_else(|| DepsErr::CrateNotFound(Origin::from_crates_io_name(name)))
     }
 
-    pub fn crate_highest_version(&self, name: &str, stable_only: bool) -> Result<&Version, KitchenSinkErr> {
+    pub fn crate_highest_version(&self, name: &str, stable_only: bool) -> Result<&Version, DepsErr> {
         debug_assert_eq!(name, name.to_ascii_lowercase());
         Ok(Self::highest_crates_io_version(self.crates_io_crate_by_lowercase_name(name)?, stable_only))
     }
@@ -215,12 +211,12 @@ impl Index {
             .unwrap_or_else(|| krate.latest_version()) // latest_version = most recently published version
     }
 
-    pub(crate) fn deps_of_crate(&self, krate: &impl ICrate, query: DepQuery) -> Result<Dep, KitchenSinkErr> {
+    pub fn deps_of_crate(&self, krate: &impl ICrate, query: DepQuery) -> Result<Dep, DepsErr> {
         let (latest, features) = krate.latest_version_with_features(query.all_optional);
         self.deps_of_crate_int(latest, features, query)
     }
 
-    fn deps_of_crate_int(&self, latest: &impl IVersion, features: Box<[Box<str>]>, DepQuery { default, all_optional, dev }: DepQuery) -> Result<Dep, KitchenSinkErr> {
+    fn deps_of_crate_int(&self, latest: &impl IVersion, features: Box<[Box<str>]>, DepQuery { default, all_optional, dev }: DepQuery) -> Result<Dep, DepsErr> {
         Ok(Dep {
             semver: semver_parse(latest.version()).into(),
             runtime: self.deps_of_ver(latest, Features {
@@ -240,7 +236,7 @@ impl Index {
         })
     }
 
-    pub(crate) fn deps_of_ver<'a>(&self, ver: &'a impl IVersion, wants: Features) -> Result<ArcDepSet, KitchenSinkErr> {
+    pub fn deps_of_ver<'a>(&self, ver: &'a impl IVersion, wants: Features) -> Result<ArcDepSet, DepsErr> {
         let key = (format!("{}-{}", ver.name(), ver.version()).into(), wants);
         if let Some(cached) = self.cache.read().get(&key) {
             return Ok(cached.clone());
@@ -314,7 +310,7 @@ impl Index {
                 continue;
             }
 
-            let req = VersionReq::parse(requirement).map_err(|_| KitchenSinkErr::SemverParsingError)?;
+            let req = VersionReq::parse(requirement).map_err(|_| DepsErr::SemverParsingError)?;
             let krate = match self.crates_io_crate_by_lowercase_name(&crate_name) {
                 Ok(k) => k,
                 Err(e) => {
@@ -389,7 +385,7 @@ impl Index {
     /// For crate being outdated. Returns (is_latest, popularity)
     /// 0 = not used *or deprecated*
     /// 1 = everyone uses it
-    pub async fn version_popularity(&self, crate_name: &str, requirement: &VersionReq) -> Result<Option<(bool, f32)>, KitchenSinkErr> {
+    pub async fn version_popularity(&self, crate_name: &str, requirement: &VersionReq) -> Result<Option<(bool, f32)>, DepsErr> {
         if is_deprecated(crate_name) {
             return Ok(Some((false, 0.)));
         }
@@ -425,7 +421,7 @@ impl Index {
     }
 
     /// How likely it is that this exact crate will be installed in any project
-    pub async fn version_global_popularity(&self, crate_name: &str, version: &MiniVer) -> Result<Option<f32>, KitchenSinkErr> {
+    pub async fn version_global_popularity(&self, crate_name: &str, version: &MiniVer) -> Result<Option<f32>, DepsErr> {
         match crate_name {
             // bindings' SLoC looks heavier than actual overhead of standard system libs
             "libc" | "winapi" | "kernel32-sys" | "winapi-i686-pc-windows-gnu" | "winapi-x86_64-pc-windows-gnu" => return Ok(Some(0.99)),
@@ -511,12 +507,3 @@ pub struct DepQuery {
     pub all_optional: bool,
     pub dev: bool,
 }
-
-#[tokio::test]
-async fn index_test() {
-    let idx = Index::new_default().unwrap();
-    let stats = idx.deps_stats().await.unwrap();
-    assert!(stats.total > 13800);
-    let lode = stats.counts.get("lodepng").unwrap();
-    assert_eq!(12, lode.runtime.def);
-}
diff --git a/deps_index/src/lib.rs b/deps_index/src/lib.rs
new file mode 100644
index 0000000..b6e6ffd
--- /dev/null
+++ b/deps_index/src/lib.rs
@@ -0,0 +1,33 @@
+mod index;
+use rich_crate::Origin;
+use std::path::PathBuf;
+pub use index::*;
+use failure::Fail;
+
+mod deps_stats;
+mod git_crates_index;
+pub use deps_stats::*;
+pub use crates_index::Crate as CratesIndexCrate;
+pub use crates_index::Version as CratesIndexVersion;
+
+#[derive(Debug, Clone, Fail)]
+pub enum DepsErr {
+
+    #[fail(display = "crate not found: {:?}", _0)]
+    CrateNotFound(Origin),
+    #[fail(display = "crate {} not found in repo {}", _0, _1)]
+    CrateNotFoundInRepo(String, String),
+    #[fail(display = "crate is not a package: {:?}", _0)]
+    NotAPackage(Origin),
+
+    #[fail(display = "Error when parsing verison")]
+    SemverParsingError,
+    #[fail(display = "Stopped")]
+    Stopped,
+    #[fail(display = "Deps stats timeout")]
+    DepsNotAvailable,
+    #[fail(display = "Crate timeout")]
+    GitIndexFile(PathBuf, String),
+    #[fail(display = "Git crate '{:?}' can't be indexed, because it's not on the list", _0)]
+    GitCrateNotAllowed(Origin),
+}
diff --git a/front_end/Cargo.toml b/front_end/Cargo.toml
index 8f5deb6..29f1361 100644
--- a/front_end/Cargo.toml
+++ b/front_end/Cargo.toml
@@ -13,7 +13,7 @@ path = "src/front_end.rs"
 ructe = "0.9"
 
 [dependencies]
-kitchen_sink = { path = "../kitchen_sink", version = "0.8" }
+kitchen_sink = { path = "../kitchen_sink", version = "0.9" }
 rich_crate = { path = "../rich_crate" }
 render_readme = { git = "https://gitlab.com/crates.rs/render_readme.git", version = "0.6.0" }
 categories = { path = "../categories" }
diff --git a/kitchen_sink/Cargo.toml b/kitchen_sink/Cargo.toml
index 824e476..4d01659 100644
--- a/kitchen_sink/Cargo.toml
+++ b/kitchen_sink/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 edition = "2018"
 name = "kitchen_sink"
-version = "0.8.3"
+version = "0.9.0"
 authors = ["Kornel <kornel@geekhood.net>"]
 publish = false
 
@@ -10,8 +10,8 @@ name = "kitchen_sink"
 path = "src/lib_kitchen_sink.rs"
 
 [dependencies]
-crates-index = "0.13.3"
 crates_io_client = { path = "../crates_io_client" }
+deps_index = { path = "../deps_index" }
 docs_rs_client = { git = "https://gitlab.com/crates.rs/docs_rs_client.git", version = "0.4.0" }
 github_info = { path = "../github_info", version = "0.8.0" }
 crate_git_checkout = { git = "https://gitlab.com/crates.rs/crate_git_checkout.git", version = "0.4.3" }
diff --git a/kitchen_sink/src/lib_kitchen_sink.rs b/kitchen_sink/src/lib_kitchen_sink.rs
index 72f7f44..1fe68f5 100644
--- a/kitchen_sink/src/lib_kitchen_sink.rs
+++ b/kitchen_sink/src/lib_kitchen_sink.rs
@@ -3,23 +3,18 @@
 #[macro_use]
 extern crate serde_derive;
 
-mod index;
-pub use crate::index::*;
 use futures::stream::StreamExt;
 mod yearly;
 pub use crate::yearly::*;
-mod deps_stats;
-pub use crate::deps_stats::*;
+pub use deps_index::*;
 pub mod filter;
 
 mod ctrlcbreak;
-mod git_crates_index;
 mod tarball;
 pub use crate::ctrlcbreak::*;
 
 pub use crate_db::builddb::Compat;
 pub use crate_db::builddb::CompatibilityInfo;
-pub use crates_index::Crate as CratesIndexCrate;
 use crates_io_client::CrateMetaFile;
 pub use crates_io_client::CrateDepKind;
 pub use crates_io_client::CrateDependency;
@@ -140,6 +135,8 @@ pub enum KitchenSinkErr {
     GitIndexFile(PathBuf, String),
     #[fail(display = "Git crate '{:?}' can't be indexed, because it's not on the list", _0)]
     GitCrateNotAllowed(Origin),
+    #[fail(display = "Deps err: {}", _0)]
+    Deps(DepsErr),
 }
 
 #[derive(Debug, Clone)]
@@ -781,7 +778,7 @@ impl KitchenSink {
         Ok(tarball)
     }
 
-    async fn rich_crate_version_data_from_crates_io(&self, latest: &crates_index::Version) -> CResult<(CrateVersionSourceData, Manifest, Warnings)> {
+    async fn rich_crate_version_data_from_crates_io(&self, latest: &CratesIndexVersion) -> CResult<(CrateVersionSourceData, Manifest, Warnings)> {
         let _f = self.throttle.acquire().await;
 
         let mut warnings = HashSet::new();
@@ -1184,10 +1181,10 @@ impl KitchenSink {
     pub fn all_dependencies_flattened(&self, krate: &RichCrateVersion) -> Result<DepInfMap, KitchenSinkErr> {
         match krate.origin() {
             Origin::CratesIo(name) => {
-                self.index.all_dependencies_flattened(self.index.crates_io_crate_by_lowercase_name(name)?)
+                self.index.all_dependencies_flattened(self.index.crates_io_crate_by_lowercase_name(name).map_err(KitchenSinkErr::Deps)?).map_err(KitchenSinkErr::Deps)
             },
             _ => {
-                self.index.all_dependencies_flattened(krate)
+                self.index.all_dependencies_flattened(krate).map_err(KitchenSinkErr::Deps)
             }
         }
     }
@@ -1203,7 +1200,7 @@ impl KitchenSink {
 
     pub async fn crates_io_dependents_stats_of(&self, origin: &Origin) -> Result<Option<&RevDependencies>, KitchenSinkErr> {
         match origin {
-            Origin::CratesIo(crate_name) => Ok(self.index.deps_stats().await?.counts.get(crate_name)),
+            Origin::CratesIo(crate_name) => Ok(self.index.deps_stats().await.map_err(KitchenSinkErr::Deps)?.counts.get(crate_name)),
             _ => Ok(None),
         }
     }
@@ -1212,7 +1209,7 @@ impl KitchenSink {
     /// 0 = not used
     /// 1 = everyone uses it
     pub async fn version_popularity(&self, crate_name: &str, requirement: &VersionReq) -> Result<Option<(bool, f32)>, KitchenSinkErr> {
-        self.index.version_popularity(crate_name, requirement).await
+        self.index.version_popularity(crate_name, requirement).await.map_err(KitchenSinkErr::Deps)
     }
 
     /// "See also"
@@ -2110,3 +2107,12 @@ fn fetch_uppercase_name() {
     })).unwrap();
 }
 
+
+#[tokio::test]
+async fn index_test() {
+    let idx = Index::new(&KitchenSink::data_path().unwrap()).unwrap();
+    let stats = idx.deps_stats().await.unwrap();
+    assert!(stats.total > 13800);
+    let lode = stats.counts.get("lodepng").unwrap();
+    assert_eq!(12, lode.runtime.def);
+}
diff --git a/reindex/Cargo.toml b/reindex/Cargo.toml
index a90c6ec..6930d95 100644
--- a/reindex/Cargo.toml
+++ b/reindex/Cargo.toml
@@ -7,7 +7,7 @@ authors = ["Kornel <kornel@geekhood.net>"]
 [dependencies]
 crate_db = { path = "../crate_db", version = "0.4.0" }
 github_info = { path = "../github_info", version = "0.8.0" }
-kitchen_sink = { path = "../kitchen_sink", version = "0.8" }
+kitchen_sink = { path = "../kitchen_sink", version = "0.9" }
 repo_url = { path = "../repo_url" }
 user_db = { path = "../user_db", version = "0.3" }
 failure = "0.1.6"
diff --git a/render_readme b/render_readme
-Subproject 79660ad6bc83c199782c06d35f0b640b6b781e8
+Subproject 90b6dee02e393d6149d53eb6477cff797d00026
diff --git a/server/Cargo.toml b/server/Cargo.toml
index 8654887..c046be5 100644
--- a/server/Cargo.toml
+++ b/server/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "crates-server"
-version = "0.12.2"
+version = "0.12.3"
 authors = ["Kornel <kornel@geekhood.net>"]
 edition = "2018"
 description = "Crates.rs web server"
@@ -22,7 +22,7 @@ log = "0.4.6"
 render_readme = { git = "https://gitlab.com/crates.rs/render_readme.git", version = "0.6.1" }
 search_index = { path = "../search_index" }
 repo_url = { path = "../repo_url" }
-kitchen_sink = { path = "../kitchen_sink", version = "0.8" }
+kitchen_sink = { path = "../kitchen_sink", version = "0.9" }
 front_end = { path = "../front_end", version = "0.4" }
 urlencoding = "1.0.0"
 failure = "0.1.5"