Crates-io-isms

4 files changed, 72 insertions, 28 deletions

diff --git a/front_end/src/crate_page.rs b/front_end/src/crate_page.rs
index 4b0770e..54725c1 100644
--- a/front_end/src/crate_page.rs
+++ b/front_end/src/crate_page.rs
@@ -167,7 +167,7 @@ impl<'a> CratePage<'a> {
     }
 
     pub fn dependents_stats(&self) -> Option<(u32, u32)> {
-        self.kitchen_sink.dependents_stats_of(self.ver.origin()).expect("deps").map(|d| (
+        self.kitchen_sink.crates_io_dependents_stats_of(self.ver.origin()).expect("deps").map(|d| (
             d.runtime.def as u32 + d.runtime.opt as u32 +
             d.build.def as u32  + d.build.opt as u32 +
             d.dev as u32, d.direct as u32))
@@ -285,7 +285,12 @@ impl<'a> CratePage<'a> {
     }
 
     pub fn up_to_date_class(&self, richdep: &RichDep) -> &str {
-        let (matches_latest, pop) = richdep.dep.req().parse().ok().and_then(|req| self.kitchen_sink.version_popularity(&richdep.package, &req).expect("deps")).unwrap_or((false, 0.));
+        let (matches_latest, pop) = richdep.dep.req().parse().ok().and_then(|req| {
+            if !richdep.dep.is_crates_io() {
+                return None;
+            }
+            self.kitchen_sink.version_popularity(&richdep.package, &req).expect("deps")
+        }).unwrap_or((false, 0.));
         match pop {
             x if x >= 0.5 && matches_latest => "top",
             x if x >= 0.75 || matches_latest => "common",
@@ -568,12 +573,12 @@ impl<'a> CratePage<'a> {
     }
 
     pub fn all_versions(&self) -> impl Iterator<Item = Version<'a>> {
-        self.all.versions().iter().map(|v| Version {
+        self.all.versions().iter().filter_map(|v| Some(Version {
             yanked: v.yanked,
             num: &v.num,
-            semver: SemVer::parse(&v.num).expect("semver parse"),
+            semver: SemVer::parse(&v.num).map_err(|e| eprintln!("semver parse {} {:?}", e, v.num)).ok()?,
             created_at: DateTime::parse_from_rfc3339(&v.created_at).expect("created_at parse"),
-        })
+        }))
     }
 
     pub fn published_date(&self) -> DateTime<FixedOffset> {
diff --git a/front_end/src/urler.rs b/front_end/src/urler.rs
index cda9be2..d3ab409 100644
--- a/front_end/src/urler.rs
+++ b/front_end/src/urler.rs
@@ -19,7 +19,11 @@ impl Urler {
 
     /// Link to a dependency of a crate
     pub fn dependency(&self, dep: &RichDep) -> String {
-        format!("/crates/{}", encode(&dep.package))
+        if let Some(git) = dep.dep.git() {
+            git.to_string()
+        } else {
+            format!("/crates/{}", encode(&dep.package))
+        }
     }
 
     /// Summary of all dependencies
diff --git a/kitchen_sink/src/lib_kitchen_sink.rs b/kitchen_sink/src/lib_kitchen_sink.rs
index d8813cb..913a3e5 100644
--- a/kitchen_sink/src/lib_kitchen_sink.rs
+++ b/kitchen_sink/src/lib_kitchen_sink.rs
@@ -342,6 +342,19 @@ impl KitchenSink {
                 let host = RepoHost::GitHub(repo.clone()).try_into().map_err(|_| KitchenSinkErr::CrateNotFound(origin.clone())).context("ghrepo host bad")?;
                 let cachebust = self.cachebust_string_for_repo(&host).context("ghrepo")?;
                 let gh = self.gh.repo(repo, &cachebust)?.ok_or_else(|| KitchenSinkErr::CrateNotFound(origin.clone())).context("ghrepo not found")?;
+                let releases = self.gh.releases(repo, &cachebust)?.ok_or_else(|| KitchenSinkErr::CrateNotFound(origin.clone())).context("releases not found")?;
+                let versions = releases.into_iter().filter_map(|r| {
+                    let date = r.published_at.or(r.created_at)?;
+                    Some(CrateVersion {
+                        num: r.tag_name?.trim_start_matches(|c:char| !c.is_numeric()).to_string(),
+                        yanked: r.draft.unwrap_or(false),
+                        updated_at: date.clone(),
+                        created_at: date,
+                    })
+                }).collect::<Vec<_>>();
+                if versions.is_empty() {
+                    eprintln!("No versions found for {:?}", origin);
+                }
                 Ok(RichCrate::new(origin.clone(), gh.owner.into_iter().map(|o| {
                     CrateOwner {
                         id: 0,
@@ -353,7 +366,7 @@ impl KitchenSink {
                     }
                 }).collect(),
                 format!("{}/{}/{}", repo.owner, repo.repo, package),
-                vec![]))
+                versions))
             }
         }
     }
@@ -494,10 +507,23 @@ impl KitchenSink {
 
         let checkout = crate_git_checkout::checkout(&repo, &self.git_checkout_path)?;
         let (path_in_repo, tree_id, manifest) = crate_git_checkout::path_in_repo(&checkout, package)?
-            .ok_or_else(|| KitchenSinkErr::CrateNotFoundInRepo(repo.canonical_git_url().to_string(), package.to_string()))?;
+            .ok_or_else(|| {
+                let (has, err) = crate_git_checkout::find_manifests(&checkout).unwrap_or_default();
+                for e in err {
+                    eprintln!("parse err: {}", e.0);
+                }
+                for h in has {
+                    eprintln!("has: {} -> {}", h.0, h.2.package.as_ref().map(|p| p.name.as_str()).unwrap_or("?"));
+                }
+                KitchenSinkErr::CrateNotFoundInRepo(package.to_string(), repo.canonical_git_url().into_owned())
+            })?;
 
-        let meta = tarball::read_repo(&checkout, tree_id)?;
-        assert_eq!(meta.manifest.package, manifest.package);
+        let mut meta = tarball::read_repo(&checkout, tree_id)?;
+        debug_assert_eq!(meta.manifest.package, manifest.package);
+        let package = meta.manifest.package.as_mut().ok_or_else(|| KitchenSinkErr::NotAPackage(origin.clone()))?;
+        if package.repository.is_none() {
+            package.repository = Some(repo.canonical_git_url().into_owned());
+        }
 
         self.rich_crate_version_data_common(origin.clone(), meta, Some(path_in_repo), 0, false, HashSet::new())
     }
@@ -763,7 +789,7 @@ impl KitchenSink {
     }
 
     pub fn is_build_or_dev(&self, k: &Origin) -> Result<(bool, bool), KitchenSinkErr> {
-        Ok(self.dependents_stats_of(k)?
+        Ok(self.crates_io_dependents_stats_of(k)?
         .map(|d| {
             let is_build = d.build.def > 3 * (d.runtime.def + d.runtime.opt + 5);
             let is_dev = !is_build && d.dev > (3 * d.runtime.def + d.runtime.opt + 3 * d.build.def + d.build.opt + 5);
@@ -898,7 +924,10 @@ impl KitchenSink {
             Origin::CratesIo(name) => {
                 self.index.all_dependencies_flattened(self.index.crates_io_crate_by_name(name)?)
             },
-            _ => unimplemented!()
+            _ => {
+                eprintln!("deps unimplemented!()");
+                return Ok(Default::default())
+            }
         }
     }
 
@@ -911,10 +940,10 @@ impl KitchenSink {
         let _ = self.index.deps_stats();
     }
 
-    pub fn dependents_stats_of(&self, origin: &Origin) -> Result<Option<RevDependencies>, KitchenSinkErr> {
+    pub fn crates_io_dependents_stats_of(&self, origin: &Origin) -> Result<Option<RevDependencies>, KitchenSinkErr> {
         match origin {
             Origin::CratesIo(crate_name) => Ok(self.index.deps_stats()?.counts.get(crate_name).cloned()),
-            _ => unimplemented!(),
+            _ => Ok(None),
         }
     }
 
@@ -999,12 +1028,9 @@ impl KitchenSink {
         if stopped() {Err(KitchenSinkErr::Stopped)?;}
         let (res1, res2) = rayon::join(|| -> CResult<()> {
             let origin = k.origin();
-            match origin {
-                Origin::CratesIo(name) => {
-                    let meta = self.crates_io_meta(name, true)?;
-                    self.index_crate_downloads(name, &meta)?;
-                },
-                _ => {},
+            if let Origin::CratesIo(name) = origin {
+                let meta = self.crates_io_meta(name, true)?;
+                self.index_crate_downloads(name, &meta)?;
             }
             Ok(())
         }, || self.crate_db.index_versions(k, score, self.downloads_recent(k.origin())?));
@@ -1066,13 +1092,18 @@ impl KitchenSink {
         if stopped() {Err(KitchenSinkErr::Stopped)?;}
 
         let origin = k.origin();
-        let ver = match origin {
-            Origin::CratesIo(ref name) => self.index.crate_version_latest_unstable(name).context("rich_crate_version2")?,
-            _ => unimplemented!(),
+        let (v, is_yanked) = match origin {
+            Origin::CratesIo(ref name) => {
+                let ver = self.index.crate_version_latest_unstable(name).context("rich_crate_version2")?;
+                let (v, _warn) = self.rich_crate_version_data_from_crates_io(ver)?;
+                (v, ver.is_yanked())
+            },
+            Origin::GitHub {..} => {
+                let (v, _warn) = self.rich_crate_version_from_repo(&origin)?;
+                (v, false)
+            },
         };
 
-        let (v, _warn) = self.rich_crate_version_data_from_crates_io(ver)?;
-
         // direct deps are used as extra keywords for similarity matching,
         // but we're taking only niche deps to group similar niche crates together
         let raw_deps_stats = self.index.deps_stats()?;
@@ -1103,7 +1134,7 @@ impl KitchenSink {
             origin,
             repository: repository.as_ref(),
             deps_stats: &weighed_deps,
-            is_yanked: ver.is_yanked(),
+            is_yanked,
             is_build, is_dev,
             manifest: &v.manifest,
             derived: &v.derived,
diff --git a/reindex/src/bin/reindex_crates.rs b/reindex/src/bin/reindex_crates.rs
index daa5440..3b3c7f2 100644
--- a/reindex/src/bin/reindex_crates.rs
+++ b/reindex/src/bin/reindex_crates.rs
@@ -186,7 +186,11 @@ fn crate_overall_score(crates: &KitchenSink, all: &RichCrate, k: &RichCrateVersi
     let dependency_freshness = if let Ok((runtime, _, build)) = k.direct_dependencies() {
         // outdated dev deps don't matter
         runtime.iter().chain(&build).filter_map(|richdep| {
-            richdep.dep.req().parse().ok().map(|req| (richdep.is_optional(), crates.version_popularity(&richdep.package, &req).expect("verpop").expect("verpop")))
+            if !richdep.dep.is_crates_io() {
+                return None;
+            }
+            let req = richdep.dep.req().parse().ok()?;
+            Some((richdep.is_optional(), crates.version_popularity(&richdep.package, &req).expect("ver1pop").expect("ver2pop")))
         })
         .map(|(is_optional, (is_latest, popularity))| {
             if is_latest {1.0} // don't penalize pioneers
@@ -213,7 +217,7 @@ fn crate_overall_score(crates: &KitchenSink, all: &RichCrate, k: &RichCrateVersi
 
     let mut direct_rev_deps = 0;
     let mut indirect_reverse_optional_deps = 0;
-    if let Some(deps) = crates.dependents_stats_of(k.origin()).expect("depsstats") {
+    if let Some(deps) = crates.crates_io_dependents_stats_of(k.origin()).expect("depsstats") {
         direct_rev_deps = deps.direct as u32;
         indirect_reverse_optional_deps = (deps.runtime.def as u32 + deps.runtime.opt as u32)
             .max(deps.dev as u32)