diff options
12 files changed, 73 insertions, 15 deletions
diff --git a/cargo_toml b/cargo_toml -Subproject 4215f6874c2aa116a86800a561c801d45304c39 +Subproject 5618b16425d96a104dbe3e903e5e6ffebe2b223 diff --git a/crate_git_checkout b/crate_git_checkout -Subproject 9ef8ae038dbccdbefb909685ab254330568e830 +Subproject c4b6bef8c387ca8b67376bc0901aa332dafc15c diff --git a/kitchen_sink/src/lib_kitchen_sink.rs b/kitchen_sink/src/lib_kitchen_sink.rs index 7f59895..d8813cb 100644 --- a/kitchen_sink/src/lib_kitchen_sink.rs +++ b/kitchen_sink/src/lib_kitchen_sink.rs @@ -92,6 +92,8 @@ pub enum Warning { ErrorCloning(String), #[fail(display = "{} URL is a broken link: {}", _0, _1)] BrokenLink(String, String), + #[fail(display = "Error parsing manifest: {}", _0)] + ManifestParseError(String), } #[derive(Debug, Clone, Fail)] @@ -104,6 +106,8 @@ pub enum KitchenSinkErr { CategoryQueryFailed, #[fail(display = "crate not found: {:?}", _0)] CrateNotFound(Origin), + #[fail(display = "crate {} not found in repo {}", _0, _1)] + CrateNotFoundInRepo(String, String), #[fail(display = "crate is not a package: {:?}", _0)] NotAPackage(Origin), #[fail(display = "data not found, wanted {}", _0)] @@ -433,8 +437,9 @@ impl KitchenSink { let ver = self.index.crate_version_latest_unstable(name).context("rich_crate_version1")?; self.rich_crate_version_from_crates_io(ver).map(|(krate, _)| krate)? }, - _ => { - unimplemented!() + o => { + let d = self.rich_crate_version_from_repo(o).map(|(krate, _)| krate)?; + RichCrateVersion::new(origin.clone(), d.manifest, d.derived) } } }, @@ -479,6 +484,24 @@ impl KitchenSink { None } + fn rich_crate_version_from_repo(&self, origin: &Origin) -> CResult<(RichCrateVersionCacheData, Warnings)> { + let (repo, package) = match origin { + Origin::GitHub {repo, package} => { + (RepoHost::GitHub(repo.clone()).try_into().expect("repohost"), &**package) + }, + _ => unreachable!() + }; + + let checkout = crate_git_checkout::checkout(&repo, &self.git_checkout_path)?; + let (path_in_repo, tree_id, manifest) = crate_git_checkout::path_in_repo(&checkout, package)? + .ok_or_else(|| KitchenSinkErr::CrateNotFoundInRepo(repo.canonical_git_url().to_string(), package.to_string()))?; + + let meta = tarball::read_repo(&checkout, tree_id)?; + assert_eq!(meta.manifest.package, manifest.package); + + self.rich_crate_version_data_common(origin.clone(), meta, Some(path_in_repo), 0, false, HashSet::new()) + } + fn rich_crate_version_data_from_crates_io(&self, latest: &crates_index::Version) -> CResult<(RichCrateVersionCacheData, Warnings)> { let mut warnings = HashSet::new(); @@ -1158,7 +1181,7 @@ impl KitchenSink { for warn in warnings { eprintln!("warning: {}", warn.0); } - let manif = manif.into_iter().filter_map(|(subpath, manifest)| { + let manif = manif.into_iter().filter_map(|(subpath, _, manifest)| { manifest.package.map(|p| (subpath, p.name)) }); self.crate_db.index_repo_crates(repo, manif).context("index rev repo")?; diff --git a/kitchen_sink/src/tarball.rs b/kitchen_sink/src/tarball.rs index 4156b74..d5cc35c 100644 --- a/kitchen_sink/src/tarball.rs +++ b/kitchen_sink/src/tarball.rs @@ -57,15 +57,12 @@ enum ReadAs { const MAX_FILE_SIZE: u64 = 50_000_000; -pub fn read_repo(repo: &crate_git_checkout::Repository, path_in_tree: &Path) -> Result<CrateFile, failure::Error> { +pub fn read_repo(repo: &crate_git_checkout::Repository, path_in_tree: crate_git_checkout::Oid) -> Result<CrateFile, failure::Error> { let mut collect = Collector::new(); - crate_git_checkout::iter_blobs(repo, |path, name, blob| { - let path = Path::new(path); + crate_git_checkout::iter_blobs(repo, Some(path_in_tree), |path, _, name, blob| { // FIXME: skip directories that contain other crates - if let Ok(p) = path.strip_prefix(path_in_tree) { - let mut blob_content = blob.content(); - collect.add(p.join(name), blob_content.len() as u64, &mut blob_content)?; - } + let mut blob_content = blob.content(); + collect.add(Path::new(path).join(name), blob_content.len() as u64, &mut blob_content)?; Ok(()) })?; Ok(collect.finish()?) @@ -289,3 +286,27 @@ fn unpack_crate() { assert!(d.language_stats.langs.get(&udedokei::Language::Bash).is_none()); assert_eq!(d.decompressed_size, 161); } + +#[test] +fn unpack_repo() { + let test_repo_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("test.repo"); + let repo = Repo::new("http://example.invalid/foo.git").unwrap(); + let checkout = crate_git_checkout::checkout(&repo, &test_repo_path).unwrap(); + let (_path_in_repo, tree_id, manifest) = crate_git_checkout::path_in_repo(&checkout, "crates-server").unwrap().unwrap(); + + let d = read_repo(&checkout, tree_id).unwrap(); + assert_eq!(d.manifest.package, manifest.package); + + assert_eq!(d.manifest.package.as_ref().unwrap().name, "crates-server"); + assert_eq!(d.manifest.package.as_ref().unwrap().version, "0.5.1"); + assert!(d.lib_file.unwrap().contains("fn nothing")); + assert_eq!(d.files.len(), 5); + assert!(match d.readme.unwrap().markup { + Markup::Rst(a) => a == "o hi\n", _ => false, + }); + assert_eq!(d.language_stats.langs.get(&udedokei::Language::Rust).unwrap().code, 1); + assert_eq!(d.language_stats.langs.get(&udedokei::Language::C).unwrap().code, 1); + assert_eq!(d.language_stats.langs.get(&udedokei::Language::JavaScript).unwrap().code, 0); + assert!(d.language_stats.langs.get(&udedokei::Language::Bash).is_none()); + assert_eq!(d.decompressed_size, 161); +} diff --git a/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/HEAD b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/HEAD new file mode 100644 index 0000000..cb089cd --- /dev/null +++ b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/HEAD @@ -0,0 +1 @@ +ref: refs/heads/master diff --git a/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/config b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/config new file mode 100644 index 0000000..0ce5c57 --- /dev/null +++ b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/config @@ -0,0 +1,8 @@ +[core] + repositoryformatversion = 0 + filemode = true + bare = true + ignorecase = true + precomposeunicode = true +[remote "origin"] + url = /Users/kornel/www/crates.rs/kitchen_sink/test.repo/.git diff --git a/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/info/refs b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/info/refs new file mode 100644 index 0000000..7eea1fa --- /dev/null +++ b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/info/refs @@ -0,0 +1 @@ +e9194dfd37caa5ff5d753f70993d68ac679eaebd refs/heads/master diff --git a/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/info/packs b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/info/packs new file mode 100644 index 0000000..dedd9d1 --- /dev/null +++ b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/info/packs @@ -0,0 +1,2 @@ +P pack-7b4d762e5433b85897372d64e8c489002696940c.pack + diff --git a/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.bitmap b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.bitmap Binary files differnew file mode 100644 index 0000000..c1e9422 --- /dev/null +++ b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.bitmap diff --git a/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.idx b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.idx Binary files differnew file mode 100644 index 0000000..0a83978 --- /dev/null +++ b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.idx diff --git a/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.pack b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.pack Binary files differnew file mode 100644 index 0000000..d989ccc --- /dev/null +++ b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/objects/pack/pack-7b4d762e5433b85897372d64e8c489002696940c.pack diff --git a/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/packed-refs b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/packed-refs new file mode 100644 index 0000000..fda73bc --- /dev/null +++ b/kitchen_sink/test.repo/http%3A%2F%2Fexample.invalid%2Ffoo.git/packed-refs @@ -0,0 +1,2 @@ +# pack-refs with: peeled fully-peeled sorted +e9194dfd37caa5ff5d753f70993d68ac679eaebd refs/heads/master |