path: root/kitchen_sink/src/tarball.rs

use cargo_toml::Manifest;
use cargo_toml::Package;
use libflate::gzip::Decoder;
use render_readme::Markup;
use std::collections::HashSet;
use std::io;
use std::io::Read;
use std::path::Path;
use std::path::PathBuf;
use tar::{Archive, Entry, EntryType};
use udedokei;
use udedokei::LanguageExt;

#[derive(Debug, Fail)]
pub enum UnarchiverError {
    #[fail(display = "Cargo.toml not found. Got files: {}", _0)]
    TomlNotFound(String),
    #[fail(display = "I/O error during unarchiving: {}", _0)]
    Io(io::Error),
    #[fail(display = "Cargo.toml parsing error: {}", _0)]
    Toml(cargo_toml::Error),
}

impl From<io::Error> for UnarchiverError {
    fn from(i: io::Error) -> Self {
        UnarchiverError::Io(i)
    }
}

impl From<cargo_toml::Error> for UnarchiverError {
    fn from(i: cargo_toml::Error) -> Self {
        UnarchiverError::Toml(i)
    }
}

fn read_archive_files<R: Read>(archive: R, mut cb: impl FnMut(Entry<'_, Decoder<R>>) -> Result<(), UnarchiverError>) -> Result<(), UnarchiverError> {
    let mut archive = Archive::new(Decoder::new(archive)?);
    let entries = archive.entries()?;
    for entry in entries {
        cb(entry?)?
    }
    Ok(())
}

#[derive(Eq, PartialEq)]
enum ReadAs {
    Toml,
    ReadmeMarkdown(String),
    ReadmeRst(String),
    Lib,
    GetStatsOfFile(udedokei::Language),
    Skip,
}

const MAX_FILE_SIZE: u64 = 50_000_000;

pub fn read_repo(repo: &crate_git_checkout::Repository, path_in_tree: crate_git_checkout::Oid) -> Result<CrateFile, failure::Error> {
    let mut collect = Collector::new();
    crate_git_checkout::iter_blobs(repo, Some(path_in_tree), |path, _, name, blob| {
        // FIXME: skip directories that contain other crates
        let mut blob_content = blob.content();
        collect.add(Path::new(path).join(name), blob_content.len() as u64, &mut blob_content)?;
        Ok(())
    })?;
    Ok(collect.finish()?)
}

pub fn read_archive(archive: impl Read, name: &str, ver: &str) -> Result<CrateFile, UnarchiverError> {
    let prefix = PathBuf::from(format!("{}-{}", name, ver));
    let mut collect = Collector::new();
    read_archive_files(archive, |mut file| {
        let header = file.header();
        match header.entry_type() {
            EntryType::Regular | EntryType::Char => {
                let path = header.path()?;
                if let Ok(relpath) = path.strip_prefix(&prefix) {
                    return collect.add(relpath.to_path_buf(), header.size()?, &mut file);
                }
            },
            _ => {},
        }
        Ok(())
    })?;
    collect.finish()
}

struct Collector {
    manifest: Option<Manifest>,
    markup: Option<(String, Markup)>,
    files: Vec<PathBuf>,
    lib_file: Option<String>,
    stats: udedokei::Collect,
    decompressed_size: usize,
    is_nightly: bool,
}

impl Collector {
    pub fn new() -> Self {
        Self {
            manifest: None,
            markup: None,
            files: Vec::new(),
            lib_file: None,
            stats: udedokei::Collect::new(),
            decompressed_size: 0,
            is_nightly: false,
        }
    }

    pub fn add(&mut self, relpath: PathBuf, size: u64, file_data: &mut dyn Read) -> Result<(), UnarchiverError> {
        let path_match = {
            match &relpath {
                p if p == Path::new("Cargo.toml") || p == Path::new("cargo.toml") => ReadAs::Toml,
                p if p == Path::new("src/lib.rs") => ReadAs::Lib,
                p if is_readme_filename(p, self.manifest.as_ref().and_then(|m| m.package.as_ref())) => {
                    let path_prefix = p.parent().unwrap().display().to_string();
                    if p.extension().map_or(false, |e| e == "rst") {
                        ReadAs::ReadmeRst(path_prefix)
                    } else {
                        ReadAs::ReadmeMarkdown(path_prefix)
                    }
                },
                p => {
                    if let Some(lang) = is_source_code_file(p) {
                        if lang.is_code() {
                            ReadAs::GetStatsOfFile(lang)
                        } else {
                            ReadAs::Skip
                        }
                    } else {
                        ReadAs::Skip
                    }
                },
            }
        };
        self.files.push(relpath);
        if path_match == ReadAs::Skip {
            return Ok(());
        }

        let mut data = Vec::with_capacity(size.min(MAX_FILE_SIZE) as usize);
        file_data.take(MAX_FILE_SIZE).read_to_end(&mut data)?;
        self.decompressed_size += data.len();
        let data = String::from_utf8_lossy(&data);

        match path_match {
            ReadAs::Lib => {
                self.stats.add_to_stats(udedokei::from_path("lib.rs").unwrap(), &data);
                if check_if_uses_nightly_features(&data) {
                    self.is_nightly = true;
                }
                self.lib_file = Some(data.to_string());
            },
            ReadAs::Toml => {
                self.manifest = Some(Manifest::from_slice(data.as_bytes())?);
            },
            ReadAs::ReadmeMarkdown(path_prefix) => {
                self.markup = Some((path_prefix, Markup::Markdown(data.to_string())));
            },
            ReadAs::ReadmeRst(path_prefix) => {
                self.markup = Some((path_prefix, Markup::Rst(data.to_string())));
            },
            ReadAs::GetStatsOfFile(lang) => {
                self.stats.add_to_stats(lang, &data);
            },
            ReadAs::Skip => unreachable!(),
        }
        Ok(())
    }

    fn finish(self) -> Result<CrateFile, UnarchiverError> {
        let mut manifest = match self.manifest {
            Some(m) => m,
            None => return Err(UnarchiverError::TomlNotFound(self.files.iter().map(|p| p.display().to_string()).collect::<Vec<_>>().join(", "))),
        };

        manifest.complete_from_abstract_filesystem(FilesFs(&self.files))?;

        Ok(CrateFile {
            decompressed_size: self.decompressed_size,
            readme: self.markup,
            manifest,
            files: self.files,
            lib_file: self.lib_file,
            language_stats: self.stats.finish(),
            is_nightly: self.is_nightly,
        })
    }
}

struct FilesFs<'a>(&'a [PathBuf]);

impl cargo_toml::AbstractFilesystem for FilesFs<'_> {
    fn file_names_in(&self, dir: &str) -> io::Result<HashSet<Box<str>>> {
        Ok(self.0.iter().filter_map(|p| {
            p.strip_prefix(dir).ok()
        })
        .filter_map(|p| p.to_str())
        .map(From::from)
        .collect())
    }
}

fn check_if_uses_nightly_features(lib_source: &str) -> bool {
    lib_source.lines()
        .take(1000)
        .map(|line| line.find('"').map(|pos| &line[0..pos]).unwrap_or(line)) // half-assed effort to ignore feature in strings
        .map(|line| line.find("//").map(|pos| &line[0..pos]).unwrap_or(line)) // half-assed effort to remove comments
        .any(|line| line.contains("#![feature("))
}

fn is_source_code_file(path: &Path) -> Option<udedokei::Language> {
    use std::os::unix::ffi::OsStrExt;

    if path.starts_with("tests") || path.starts_with("benches")<