Simpler builder

5 files changed, 78 insertions, 72 deletions

diff --git a/builder/Cargo.toml b/builder/Cargo.toml
index f20a5b3..0d23bf7 100644
--- a/builder/Cargo.toml
+++ b/builder/Cargo.toml
@@ -16,3 +16,4 @@ serde = "1.0.45"
 serde_json = "1.0.17"
 serde_derive = "1.0.45"
 scopeguard = "1.0.0"
+rand = "0.7.0"
diff --git a/builder/docker/Dockerfile b/builder/docker/Dockerfile
index 8c987bd..dcacab7 100644
--- a/builder/docker/Dockerfile
+++ b/builder/docker/Dockerfile
@@ -22,13 +22,3 @@ ENV CARGO_MANIFEST_DIR=/crate
 COPY run.sh /tmp/run-crate-tests.sh
 RUN chmod 0550 /tmp/run-crate-tests.sh
 
-ARG crate_tarball
-RUN test -n "$crate_tarball"
-COPY ${crate_tarball} /tmp/crate.tar.gz
-RUN tar -xvz --one-file-system --no-acls --no-same-permissions --no-same-owner --exclude-vcs --group=nobody --no-wildcards --exclude='.*' --exclude='*/.*' --no-xattrs --strip-components=1 -f /tmp/crate.tar.gz
-RUN rm -rf rust-toolchain Cargo.lock .cargo
-RUN ls -lAR
-ARG deps_date
-RUN cargo lts ${deps_date}
-RUN cargo generate-lockfile
-RUN cargo fetch --locked
diff --git a/builder/docker/run.sh b/builder/docker/run.sh
index 8b8d4a2..b4cd30c 100755
--- a/builder/docker/run.sh
+++ b/builder/docker/run.sh
@@ -1,10 +1,21 @@
 #!/bin/bash
 set -exuo pipefail
 
-echo "----SNIP----"; echo >&2 "----SNIP----";
+for crate in "$@"
+do
+    echo > Cargo.toml "
+[package]
+name = "\""______"\""
+version = "\""0.0.0"\""
 
-rustup show
-time cargo check --locked --message-format=json
+[lib]
+path = "\""/dev/null"\""
+
+[dependencies]
+$crate
+"
+
+cargo fetch
 
 echo "----SNIP----"; echo >&2 "----SNIP----";
 
@@ -20,4 +31,4 @@ time cargo check --locked --message-format=json
 
 echo "----SNIP----"; echo >&2 "----SNIP----";
 
-cargo +stable clippy
+done
diff --git a/builder/src/main.rs b/builder/src/main.rs
index f3bd7b6..3f9f647 100644
--- a/builder/src/main.rs
+++ b/builder/src/main.rs
@@ -1,11 +1,15 @@
+
+use rand::thread_rng;
+use rand::seq::SliceRandom;
+
 mod db;
 mod parse;
 
-use chrono::prelude::*;
-use crate::db::Compat;
+
+
 use kitchen_sink::*;
 use parse::*;
-use std::fs;
+
 use std::path::Path;
 use std::process::Command;
 
@@ -14,89 +18,56 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 
     let db = db::BuildDb::new(crates.main_cache_dir().join("builds.db"))?;
     let docker_root = crates.main_cache_dir().join("docker");
-    let tarball_reldir = Path::new("tarballs-tmp");
-    let tarball_absdir = docker_root.join(tarball_reldir);
-    let _ = fs::create_dir_all(&tarball_absdir);
+    prepare_docker(&docker_root)?;
 
-    for (name, _) in crates.all_crates_io_crates() {
+    for (_, all) in crates.all_crates_io_crates() {
         if stopped() {
             break;
         }
-        let origin = Origin::from_crates_io_name(name);
-        if let Err(e) = analyze_crate(&origin, &db, &crates, &docker_root, tarball_reldir, &tarball_absdir) {
-            eprintln!("•• {}: {}", name, e);
+        if let Err(e) = analyze_crate(&all, &db, &crates, &docker_root) {
+            eprintln!("•• {}: {}", all.name(), e);
             continue;
         }
     }
     Ok(())
 }
 
-fn analyze_crate(origin: &Origin, db: &db::BuildDb, crates: &KitchenSink, docker_root: &Path, tarball_reldir: &Path, tarball_absdir: &Path) -> Result<(), Box<dyn std::error::Error>> {
-    let all = crates.rich_crate(origin)?;
-    let k = crates.rich_crate_version(origin)?;
+fn analyze_crate(all: &CratesIndexCrate, db: &db::BuildDb, crates: &KitchenSink, docker_root: &Path) -> Result<(), Box<dyn std::error::Error>> {
+    let ref origin = Origin::from_crates_io_name(all.name());
+    let ver = all.latest_version();
 
-    if k.edition() == Edition::E2018 {
-        db.set_compat(k.origin(), k.version(), "1.30.1", Compat::Incompatible)?;
-        return Ok(());
-    }
-
-    let compat_info = db.get_compat(k.origin(), k.version())?;
+    let compat_info = db.get_compat(origin, ver.version())?;
     if !compat_info.is_empty() {
-        println!("got it {:?}", compat_info);
+        println!("{} got it {:?}", ver.name(), compat_info);
         return Ok(());
     }
 
-    let (d1,d2,d3) = k.direct_dependencies()?;
-    if d1.len() + d2.len() + d3.len() > 4 {
-        eprintln!("Too many deps {}", k.short_name());
-        return Ok(());
-    }
-
-    let res = db.get_raw_build_info(origin, k.version())?;
+    let res = db.get_raw_build_info(origin, ver.version())?;
     let builds = match res {
         Some(res) => res,
         None => {
-            let (stdout, stderr) = do_builds(&crates, &all, &k, &tarball_reldir, &tarball_absdir, &docker_root)?;
-            db.set_raw_build_info(origin, k.version(), &stdout, &stderr)?;
+            let (stdout, stderr) = do_builds(&crates, &all, &docker_root)?;
+            db.set_raw_build_info(origin, ver.version(), &stdout, &stderr)?;
             (stdout, stderr)
         },
     };
     for f in parse_analyses(&builds.0, &builds.1) {
+        println!("{:#?}", f);
         if let Some(rustc_version) = f.rustc_version {
-            for (name, version, compat) in f.crates {
-                db.set_compat(&Origin::from_crates_io_name(&name), &version, &rustc_version, compat)?;
+            for (rustc_override, name, version, compat) in f.crates {
+                let rustc_version = rustc_override.unwrap_or(&rustc_version);
+                db.set_compat(&Origin::from_crates_io_name(&name), &version, rustc_version, compat)?;
             }
         }
     }
     Ok(())
 }
 
-fn do_builds(crates: &KitchenSink, all: &RichCrate, k: &RichCrateVersion, tarball_reldir: &Path, tarball_absdir: &Path, docker_root: &Path) -> Result<(String, String), Box<dyn std::error::Error>> {
-    let tarball_data = crates.tarball(k.short_name(), k.version())?;
-    let filename = format!("crate-{}-{}.tar.gz", k.short_name(), k.version());
-    // has to be relative, because docker
-    let tarball_relpath = tarball_reldir.join(&filename);
-    let tarball_abspath = tarball_absdir.join(&filename);
-    fs::write(&tarball_abspath, tarball_data)?;
-    scopeguard::guard((), |_| {
-        let _ = fs::remove_file(tarball_abspath);
-    });
-
-    let version_info = all.versions().iter().find(|v| v.num == k.version()).ok_or("Bad version")?;
-    // use cargo-lts to rewind deps to a week after publication of this crate
-    // (it can't be exact date, because timezones, plus crates may rely on sibling crates or bugfixes released shortly after)
-    let deps_date = DateTime::parse_from_rfc3339(&version_info.created_at)? + chrono::Duration::days(7);
-    let deps_cutoff = if deps_date.year() < 2018 {
-        "2018-02-27".to_string() // oldest compiler version we test
-    } else {
-        deps_date.format("%Y-%m-%d").to_string()
-    };
+fn prepare_docker(docker_root: &Path) -> Result<(), Box<dyn std::error::Error>> {
 
     let res = Command::new("docker")
         .current_dir(docker_root)
         .arg("build")
-        .arg("--build-arg").arg(format!("crate_tarball={}", tarball_relpath.display()))
-        .arg("--build-arg").arg(format!("deps_date={}", deps_cutoff))
         .arg("-t").arg("testing1")
         .arg(".")
         .status()?;
@@ -104,14 +75,27 @@ fn do_builds(crates: &KitchenSink, all: &RichCrate, k: &RichCrateVersion, tarbal
     if !res.success() {
         Err("failed build")?;
     }
+    Ok(())
+}
+
+fn do_builds(_crates: &KitchenSink, all: &CratesIndexCrate, docker_root: &Path) -> Result<(String, String), Box<dyn std::error::Error>> {
 
-    let out = Command::new("docker")
+    let mut rng = thread_rng();
+    let ver = all.versions();
+    let versions = ver.choose_multiple(&mut rng, (ver.len()/3).max(1).min(8));
+
+    let mut cmd = Command::new("docker");
+    cmd
         .current_dir(docker_root)
         .arg("run")
         .arg("--rm")
         .arg("-m1500m")
         .arg("testing1")
-        .arg("/tmp/run-crate-tests.sh")
+        .arg("/tmp/run-crate-tests.sh");
+    for ver in versions {
+        cmd.arg(format!("{}=\"{}\"\n", all.name(), ver.version()));
+    }
+    let out = cmd
         .output()?;
 
     let stdout = String::from_utf8_lossy(&out.stdout).into_owned();
diff --git a/builder/src/parse.rs b/builder/src/parse.rs
index 2204aab..d63ab5d 100644
--- a/builder/src/parse.rs
+++ b/builder/src/parse.rs
@@ -9,15 +9,25 @@ pub struct CompilerMessageInner {
 }
 
 #[derive(Deserialize)]
+pub struct CompilerMessageTarget {
+    #[serde(default)]
+    kind: Vec<String>,
+    edition: Option<String>,
+}
+
+#[derive(Deserialize)]
 pub struct CompilerMessage {
+    target: Option<CompilerMessageTarget>,
     message: Option<CompilerMessageInner>,
     reason: Option<String>,
     package_id: String,
+    #[serde(default)]
+    filenames: Vec<String>,
 }
 
 #[derive(Default, Debug)]
 pub struct Findings {
-    pub crates: HashSet<(String, String, Compat)>,
+    pub crates: HashSet<(Option<&'static str>, String, String, Compat)>,
     pub rustc_version: Option<String>,
     pub check_time: Option<f32>,
 }
@@ -48,13 +58,23 @@ fn parse_analysis(stdout: &str, stderr: &str) -> Option<Findings> {
         if line.starts_with('{') {
             if let Ok(msg) = serde_json::from_str::<CompilerMessage>(line) {
                 if let Some((name, ver)) = parse_package_id(&msg.package_id) {
+                    if name == "______" || name == "_____" {
+                        continue;
+                    }
                     let level = msg.message.as_ref().map(|m| m.level.as_str()).unwrap_or("");
                     let reason = msg.reason.as_ref().map(|s| s.as_str()).unwrap_or("");
+                    // not an achievement, ignore
+                    if msg.filenames.iter().any(|f| f.contains("/build-script-build")) {
+                        continue;
+                    }
+                    if msg.target.as_ref().and_then(|t| t.edition.as_ref()).map_or(false, |e| e == "2018") {
+                        findings.crates.insert((Some("1.30.1"), name.clone(), ver.clone(), Compat::Incompatible));
+                    }
                     if level == "error" {
-                        findings.crates.insert((name, ver, Compat::Incompatible));
+                        findings.crates.insert((None, name, ver, Compat::Incompatible));
                     }
                     else if reason == "compiler-artifact" {
-                        findings.crates.insert((name, ver, Compat::VerifiedWorks));
+                        findings.crates.insert((None, name, ver, Compat::VerifiedWorks));
                     } else {
                         if level != "warning" && reason != "build-script-executed" && !(level == "" && reason == "compiler-message") {
                             eprintln!("unknown line {} {} {}", level, reason, line);