diff options
author | Thomas Linford <tlinford@users.noreply.github.com> | 2023-05-03 21:16:38 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-05-03 21:16:38 +0200 |
commit | f598ca373829c99c6e2b2c6064047ed07c139ff7 (patch) | |
tree | 3f409b9218d9eaec5f7ce087408f75c29914fb07 | |
parent | ecd63a100b9d927d78c64cda73bae5ec16cac419 (diff) |
improve build/ci times (#2396)
- avoid building all workspace crates with `cargo x build` (only plugins and main binary)
- only set the target triple in tests for plugins
- add new profile for `cargo x run` to build with optimized dependencies => FAST plugins when developing (thanks [Bevy Book](https://bevyengine.org/learn/book/getting-started/setup/#compile-with-performance-optimizations) for the idea)
- use https://github.com/Swatinem/rust-cache to avoid rebuilding dependencies every time in ci
- split `Build & Test` job into two so they run in parallel
- hopefully improve the flaky tests situation, this also makes the e2e tests run much faster (some tests produced correct snapshots but had some logic errors causing them to loop for much longer than necessary). Add some output to the tests so it is easier to see if something goes wrong.
- remove verbose build output from e2e test build
-rw-r--r-- | .github/workflows/e2e.yml | 8 | ||||
-rw-r--r-- | .github/workflows/rust.yml | 80 | ||||
-rw-r--r-- | Cargo.toml | 6 | ||||
-rw-r--r-- | src/tests/e2e/cases.rs | 31 | ||||
-rw-r--r-- | src/tests/e2e/remote_runner.rs | 15 | ||||
-rw-r--r-- | xtask/src/build.rs | 19 | ||||
-rw-r--r-- | xtask/src/ci.rs | 26 | ||||
-rw-r--r-- | xtask/src/clippy.rs | 10 | ||||
-rw-r--r-- | xtask/src/flags.rs | 3 | ||||
-rw-r--r-- | xtask/src/format.rs | 10 | ||||
-rw-r--r-- | xtask/src/main.rs | 27 | ||||
-rw-r--r-- | xtask/src/pipelines.rs | 24 | ||||
-rw-r--r-- | xtask/src/test.rs | 25 |
13 files changed, 167 insertions, 117 deletions
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 43f6d4a88..76e38d3cf 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -10,7 +10,7 @@ env: CARGO_TERM_COLOR: always jobs: - test: + test-e2e: name: Build generic binary and run tests on it runs-on: ubuntu-latest environment: cachix @@ -37,8 +37,10 @@ jobs: - name: Add musl target run: rustup target add x86_64-unknown-linux-musl #run: cargo install --debug cargo-make - - name: Build asset - run: cargo xtask ci e2e --build + - uses: Swatinem/rust-cache@v2 + # ensure the target folder exists, otherwise fixtures won't be in the right place + - name: Create target folder + run: mkdir -p ${{ github.workspace }}/target # we copy this manually into the target folder instead of mounting it because # github actions creates the service first, and if it has a mount that is part # of your yet unchecked out code, you cannot checkout the code after the mount diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 06db6d82d..cc0ee2ab9 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -2,71 +2,69 @@ name: Rust on: push: - branches: [ main ] + branches: [main] pull_request: - branches: [ main ] + branches: [main] env: CARGO_TERM_COLOR: always jobs: build: - name: Build & Test + name: Build strategy: fail-fast: false matrix: - os: [ ubuntu-latest, macos-latest ] + os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Add WASM target - run: rustup target add wasm32-wasi - - name: Build - run: cargo xtask build - - name: Test - run: cargo xtask test + - uses: actions/checkout@v3 + - name: Setup toolchain + run: rustup show + - uses: Swatinem/rust-cache@v2 + - name: Build + run: cargo xtask build + + test: + name: Test + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + - name: Setup toolchain + run: rustup show + - uses: Swatinem/rust-cache@v2 + - name: Test + run: cargo xtask test format: name: Check Formatting runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Check Format - run: cargo xtask format --check + - uses: actions/checkout@v3 + - name: Setup toolchain + run: rustup show + - name: Check Format + run: cargo xtask format --check clippy: name: Check Clippy Lints runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/cache@v3 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Check clippy lints - run: cargo xtask clippy + - uses: actions/checkout@v3 + - name: Setup toolchain + run: rustup show + - uses: Swatinem/rust-cache@v2 + - name: Check clippy lints + run: cargo xtask clippy diff --git a/Cargo.toml b/Cargo.toml index 484bacf2c..f73af05a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,12 @@ members = [ ".", ] +[profile.dev-opt] +inherits = "dev" + +[profile.dev-opt.package."*"] +opt-level = 3 + [profile.release] lto = true strip = true diff --git a/src/tests/e2e/cases.rs b/src/tests/e2e/cases.rs index d2b7086ac..b34138b95 100644 --- a/src/tests/e2e/cases.rs +++ b/src/tests/e2e/cases.rs @@ -190,8 +190,7 @@ pub fn cannot_split_terminals_vertically_when_active_terminal_is_too_small() { name: "Split pane to the right", instruction: |mut remote_terminal: RemoteTerminal| -> bool { let mut step_is_complete = false; - if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(3, 2) - { + if remote_terminal.cursor_position_is(3, 2) { remote_terminal.send_key(&PANE_MODE); remote_terminal.send_key(&SPLIT_RIGHT_IN_PANE_MODE); // back to normal mode after split @@ -205,7 +204,12 @@ pub fn cannot_split_terminals_vertically_when_active_terminal_is_too_small() { name: "Make sure only one pane appears", instruction: |remote_terminal: RemoteTerminal| -> bool { let mut step_is_complete = false; - if remote_terminal.cursor_position_is(3, 2) { + if remote_terminal.cursor_position_is(3, 2) + //two empty lines at the bottom to make sure there is no plugin output + && remote_terminal + .current_snapshot() + .ends_with(" \n ") + { // ... is the truncated tip line step_is_complete = true; } @@ -967,7 +971,7 @@ pub fn detach_and_attach_session() { name: "Wait for session to be attached", instruction: |remote_terminal: RemoteTerminal| -> bool { let mut step_is_complete = false; - if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(3, 2) + if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(77, 2) { // we're back inside the session step_is_complete = true; @@ -1003,9 +1007,7 @@ pub fn status_bar_loads_custom_keybindings() { name: "Wait for app to load", instruction: |remote_terminal: RemoteTerminal| -> bool { let mut step_is_complete = false; - if remote_terminal.cursor_position_is(3, 1) - && remote_terminal.snapshot_contains("$ █ ││$") - && remote_terminal.snapshot_contains("$ ") { + if remote_terminal.cursor_position_is(3, 2) && remote_terminal.tip_appears() { step_is_complete = true; } step_is_complete @@ -1316,7 +1318,7 @@ pub fn mirrored_sessions() { name: "take snapshot after", instruction: |remote_terminal: RemoteTerminal| -> bool { let mut step_is_complete = false; - if remote_terminal.cursor_position_is(3, 2) + if remote_terminal.cursor_position_is(63, 2) && remote_terminal.snapshot_contains("┐┌") { // cursor is back in the first tab @@ -1329,7 +1331,7 @@ pub fn mirrored_sessions() { name: "take snapshot after", instruction: |remote_terminal: RemoteTerminal| -> bool { let mut step_is_complete = false; - if remote_terminal.cursor_position_is(3, 2) + if remote_terminal.cursor_position_is(63, 2) && remote_terminal.snapshot_contains("┐┌") { // cursor is back in the first tab @@ -1592,6 +1594,7 @@ pub fn multiple_users_in_different_tabs() { let mut step_is_complete = false; if remote_terminal.cursor_position_is(3, 2) && remote_terminal.tip_appears() + && remote_terminal.snapshot_contains("Tab #1 [ ]") && remote_terminal.snapshot_contains("Tab #2") && remote_terminal.status_bar_appears() { @@ -1608,7 +1611,7 @@ pub fn multiple_users_in_different_tabs() { let mut step_is_complete = false; if remote_terminal.cursor_position_is(3, 2) && remote_terminal.tip_appears() - && remote_terminal.snapshot_contains("Tab #2") + && remote_terminal.snapshot_contains("Tab #2 [ ]") && remote_terminal.status_bar_appears() { // cursor is in the newly opened second tab @@ -1649,7 +1652,9 @@ pub fn bracketed_paste() { name: "Send pasted text followed by normal text", instruction: |mut remote_terminal: RemoteTerminal| -> bool { let mut step_is_complete = false; - if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(3, 2) + if remote_terminal.status_bar_appears() + && remote_terminal.tab_bar_appears() + && remote_terminal.cursor_position_is(3, 2) { remote_terminal.send_key(&BRACKETED_PASTE_START); remote_terminal.send_key(&TAB_MODE); @@ -1717,7 +1722,9 @@ pub fn toggle_floating_panes() { name: "Wait for new pane to appear", instruction: |remote_terminal: RemoteTerminal| -> bool { let mut step_is_complete = false; - if remote_terminal.cursor_position_is(33, 7) && remote_terminal.tip_appears() { + if remote_terminal.cursor_position_is(33, 7) + && remote_terminal.snapshot_contains("FLOATING PANES VISIBLE") + { // cursor is in the newly opened second pane step_is_complete = true; } diff --git a/src/tests/e2e/remote_runner.rs b/src/tests/e2e/remote_runner.rs index bafd9df76..e5c7d2ef2 100644 --- a/src/tests/e2e/remote_runner.rs +++ b/src/tests/e2e/remote_runner.rs @@ -299,11 +299,15 @@ impl RemoteTerminal { x == self.cursor_x && y == self.cursor_y } pub fn tip_appears(&self) -> bool { - self.last_snapshot.lock().unwrap().contains("Tip:") + let snapshot = self.last_snapshot.lock().unwrap(); + snapshot.contains("Tip:") || snapshot.contains("QuickNav:") } pub fn status_bar_appears(&self) -> bool { self.last_snapshot.lock().unwrap().contains("Ctrl +") } + pub fn tab_bar_appears(&self) -> bool { + self.last_snapshot.lock().unwrap().contains("Tab #1") + } pub fn snapshot_contains(&self, text: &str) -> bool { self.last_snapshot.lock().unwrap().contains(text) } @@ -625,6 +629,10 @@ impl RemoteRunner { } pub fn run_next_step(&mut self) { if let Some(next_step) = self.steps.get(self.current_step_index) { + println!( + "running step: {}, retries left: {}", + next_step.name, self.retries_left + ); let (cursor_x, cursor_y) = *self.cursor_coordinates.lock().unwrap(); let remote_terminal = RemoteTerminal { cursor_x, @@ -650,6 +658,10 @@ impl RemoteRunner { let mut retries_left = RETRIES; let instruction = step.instruction; loop { + println!( + "taking snapshot: {}, retries left: {}", + step.name, retries_left + ); if retries_left == 0 { self.test_timed_out = true; return self.last_snapshot.lock().unwrap().clone(); @@ -671,6 +683,7 @@ impl RemoteRunner { } } pub fn run_all_steps(&mut self) { + println!(); loop { self.run_next_step(); if !self.steps_left() { diff --git a/xtask/src/build.rs b/xtask/src/build.rs index 1147eeee6..5bdf6ec88 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -4,7 +4,7 @@ //! //! - [`build`]: Builds general cargo projects (i.e. zellij components) with `cargo build` //! - [`manpage`]: Builds the manpage with `mandown` -use crate::flags; +use crate::{flags, WorkspaceMember}; use anyhow::Context; use std::path::{Path, PathBuf}; use xshell::{cmd, Shell}; @@ -22,10 +22,13 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> { std::process::exit(1); } - for subcrate in crate::WORKSPACE_MEMBERS.iter() { - let err_context = || format!("failed to build '{subcrate}'"); + for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS + .iter() + .filter(|member| member.build) + { + let err_context = || format!("failed to build '{crate_name}'"); - if subcrate.contains("plugins") { + if crate_name.contains("plugins") { if flags.no_plugins { continue; } @@ -35,10 +38,10 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> { } } - let _pd = sh.push_dir(Path::new(subcrate)); + let _pd = sh.push_dir(Path::new(crate_name)); // Tell the user where we are now println!(); - let msg = format!(">> Building '{subcrate}'"); + let msg = format!(">> Building '{crate_name}'"); crate::status(&msg); println!("{}", msg); @@ -48,8 +51,8 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> { } base_cmd.run().with_context(err_context)?; - if subcrate.contains("plugins") { - let (_, plugin_name) = subcrate + if crate_name.contains("plugins") { + let (_, plugin_name) = crate_name .rsplit_once('/') .context("Cannot determine plugin name from '{subcrate}'")?; diff --git a/xtask/src/ci.rs b/xtask/src/ci.rs index bff00792b..b756052a9 100644 --- a/xtask/src/ci.rs +++ b/xtask/src/ci.rs @@ -90,7 +90,7 @@ fn e2e_build(sh: &Shell) -> anyhow::Result<()> { .and_then(|cargo| { cmd!( sh, - "{cargo} build --verbose --release --target x86_64-unknown-linux-musl" + "{cargo} build --release --target x86_64-unknown-linux-musl" ) .run() .map_err(anyhow::Error::new) @@ -101,26 +101,20 @@ fn e2e_build(sh: &Shell) -> anyhow::Result<()> { fn e2e_test(sh: &Shell, args: Vec<OsString>) -> anyhow::Result<()> { let err_context = "failed to run E2E tests"; - let _pd = sh.push_dir(crate::project_root()); e2e_build(sh).context(err_context)?; - // Build debug plugins for test binary - build::build( - sh, - flags::Build { - release: false, - no_plugins: false, - plugins_only: true, - }, - ) - .context(err_context)?; + let _pd = sh.push_dir(crate::project_root()); + // set --no-default-features so the test binary gets built with the plugins from assets/plugins that just got built crate::cargo() .and_then(|cargo| { - cmd!(sh, "{cargo} test -- --ignored --nocapture --test-threads 1") - .args(args) - .run() - .map_err(anyhow::Error::new) + cmd!( + sh, + "{cargo} test --no-default-features -- --ignored --nocapture --test-threads 1" + ) + .args(args) + .run() + .map_err(anyhow::Error::new) }) .context(err_context) } diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs index a19f8e5a5..b5e2cfe57 100644 --- a/xtask/src/clippy.rs +++ b/xtask/src/clippy.rs @@ -1,5 +1,5 @@ //! Handle running `cargo clippy` on the sources. -use crate::{build, flags}; +use crate::{build, flags, WorkspaceMember}; use anyhow::Context; use std::path::{Path, PathBuf}; use xshell::{cmd, Shell}; @@ -21,17 +21,17 @@ pub fn clippy(sh: &Shell, _flags: flags::Clippy) -> anyhow::Result<()> { .and_then(|_| crate::cargo()) .context("failed to run task 'clippy'")?; - for subcrate in crate::WORKSPACE_MEMBERS.iter() { - let _pd = sh.push_dir(Path::new(subcrate)); + for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() { + let _pd = sh.push_dir(Path::new(crate_name)); // Tell the user where we are now println!(); - let msg = format!(">> Running clippy on '{subcrate}'"); + let msg = format!(">> Running clippy on '{crate_name}'"); crate::status(&msg); println!("{}", msg); cmd!(sh, "{cargo} clippy --all-targets --all-features") .run() - .with_context(|| format!("failed to run task 'clippy' on '{subcrate}'"))?; + .with_context(|| format!("failed to run task 'clippy' on '{crate_name}'"))?; } Ok(()) } diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs index 4630a8938..75ef19e4d 100644 --- a/xtask/src/flags.rs +++ b/xtask/src/flags.rs @@ -69,6 +69,8 @@ xflags::xflags! { optional --data-dir path: PathBuf /// Enable the singlepass compiler for WASM plugins optional --singlepass + /// Disable optimizing dependencies + optional --disable-deps-optimize /// Arguments to pass after `cargo run --` repeated args: OsString } @@ -182,6 +184,7 @@ pub struct Run { pub data_dir: Option<PathBuf>, pub singlepass: bool, + pub disable_deps_optimize: bool, } #[derive(Debug)] diff --git a/xtask/src/format.rs b/xtask/src/format.rs index 2aed93bdc..4084a1f3c 100644 --- a/xtask/src/format.rs +++ b/xtask/src/format.rs @@ -1,5 +1,5 @@ //! Handle running `cargo fmt` on the sources. -use crate::flags; +use crate::{flags, WorkspaceMember}; use anyhow::Context; use std::path::{Path, PathBuf}; use xshell::{cmd, Shell}; @@ -11,11 +11,11 @@ pub fn format(sh: &Shell, flags: flags::Format) -> anyhow::Result<()> { .and_then(|_| crate::cargo()) .context("failed to run task 'format'")?; - for subcrate in crate::WORKSPACE_MEMBERS.iter() { - let _pd = sh.push_dir(Path::new(subcrate)); + for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() { + let _pd = sh.push_dir(Path::new(crate_name)); // Tell the user where we are now println!(); - let msg = format!(">> Formatting '{subcrate}'"); + let msg = format!(">> Formatting '{crate_name}'"); crate::status(&msg); println!("{}", msg); @@ -24,7 +24,7 @@ pub fn format(sh: &Shell, flags: flags::Format) -> anyhow::Result<()> { cmd = cmd.arg("--check"); } cmd.run() - .with_context(|| format!("Failed to format '{subcrate}'"))?; + .with_context(|| format!("Failed to format '{crate_name}'"))?; } Ok(()) } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 209caa5e7..9759d3a64 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -23,18 +23,23 @@ use std::{ }; use xshell::Shell; +pub struct WorkspaceMember { + crate_name: &'static str, + build: bool, +} + lazy_static::lazy_static! { - pub static ref WORKSPACE_MEMBERS: Vec<&'static str> = vec![ - "default-plugins/compact-bar", - "default-plugins/status-bar", - "default-plugins/strider", - "default-plugins/tab-bar", - "zellij-utils", - "zellij-tile-utils", - "zellij-tile", - "zellij-client", - "zellij-server", - ".", + pub static ref WORKSPACE_MEMBERS: Vec<WorkspaceMember> = vec![ + WorkspaceMember{crate_name: "default-plugins/compact-bar", build: true}, + WorkspaceMember{crate_name: "default-plugins/status-bar", build: true}, + WorkspaceMember{crate_name: "default-plugins/strider", build: true}, + WorkspaceMember{crate_name: "default-plugins/tab-bar", build: true}, + WorkspaceMember{crate_name: "zellij-utils", build: false}, + WorkspaceMember{crate_name: "zellij-tile-utils", build: false}, + WorkspaceMember{crate_name: "zellij-tile", build: false}, + WorkspaceMember{crate_name: "zellij-client", build: false}, + WorkspaceMember{crate_name: "zellij-server", build: false}, + WorkspaceMember{crate_name: ".", build: true}, ]; } diff --git a/xtask/src/pipelines.rs b/xtask/src/pipelines.rs index ad21bffe5..b89d7c0b6 100644 --- a/xtask/src/pipelines.rs +++ b/xtask/src/pipelines.rs @@ -1,8 +1,8 @@ //! Composite pipelines for the build system. //! //! Defines multiple "pipelines" that run specific individual steps in sequence. -use crate::flags; use crate::{build, clippy, format, test}; +use crate::{flags, WorkspaceMember}; use anyhow::Context; use xshell::{cmd, Shell}; @@ -95,6 +95,12 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> { let singlepass = flags.singlepass.then_some(["--features", "singlepass"]); + let profile = if flags.disable_deps_optimize { + "dev" + } else { + "dev-opt" + }; + if let Some(ref data_dir) = flags.data_dir { let data_dir = sh.current_dir().join(data_dir); @@ -105,6 +111,7 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> { .arg("--no-default-features") .args(["--features", "disable_automatic_asset_installation"]) .args(singlepass.iter().flatten()) + .args(["--profile", profile]) .args(["--", "--data-dir", &format!("{}", data_dir.display())]) .args(&flags.args) .run() @@ -124,6 +131,7 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> { .and_then(|cargo| { cmd!(sh, "{cargo} run") .args(singlepass.iter().flatten()) + .args(["--profile", profile]) .args(["--"]) .args(&flags.args) .run() @@ -288,18 +296,18 @@ pub fn publish(sh: &Shell, flags: flags::Publish) -> anyhow::Result<()> { } // Publish all the crates - for member in crate::WORKSPACE_MEMBERS.iter() { - if member.contains("plugin") || member.contains("xtask") { + for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() { + if crate_name.contains("plugin") || crate_name.contains("xtask") { continue; } - let _pd = sh.push_dir(project_dir.join(member)); + let _pd = sh.push_dir(project_dir.join(crate_name)); loop { - let msg = format!(">> Publishing '{member}'"); + let msg = format!(">> Publishing '{crate_name}'"); crate::status(&msg); println!("{}", msg); - let more_args = match *member { + let more_args = match *crate_name { // This is needed for |