improve build/ci times (#2396)

- avoid building all workspace crates with `cargo x build` (only plugins and main binary)
- only set the target triple in tests for plugins
- add new profile for `cargo x run` to build with optimized dependencies => FAST plugins when developing (thanks [Bevy Book](https://bevyengine.org/learn/book/getting-started/setup/#compile-with-performance-optimizations) for the idea)
- use https://github.com/Swatinem/rust-cache to avoid rebuilding dependencies every time in ci
- split `Build & Test` job into two so they run in parallel
- hopefully improve the flaky tests situation, this also makes the e2e tests run much faster (some tests produced correct snapshots but had some logic errors causing them to loop for much longer than necessary). Add some output to the tests so it is easier to see if something goes wrong.
- remove verbose build output from e2e test build

8 files changed, 84 insertions, 60 deletions

diff --git a/xtask/src/build.rs b/xtask/src/build.rs
index 1147eeee6..5bdf6ec88 100644
--- a/xtask/src/build.rs
+++ b/xtask/src/build.rs
@@ -4,7 +4,7 @@
 //!
 //! - [`build`]: Builds general cargo projects (i.e. zellij components) with `cargo build`
 //! - [`manpage`]: Builds the manpage with `mandown`
-use crate::flags;
+use crate::{flags, WorkspaceMember};
 use anyhow::Context;
 use std::path::{Path, PathBuf};
 use xshell::{cmd, Shell};
@@ -22,10 +22,13 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> {
         std::process::exit(1);
     }
 
-    for subcrate in crate::WORKSPACE_MEMBERS.iter() {
-        let err_context = || format!("failed to build '{subcrate}'");
+    for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS
+        .iter()
+        .filter(|member| member.build)
+    {
+        let err_context = || format!("failed to build '{crate_name}'");
 
-        if subcrate.contains("plugins") {
+        if crate_name.contains("plugins") {
             if flags.no_plugins {
                 continue;
             }
@@ -35,10 +38,10 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> {
             }
         }
 
-        let _pd = sh.push_dir(Path::new(subcrate));
+        let _pd = sh.push_dir(Path::new(crate_name));
         // Tell the user where we are now
         println!();
-        let msg = format!(">> Building '{subcrate}'");
+        let msg = format!(">> Building '{crate_name}'");
         crate::status(&msg);
         println!("{}", msg);
 
@@ -48,8 +51,8 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> {
         }
         base_cmd.run().with_context(err_context)?;
 
-        if subcrate.contains("plugins") {
-            let (_, plugin_name) = subcrate
+        if crate_name.contains("plugins") {
+            let (_, plugin_name) = crate_name
                 .rsplit_once('/')
                 .context("Cannot determine plugin name from '{subcrate}'")?;
 
diff --git a/xtask/src/ci.rs b/xtask/src/ci.rs
index bff00792b..b756052a9 100644
--- a/xtask/src/ci.rs
+++ b/xtask/src/ci.rs
@@ -90,7 +90,7 @@ fn e2e_build(sh: &Shell) -> anyhow::Result<()> {
         .and_then(|cargo| {
             cmd!(
                 sh,
-                "{cargo} build --verbose --release --target x86_64-unknown-linux-musl"
+                "{cargo} build --release --target x86_64-unknown-linux-musl"
             )
             .run()
             .map_err(anyhow::Error::new)
@@ -101,26 +101,20 @@ fn e2e_build(sh: &Shell) -> anyhow::Result<()> {
 fn e2e_test(sh: &Shell, args: Vec<OsString>) -> anyhow::Result<()> {
     let err_context = "failed to run E2E tests";
 
-    let _pd = sh.push_dir(crate::project_root());
     e2e_build(sh).context(err_context)?;
 
-    // Build debug plugins for test binary
-    build::build(
-        sh,
-        flags::Build {
-            release: false,
-            no_plugins: false,
-            plugins_only: true,
-        },
-    )
-    .context(err_context)?;
+    let _pd = sh.push_dir(crate::project_root());
 
+    // set --no-default-features so the test binary gets built with the plugins from assets/plugins that just got built
     crate::cargo()
         .and_then(|cargo| {
-            cmd!(sh, "{cargo} test -- --ignored --nocapture --test-threads 1")
-                .args(args)
-                .run()
-                .map_err(anyhow::Error::new)
+            cmd!(
+                sh,
+                "{cargo} test --no-default-features -- --ignored --nocapture --test-threads 1"
+            )
+            .args(args)
+            .run()
+            .map_err(anyhow::Error::new)
         })
         .context(err_context)
 }
diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs
index a19f8e5a5..b5e2cfe57 100644
--- a/xtask/src/clippy.rs
+++ b/xtask/src/clippy.rs
@@ -1,5 +1,5 @@
 //! Handle running `cargo clippy` on the sources.
-use crate::{build, flags};
+use crate::{build, flags, WorkspaceMember};
 use anyhow::Context;
 use std::path::{Path, PathBuf};
 use xshell::{cmd, Shell};
@@ -21,17 +21,17 @@ pub fn clippy(sh: &Shell, _flags: flags::Clippy) -> anyhow::Result<()> {
         .and_then(|_| crate::cargo())
         .context("failed to run task 'clippy'")?;
 
-    for subcrate in crate::WORKSPACE_MEMBERS.iter() {
-        let _pd = sh.push_dir(Path::new(subcrate));
+    for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() {
+        let _pd = sh.push_dir(Path::new(crate_name));
         // Tell the user where we are now
         println!();
-        let msg = format!(">> Running clippy on '{subcrate}'");
+        let msg = format!(">> Running clippy on '{crate_name}'");
         crate::status(&msg);
         println!("{}", msg);
 
         cmd!(sh, "{cargo} clippy --all-targets --all-features")
             .run()
-            .with_context(|| format!("failed to run task 'clippy' on '{subcrate}'"))?;
+            .with_context(|| format!("failed to run task 'clippy' on '{crate_name}'"))?;
     }
     Ok(())
 }
diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs
index 4630a8938..75ef19e4d 100644
--- a/xtask/src/flags.rs
+++ b/xtask/src/flags.rs
@@ -69,6 +69,8 @@ xflags::xflags! {
             optional --data-dir path: PathBuf
             /// Enable the singlepass compiler for WASM plugins
             optional --singlepass
+            /// Disable optimizing dependencies
+            optional --disable-deps-optimize
             /// Arguments to pass after `cargo run --`
             repeated args: OsString
         }
@@ -182,6 +184,7 @@ pub struct Run {
 
     pub data_dir: Option<PathBuf>,
     pub singlepass: bool,
+    pub disable_deps_optimize: bool,
 }
 
 #[derive(Debug)]
diff --git a/xtask/src/format.rs b/xtask/src/format.rs
index 2aed93bdc..4084a1f3c 100644
--- a/xtask/src/format.rs
+++ b/xtask/src/format.rs
@@ -1,5 +1,5 @@
 //! Handle running `cargo fmt` on the sources.
-use crate::flags;
+use crate::{flags, WorkspaceMember};
 use anyhow::Context;
 use std::path::{Path, PathBuf};
 use xshell::{cmd, Shell};
@@ -11,11 +11,11 @@ pub fn format(sh: &Shell, flags: flags::Format) -> anyhow::Result<()> {
         .and_then(|_| crate::cargo())
         .context("failed to run task 'format'")?;
 
-    for subcrate in crate::WORKSPACE_MEMBERS.iter() {
-        let _pd = sh.push_dir(Path::new(subcrate));
+    for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() {
+        let _pd = sh.push_dir(Path::new(crate_name));
         // Tell the user where we are now
         println!();
-        let msg = format!(">> Formatting '{subcrate}'");
+        let msg = format!(">> Formatting '{crate_name}'");
         crate::status(&msg);
         println!("{}", msg);
 
@@ -24,7 +24,7 @@ pub fn format(sh: &Shell, flags: flags::Format) -> anyhow::Result<()> {
             cmd = cmd.arg("--check");
         }
         cmd.run()
-            .with_context(|| format!("Failed to format '{subcrate}'"))?;
+            .with_context(|| format!("Failed to format '{crate_name}'"))?;
     }
     Ok(())
 }
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index 209caa5e7..9759d3a64 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -23,18 +23,23 @@ use std::{
 };
 use xshell::Shell;
 
+pub struct WorkspaceMember {
+    crate_name: &'static str,
+    build: bool,
+}
+
 lazy_static::lazy_static! {
-    pub static ref WORKSPACE_MEMBERS: Vec<&'static str> = vec![
-        "default-plugins/compact-bar",
-        "default-plugins/status-bar",
-        "default-plugins/strider",
-        "default-plugins/tab-bar",
-        "zellij-utils",
-        "zellij-tile-utils",
-        "zellij-tile",
-        "zellij-client",
-        "zellij-server",
-        ".",
+    pub static ref WORKSPACE_MEMBERS: Vec<WorkspaceMember> = vec![
+        WorkspaceMember{crate_name: "default-plugins/compact-bar", build: true},
+        WorkspaceMember{crate_name: "default-plugins/status-bar", build: true},
+        WorkspaceMember{crate_name: "default-plugins/strider", build: true},
+        WorkspaceMember{crate_name: "default-plugins/tab-bar", build: true},
+        WorkspaceMember{crate_name: "zellij-utils", build: false},
+        WorkspaceMember{crate_name: "zellij-tile-utils", build: false},
+        WorkspaceMember{crate_name: "zellij-tile", build: false},
+        WorkspaceMember{crate_name: "zellij-client", build: false},
+        WorkspaceMember{crate_name: "zellij-server", build: false},
+        WorkspaceMember{crate_name: ".", build: true},
     ];
 }
 
diff --git a/xtask/src/pipelines.rs b/xtask/src/pipelines.rs
index ad21bffe5..b89d7c0b6 100644
--- a/xtask/src/pipelines.rs
+++ b/xtask/src/pipelines.rs
@@ -1,8 +1,8 @@
 //! Composite pipelines for the build system.
 //!
 //! Defines multiple "pipelines" that run specific individual steps in sequence.
-use crate::flags;
 use crate::{build, clippy, format, test};
+use crate::{flags, WorkspaceMember};
 use anyhow::Context;
 use xshell::{cmd, Shell};
 
@@ -95,6 +95,12 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> {
 
     let singlepass = flags.singlepass.then_some(["--features", "singlepass"]);
 
+    let profile = if flags.disable_deps_optimize {
+        "dev"
+    } else {
+        "dev-opt"
+    };
+
     if let Some(ref data_dir) = flags.data_dir {
         let data_dir = sh.current_dir().join(data_dir);
 
@@ -105,6 +111,7 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> {
                     .arg("--no-default-features")
                     .args(["--features", "disable_automatic_asset_installation"])
                     .args(singlepass.iter().flatten())
+                    .args(["--profile", profile])
                     .args(["--", "--data-dir", &format!("{}", data_dir.display())])
                     .args(&flags.args)
                     .run()
@@ -124,6 +131,7 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> {
         .and_then(|cargo| {
             cmd!(sh, "{cargo} run")
                 .args(singlepass.iter().flatten())
+                .args(["--profile", profile])
                 .args(["--"])
                 .args(&flags.args)
                 .run()
@@ -288,18 +296,18 @@ pub fn publish(sh: &Shell, flags: flags::Publish) -> anyhow::Result<()> {
         }
 
         // Publish all the crates
-        for member in crate::WORKSPACE_MEMBERS.iter() {
-            if member.contains("plugin") || member.contains("xtask") {
+        for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() {
+            if crate_name.contains("plugin") || crate_name.contains("xtask") {
                 continue;
             }
 
-            let _pd = sh.push_dir(project_dir.join(member));
+            let _pd = sh.push_dir(project_dir.join(crate_name));
             loop {
-                let msg = format!(">> Publishing '{member}'");
+                let msg = format!(">> Publishing '{crate_name}'");
                 crate::status(&msg);
                 println!("{}", msg);
 
-                let more_args = match *member {
+                let more_args = match *crate_name {
                     // This is needed for zellij to pick up the plugins from the assets included in
                     // the released zellij-utils binary
                     "." => Some("--no-default-features"),
@@ -314,7 +322,7 @@ pub fn publish(sh: &Shell, flags: flags::Publish) -> anyhow::Result<()> {
                 .context(err_context)
                 {
                     println!();
-                    println!("Publishing crate '{member}' failed with error:");
+                    println!("Publishing crate '{crate_name}' failed with error:");
                     println!("{:?}", err);
                     println!();
                     println!("Retry? [y/n]");
@@ -346,7 +354,7 @@ pub fn publish(sh: &Shell, flags: flags::Publish) -> anyhow::Result<()> {
                     if retry {
                         continue;
                     } else {
-                        println!("Aborting publish for crate '{member}'");
+                        println!("Aborting publish for crate '{crate_name}'");
                         return Err::<(), _>(err);
                     }
                 } else {
diff --git a/xtask/src/test.rs b/xtask/src/test.rs
index 92ea778dd..79e5ba6b2 100644
--- a/xtask/src/test.rs
+++ b/xtask/src/test.rs
@@ -1,4 +1,4 @@
-use crate::{build, flags};
+use crate::{build, flags, WorkspaceMember};
 use anyhow::{anyhow, Context};
 use std::path::Path;
 use xshell::{cmd, Shell};
@@ -20,18 +20,29 @@ pub fn test(sh: &Shell, flags: flags::Test) -> anyhow::Result<()> {
     )
     .context(err_context)?;
 
-    for subcrate in crate::WORKSPACE_MEMBERS.iter() {
-        let _pd = sh.push_dir(Path::new(subcrate));
+    for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() {
+        // the workspace root only contains e2e tests, skip it
+        if *crate_name == "." {
+            continue;
+        }
+
+        let _pd = sh.push_dir(Path::new(crate_name));
         // Tell the user where we are now
         println!("");
-        let msg = format!(">> Testing '{}'", subcrate);
+        let msg = format!(">> Testing '{}'", crate_name);
         crate::status(&msg);
         println!("{}", msg);
 
-        cmd!(sh, "{cargo} test --target {host_triple} --")
-            .args(&flags.args)
+        // Override wasm32-wasi target for plugins only
+        let cmd = if crate_name.contains("plugins") {
+            cmd!(sh, "{cargo} test --target {host_triple} --")
+        } else {
+            cmd!(sh, "{cargo} test --")
+        };
+
+        cmd.args(&flags.args)
             .run()
-            .with_context(|| format!("Failed to run tests for '{}'", subcrate))?;
+            .with_context(|| format!("Failed to run tests for '{}'", crate_name))?;
     }
     Ok(())
 }