diff options
| -rw-r--r-- | .github/workflows/e2e.yml | 2 | ||||
| -rw-r--r-- | CHANGELOG.md | 1 | ||||
| -rw-r--r-- | Cargo.toml | 9 | ||||
| -rw-r--r-- | Makefile.toml | 99 | ||||
| -rwxr-xr-x | assets/plugins/compact-bar.wasm | bin | 709146 -> 375644 bytes | |||
| -rwxr-xr-x | assets/plugins/status-bar.wasm | bin | 822127 -> 475869 bytes | |||
| -rwxr-xr-x | assets/plugins/strider.wasm | bin | 784162 -> 403004 bytes | |||
| -rwxr-xr-x | assets/plugins/tab-bar.wasm | bin | 685441 -> 351921 bytes | |||
| -rw-r--r-- | src/commands.rs | 47 | ||||
| -rw-r--r-- | src/install.rs | 51 | ||||
| -rw-r--r-- | src/main.rs | 1 | ||||
| -rw-r--r-- | zellij-server/src/wasm_vm.rs | 293 | ||||
| -rw-r--r-- | zellij-utils/Cargo.toml | 4 | ||||
| -rw-r--r-- | zellij-utils/src/consts.rs | 45 | ||||
| -rw-r--r-- | zellij-utils/src/errors.rs | 59 | ||||
| -rw-r--r-- | zellij-utils/src/input/layout.rs | 2 | ||||
| -rw-r--r-- | zellij-utils/src/input/plugins.rs | 78 | ||||
| -rw-r--r-- | zellij-utils/src/setup.rs | 82 | ||||
| -rw-r--r-- | zellij-utils/src/shared.rs | 5 |
19 files changed, 512 insertions, 266 deletions
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d1750cb18..22c243e1f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -43,6 +43,8 @@ jobs: run: rustup target add x86_64-unknown-linux-musl - name: Install cargo-make run: nix profile install nixpkgs#cargo-make + - name: Install wasm-opt + run: sudo apt-get install -y --no-install-recommends binaryen #run: cargo install --debug cargo-make - name: Build asset run: cargo make build-e2e diff --git a/CHANGELOG.md b/CHANGELOG.md index 5db3eb56c..5624849d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) * fix(themes): black and white inverted (https://github.com/zellij-org/zellij/pull/1953) * fix(stability): gracefully handle SSH timeouts and other client buffer overflow issues (https://github.com/zellij-org/zellij/pull/1955) * fix: empty session name (https://github.com/zellij-org/zellij/pull/1959) +* plugins: Cache plugins, don't load builtin plugins from disk (https://github.com/zellij-org/zellij/pull/1924) ## [0.33.0] - 2022-11-10 diff --git a/Cargo.toml b/Cargo.toml index 760a27027..26a9da0d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,15 +31,15 @@ rand = "0.8.0" [workspace] members = [ + "default-plugins/compact-bar", + "default-plugins/status-bar", + "default-plugins/strider", + "default-plugins/tab-bar", "zellij-client", "zellij-server", "zellij-utils", "zellij-tile", "zellij-tile-utils", - "default-plugins/compact-bar", - "default-plugins/status-bar", - "default-plugins/strider", - "default-plugins/tab-bar", ".", ] @@ -68,5 +68,6 @@ bin-dir = "{ bin }{ binary-ext }" pkg-fmt = "tgz" [features] +# See remarks in zellij_utils/Cargo.toml disable_automatic_asset_installation = [ "zellij-utils/disable_automatic_asset_installation" ] unstable = [ "zellij-client/unstable", "zellij-utils/unstable" ] diff --git a/Makefile.toml b/Makefile.toml index cf8e2efed..1fcdb138b 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -9,6 +9,7 @@ ZELLIJ_ASSETS_DIR = "${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/zellij-utils/asse # Add clippy to the default flow [tasks.dev-test-flow] dependencies = [ + "plugins", "format-flow", "format-toml-conditioned-flow", "pre-build", @@ -25,37 +26,21 @@ args = ["test", "--target", "${CARGO_HOST_TRIPLE}", "--", "@@split(CARGO_MAKE_TA # Running Zellij using the development data directory [tasks.run] workspace = false -dependencies = ["build-workspace", "build-dev-data-dir"] +dependencies = ["build-workspace"] run_task = "launch" [tasks.build-workspace] run_task = { name = "build", fork = true } [tasks.build] +env = { "CARGO_MAKE_WORKSPACE_SKIP_MEMBERS" = "default-plugins*" } args = ["build"] [tasks.build-release] args = ["build", "--release"] -[tasks.build-dev-data-dir] -dependencies = ["build-plugins"] -script_runner = "@duckscript" -script = ''' -target_dir = set ${CARGO_TARGET_DIR} -data_dir = set ${target_dir}/dev-data -rm -r ${data_dir} -plugins = glob_array ${target_dir}/wasm32-wasi/debug/*.wasm -mkdir ${data_dir} -mkdir ${data_dir}/plugins -for plugin in ${plugins} - plugin_name = basename ${plugin} - cp ${plugin} ${data_dir}/plugins/${plugin_name} -end -writefile ${data_dir}/VERSION ${CARGO_MAKE_CRATE_VERSION} -''' - [tasks.build-e2e-data-dir] -dependencies = ["build-plugins-release"] +dependencies = ["plugins-release"] script_runner = "@duckscript" script = ''' target_dir = set ${CARGO_TARGET_DIR} @@ -83,6 +68,7 @@ args = [ # Simple clippy tweak [tasks.clippy] +dependencies = ["plugins"] args = ["clippy", "--all-targets", "--all-features", "@@split(CARGO_MAKE_TASK_ARGS,;)"] # Release building and installing Zellij @@ -98,23 +84,40 @@ else end ''' -[tasks.build-plugins-release] -env = { "CARGO_MAKE_WORKSPACE_INCLUDE_MEMBERS" = [ - "default-plugins/compact-bar", - "default-plugins/status-bar", - "default-plugins/strider", - "default-plugins/tab-bar", -] } -run_task = { name = "build-release", fork = true } - -[tasks.build-plugins] -env = { "CARGO_MAKE_WORKSPACE_INCLUDE_MEMBERS" = [ - "default-plugins/compact-bar", - "default-plugins/status-bar", - "default-plugins/strider", - "default-plugins/tab-bar", -] } -run_task = { name = "build", fork = true } +[tasks.wasm-opt-plugins] +alias = "plugins-release" + +[tasks.plugins-release] +workspace = false +script_runner = "@duckscript" +script = ''' +plugins = glob_array ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/default-plugins/* +out_dir = set ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/assets/plugins/ +mkdir ${out_dir} + +for plugin in ${plugins} + cd ${plugin} + exec cargo build --release + plugin_name = basename ${plugin} + plugin_in = set ${CARGO_TARGET_DIR}/wasm32-wasi/release/${plugin_name}.wasm + plugin_out = set ${out_dir}/${plugin_name}.wasm + exec wasm-opt -O ${plugin_in} -o ${plugin_out} + cd .. +end +''' + +[tasks.plugins] +workspace = false +script_runner = "@duckscript" +script = ''' +plugins = glob_array ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/default-plugins/* + +for plugin in ${plugins} + cd ${plugin} + exec cargo build + cd .. +end +''' [tasks.get-host-triple] script_runner = "@duckscript" @@ -135,20 +138,6 @@ if not is_empty ${triple} end ''' -[tasks.wasm-opt-plugins] -dependencies = ["build-plugins-release"] -script_runner = "@duckscript" -script = ''' -plugins = glob_array ${CARGO_TARGET_DIR}/wasm32-wasi/release/*.wasm - -for plugin in ${plugins} - mkdir ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/assets/plugins/ - plugin_name = basename ${plugin} - plugin_out = set ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/assets/plugins/${plugin_name} - exec wasm-opt -O ${plugin} -o ${plugin_out} -end -''' - [tasks.manpage] workspace = false description = "Use mandown crate to create or update man entry from docs/MANPAGES.md" @@ -177,9 +166,8 @@ cp ${ZELLIJ_ASSETS_DIR}/config/default.kdl ${ZELLIJ_EXAMPLE_DIR}/default.kdl [tasks.ci-build-release] workspace = false dependencies = [ + "plugins-release", "setup-cross-compilation", - "build-plugins-release", - "wasm-opt-plugins", "manpage", ] command = "cross" @@ -194,7 +182,7 @@ args = [ # Build e2e asset [tasks.build-e2e] workspace = false -dependencies = ["build-plugins-release", "build-e2e-data-dir"] +dependencies = ["wasm-opt-plugins", "build-e2e-data-dir"] command = "cargo" args = [ "build", @@ -207,7 +195,7 @@ args = [ # Run e2e tests - we mark the e2e tests as "ignored" so they will not be run with the normal ones [tasks.e2e-test] workspace = false -dependencies = ["build-e2e"] +dependencies = ["build-e2e", "plugins"] command = "cargo" args = [ "test", @@ -228,9 +216,8 @@ args = ["install", "cross"] clear = true workspace = false dependencies = [ + "plugins-release", "update-default-config", - "build-plugins-release", - "wasm-opt-plugins", "release-commit", ] run_task = "publish-zellij" diff --git a/assets/plugins/compact-bar.wasm b/assets/plugins/compact-bar.wasm Binary files differindex 19c7ff8e3..2dbafb958 100755 --- a/assets/plugins/compact-bar.wasm +++ b/assets/plugins/compact-bar.wasm diff --git a/assets/plugins/status-bar.wasm b/assets/plugins/status-bar.wasm Binary files differindex 18c165a21..ac88ec3ba 100755 --- a/assets/plugins/status-bar.wasm +++ b/assets/plugins/status-bar.wasm diff --git a/assets/plugins/strider.wasm b/assets/plugins/strider.wasm Binary files differindex fcd624aae..a62576470 100755 --- a/assets/plugins/strider.wasm +++ b/assets/plugins/strider.wasm diff --git a/assets/plugins/tab-bar.wasm b/assets/plugins/tab-bar.wasm Binary files differindex 58dc85ced..d09286257 100755 --- a/assets/plugins/tab-bar.wasm +++ b/assets/plugins/tab-bar.wasm diff --git a/src/commands.rs b/src/commands.rs index 93ea21fdd..bf7cecb37 100644 --- a/src/commands.rs +++ b/src/commands.rs @@ -1,33 +1,28 @@ -use crate::install::populate_data_dir; -use crate::sessions::kill_session as kill_session_impl; +use dialoguer::Confirm; +use miette::{Report, Result}; +use std::{fs::File, io::prelude::*, path::PathBuf, process}; + use crate::sessions::{ assert_session, assert_session_ne, get_active_session, get_sessions, - get_sessions_sorted_by_mtime, match_session_name, print_sessions, print_sessions_with_index, - session_exists, ActiveSession, SessionNameMatch, + get_sessions_sorted_by_mtime, kill_session as kill_session_impl, match_session_name, + print_sessions, print_sessions_with_index, session_exists, ActiveSession, SessionNameMatch, }; -use dialoguer::Confirm; -use miette::{Report, Result}; -use std::path::PathBuf; -use std::process; -use zellij_client::old_config_converter::{ - config_yaml_to_config_kdl, convert_old_yaml_files, layout_yaml_to_layout_kdl, +use zellij_client::{ + old_config_converter::{ + config_yaml_to_config_kdl, convert_old_yaml_files, layout_yaml_to_layout_kdl, + }, + os_input_output::get_client_os_input, + start_client as start_client_impl, ClientInfo, }; -use zellij_client::start_client as start_client_impl; -use zellij_client::{os_input_output::get_client_os_input, ClientInfo}; -use zellij_server::os_input_output::get_server_os_input; -use zellij_server::start_server as start_server_impl; -use zellij_utils::input::actions::Action; -use zellij_utils::input::config::ConfigError; -use zellij_utils::input::options::Options; -use zellij_utils::nix; +use zellij_server::{os_input_output::get_server_os_input, start_server as start_server_impl}; use zellij_utils::{ cli::{CliArgs, Command, SessionCommand, Sessions}, envs, - setup::{get_default_data_dir, Setup}, + input::{actions::Action, config::ConfigError, options::Options}, + nix, + setup::Setup, }; -use std::{fs::File, io::prelude::*}; - pub(crate) use crate::sessions::list_sessions; pub(crate) fn kill_all_sessions(yes: bool) { @@ -97,11 +92,6 @@ fn create_new_client() -> ClientInfo { ClientInfo::New(names::Generator::default().next().unwrap()) } -fn install_default_assets(opts: &CliArgs) { - let data_dir = opts.data_dir.clone().unwrap_or_else(get_default_data_dir); - populate_data_dir(&data_dir); -} - fn find_indexed_session( sessions: Vec<String>, config_options: Options, @@ -364,10 +354,6 @@ pub(crate) fn start_client(opts: CliArgs) { ClientInfo::New(_) => Some(layout), }; - if create { - install_default_assets(&opts); - } - start_client_impl( Box::new(os_input), opts, @@ -379,7 +365,6 @@ pub(crate) fn start_client(opts: CliArgs) { } else { let start_client_plan = |session_name: std::string::String| { assert_session_ne(&session_name); - install_default_assets(&opts); }; if let Some(session_name) = opts.session.clone() { diff --git a/src/install.rs b/src/install.rs deleted file mode 100644 index 53c7122ac..000000000 --- a/src/install.rs +++ /dev/null @@ -1,51 +0,0 @@ -#[cfg(not(feature = "disable_automatic_asset_installation"))] -use std::fs; -use std::path::Path; -#[cfg(not(feature = "disable_automatic_asset_installation"))] -use zellij_utils::{consts::VERSION, shared::set_permissions}; - -#[cfg(not(feature = "disable_automatic_asset_installation"))] -macro_rules! asset_map { - ($($src:literal => $dst:literal),+ $(,)?) => { - { - let mut assets = std::collections::HashMap::new(); - $( - assets.insert($dst, include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/", $src)).to_vec()); - )+ - assets - } - } -} - -#[cfg(not(feature = "disable_automatic_asset_installation"))] -pub(crate) fn populate_data_dir(data_dir: &Path) { - let mut assets = asset_map! { - "assets/plugins/compact-bar.wasm" => "plugins/compact-bar.wasm", - "assets/plugins/status-bar.wasm" => "plugins/status-bar.wasm", - "assets/plugins/tab-bar.wasm" => "plugins/tab-bar.wasm", - "assets/plugins/strider.wasm" => "plugins/strider.wasm", - }; - assets.insert("VERSION", VERSION.as_bytes().to_vec()); - - let last_version = fs::read_to_string(data_dir.join("VERSION")).unwrap_or_default(); - let out_of_date = VERSION != last_version; - - for (path, bytes) in assets { - let path = data_dir.join(path); - // TODO: Is the [path.parent()] really necessary here? - // We already have the path and the parent through `data_dir` - if let Some(parent_path) = path.parent() { - fs::create_dir_all(parent_path).unwrap_or_else(|e| log::error!("{:?}", e)); - set_permissions(parent_path, 0o700).unwrap_or_else(|e| log::error!("{:?}", e)); - if out_of_date || !path.exists() { - fs::write(path, bytes) - .unwrap_or_else(|e| log::error!("Failed to install default assets! {:?}", e)); - } - } else { - log::error!("The path {:?} has no parent directory", path); - } - } -} - -#[cfg(feature = "disable_automatic_asset_installation")] -pub(crate) fn populate_data_dir(_data_dir: &Path) {} diff --git a/src/main.rs b/src/main.rs index 94a6eb4b2..77c3256d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,4 @@ mod commands; -mod install; mod sessions; #[cfg(test)] mod tests; diff --git a/zellij-server/src/wasm_vm.rs b/zellij-server/src/wasm_vm.rs index ed9f11e4b..8bc55ab1d 100644 --- a/zellij-server/src/wasm_vm.rs +++ b/zellij-server/src/wasm_vm.rs @@ -29,7 +29,7 @@ use crate::{ }; use zellij_utils::{ - consts::{DEBUG_MODE, VERSION, ZELLIJ_CACHE_DIR, ZELLIJ_PROJ_DIR, ZELLIJ_TMP_DIR}, + consts::{DEBUG_MODE, VERSION, ZELLIJ_CACHE_DIR, ZELLIJ_TMP_DIR}, data::{Event, EventType, PluginIds}, errors::{prelude::*, ContextType, PluginContext}, input::{ @@ -51,26 +51,41 @@ pub struct VersionMismatchError { zellij_version: String, plugin_version: String, plugin_path: PathBuf, + // true for builtin plugins + builtin: bool, } impl std::error::Error for VersionMismatchError {} impl VersionMismatchError { - pub fn new(zellij_version: &str, plugin_version: &str, plugin_path: &PathBuf) -> Self { + pub fn new( + zellij_version: &str, + plugin_version: &str, + plugin_path: &PathBuf, + builtin: bool, + ) -> Self { VersionMismatchError { zellij_version: zellij_version.to_owned(), plugin_version: plugin_version.to_owned(), plugin_path: plugin_path.to_owned(), + builtin, } } } impl fmt::Display for VersionMismatchError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let first_line = if self.builtin { + "It seems your version of zellij was built with outdated core plugins." + } else { + "If you're seeing this error a plugin version doesn't match the current +zellij version." + }; + write!( f, - "If you're seeing this error the plugin versions don't match the current -zellij version. Detected versions: + "{} +Detected versions: - Plugin version: {} - Zellij version: {} @@ -81,15 +96,13 @@ If you're a user: to them. If you're a developer: - Please run zellij with the updated plugins. The easiest way to achieve this + Please run zellij with updated plugins. The easiest way to achieve this is to build zellij with `cargo make install`. Also refer to the docs: https://github.com/zellij-org/zellij/blob/main/CONTRIBUTING.md#building - -A possible fix for this error is to remove all contents of the 'PLUGIN DIR' -folder from the output of the `zellij setup --check` command. ", - self.plugin_version, - self.zellij_version, + first_line, + self.plugin_version.trim_end(), + self.zellij_version.trim_end(), self.plugin_path.display() ) } @@ -149,11 +162,10 @@ pub(crate) fn wasm_thread_main( let mut connected_clients: Vec<ClientId> = vec![]; let plugin_dir = data_dir.join("plugins/"); let plugin_global_data_dir = plugin_dir.join("data"); - - #[cfg(not(feature = "disable_automatic_asset_installation"))] - fs::create_dir_all(&plugin_global_data_dir) - .context("failed to create plugin asset directory") - .non_fatal(); + // Caches the "wasm bytes" of all plugins that have been loaded since zellij was started. + // Greatly decreases loading times of all plugins and avoids accesses to the hard-drive during + // "regular" operation. + let mut plugin_cache: HashMap<PathBuf, Module> = HashMap::new(); loop { let (event, mut err_ctx) = bus.recv().expect("failed to receive event on channel"); @@ -169,7 +181,14 @@ pub(crate) fn wasm_thread_main( .fatal(); let (instance, plugin_env) = start_plugin( - plugin_id, client_id, &plugin, tab_index, &bus, &store, &data_dir, + plugin_id, + client_id, + &plugin, + tab_index, + &bus, + &store, + &plugin_dir, + &mut plugin_cache, ) .with_context(err_context)?; @@ -243,7 +262,8 @@ pub(crate) fn wasm_thread_main( anyError::new(VersionMismatchError::new( VERSION, "Unavailable", - &plugin_env.plugin.path + &plugin_env.plugin.path, + plugin_env.plugin.is_builtin(), )) ), Err(e) => Err(e).with_context(err_context), @@ -353,9 +373,17 @@ pub(crate) fn wasm_thread_main( // load headless plugins for plugin in plugins.iter() { if let PluginType::Headless = plugin.run { - let (instance, plugin_env) = - start_plugin(plugin_id, client_id, plugin, 0, &bus, &store, &data_dir) - .with_context(err_context)?; + let (instance, plugin_env) = start_plugin( + plugin_id, + client_id, + plugin, + 0, + &bus, + &store, + &plugin_dir, + &mut plugin_cache, + ) + .with_context(err_context)?; headless_plugins.insert(plugin_id, (instance, plugin_env)); plugin_id += 1; } @@ -368,9 +396,17 @@ pub(crate) fn wasm_thread_main( } } info!("wasm main thread exits"); + fs::remove_dir_all(&plugin_global_data_dir) - .context("failed to cleanup plugin data directory")?; - Ok(()) + .or_else(|err| { + if err.kind() == std::io::ErrorKind::NotFound { + // I don't care... + Ok(()) + } else { + Err(err) + } + }) + .context("failed to cleanup plugin data directory") } #[allow(clippy::too_many_arguments)] @@ -381,88 +417,116 @@ fn start_plugin( tab_index: usize, bus: &Bus<PluginInstruction>, store: &Store, - data_dir: &Path, + plugin_dir: &Path, + plugin_cache: &mut HashMap<PathBuf, Module>, ) -> Result<(Instance, PluginEnv)> { let err_context = || format!("failed to start plugin {plugin:#?} for client {client_id}"); - if plugin._allow_exec_host_cmd { - info!( - "Plugin({:?}) is able to run any host command, this may lead to some security issues!", - plugin.path - ); - } - - // The plugins blob as stored on the filesystem - let wasm_bytes = plugin - .resolve_wasm_bytes(&data_dir.join("plugins/")) - .with_context(err_context) - .fatal(); - - let hash: String = PortableHash::default() - .hash256(&wasm_bytes) - .iter() - .map(ToString::to_string) - .collect(); - - let cached_path = ZELLIJ_PROJ_DIR.cache_dir().join(&hash); - - let module = unsafe { - match Module::deserialize_from_file(store, &cached_path) { - Ok(m) => m, - Err(e) => { - let inner_context = || format!("failed to recover from {e:?}"); - - let m = Module::new(store, &wasm_bytes) - .with_context(inner_context) - .with_context(err_context)?; - fs::create_dir_all(ZELLIJ_PROJ_DIR.cache_dir()) - .with_context(inner_context) - .with_context(err_context)?; - m.serialize_to_file(&cached_path) - .with_context(inner_context) - .with_context(err_context)?; - m - }, - } - }; - - let output = Pipe::new(); - let input = Pipe::new(); - let stderr = LoggingPipe::new(&plugin.location.to_string(), plugin_id); let plugin_own_data_dir = ZELLIJ_CACHE_DIR.join(Url::from(&plugin.location).to_string()); - fs::create_dir_all(&plugin_own_data_dir) - .with_context(|| format!("failed to create datadir in {plugin_own_data_dir:?}")) - .with_context(|| format!("while starting plugin {plugin:#?}")) - .non_fatal(); + let cache_hit = plugin_cache.contains_key(&plugin.path); + + // We remove the entry here and repopulate it at the very bottom, if everything went well. + // We must do that because a `get` will only give us a borrow of the Module. This suffices for + // the purpose of setting everything up, but we cannot return a &Module from the "None" match + // arm, because we create the Module from scratch there. Any reference passed outside would + // outlive the Module we create there. Hence, we remove the plugin here and reinsert it + // below... + let module = match plugin_cache.remove(&plugin.path) { + Some(module) => { + log::debug!( + "Loaded plugin '{}' from plugin cache", + plugin.path.display() + ); + module + }, + None => { + // Populate plugin module cache for this plugin! + // Is it in the cache folder already? + if plugin._allow_exec_host_cmd { + info!( + "Plugin({:?}) is able to run any host command, this may lead to some security issues!", < |