refactor(plugins): change the data flow (#1934)

* refactor(plugins): do not block render loop

* refactor(plugins): cleanup

* style(fmt): rustfmt

* fix(plugins): various rendering pipeline fixes

13 files changed, 472 insertions, 312 deletions

diff --git a/zellij-server/Cargo.toml b/zellij-server/Cargo.toml
index 22112db8a..e8af7e5fb 100644
--- a/zellij-server/Cargo.toml
+++ b/zellij-server/Cargo.toml
@@ -18,8 +18,8 @@ daemonize = "0.4.1"
 serde_json = "1.0"
 unicode-width = "0.1.8"
 url = "2.2.2"
-wasmer = "1.0.0"
-wasmer-wasi = "1.0.0"
+wasmer = "2.3.0"
+wasmer-wasi = "2.3.0"
 cassowary = "0.3.0"
 zellij-utils = { path = "../zellij-utils/", version = "0.34.0" }
 log = "0.4.17"
diff --git a/zellij-server/src/logging_pipe.rs b/zellij-server/src/logging_pipe.rs
index d54de9b09..6081b283f 100644
--- a/zellij-server/src/logging_pipe.rs
+++ b/zellij-server/src/logging_pipe.rs
@@ -114,7 +114,6 @@ impl Seek for LoggingPipe {
     }
 }
 
-#[typetag::serde]
 impl WasiFile for LoggingPipe {
     fn last_accessed(&self) -> u64 {
         0
diff --git a/zellij-server/src/panes/floating_panes/mod.rs b/zellij-server/src/panes/floating_panes/mod.rs
index c8a606812..5eb73f438 100644
--- a/zellij-server/src/panes/floating_panes/mod.rs
+++ b/zellij-server/src/panes/floating_panes/mod.rs
@@ -1,6 +1,7 @@
 mod floating_pane_grid;
 use zellij_utils::position::Position;
 
+use crate::resize_pty;
 use crate::tab::Pane;
 use floating_pane_grid::FloatingPaneGrid;
 
@@ -8,7 +9,9 @@ use crate::{
     os_input_output::ServerOsApi,
     output::{FloatingPanesStack, Output},
     panes::{ActivePanes, PaneId},
+    thread_bus::ThreadSenders,
     ui::pane_contents_and_ui::PaneContentsAndUi,
+    wasm_vm::PluginInstruction,
     ClientId,
 };
 use std::cell::RefCell;
@@ -22,22 +25,6 @@ use zellij_utils::{
     pane_size::{Offset, PaneGeom, Size, Viewport},
 };
 
-macro_rules! resize_pty {
-    ($pane:expr, $os_input:expr) => {
-        if let PaneId::Terminal(ref pid) = $pane.pid() {
-            // FIXME: This `set_terminal_size_using_terminal_id` call would be best in
-            // `TerminalPane::reflow_lines`
-            $os_input.set_terminal_size_using_terminal_id(
-                *pid,
-                $pane.get_content_columns() as u16,
-                $pane.get_content_rows() as u16,
-            )
-        } else {
-            Ok(())
-        }
-    };
-}
-
 pub struct FloatingPanes {
     panes: BTreeMap<PaneId, Box<dyn Pane>>,
     display_area: Rc<RefCell<Size>>,
@@ -53,6 +40,7 @@ pub struct FloatingPanes {
     active_panes: ActivePanes,
     show_panes: bool,
     pane_being_moved_with_mouse: Option<(PaneId, Position)>,
+    senders: ThreadSenders,
 }
 
 #[allow(clippy::borrowed_box)]
@@ -68,6 +56,7 @@ impl FloatingPanes {
         default_mode_info: ModeInfo,
         style: Style,
         os_input: Box<dyn ServerOsApi>,
+        senders: ThreadSenders,
     ) -> Self {
         FloatingPanes {
             panes: BTreeMap::new(),
@@ -84,6 +73,7 @@ impl FloatingPanes {
             show_panes: false,
             active_panes: ActivePanes::new(&os_input),
             pane_being_moved_with_mouse: None,
+            senders,
         }
     }
     pub fn stack(&self) -> Option<FloatingPanesStack> {
@@ -241,7 +231,7 @@ impl FloatingPanes {
             } else {
                 pane.set_content_offset(Offset::default());
             }
-            resize_pty!(pane, os_api).unwrap();
+            resize_pty!(pane, os_api, self.senders).unwrap();
         }
     }
     pub fn render(&mut self, output: &mut Output) -> Result<()> {
@@ -321,7 +311,7 @@ impl FloatingPanes {
     }
     pub fn resize_pty_all_panes(&mut self, os_api: &mut Box<dyn ServerOsApi>) {
         for pane in self.panes.values_mut() {
-            resize_pty!(pane, os_api).unwrap();
+            resize_pty!(pane, os_api, self.senders).unwrap();
         }
     }
     pub fn resize_active_pane_left(
@@ -341,7 +331,7 @@ impl FloatingPanes {
             );
             floating_pane_grid.resize_pane_left(active_floating_pane_id);
             for pane in self.panes.values_mut() {
-                resize_pty!(pane, os_api).unwrap();
+                resize_pty!(pane, os_api, self.senders).unwrap();
             }
             self.set_force_render();
             return true;
@@ -365,7 +355,7 @@ impl FloatingPanes {
             );
             floating_pane_grid.resize_pane_right(active_floating_pane_id);
             for pane in self.panes.values_mut() {
-                resize_pty!(pane, os_api).unwrap();
+                resize_pty!(pane, os_api, self.senders).unwrap();
             }
             self.set_force_render();
             return true;
@@ -389,7 +379,7 @@ impl FloatingPanes {
             );
             floating_pane_grid.resize_pane_down(active_floating_pane_id);
             for pane in self.panes.values_mut() {
-                resize_pty!(pane, os_api).unwrap();
+                resize_pty!(pane, os_api, self.senders).unwrap();
             }
             self.set_force_render();
             return true;
@@ -413,7 +403,7 @@ impl FloatingPanes {
             );
             floating_pane_grid.resize_pane_up(active_floating_pane_id);
             for pane in self.panes.values_mut() {
-                resize_pty!(pane, os_api).unwrap();
+                resize_pty!(pane, os_api, self.senders).unwrap();
             }
             self.set_force_render();
             return true;
@@ -437,7 +427,7 @@ impl FloatingPanes {
             );
             floating_pane_grid.resize_increase(active_floating_pane_id);
             for pane in self.panes.values_mut() {
-                resize_pty!(pane, os_api).unwrap();
+                resize_pty!(pane, os_api, self.senders).unwrap();
             }
             self.set_force_render();
             return true;
@@ -461,7 +451,7 @@ impl FloatingPanes {
             );
             floating_pane_grid.resize_decrease(active_floating_pane_id);
             for pane in self.panes.values_mut() {
-                resize_pty!(pane, os_api).unwrap();
+                resize_pty!(pane, os_api, self.senders).unwrap();
             }
             self.set_force_render();
             return true;
diff --git a/zellij-server/src/panes/grid.rs b/zellij-server/src/panes/grid.rs
index f570a4460..0bc2b7671 100644
--- a/zellij-server/src/panes/grid.rs
+++ b/zellij-server/src/panes/grid.rs
@@ -4,6 +4,8 @@ use std::cell::RefCell;
 use std::collections::HashMap;
 use std::rc::Rc;
 use unicode_width::UnicodeWidthChar;
+use zellij_utils::data::Style;
+use zellij_utils::errors::prelude::*;
 use zellij_utils::regex::Regex;
 
 use std::{
@@ -549,7 +551,7 @@ impl Grid {
     pub fn cursor_shape(&self) -> CursorShape {
         self.cursor.get_shape()
     }
-    pub fn scrollback_position_and_length(&mut self) -> (usize, usize) {
+    pub fn scrollback_position_and_length(&self) -> (usize, usize) {
         // (position, length)
         (
             self.lines_below.len(),
@@ -979,6 +981,71 @@ impl Grid {
 
         (changed_character_chunks, changed_sixel_image_chunks)
     }
+    pub fn render(
+        &mut self,
+        content_x: usize,
+        content_y: usize,
+        style: &Style,
+    ) -> Result<Option<(Vec<CharacterChunk>, Option<String>, Vec<SixelImageChunk>)>> {
+        let mut raw_vte_output = String::new();
+
+        let (mut character_chunks, sixel_image_chunks) = self.read_changes(content_x, content_y);
+        for character_chunk in character_chunks.iter_mut() {
+            character_chunk.add_changed_colors(self.changed_colors);
+            if self
+                .selection
+                .contains_row(character_chunk.y.saturating_sub(content_y))
+            {
+                let background_color = match style.colors.bg {
+                    PaletteColor::Rgb(rgb) => AnsiCode::RgbCode(rgb),
+                    PaletteColor::EightBit(col) => AnsiCode::ColorIndex(col),
+                };
+                character_chunk.add_selection_and_colors(
+                    self.selection,
+                    background_color,
+                    None,
+                    content_x,
+                    content_y,
+                );
+            } else if !self.search_results.selections.is_empty() {
+                for res in self.search_results.selections.iter() {
+                    if res.contains_row(character_chunk.y.saturating_sub(content_y)) {
+                        let (select_background_palette, select_foreground_palette) =
+                            if Some(res) == self.search_results.active.as_ref() {
+                                (style.colors.orange, style.colors.black)
+                            } else {
+                                (style.colors.green, style.colors.black)
+                            };
+                        let background_color = match select_background_palette {
+                            PaletteColor::Rgb(rgb) => AnsiCode::RgbCode(rgb),
+                            PaletteColor::EightBit(col) => AnsiCode::ColorIndex(col),
+                        };
+                        let foreground_color = match select_foreground_palette {
+                            PaletteColor::Rgb(rgb) => AnsiCode::RgbCode(rgb),
+                            PaletteColor::EightBit(col) => AnsiCode::ColorIndex(col),
+                        };
+                        character_chunk.add_selection_and_colors(
+                            *res,
+                            background_color,
+                            Some(foreground_color),
+                            content_x,
+                            content_y,
+                        );
+                    }
+                }
+            }
+        }
+        if self.ring_bell {
+            let ring_bell = '\u{7}';
+            raw_vte_output.push(ring_bell);
+            self.ring_bell = false;
+        }
+        return Ok(Some((
+            character_chunks,
+            Some(raw_vte_output),
+            sixel_image_chunks,
+        )));
+    }
     pub fn cursor_coordinates(&self) -> Option<(usize, usize)> {
         if self.cursor_is_hidden {
             None
@@ -1959,6 +2026,14 @@ impl Grid {
             None
         }
     }
+    pub fn delete_viewport_and_scroll(&mut self) {
+        self.lines_above.clear();
+        self.viewport.clear();
+        self.lines_below.clear();
+    }
+    pub fn reset_cursor_position(&mut self) {
+        self.cursor = Cursor::new(0, 0);
+    }
 }
 
 impl Perform for Grid {
@@ -2563,7 +2638,7 @@ impl Perform for Grid {
             };
             if first_intermediate_is_questionmark {
                 let query_type = params_iter.next();
-                let is_query = params_iter.next() == Some(&[1]);
+                let is_query = matches!(params_iter.next(), Some(&[1]));
                 if is_query {
                     // XTSMGRAPHICS
                     match query_type {
diff --git a/zellij-server/src/panes/plugin_pane.rs b/zellij-server/src/panes/plugin_pane.rs
index 81610188f..9353b5114 100644
--- a/zellij-server/src/panes/plugin_pane.rs
+++ b/zellij-server/src/panes/plugin_pane.rs
@@ -1,28 +1,50 @@
-use std::fmt::Write;
-use std::sync::mpsc::channel;
+use std::collections::HashMap;
 use std::time::Instant;
 
 use crate::output::{CharacterChunk, SixelImageChunk};
-use crate::panes::PaneId;
+use crate::panes::{grid::Grid, sixel::SixelImageStore, LinkHandler, PaneId};
 use crate::pty::VteBytes;
 use crate::tab::Pane;
 use crate::ui::pane_boundaries_frame::{FrameParams, PaneFrame};
 use crate::wasm_vm::PluginInstruction;
 use crate::ClientId;
-use zellij_utils::pane_size::Offset;
+use std::cell::RefCell;
+use std::rc::Rc;
+use zellij_utils::pane_size::{Offset, SizeInPixels};
 use zellij_utils::position::Position;
-use zellij_utils::shared::ansi_len;
 use zellij_utils::{
     channels::SenderWithContext,
-    data::{Event, InputMode, Mouse, PaletteColor},
+    data::{Event, InputMode, Mouse, Palette, PaletteColor, Style},
     errors::prelude::*,
     pane_size::{Dimension, PaneGeom},
     shared::make_terminal_title,
+    vte,
 };
 
+macro_rules! get_or_create_grid {
+    ($self:ident, $client_id:ident) => {{
+        let rows = $self.get_content_rows();
+        let cols = $self.get_content_columns();
+
+        $self.grids.entry($client_id).or_insert_with(|| {
+            let mut grid = Grid::new(
+                rows,
+                cols,
+                $self.terminal_emulator_colors.clone(),
+                $self.terminal_emulator_color_codes.clone(),
+                $self.link_handler.clone(),
+                $self.character_cell_size.clone(),
+                $self.sixel_image_store.clone(),
+            );
+            grid.hide_cursor();
+            grid
+        })
+    }};
+}
+
 pub(crate) struct PluginPane {
     pub pid: u32,
-    pub should_render: bool,
+    pub should_render: HashMap<ClientId, bool>,
     pub selectable: bool,
     pub geom: PaneGeom,
     pub geom_override: Option<PaneGeom>,
@@ -31,8 +53,16 @@ pub(crate) struct PluginPane {
     pub active_at: Instant,
     pub pane_title: String,
     pub pane_name: String,
+    pub style: Style,
+    sixel_image_store: Rc<RefCell<SixelImageStore>>,
+    terminal_emulator_colors: Rc<RefCell<Palette>>,
+    terminal_emulator_color_codes: Rc<RefCell<HashMap<usize, String>>>,
+    link_handler: Rc<RefCell<LinkHandler>>,
+    character_cell_size: Rc<RefCell<Option<SizeInPixels>>>,
+    vte_parsers: HashMap<ClientId, vte::Parser>,
+    grids: HashMap<ClientId, Grid>,
     prev_pane_name: String,
-    frame: bool,
+    frame: HashMap<ClientId, PaneFrame>,
     borderless: bool,
 }
 
@@ -43,21 +73,35 @@ impl PluginPane {
         send_plugin_instructions: SenderWithContext<PluginInstruction>,
         title: String,
         pane_name: String,
+        sixel_image_store: Rc<RefCell<SixelImageStore>>,
+        terminal_emulator_colors: Rc<RefCell<Palette>>,
+        terminal_emulator_color_codes: Rc<RefCell<HashMap<usize, String>>>,
+        link_handler: Rc<RefCell<LinkHandler>>,
+        character_cell_size: Rc<RefCell<Option<SizeInPixels>>>,
+        style: Style,
     ) -> Self {
         Self {
             pid,
-            should_render: true,
+            should_render: HashMap::new(),
             selectable: true,
             geom: position_and_size,
             geom_override: None,
             send_plugin_instructions,
             active_at: Instant::now(),
-            frame: false,
+            frame: HashMap::new(),
             content_offset: Offset::default(),
             pane_title: title,
             borderless: false,
             pane_name: pane_name.clone(),
             prev_pane_name: pane_name,
+            terminal_emulator_colors,
+            terminal_emulator_color_codes,
+            link_handler,
+            character_cell_size,
+            sixel_image_store,
+            vte_parsers: HashMap::new(),
+            grids: HashMap::new(),
+            style,
         }
     }
 }
@@ -98,18 +142,37 @@ impl Pane for PluginPane {
     }
     fn reset_size_and_position_override(&mut self) {
         self.geom_override = None;
-        self.should_render = true;
+        self.resize_grids();
+        self.set_should_render(true);
     }
     fn set_geom(&mut self, position_and_size: PaneGeom) {
         self.geom = position_and_size;
-        self.should_render = true;
+        self.resize_grids();
+        self.set_should_render(true);
     }
     fn set_geom_override(&mut self, pane_geom: PaneGeom) {
         self.geom_override = Some(pane_geom);
-        self.should_render = true;
+        self.resize_grids();
+        self.set_should_render(true);
     }
-    fn handle_pty_bytes(&mut self, _event: VteBytes) {
-        // noop
+    fn handle_plugin_bytes(&mut self, client_id: ClientId, bytes: VteBytes) {
+        self.set_client_should_render(client_id, true);
+        let grid = get_or_create_grid!(self, client_id);
+
+        // this is part of the plugin contract, whenever we update the plugin and call its render function, we delete the existing viewport
+        // and scroll, reset the cursor position and make sure all the viewport is rendered
+        grid.delete_viewport_and_scroll();
+        grid.reset_cursor_position();
+        grid.render_full_viewport();
+
+        let vte_parser = self
+            .vte_parsers
+            .entry(client_id)
+            .or_insert_with(|| vte::Parser::new());
+        for &byte in &bytes {
+            vte_parser.advance(grid, byte);
+        }
+        self.should_render.insert(client_id, true);
     }
     fn cursor_coordinates(&self) -> Option<(usize, usize)> {
         None
@@ -124,10 +187,21 @@ impl Pane for PluginPane {
         self.geom_override
     }
     fn should_render(&self) -> bool {
-        self.should_render
+        // set should_render for all clients
+        self.should_render.values().any(|v| *v)
     }
     fn set_should_render(&mut self, should_render: bool) {
-        self.should_render = should_render;
+        self.should_render
+            .values_mut()
+            .for_each(|v| *v = should_render);
+    }
+    fn render_full_viewport(&mut self) {
+        // this marks the pane for a full re-render, rather than just rendering the
+        // diff as it usually does with the OutputBuffer
+        self.frame.clear();
+        for grid in self.grids.values_mut() {
+            grid.render_full_viewport();
+        }
     }
     fn selectable(&self) -> bool {
         self.selectable
@@ -139,127 +213,86 @@ impl Pane for PluginPane {
         &mut self,
         client_id: Option<ClientId>,
     ) -> Result<Option<(Vec<CharacterChunk>, Option<String>, Vec<SixelImageChunk>)>> {
-        // this is a bit of a hack but works in a pinch
-        let client_id = match client_id {
-            Some(id) => id,
-            None => return Ok(None),
-        };
-        // if self.should_render {
-        if true {
-            let err_context = || format!("failed to render plugin panes for client {client_id}");
-
-            // while checking should_render rather than rendering each pane every time
-            // is more performant, it causes some problems when the pane to the left should be
-            // rendered and has wide characters (eg. Chinese characters or emoji)
-            // as a (hopefully) temporary hack, we render all panes until we find a better solution
-            let mut vte_output = String::new();
-            let (buf_tx, buf_rx) = channel();
-
-            self.send_plugin_instructions
-                .send(PluginInstruction::Render(
-                    buf_tx,
-                    self.pid,
-                    client_id,
-                    self.get_content_rows(),
-                    self.get_content_columns(),
-                ))
-                .to_anyhow()
-                .with_context(err_context)?;
-
-            self.should_render = false;
-            // This is where we receive the text to render from the plugins.
-            let contents = buf_rx
-                .recv()
-                .with_context(err_context)
-                .to_log()
-                .unwrap_or("No output from plugin received. See logs".to_string());
-            for (index, line) in contents.lines().enumerate() {
-                let actual_len = ansi_len(line);
-                let line_to_print = if actual_len > self.get_content_columns() {
-                    let mut line = String::from(line);
-                    line.truncate(self.get_content_columns());
-                    line
-                } else {
-                    [
-                        line,
-                        &str::repeat(" ", self.get_content_columns() - ansi_len(line)),
-                    ]
-                    .concat()
-                };
-
-                write!(
-                    &mut vte_output,
-                    "\u{1b}[{};{}H\u{1b}[m{}",
-                    self.get_content_y() + 1 + index,
-                    self.get_content_x() + 1,
-                    line_to_print,
-                )
-                .with_context(err_context)?; // goto row/col and reset styles
-                let line_len = line_to_print.len();
-                if line_len < self.get_content_columns() {
-                    // pad line
-                    for _ in line_len..self.get_content_columns() {
-                        vte_output.push(' ');
-                    }
-                }
-            }
-            let total_line_count = contents.lines().count();
-            if total_line_count < self.get_content_rows() {
-                // pad lines
-                for line_index in total_line_count..self.get_content_rows() {
-                    let x = self.get_content_x();
-                    let y = self.get_content_y();
-                    write!(
-                        &mut vte_output,
-                        "\u{1b}[{};{}H\u{1b}[m",
-                        y + line_index + 1,
-                        x + 1
-                    )
-                    .with_context(err_context)?; // goto row/col and reset styles
-                    for _col_index in 0..self.get_content_columns() {
-                        vte_output.push(' ');
+        if client_id.is_none() {
+            return Ok(None);
+        }
+        if let Some(client_id) = client_id {
+            if self.should_render.get(&client_id).copied().unwrap_or(false) {
+                let content_x = self.get_content_x();
+                let content_y = self.get_content_y();
+                if let Some(grid) = self.grids.get_mut(&client_id) {
+                    match grid.render(content_x, content_y, &self.style) {
+                        Ok(rendered_assets) => {
+                            self.should_render.insert(client_id, false);
+                            return Ok(rendered_assets);
+                        },
+                        e => return e,
                     }
                 }
             }
-            Ok(Some((vec![], Some(vte_output), vec![]))) // TODO: PluginPanes should have their own grid so that we can return the non-serialized TerminalCharacters and have them participate in the render buffer
-        } else {
-            Ok(None)
         }
+        Ok(None)
     }
     fn render_frame(
         &mut self,
-        _client_id: ClientId,
+        client_id: ClientId,
         frame_params: FrameParams,
         input_mode: InputMode,
     ) -> Result<Option<(Vec<CharacterChunk>, Option<String>)>> {
-        // FIXME: This is a hack that assumes all fixed-size panes are borderless. This
-        // will eventually need fixing!
-        let res = if self.frame && !(self.geom.rows.is_fixed() || self.geom.cols.is_fixed()) {
+        if self.borderless {
+            return Ok(None);
+        }
+        if let Some(grid) = self.grids.get(&client_id) {
+            let err_context = || format!("failed to render frame for client {client_id}");
             let pane_title = if self.pane_name.is_empty()
                 && input_mode == InputMode::RenamePane
                 && frame_params.is_main_client
             {
                 String::from("Enter name...")
             } else if self.pane_name.is_empty() {
-                self.pane_title.clone()
+                grid.title
+                    .clone()
+                    .unwrap_or_else(|| self.pane_title.clone())
             } else {
                 self.pane_name.clone()
             };
+
             let frame = PaneFrame::new(
                 self.current_geom().into(),
-                (0, 0), // scroll position
+                grid.scrollback_position_and_length(),
                 pane_title,
                 frame_params,
             );
-            Some(
-                frame
-                    .render()
-                    .with_context(|| format!("failed to render frame for client {_client_id}"))?,
-            )
+
+            let res = match self.fram