From ef29c47e88f13d2931a2c571a579bf0bf0dacc8c Mon Sep 17 00:00:00 2001 From: rabite Date: Mon, 23 Dec 2019 02:08:50 +0100 Subject: fixed hangs when updating large directories (>10k files) --- Cargo.lock | 68 +++++++----- Cargo.toml | 3 +- src/file_browser.rs | 52 ++------- src/files.rs | 297 +++++++++++++++++++++++++++++++++++++--------------- src/fscache.rs | 268 ++++++++++++++++++++++++++++++++--------------- src/listview.rs | 169 ++++++++++++++++++------------ src/main.rs | 2 + src/preview.rs | 17 --- 8 files changed, 555 insertions(+), 321 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b63584..b862004 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -258,6 +258,16 @@ dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "derivative" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "dirs" version = "1.0.5" @@ -410,7 +420,7 @@ dependencies = [ "glib 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "muldiv 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -427,21 +437,21 @@ dependencies = [ "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-app-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-app-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "gstreamer-app-sys" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -456,19 +466,19 @@ dependencies = [ "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "gstreamer-base-sys" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -484,7 +494,7 @@ dependencies = [ "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer-player-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer-video 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -496,15 +506,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "gstreamer-sys" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -524,22 +534,22 @@ dependencies = [ "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)", "gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "gstreamer-video-sys" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -554,7 +564,7 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", @@ -569,6 +579,7 @@ dependencies = [ "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "dirs-2 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -908,7 +919,7 @@ name = "num_cpus" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1562,6 +1573,7 @@ dependencies = [ "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" "checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4" "checksum deflate 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "707b6a7b384888a70c8d2e8650b3e60170dfc6a67bb4aa67b6dfca57af4bedb4" +"checksum derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "942ca430eef7a3806595a6737bc388bf51adb888d3fc0dd1b50f1c170167ee3a" "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" "checksum dirs-2 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50b7e2b65c73137ec48935d50a5ae89b03150df566b7e14a1371df044e76765c" "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" @@ -1581,16 +1593,16 @@ dependencies = [ "checksum gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31d1a804f62034eccf370006ccaef3708a71c31d561fee88564abe71177553d9" "checksum gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa91e470b0cd4b05611f7d0e89caf76e39752156440877f04c23ad34ffc9761c" "checksum gstreamer-app 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a85485c2db4149ccb24d0b3c6598725743dec254bf757ac7a3684e62b9822c27" -"checksum gstreamer-app-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "41c85ef44d827b9292833203f6623cf6592d5eda06ad1eeefa63bca0cc38ce71" +"checksum gstreamer-app-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bf869ce152c23bca5d761ab62146b47f750d0b28d4d499731857532897d48167" "checksum gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)" = "37ed3ddb8f1a65e401340afb5657d2107d78c82f06a081e19393fc8d4c5ee720" -"checksum gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9ba1955ea091323c17fdf8ff54fd7cf3dfed1a6035193ba08f85eb76bf549056" +"checksum gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ba384f52174b3c586593fca32642680a9e67961fea9f4cd8419f678965023bed" "checksum gstreamer-player 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "78395f87de2b954ca3e33a594a4eb85e68246f5f41a70bf0ab52187aacb4d3a9" "checksum gstreamer-player-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c3608d3e96c8977f31b9b8db7da0b8d0e96758b060e3f05fc3ee9626d75ab1c5" -"checksum gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bdfc2f6cc9b6a1f5159bfd500310fe431cfb0b74b3af17ce3fdf8353cf586975" +"checksum gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1d18da01b97d0ab5896acd5151e4c155acefd0e6c03c3dd24dd133ba054053db" "checksum gstreamer-video 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0b48c3edfa6c0419ad7587a557bbed716caf5585c3be65bfbe3584703735df51" -"checksum gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8fcb1e577de93d1ad1e5117234ce64d40f215143d752140719923651608983" +"checksum gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "615f4298f842f4b4581606e13cf52e1710e2130d989bb99161a5665aa3ccb7cc" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" -"checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120" +"checksum hermit-abi 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f629dc602392d3ec14bfc8a09b5e644d7ffd725102b48b81e59f90f2633621d7" "checksum image 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)" = "35371e467cd7b0b3d1d6013d619203658467df12d61b0ca43cd67b743b1965eb" "checksum inflate 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff" "checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718" diff --git a/Cargo.toml b/Cargo.toml index dc853cb..fe0898e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ chrono = "0.4" libc = "0.2.51" failure = "0.1.5" failure_derive = "0.1.1" -notify = "4.0.12" +notify = "4.0.14" parse-ansi = "0.1.6" signal-notify = "0.1.3" systemstat = "0.1.5" @@ -41,6 +41,7 @@ base64 = "0.10.1" strum = "0.15" strum_macros = "0.15" rust-ini = "0.13" +derivative = "1.0.3" image = { version = "0.21.1", optional = true } diff --git a/src/file_browser.rs b/src/file_browser.rs index 257ca32..43394ac 100644 --- a/src/file_browser.rs +++ b/src/file_browser.rs @@ -168,21 +168,6 @@ impl Tabbable for TabView { } fn on_refresh(&mut self) -> HResult<()> { - let fs_changes = self.active_tab_() - .fs_cache - .fs_changes - .write()? - .drain(..) - .collect::>(); - - for tab in &mut self.widgets { - for (dir, old_file, new_file) in fs_changes.iter() { - tab.replace_file(&dir, - old_file.as_ref(), - new_file.as_ref()).log() - } - } - let open_dirs = self.widgets .iter() .fold(HashSet::new(), |mut dirs, tab| { @@ -503,7 +488,7 @@ impl FileBrowser { } pub fn main_widget_goto(&mut self, dir: &File) -> HResult<()> { - self.cache_files().log(); + self.save_tab_settings().log(); let dir = dir.clone(); let cache = self.fs_cache.clone(); @@ -733,22 +718,15 @@ impl FileBrowser { Ok(&self.left_widget()?.content) } - pub fn cache_files(&mut self) -> HResult<()> { - if self.main_widget().is_ok() { + pub fn save_tab_settings(&mut self) -> HResult<()> { + if !self.main_async_widget_mut()?.ready() { return Ok(()) } + + if self.main_widget()?.content.len() > 0 { let files = self.get_files()?; let selected_file = self.selected_file().ok(); - self.fs_cache.put_files(files, selected_file).log(); - self.main_widget_mut()?.content.meta_updated = false; + self.fs_cache.save_settings(files, selected_file).log(); } - - // if self.cwd.parent().is_some() { - // let left_selection = self.left_widget()?.clone_selected_file(); - // let left_files = self.get_left_files()?; - // self.fs_cache.put_files(left_files, Some(left_selection)).log(); - // self.left_widget_mut()?.content.meta_updated = false; - // } - Ok(()) } @@ -769,22 +747,6 @@ impl FileBrowser { Ok(dir) } - fn replace_file(&mut self, - dir: &File, - old: Option<&File>, - new: Option<&File>) -> HResult<()> { - if &self.cwd == dir { - self.main_widget_mut()?.content.replace_file(old, new.cloned()).log(); - } - - self.preview_widget_mut()?.replace_file(dir, old, new).ok(); - - if &self.left_dir()? == &dir { - self.left_widget_mut()?.content.replace_file(old, new.cloned()).log(); - } - Ok(()) - } - pub fn selected_file(&self) -> HResult { let widget = self.main_widget()?; let file = widget.selected_file().clone(); @@ -1288,7 +1250,7 @@ impl Widget for FileBrowser { self.set_cwd().log(); if !self.columns.zoom_active { self.update_preview().log(); } self.columns.refresh().log(); - self.cache_files().log(); + self.save_tab_settings().log(); Ok(()) } diff --git a/src/files.rs b/src/files.rs index d8558af..bafffc9 100644 --- a/src/files.rs +++ b/src/files.rs @@ -1,4 +1,5 @@ use std::cmp::{Ord, Ordering}; +use std::collections::{HashMap, HashSet}; use std::ops::Index; use std::fs::Metadata; use std::os::unix::fs::MetadataExt; @@ -16,7 +17,6 @@ use users::{get_current_username, get_group_by_gid}; use chrono::TimeZone; use failure::Error; -use notify::DebouncedEvent; use rayon::{ThreadPool, ThreadPoolBuilder}; use alphanumeric_sort::compare_str; use mime_guess; @@ -28,6 +28,7 @@ use crate::fail::{HResult, HError, ErrorLog}; use crate::dirty::{AsyncDirtyBit, DirtyBit, Dirtyable}; use crate::widget::Events; use crate::icon::Icons; +use crate::fscache::FsEvent; lazy_static! { @@ -99,11 +100,147 @@ pub fn tags_loaded() -> HResult<()> { #[derive(PartialEq, Eq, Hash, Clone, Debug)] +pub struct RefreshPackage { + pub new_files: Option>, + pub new_buffer: Option>, +} + + + + +impl RefreshPackage { + fn new(mut files: Files, + old_buffer: Vec, + events: Vec, + render_fn: impl Fn(&File) -> String) -> RefreshPackage { + use FsEvent::*; + + // If there is only a placeholder at this point, remove it now + if files.len() == 1 { + files.remove_placeholder(); + } + + //To preallocate collections + let event_count = events.len(); + + // Need at least one copy for the hashmaps + let static_files = files.clone(); + + // Optimization to speed up access to array + let file_pos_map: HashMap<&File, usize> = static_files + .files + .iter() + .enumerate() + .map(|(i, file)| (file, i)) + .collect(); + + + // Need to know which line of the ListView buffer belongs to which file + let list_pos_map: HashMap<&File, usize> = static_files + .iter_files() + .enumerate() + .take_while(|&(i, _)| i < old_buffer.len()) + .map(|(i, file)| (file, i)) + .collect(); + + // Save new files to add them all at once later + let mut new_files = Vec::with_capacity(event_count); + + // Files that need rerendering to make all changes visible (size, etc.) + let mut changed_files = HashSet::with_capacity(event_count); + + // Save deletions to delete them efficiently later + let mut deleted_files = HashSet::with_capacity(event_count); + + for event in events.into_iter() { + match event { + Create(mut file) => { + let dirty_meta = files.dirty_meta.clone(); + file.dirty_meta = Some(dirty_meta); + file.meta_sync().log(); + new_files.push(file); + } + Change(file) => { + if let Some(&fpos) = file_pos_map.get(&file) { + files.files[fpos].meta_sync().log(); + changed_files.insert(file); + } + } + Rename(old, new) => { + if let Some(&fpos) = file_pos_map.get(&old) { + files.files[fpos].rename(&new.path).log(); + files.files[fpos].meta_sync().log(); + } + } + Remove(file) => { + if let Some(_) = file_pos_map.get(&file) { + deleted_files.insert(file); + } + } + } + } + + if deleted_files.len() > 0 { + files.files.retain(|file| !deleted_files.contains(file)); + } + + // Finally add all new files + files.files.extend(new_files); + + // Need to unpack this to prevent issue with recursive Files type + // Also, if no files remain add placeholder + let files = if files.len() > 0 { + // Sort again to make sure new/changed files are in correct order + files.sort(); + files.files + } else { + let placeholder = File::new_placeholder(&files.directory.path).unwrap(); + files.files.push(placeholder); + + // Need to sort because of possible hidden files + files.sort(); + files.files + }; + + let mut old_buffer = old_buffer; + + // Prerender new buffer in current thread + let new_buffer = files.iter() + .map(|file| { + match list_pos_map.get(&file) { + Some(&old_pos) => + match changed_files.contains(&file) { + true => render_fn(&file), + false => std::mem::take(&mut old_buffer[old_pos]) + } + None => render_fn(&file) + } + }).collect(); + + + + RefreshPackage { + new_files: Some(files), + new_buffer: Some(new_buffer), + } + } +} + + +#[derive(Derivative)] +#[derivative(PartialEq, Eq, Hash, Clone, Debug)] pub struct Files { pub directory: File, pub files: Vec, + #[derivative(Debug="ignore")] + #[derivative(PartialEq="ignore")] + #[derivative(Hash="ignore")] + pub pending_events: Arc>>, + #[derivative(Debug="ignore")] + #[derivative(PartialEq="ignore")] + #[derivative(Hash="ignore")] + pub refresh: Option>, pub meta_upto: Option, - pub meta_updated: bool, pub sort: SortBy, pub dirs_first: bool, pub reverse: bool, @@ -158,8 +295,9 @@ impl Files { let mut files = Files { directory: File::new_from_path(&path, None)?, files: files, + pending_events: Arc::new(RwLock::new(vec![])), + refresh: None, meta_upto: None, - meta_updated: false, sort: SortBy::Name, dirs_first: true, reverse: false, @@ -216,8 +354,9 @@ impl Files { let mut files = Files { directory: File::new_from_path(&path, None)?, files: files, + pending_events: Arc::new(RwLock::new(vec![])), + refresh: None, meta_upto: None, - meta_updated: false, sort: SortBy::Name, dirs_first: true, reverse: false, @@ -375,77 +514,69 @@ impl Files { } } - pub fn replace_file(&mut self, - old: Option<&File>, - new: Option) -> HResult<()> { - let (tag, selected) = if let Some(old) = old { - if let Some(old) = self.find_file_with_path(&old.path) { - (old.tag, old.selected) - } else { - (None, false) - } - } else { - (None, false) - }; - old.map(|old| self.files.remove_item(old)); - new.map(|mut new| { - new.tag = tag; - new.selected = selected; - self.files.push(new); - }); + fn remove_placeholder(&mut self) { + let dirpath = self.directory.path.clone(); + self.find_file_with_path(&dirpath).cloned() + .map(|placeholder| self.files.remove_item(&placeholder)); + } - self.sort(); + pub fn ready_to_refresh(&self) -> HResult { + let pending = self.pending_events.read()?.len(); + let running = self.refresh.is_some(); + Ok(pending > 0 && !running) + } - if self.len() == 0 { - let placeholder = File::new_placeholder(&self.directory.path)?; - self.files.push(placeholder); - } else { - self.remove_placeholder(); + pub fn get_refresh(&mut self) -> HResult> { + if let Some(mut refresh) = self.refresh.take() { + if refresh.is_ready() { + refresh.pull_async()?; + let mut refresh = refresh.value?; + self.files = refresh.new_files.take()?; + return Ok(Some(refresh)); + } else { + self.refresh.replace(refresh); + } } - Ok(()) + return Ok(None) } - fn remove_placeholder(&mut self) { - let dirpath = self.directory.path.clone(); - self.find_file_with_path(&dirpath).cloned() - .map(|placeholder| self.files.remove_item(&placeholder)); - } + pub fn process_fs_events(&mut self, + buffer: Vec, + sender: Sender, + render_fn: impl Fn(&File) -> String + Send + 'static) + -> HResult<()> { + let pending = self.pending_events.read()?.len(); + if pending > 0 { + let pending = if pending >= 1000 { + 1000 + } else { + pending + }; + + let events = self.pending_events + .write()? + .drain(0..pending) + .collect::>(); + let files = self.clone(); + + let mut refresh = Async::new(move |_| { + let refresh = RefreshPackage::new(files, + buffer, + events, + render_fn); + Ok(refresh) + }); + + refresh.on_ready(move |_,_| { + Ok(sender.send(Events::WidgetReady)?) + })?; + + refresh.run()?; - pub fn handle_event(&mut self, - event: &DebouncedEvent) -> HResult<()> { - match event { - DebouncedEvent::Create(path) => { - self.path_in_here(&path)?; - let file = File::new_from_path(&path, - Some(self.dirty_meta.clone()))?; - self.files.push(file); - self.sort(); - }, - DebouncedEvent::Write(path) | DebouncedEvent::Chmod(path) => { - self.path_in_here(&path)?; - let file = self.find_file_with_path(&path)?; - file.reload_meta()?; - }, - DebouncedEvent::Remove(path) => { - self.path_in_here(&path)?; - let file = self.find_file_with_path(&path)?.clone(); - self.files.remove_item(&file); - }, - DebouncedEvent::Rename(old_path, new_path) => { - self.path_in_here(&new_path)?; - let mut file = self.find_file_with_path(&old_path)?; - file.name = new_path.file_name()?.to_string_lossy().to_string(); - file.path = new_path.into(); - file.reload_meta()?; - }, - DebouncedEvent::Error(err, _path) => { - // Never seen this happen. Should reload affected dirs - HError::log::<()>(&format!("{}", err))?; - }, - _ => {}, + self.refresh = Some(refresh); } - self.set_dirty(); + Ok(()) } @@ -465,17 +596,16 @@ impl Files { } pub fn find_file_with_path(&mut self, path: &Path) -> Option<&mut File> { - self.files.iter_mut().find(|file| file.path == path) + self.iter_files_mut().find(|file| file.path == path) } pub fn meta_all_sync(&mut self) -> HResult<()> { - for file in self.files.iter_mut() { + for file in self.iter_files_mut() { if !file.meta_processed { file.meta_sync().log(); } } self.set_dirty(); - self.meta_updated = true; Ok(()) } @@ -508,10 +638,10 @@ impl Files { .filter(|f| !(!show_hidden && f.name.starts_with("."))) .take(meta_files) { if !file.meta_processed { - file.take_meta(&meta_pool, &mut self.meta_updated).ok(); + file.take_meta(&meta_pool).ok(); } if file.is_dir() { - file.take_dirsize(&meta_pool, &mut self.meta_updated).ok(); + file.take_dirsize(&meta_pool).ok(); } } @@ -519,7 +649,7 @@ impl Files { } pub fn meta_set_fresh(&self) -> HResult<()> { - self.files.get(0)?.meta.set_fresh()?; + self.iter_files().nth(0)?.meta.set_fresh()?; Ok(()) } @@ -697,6 +827,12 @@ impl File { Ok(file) } + pub fn rename(&mut self, new_path: &Path) -> HResult<()> { + self.name = new_path.file_name()?.to_string_lossy().to_string(); + self.path = new_path.into(); + Ok(()) + } + pub fn meta_sync(&mut self) -> HResult<()> { let stale = self.meta.get_stale(); let meta = std::fs::metadata(&self.path)?; @@ -754,36 +890,32 @@ impl File { } fn take_dirsize(&mut self, - pool: &ThreadPool, - meta_updated: &mut bool) -> HResult<()> { + pool: &ThreadPool) -> HResult<()> { let dirsize = self.dirsize.as_mut()?; if let Ok(_) = dirsize.value { return Ok(()) } - if !dirsize.is_running() { + if !dirsize.is_running() && !dirsize.is_ready() { dirsize.run_pooled(Some(&*pool))?; } if dirsize.is_ready() { dirsize.pull_async()?; - *meta_updated = true; } Ok(()) } pub fn take_meta(&mut self, - pool: &ThreadPool, - meta_updated: &mut bool) -> HResult<()> { + pool: &ThreadPool) -> HResult<()> { if self.meta_processed { return Ok(()) } - if !self.meta.is_running() { + if !self.meta.is_running() && !self.meta.is_ready() { self.meta.run_pooled(Some(&*pool))?; } if self.meta.is_ready() { self.meta.pull_async()?; self.process_meta()?; - *meta_updated = true; } Ok(()) @@ -857,10 +989,11 @@ impl File { mime } else { // Fix crash in tree_magic when called on non-regular file + // Also fix crash when a file doesn't exist any more self.meta() .ok() .and_then(|meta| { - if meta.is_file() { + if meta.is_file() && self.path.exists() { let mime = tree_magic::from_filepath(&self.path); mime::Mime::from_str(&mime).ok() } else { None } diff --git a/src/fscache.rs b/src/fscache.rs index 8fb991e..7f4ce24 100644 --- a/src/fscache.rs +++ b/src/fscache.rs @@ -2,7 +2,7 @@ use notify::{RecommendedWatcher, Watcher, DebouncedEvent, RecursiveMode}; use async_value::{Async, Stale}; -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, RwLock, Weak}; use std::sync::mpsc::{channel, Sender, Receiver}; use std::collections::{HashMap, HashSet}; use std::time::Duration; @@ -12,6 +12,8 @@ use crate::files::{Files, File, SortBy}; use crate::widget::Events; use crate::fail::{HResult, HError, ErrorLog, Backtrace, ArcBacktrace}; +pub type CachedFiles = (Option, Async); + #[derive(Debug, Clone)] pub struct DirSettings { @@ -64,6 +66,63 @@ impl std::fmt::Debug for FsCache { } } +#[derive(Clone)] +struct FsEventDispatcher { + targets: Arc>>>>>> +} + +impl FsEventDispatcher { + fn new() -> Self { + FsEventDispatcher { + targets: Arc::new(RwLock::new(HashMap::new())) + } + } + + fn add_target(&self, + dir: &File, + target: &Arc>>) -> HResult<()> { + let target = Arc::downgrade(target); + + self.targets + .write() + .map(|mut targets| { + match targets.get_mut(dir) { + Some(targets) => targets.push(target), + None => { targets.insert(dir.clone(), vec![target]); } + } + })?; + Ok(()) + } + + fn remove_target(&self, dir: &File) -> HResult<()> { + self.targets + .write()? + .get_mut(dir) + .map(|targets| { + targets.retain(|t| t.upgrade().is_some()); + }); + Ok(()) + } + + fn dispatch(&self, events: HashMap>) -> HResult<()> { + for (dir, events) in events { + for target_dirs in self.targets + .read()? + .get(&dir) { + for target in target_dirs { + if let Some(target) = target.upgrade() { + let events = events.clone(); + + target.write()?.extend(events) + } + } + } + } + Ok(()) + } + + // fn remove_unnecessary +} #[derive(Clone)] pub struct FsCache { @@ -71,14 +130,14 @@ pub struct FsCache { pub tab_settings: Arc>>, watched_dirs: Arc>>, watcher: Arc>, - pub fs_changes: Arc, Option)>>>, + fs_event_dispatcher: FsEventDispatcher } impl FsCache { pub fn new(sender: Sender) -> FsCache { let (tx_fs_event, rx_fs_event) = channel(); let watcher = RecommendedWatcher::new(tx_fs_event, - Duration::from_secs(2)).unwrap(); + Duration::from_secs(2)).unwrap(); let fs_cache = FsCache { @@ -86,12 +145,11 @@ impl FsCache { tab_settings: Arc::new(RwLock::new(HashMap::new())), watched_dirs: Arc::new(RwLock::new(HashSet::new())), watcher: Arc::new(RwLock::new(watcher)), - fs_changes: Arc::new(RwLock::new(vec![])), + fs_event_dispatcher: FsEventDispatcher::new() }; watch_fs(rx_fs_event, - fs_cache.files.clone(), - fs_cache.fs_changes.clone(), + fs_cache.fs_event_dispatcher.clone(), sender); fs_cache @@ -104,8 +162,6 @@ impl FsCache { } } -pub type CachedFiles = (Option, Async); - impl FsCache { pub fn get_files(&self, dir: &File, stale: Stale) -> HResult { if self.files.read()?.contains_key(dir) { @@ -117,6 +173,8 @@ impl FsCache { let files = Async::new(move |_| { let mut files = Files::new_from_path_cancellable(&dir.path, stale)?; cache.add_watch(&dir).log(); + cache.fs_event_dispatcher.add_target(&dir, + &files.pending_events).log(); FsCache::apply_settingss(&cache, &mut files).ok(); Ok(files) }); @@ -129,7 +187,6 @@ impl FsCache { let mut files = files.run_sync()?; FsCache::apply_settingss(&self, &mut files).ok(); let files = FsCache::ensure_not_empty(files)?; - self.add_watch(&dir).log(); Ok(files) } @@ -144,28 +201,6 @@ impl FsCache { Ok(()) } - pub fn put_files(&self, files: &Files, selection: Option) -> HResult<()> { - let dir = files.directory.clone(); - - let tab_settings = FsCache::extract_tab_settings(&files, selection); - - self.tab_settings.write()?.insert(dir.clone(), tab_settings); - - // let mut file_cache = self.files.write()?; - - // if file_cache.contains_key(&files.directory) { - // if files.meta_updated { - // let mut files = files.clone(); - // files.meta_updated = false; - // file_cache.insert(dir, files); - // } - // } else { - // file_cache.insert(dir, files.clone()); - // } - - Ok(()) - } - pub fn is_cached(&self, dir: &File) -> HResult { Ok(self.files.read()?.contains_key(dir)) } @@ -298,65 +333,134 @@ impl FsCache { } +#[derive(PartialEq, Eq, Hash, Clone, Debug)] +pub enum FsEvent { + Create(File), + Change(File), + Rename(File, File), + Remove(File) +} + +impl FsEvent { + pub fn file(&self) -> &File { + use FsEvent::*; + match self { + Create(event_file) | + Change(event_file) | + Remove(event_file) | + Rename(_, event_file) => &event_file + } + } + + pub fn for_file(&self, file: &File) -> bool { + use FsEvent::*; + match self { + Create(event_file) | + Change(event_file) | + Remove(event_file) | + Rename(_, event_file) => event_file.path == file.path + } + } +} + +use std::convert::TryFrom; +impl TryFrom for FsEvent { + type Error = HError; + + fn try_from(event: DebouncedEvent) -> HResult { + let event = match event { + DebouncedEvent::Create(path) + => FsEvent::Create(File::new_from_path(&path, None)?), + + DebouncedEvent::Remove(path) + => FsEvent::Remove(File::new_from_path(&path, None)?), + + DebouncedEvent::Write(path) | + DebouncedEvent::Chmod(path) + => FsEvent::Change(File::new_from_path(&path, None)?), + + DebouncedEvent::Rename(old_path, new_path) + => FsEvent::Rename(File::new_from_path(&old_path, None)?, + File::new_from_path(&new_path, None)?), + + DebouncedEvent::Error(err, path) + => Err(HError::INotifyError(format!("{}, {:?}", err, path), + Backtrace::new_arced()))?, + DebouncedEvent::Rescan + => Err(HError::INotifyError("Need to rescan".to_string(), + Backtrace::new_arced()))?, + // Ignore NoticeRemove/NoticeWrite + _ => None?, + }; + + Ok(event) + } +} + + fn watch_fs(rx_fs_events: Receiver, - fs_cache: Arc>>, - fs_changes: Arc, Option)>>>, + fs_event_dispatcher: FsEventDispatcher, sender: Sender) { std::thread::spawn(move || -> HResult<()> { - for event in rx_fs_events.iter() { - apply_event(&fs_cache, &fs_changes, event).log(); + let transform_event = + move |event: DebouncedEvent| -> HResult<(File, FsEvent)> { + let path = event.get_source_path()?; + let dirpath = path.parent() + .map(|path| path) + .unwrap_or(std::path::Path::new("/")); + let dir = File::new_from_path(&dirpath, None)?; + let event = FsEvent::try_from(event)?; + Ok((dir, event)) + }; + + let collect_events = + move || -> HResult>> { + let event = loop { + use DebouncedEvent::*; + + let event = rx_fs_events.recv()?; + match event { + NoticeWrite(_) => continue, + NoticeRemove(_) => continue, + _ => break std::iter::once(event) + } + }; + + // Wait a bit to batch up more events + std::thread::sleep(std::time::Duration::from_millis(100)); - sender.send(Events::WidgetReady).ok(); + // Batch up all other remaining events received so far + let events = event.chain(rx_fs_events.try_iter()) + .map(transform_event) + .flatten() + .fold(HashMap::with_capacity(1000), |mut events, (dir, event)| { + events.entry(dir) + .or_insert(vec![]) + .push(event); + + events + }); + + Ok(events) + }; + + + let dispatch_events = + move |events| -> HResult<()> { + fs_event_dispatcher.dispatch(events)?; + sender.send(Events::WidgetReady)?; + Ok(()) + }; + + loop { + if let Ok(events) = collect_events().log_and() { + dispatch_events(events).log(); + } } - Ok(()) }); } -fn apply_event(_fs_cache: &Arc>>, - fs_changes: &Arc, Option)>>>, - event: DebouncedEvent) - -> HResult<()> { - let path = &event.get_source_path()?; - - let dirpath = path.parent() - .map(|path| path.to_path_buf()) - .unwrap_or_else(|| PathBuf::from("/")); - let dir = File::new_from_path(&dirpath, None)?; - - let old_file = File::new_from_path(&path, None)?; - let mut new_file = match event { - DebouncedEvent::Remove(_) => None, - _ => Some(File::new_from_path(&path, None)?) - }; - - new_file.as_mut().map(|file| file.meta_sync()); - - fs_changes.write()?.push((dir, - Some(old_file), - new_file)); - - // for dir in fs_cache.write()?.values_mut() { - // if dir.path_in_here(&path).unwrap_or(false) { - // let old_file = dir.find_file_with_path(&path).cloned(); - // let dirty_meta = old_file - // .as_ref() - // .map(|f| f.dirty_meta.clone()) - // .unwrap_or(None); - // let mut new_file = match event { - // DebouncedEvent::Remove(_) => None, - // _ => Some(File::new_from_path(&path, dirty_meta)?) - // }; - - // new_file.as_mut().map(|file| file.meta_sync()); - // dir.replace_file(old_file.as_ref(), new_file.clone()).log(); - - // fs_changes.write()?.push((dir.directory.clone(), - // old_file, - // new_file)); - // } - // } - Ok(()) -} + trait PathFromEvent { fn get_source_path(&self) -> HResult<&PathBuf>; diff --git a/src/listview.rs b/src/listview.rs index f44a705..c744f4f 100644 --- a/src/listview.rs +++ b/src/listview.rs @@ -97,6 +97,8 @@ impl Listable for ListView { self.content.meta_upto(visible_files, Some(sender.clone())); + self.refresh_files().log(); + if self.content.is_dirty() { self.content.set_clean(); self.core.set_dirty(); @@ -239,7 +241,7 @@ impl ListView // Work around annoying restriction until polonius borrow checker becomes default - // Since we only ever return one mutable borrow this is perfectly safe + // Since only ever one mutable borrow is returned this is perfectly safe // See also: https://github.com/rust-lang/rust/issues/21906 match file { Some(file) => unsafe { return file.as_mut().unwrap() }, @@ -581,77 +583,109 @@ impl ListView } fn render_line(&self, file: &File) -> String { - let icon = if self.core.config().icons { - file.icon() - } else { "" }; - - let name = String::from(icon) + &file.name; - let (size, unit) = file.calculate_size().unwrap_or((0, "".to_string())); - - - - let tag = match file.is_tagged() { - Ok(true) => term::color_red() + "*", - _ => "".to_string() - }; - let tag_len = if tag != "" { 1 } else { 0 }; - - let selection_gap = " ".to_string(); - let (name, selection_color) = if file.is_selected() { - (selection_gap + &name, crate::term::color_yellow()) - } else { (name.clone(), "".to_string()) }; - - let (link_indicator, link_indicator_len) = if file.target.is_some() { - (format!("{}{}{}", - term::color_yellow(), - "--> ".to_string(), - term::highlight_color()), - 4) - } else { ("".to_string(), 0) }; + let render_fn = self.render_line_fn(); + render_fn(file) + } + #[allow(trivial_bounds)] + fn render_line_fn(&self) -> impl Fn(&File) -> String { let xsize = self.get_coordinates().unwrap().xsize(); - let sized_string = term::sized_string(&name, xsize); - let size_pos = xsize - (size.to_string().len() as u16 - + unit.to_string().len() as u16 - + link_indicator_len); - let padding = sized_string.len() - sized_string.width_cjk(); - let padding = xsize - padding as u16; - let padding = padding - tag_len; - - format!( - "{}{}{}{}{}{}{}{}", - termion::cursor::Save, - match &file.color { - Some(color) => format!("{}{}{}{:padding$}{}", - tag, - term::from_lscolor(color), - selection_color, - &sized_string, - term::normal_color(), - padding = padding as usize), - _ => format!("{}{}{}{:padding$}{}", - tag, - term::normal_color(), - selection_color, - &sized_string, - term::normal_color(), - padding = padding as usize), - } , - termion::cursor::Restore, - termion::cursor::Right(size_pos), - link_indicator, - term::highlight_color(), - size, - unit - ) + let icons = self.core.config().icons; + + move |file| -> String { + let icon = if icons { + file.icon() + } else { "" }; + + let name = String::from(icon) + &file.name; + let (size, unit) = file.calculate_size().unwrap_or((0, "".to_string())); + + + + let tag = match file.is_tagged() { + Ok(true) => term::color_red() + "*", + _ => "".to_string() + }; + let tag_len = if tag != "" { 1 } else { 0 }; + + let selection_gap = " ".to_string(); + let (name, selection_color) = if file.is_selected() { + (selection_gap + &name, crate::term::color_yellow()) + } else { (name.clone(), "".to_string()) }; + + let (link_indicator, link_indicator_len) = if file.target.is_some() { + (format!("{}{}{}", + term::color_yellow(), + "--> ".to_string(), + term::highlight_color()), + 4) + } else { ("".to_string(), 0) }; + + + let sized_string = term::sized_string(&name, xsize); + let size_pos = xsize - (size.to_string().len() as u16 + + unit.to_string().len() as u16 + + link_indicator_len); + let padding = sized_string.len() - sized_string.width_cjk(); + let padding = xsize - padding as u16; + let padding = padding - tag_len; + + format!( + "{}{}{}{}{}{}{}{}", + termion::cursor::Save, + match &file.color { + Some(color) => format!("{}{}{}{:padding$}{}", + tag, + term::from_lscolor(color), + selection_color, + &sized_string, + term::normal_color(), + padding = padding as usize), + _ => format!("{}{}{}{:padding$}{}", + tag, + term::normal_color(), + selection_color, + &sized_string, + term::normal_color(), + padding = padding as usize), + } , + termion::cursor::Restore, + termion::cursor::Right(size_pos), + link_indicator, + term::highlight_color(), + size, + unit + ) + } } + fn render(&self) -> Vec { self.content .iter_files() .map(|file| self.render_line(file)) .collect() } + + fn refresh_files(&mut self) -> HResult<()> { + if let Ok(Some(mut refresh)) = self.content.get_refresh() { + let file = self.clone_selected_file(); + + self.buffer = refresh.new_buffer.take()?; + self.lines = self.buffer.len() - 1; + + self.select_file(&file); + } + + if self.content.ready_to_refresh()? { + let render_fn = self.render_line_fn(); + self.content.process_fs_events(self.buffer.clone(), + self.core.get_sender(), + render_fn)?; + } + + Ok(()) + } } @@ -664,13 +698,16 @@ impl Widget for ListView where ListView: Listable { } fn refresh(&mut self) -> HResult<()> { self.on_refresh().log(); - self.lines = self.len(); - if self.selection >= self.len() && self.selection != 0 { - self.selection = self.len() - 1; + let buffer_len = self.buffer.len(); + + self.lines = buffer_len; + + if self.selection >= self.buffer.len() && self.buffer.len() != 0 { + self.selection = self.buffer.len() - 1; } - if self.core.is_dirty() || self.buffer.len() != self.len() { + if self.core.is_dirty() || buffer_len != self.len() { self.buffer = self.render(); self.core.set_clean(); } diff --git a/src/main.rs b/src/main.rs index 24d23f7..a70e0c1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -27,6 +27,8 @@ extern crate clap; extern crate strum; #[macro_use] extern crate strum_macros; +#[macro_use] +extern crate derivative; extern crate osstrtools; extern crate pathbuftools; diff --git a/src/preview.rs b/src/preview.rs index ef1fa48..582dcb9 100644 --- a/src/preview.rs +++ b/src/preview.rs @@ -324,23 +324,6 @@ impl Previewer { } } - pub fn replace_file(&mut self, dir: &File, - old: Option<&File>, - new: Option<&File>) -> HResult<()> { - if self.file.as_ref() != Some(dir) { return Ok(()) } - self.widget.widget_mut().map(|widget| { - match widget { - PreviewWidget::FileList(filelist) => { - filelist.content.replace_file(old, new.cloned()).map(|_| { - filelist.refresh().ok(); - }).ok(); - - } - _ => {} - } - }) - } - pub fn put_preview_files(&mut self, files: Files) { let core = self.core.clone(); let dir = files.directory.clone(); -- cgit v1.2.3