fixed hangs when updating large directories (>10k files)

8 files changed, 555 insertions, 321 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 1b63584..b862004 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -259,6 +259,16 @@ dependencies = [
 ]
 
 [[package]]
+name = "derivative"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
 name = "dirs"
 version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -410,7 +420,7 @@ dependencies = [
  "glib 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "muldiv 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -427,21 +437,21 @@ dependencies = [
  "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-app-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-app-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "gstreamer-app-sys"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -456,19 +466,19 @@ dependencies = [
  "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "gstreamer-base-sys"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -484,7 +494,7 @@ dependencies = [
  "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer-player-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer-video 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -496,15 +506,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "gstreamer-sys"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -524,22 +534,22 @@ dependencies = [
  "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "gstreamer-video-sys"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "glib-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -554,7 +564,7 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.1.3"
+version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -569,6 +579,7 @@ dependencies = [
  "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "dirs-2 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -908,7 +919,7 @@ name = "num_cpus"
 version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hermit-abi 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1562,6 +1573,7 @@ dependencies = [
 "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
 "checksum crossbeam-utils 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce446db02cdc3165b94ae73111e570793400d0794e46125cc4056c81cbb039f4"
 "checksum deflate 0.7.20 (registry+https://github.com/rust-lang/crates.io-index)" = "707b6a7b384888a70c8d2e8650b3e60170dfc6a67bb4aa67b6dfca57af4bedb4"
+"checksum derivative 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "942ca430eef7a3806595a6737bc388bf51adb888d3fc0dd1b50f1c170167ee3a"
 "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901"
 "checksum dirs-2 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50b7e2b65c73137ec48935d50a5ae89b03150df566b7e14a1371df044e76765c"
 "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
@@ -1581,16 +1593,16 @@ dependencies = [
 "checksum gobject-sys 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31d1a804f62034eccf370006ccaef3708a71c31d561fee88564abe71177553d9"
 "checksum gstreamer 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa91e470b0cd4b05611f7d0e89caf76e39752156440877f04c23ad34ffc9761c"
 "checksum gstreamer-app 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a85485c2db4149ccb24d0b3c6598725743dec254bf757ac7a3684e62b9822c27"
-"checksum gstreamer-app-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "41c85ef44d827b9292833203f6623cf6592d5eda06ad1eeefa63bca0cc38ce71"
+"checksum gstreamer-app-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bf869ce152c23bca5d761ab62146b47f750d0b28d4d499731857532897d48167"
 "checksum gstreamer-base 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)" = "37ed3ddb8f1a65e401340afb5657d2107d78c82f06a081e19393fc8d4c5ee720"
-"checksum gstreamer-base-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9ba1955ea091323c17fdf8ff54fd7cf3dfed1a6035193ba08f85eb76bf549056"
+"checksum gstreamer-base-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ba384f52174b3c586593fca32642680a9e67961fea9f4cd8419f678965023bed"
 "checksum gstreamer-player 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)" = "78395f87de2b954ca3e33a594a4eb85e68246f5f41a70bf0ab52187aacb4d3a9"
 "checksum gstreamer-player-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c3608d3e96c8977f31b9b8db7da0b8d0e96758b060e3f05fc3ee9626d75ab1c5"
-"checksum gstreamer-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bdfc2f6cc9b6a1f5159bfd500310fe431cfb0b74b3af17ce3fdf8353cf586975"
+"checksum gstreamer-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1d18da01b97d0ab5896acd5151e4c155acefd0e6c03c3dd24dd133ba054053db"
 "checksum gstreamer-video 0.14.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0b48c3edfa6c0419ad7587a557bbed716caf5585c3be65bfbe3584703735df51"
-"checksum gstreamer-video-sys 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6b8fcb1e577de93d1ad1e5117234ce64d40f215143d752140719923651608983"
+"checksum gstreamer-video-sys 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "615f4298f842f4b4581606e13cf52e1710e2130d989bb99161a5665aa3ccb7cc"
 "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
-"checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
+"checksum hermit-abi 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f629dc602392d3ec14bfc8a09b5e644d7ffd725102b48b81e59f90f2633621d7"
 "checksum image 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)" = "35371e467cd7b0b3d1d6013d619203658467df12d61b0ca43cd67b743b1965eb"
 "checksum inflate 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff"
 "checksum inotify 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "40b54539f3910d6f84fbf9a643efd6e3aa6e4f001426c0329576128255994718"
diff --git a/Cargo.toml b/Cargo.toml
index dc853cb..fe0898e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,7 +27,7 @@ chrono = "0.4"
 libc = "0.2.51"
 failure = "0.1.5"
 failure_derive = "0.1.1"
-notify = "4.0.12"
+notify = "4.0.14"
 parse-ansi = "0.1.6"
 signal-notify = "0.1.3"
 systemstat = "0.1.5"
@@ -41,6 +41,7 @@ base64 = "0.10.1"
 strum = "0.15"
 strum_macros = "0.15"
 rust-ini = "0.13"
+derivative = "1.0.3"
 
 
 image = { version = "0.21.1", optional = true }
diff --git a/src/file_browser.rs b/src/file_browser.rs
index 257ca32..43394ac 100644
--- a/src/file_browser.rs
+++ b/src/file_browser.rs
@@ -168,21 +168,6 @@ impl Tabbable for TabView<FileBrowser> {
     }
 
     fn on_refresh(&mut self) -> HResult<()> {
-        let fs_changes = self.active_tab_()
-            .fs_cache
-            .fs_changes
-            .write()?
-            .drain(..)
-            .collect::<Vec<_>>();
-
-        for tab in &mut self.widgets {
-            for (dir, old_file, new_file) in fs_changes.iter() {
-                tab.replace_file(&dir,
-                                 old_file.as_ref(),
-                                 new_file.as_ref()).log()
-            }
-        }
-
         let open_dirs = self.widgets
             .iter()
             .fold(HashSet::new(), |mut dirs, tab| {
@@ -503,7 +488,7 @@ impl FileBrowser {
     }
 
     pub fn main_widget_goto(&mut self, dir: &File) -> HResult<()> {
-        self.cache_files().log();
+        self.save_tab_settings().log();
 
         let dir = dir.clone();
         let cache = self.fs_cache.clone();
@@ -733,22 +718,15 @@ impl FileBrowser {
         Ok(&self.left_widget()?.content)
     }
 
-    pub fn cache_files(&mut self) -> HResult<()> {
-        if self.main_widget().is_ok() {
+    pub fn save_tab_settings(&mut self) -> HResult<()> {
+        if !self.main_async_widget_mut()?.ready() { return Ok(()) }
+
+        if self.main_widget()?.content.len() > 0 {
             let files = self.get_files()?;
             let selected_file = self.selected_file().ok();
-            self.fs_cache.put_files(files, selected_file).log();
-            self.main_widget_mut()?.content.meta_updated = false;
+            self.fs_cache.save_settings(files, selected_file).log();
         }
 
-
-        // if self.cwd.parent().is_some() {
-        //     let left_selection = self.left_widget()?.clone_selected_file();
-        //     let left_files = self.get_left_files()?;
-        //     self.fs_cache.put_files(left_files, Some(left_selection)).log();
-        //     self.left_widget_mut()?.content.meta_updated = false;
-        // }
-
         Ok(())
     }
 
@@ -769,22 +747,6 @@ impl FileBrowser {
         Ok(dir)
     }
 
-    fn replace_file(&mut self,
-                    dir: &File,
-                    old: Option<&File>,
-                    new: Option<&File>) -> HResult<()> {
-        if &self.cwd == dir {
-            self.main_widget_mut()?.content.replace_file(old, new.cloned()).log();
-        }
-
-        self.preview_widget_mut()?.replace_file(dir, old, new).ok();
-
-        if &self.left_dir()? == &dir {
-            self.left_widget_mut()?.content.replace_file(old, new.cloned()).log();
-        }
-        Ok(())
-    }
-
     pub fn selected_file(&self) -> HResult<File> {
         let widget = self.main_widget()?;
         let file = widget.selected_file().clone();
@@ -1288,7 +1250,7 @@ impl Widget for FileBrowser {
         self.set_cwd().log();
         if !self.columns.zoom_active { self.update_preview().log(); }
         self.columns.refresh().log();
-        self.cache_files().log();
+        self.save_tab_settings().log();
         Ok(())
     }
 
diff --git a/src/files.rs b/src/files.rs
index d8558af..bafffc9 100644
--- a/src/files.rs
+++ b/src/files.rs
@@ -1,4 +1,5 @@
 use std::cmp::{Ord, Ordering};
+use std::collections::{HashMap, HashSet};
 use std::ops::Index;
 use std::fs::Metadata;
 use std::os::unix::fs::MetadataExt;
@@ -16,7 +17,6 @@ use users::{get_current_username,
             get_group_by_gid};
 use chrono::TimeZone;
 use failure::Error;
-use notify::DebouncedEvent;
 use rayon::{ThreadPool, ThreadPoolBuilder};
 use alphanumeric_sort::compare_str;
 use mime_guess;
@@ -28,6 +28,7 @@ use crate::fail::{HResult, HError, ErrorLog};
 use crate::dirty::{AsyncDirtyBit, DirtyBit, Dirtyable};
 use crate::widget::Events;
 use crate::icon::Icons;
+use crate::fscache::FsEvent;
 
 
 lazy_static! {
@@ -99,11 +100,147 @@ pub fn tags_loaded() -> HResult<()> {
 
 
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]
+pub struct RefreshPackage {
+    pub new_files: Option<Vec<File>>,
+    pub new_buffer: Option<Vec<String>>,
+}
+
+
+
+
+impl RefreshPackage {
+    fn new(mut files: Files,
+           old_buffer: Vec<String>,
+           events: Vec<FsEvent>,
+           render_fn: impl Fn(&File) -> String) -> RefreshPackage {
+        use FsEvent::*;
+
+        // If there is only a placeholder at this point, remove it now
+        if files.len() == 1 {
+            files.remove_placeholder();
+        }
+
+        //To preallocate collections
+        let event_count = events.len();
+
+        // Need at least one copy for the hashmaps
+        let static_files = files.clone();
+
+        // Optimization to speed up access to array
+        let file_pos_map: HashMap<&File, usize> = static_files
+            .files
+            .iter()
+            .enumerate()
+            .map(|(i, file)| (file, i))
+            .collect();
+
+
+        // Need to know which line of the ListView buffer belongs to which file
+        let list_pos_map: HashMap<&File, usize> = static_files
+            .iter_files()
+            .enumerate()
+            .take_while(|&(i, _)| i < old_buffer.len())
+            .map(|(i, file)| (file, i))
+            .collect();
+
+        // Save new files to add them all at once later
+        let mut new_files = Vec::with_capacity(event_count);
+
+        // Files that need rerendering to make all changes visible (size, etc.)
+        let mut changed_files = HashSet::with_capacity(event_count);
+
+        // Save deletions to delete them efficiently later
+        let mut deleted_files = HashSet::with_capacity(event_count);
+
+        for event in events.into_iter() {
+            match event {
+                Create(mut file) => {
+                    let dirty_meta = files.dirty_meta.clone();
+                    file.dirty_meta = Some(dirty_meta);
+                    file.meta_sync().log();
+                    new_files.push(file);
+                }
+                Change(file) => {
+                    if let Some(&fpos) = file_pos_map.get(&file) {
+                        files.files[fpos].meta_sync().log();
+                        changed_files.insert(file);
+                    }
+                }
+                Rename(old, new) => {
+                    if let Some(&fpos) = file_pos_map.get(&old) {
+                        files.files[fpos].rename(&new.path).log();
+                        files.files[fpos].meta_sync().log();
+                    }
+                }
+                Remove(file) => {
+                    if let Some(_) = file_pos_map.get(&file) {
+                        deleted_files.insert(file);
+                    }
+                }
+            }
+        }
+
+        if deleted_files.len() > 0 {
+            files.files.retain(|file| !deleted_files.contains(file));
+        }
+
+        // Finally add all new files
+        files.files.extend(new_files);
+
+        // Need to unpack this to prevent issue with recursive Files type
+        // Also, if no files remain add placeholder
+        let files = if files.len() > 0 {
+            // Sort again to make sure new/changed files are in correct order
+            files.sort();
+            files.files
+        } else {
+            let placeholder = File::new_placeholder(&files.directory.path).unwrap();
+            files.files.push(placeholder);
+
+            // Need to sort because of possible hidden files
+            files.sort();
+            files.files
+        };
+
+        let mut old_buffer = old_buffer;
+
+        // Prerender new buffer in current thread
+        let new_buffer = files.iter()
+            .map(|file| {
+                match list_pos_map.get(&file) {
+                    Some(&old_pos) =>
+                        match changed_files.contains(&file) {
+                            true => render_fn(&file),
+                            false => std::mem::take(&mut old_buffer[old_pos])
+                        }
+                    None => render_fn(&file)
+                }
+            }).collect();
+
+
+
+        RefreshPackage {
+            new_files: Some(files),
+            new_buffer: Some(new_buffer),
+        }
+    }
+}
+
+
+#[derive(Derivative)]
+#[derivative(PartialEq, Eq, Hash, Clone, Debug)]
 pub struct Files {
     pub directory: File,
     pub files: Vec<File>,
+    #[derivative(Debug="ignore")]
+    #[derivative(PartialEq="ignore")]
+    #[derivative(Hash="ignore")]
+    pub pending_events: Arc<RwLock<Vec<FsEvent>>>,
+    #[derivative(Debug="ignore")]
+    #[derivative(PartialEq="ignore")]
+    #[derivative(Hash="ignore")]
+    pub refresh: Option<Async<RefreshPackage>>,
     pub meta_upto: Option<usize>,
-    pub meta_updated: bool,
     pub sort: SortBy,
     pub dirs_first: bool,
     pub reverse: bool,
@@ -158,8 +295,9 @@ impl Files {
         let mut files = Files {
             directory: File::new_from_path(&path, None)?,
             files: files,
+            pending_events: Arc::new(RwLock::new(vec![])),
+            refresh: None,
             meta_upto: None,
-            meta_updated: false,
             sort: SortBy::Name,
             dirs_first: true,
             reverse: false,
@@ -216,8 +354,9 @@ impl Files {
         let mut files = Files {
             directory: File::new_from_path(&path, None)?,
             files: files,
+            pending_events: Arc::new(RwLock::new(vec![])),
+            refresh: None,
             meta_upto: None,
-            meta_updated: false,
             sort: SortBy::Name,
             dirs_first: true,
             reverse: false,
@@ -375,77 +514,69 @@ impl Files {
         }
     }
 
-    pub fn replace_file(&mut self,
-                        old: Option<&File>,
-                        new: Option<File>) -> HResult<()> {
-        let (tag, selected) = if let Some(old) = old {
-            if let Some(old) = self.find_file_with_path(&old.path) {
-                (old.tag, old.selected)
-            } else {
-                (None, false)
-            }
-        } else {
-            (None, false)
-        };
-        old.map(|old| self.files.remove_item(old));
-        new.map(|mut new| {
-            new.tag = tag;
-            new.selected = selected;
-            self.files.push(new);
-        });
+    fn remove_placeholder(&mut self) {
+        let dirpath = self.directory.path.clone();
+        self.find_file_with_path(&dirpath).cloned()
+            .map(|placeholder| self.files.remove_item(&placeholder));
+    }
 
-        self.sort();
+    pub fn ready_to_refresh(&self) -> HResult<bool> {
+        let pending = self.pending_events.read()?.len();
+        let running = self.refresh.is_some();
+        Ok(pending > 0 && !running)
+    }
 
-        if self.len() == 0 {
-            let placeholder = File::new_placeholder(&self.directory.path)?;
-            self.files.push(placeholder);
-        } else {
-            self.remove_placeholder();
+    pub fn get_refresh(&mut self) -> HResult<Option<RefreshPackage>> {
+        if let Some(mut refresh) = self.refresh.take() {
+            if refresh.is_ready() {
+                refresh.pull_async()?;
+                let mut refresh = refresh.value?;
+                self.files = refresh.new_files.take()?;
+                return Ok(Some(refresh));
+            } else {
+                self.refresh.replace(refresh);
+            }
         }
 
-        Ok(())
+        return Ok(None)
     }
 
-    fn remove_placeholder(&mut self) {
-        let dirpath = self.directory.path.clone();
-        self.find_file_with_path(&dirpath).cloned()
-            .map(|placeholder| self.files.remove_item(&placeholder));
-    }
+    pub fn process_fs_events(&mut self,
+                             buffer: Vec<String>,
+                             sender: Sender<Events>,
+                             render_fn: impl Fn(&File) -> String + Send + 'static)
+                             -> HResult<()> {
+        let pending = self.pending_events.read()?.len();
+        if pending > 0 {
+            let pending = if pending >= 1000 {
+                1000
+            } else {
+                pending
+            };
+
+            let events = self.pending_events
+                .write()?
+                .drain(0..pending)
+                .collect::<Vec<_>>();
+            let files = self.clone();
+
+            let mut refresh = Async::new(move |_| {
+                let refresh = RefreshPackage::new(files,
+                                                  buffer,
+                                                  events,
+                                                  render_fn);
+                Ok(refresh)
+            });
+
+            refresh.on_ready(move |_,_| {
+                Ok(sender.send(Events::WidgetReady)?)
+            })?;
+
+            refresh.run()?;
 
-    pub fn handle_event(&mut self,
-                        event: &DebouncedEvent) -> HResult<()> {
-        match event {
-            DebouncedEvent::Create(path) => {
-                self.path_in_here(&path)?;
-                let file = File::new_from_path(&path,
-                                               Some(self.dirty_meta.clone()))?;
-                self.files.push(file);
-                self.sort();
-            },
-            DebouncedEvent::Write(path) | DebouncedEvent::Chmod(path) => {
-                self.path_in_here(&path)?;
-                let file = self.find_file_with_path(&path)?;
-                file.reload_meta()?;
-            },
-            DebouncedEvent::Remove(path) => {
-                self.path_in_here(&path)?;
-                let file = self.find_file_with_path(&path)?.clone();
-                self.files.remove_item(&file);
-            },
-            DebouncedEvent::Rename(old_path, new_path) => {
-                self.path_in_here(&new_path)?;
-                let mut file = self.find_file_with_path(&old_path)?;
-                file.name = new_path.file_name()?.to_string_lossy().to_string();
-                file.path = new_path.into();
-                file.reload_meta()?;
-            },
-            DebouncedEvent::Error(err, _path) => {
-                // Never seen this happen. Should reload affected dirs
-                HError::log::<()>(&format!("{}", err))?;
-            },
-            _ => {},
+            self.refresh = Some(refresh);
         }
-        self.set_dirty();
+
         Ok(())
     }
 
@@ -465,17 +596,16 @@ impl Files {
     }
 
     pub fn find_file_with_path(&mut self, path: &Path) -> Option<&mut File> {
-        self.files.iter_mut().find(|file| file.path == path)
+        self.iter_files_mut().find(|file| file.path == path)
     }
 
     pub fn meta_all_sync(&mut self) -> HResult<()> {
-        for file in self.files.iter_mut() {
+        for file in self.iter_files_mut() {
             if !file.meta_processed {
                 file.meta_sync().log();
             }
         }
         self.set_dirty();
-        self.meta_updated = true;
         Ok(())
     }
 
@@ -508,10 +638,10 @@ impl Files {
             .filter(|f| !(!show_hidden && f.name.starts_with(".")))
             .take(meta_files) {
             if !file.meta_processed {
-                file.take_meta(&meta_pool, &mut self.meta_updated).ok();
+                file.take_meta(&meta_pool).ok();
             }
             if file.is_dir() {
-                file.take_dirsize(&meta_pool, &mut self.meta_updated).ok();
+                file.take_dirsize(&meta_pool).ok();
             }
         }
 
@@ -519,7 +649,7 @@ impl Files {
     }
 
     pub fn meta_set_fresh(&self) -> HResult<()> {
-        self.files.get(0)?.meta.set_fresh()?;
+        self.iter_files().nth(0)?.meta.set_fresh()?;
         Ok(())
     }
 
@@ -697,6 +827,12 @@ impl File {
         Ok(file)
     }
 
+    pub fn rename(&mut self, new_path: &Path) -> HResult<()> {
+        self.name = new_path.file_name()?.to_string_lossy().to_string();
+        self.path = new_path.into();
+        Ok(())
+    }
+
     pub fn meta_sync(&mut self) -> HResult<()> {
         let stale = self.meta.get_stale();
         let meta = std::fs::metadata(&self.path)?;
@@ -754,36 +890,32 @@ impl File {
     }
 
     fn take_dirsize(&mut self,
-                    pool: &ThreadPool,
-                    meta_updated: &mut bool) -> HResult<()> {
+                    pool: &ThreadPool) -> HResult<()> {
         let dirsize = self.dirsize.as_mut()?;
         if let Ok(_) = dirsize.value { return Ok(()) }
 
-        if !dirsize.is_running() {
+        if !dirsize.is_running() && !dirsize.is_ready() {
             dirsize.run_pooled(Some(&*pool))?;
         }
 
         if dirsize.is_ready() {
             dirsize.pull_async()?;
-            *meta_updated = true;
         }
 
         Ok(())
     }
 
     pub fn take_meta(&mut self,
-                     pool: &ThreadPool,
-                     meta_updated: &mut bool) -> HResult<()> {
+                     pool: &ThreadPool) -> HResult<()> {
         if self.meta_processed { return Ok(()) }
 
-        if !self.meta.is_running() {
+        if !self.meta.is_running() && !self.meta.is_ready() {
             self.meta.run_pooled(Some(&*pool))?;
         }
 
         if self.meta.is_ready() {
             self.meta.pull_async()?;
             self.process_meta()?;
-            *meta_updated = true;
         }
 
         Ok(())
@@ -857,10 +989,11 @@ impl File {
             mime
         } else {
             // Fix crash in tree_magic when called on non-regular file
+            // Also fix crash when a file doesn't exist any more
             self.meta()
                 .ok()
                 .and_then(|meta| {
-                    if meta.is_file() {
+                    if meta.is_file() && self.path.exists() {
                         let mime = tree_magic::from_filepath(&self.path);
                         mime::Mime::from_str(&mime).ok()
                     } else { None }
diff --git a/src/fscache.rs b/src/fscache.rs
index 8fb991e..7f4ce24 100644
--- a/src/fscache.rs
+++ b/src/fscache.rs
@@ -2,7 +2,7 @@ use notify::{RecommendedWatcher, Watcher, DebouncedEvent, RecursiveMode};
 
 use async_value::{Async, Stale};
 
-use std::sync::{Arc, RwLock};
+use std::sync::{Arc, RwLock, Weak};
 use std::sync::mpsc::{channel, Sender, Receiver};
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
@@ -12,6 +12,8 @@ use crate::files::{Files, File, SortBy};
 use crate::widget::Events;
 use crate::fail::{HResult, HError, ErrorLog, Backtrace, ArcBacktrace};
 
+pub type CachedFiles = (Option<File>, Async<Files>);
+
 
 #[derive(Debug, Clone)]
 pub struct DirSettings {
@@ -64,6 +66,63 @@ impl std::fmt::Debug for FsCache {
     }
 }
 
+#[derive(Clone)]
+struct FsEventDispatcher {
+    targets: Arc<RwLock<HashMap<File, Vec<Weak<RwLock<Vec<FsEvent>>>>>>>
+}
+
+impl FsEventDispatcher {
+    fn new() -> Self {
+        FsEventDispatcher {
+            targets: Arc::new(RwLock::new(HashMap::new()))
+        }
+    }
+
+    fn add_target(&self,
+                  dir: &File,
+                  target: &Arc<RwLock<Vec<FsEvent>>>) -> HResult<()> {
+        let target = Arc::downgrade(target);
+
+        self.targets
+            .write()
+            .map(|mut targets| {
+                match targets.get_mut(dir) {
+                    Some(targets) => targets.push(target),
+                    None => { targets.insert(dir.clone(), vec![target]); }
+                }
+            })?;
+        Ok(())
+    }
+
+    fn remove_target(&self, dir: &File) -> HResult<()> {
+        self.targets
+            .write()?
+            .get_mut(dir)
+            .map(|targets| {
+                targets.retain(|t| t.upgrade().is_some());
+            });
+        Ok(())
+    }
+
+    fn dispatch(&self, events: HashMap<File, Vec<FsEvent>>) -> HResult<()> {
+        for (dir, events) in events {
+            for target_dirs in self.targets
+                .read()?
+                .get(&dir) {
+                    for target in target_dirs {
+                        if let Some(target) = target.upgrade() {
+                            let events = events.clone();
+
+                            target.write()?.extend(events)
+                        }
+                    }
+                }
+        }
+        Ok(())
+    }
+
+    // fn remove_unnecessary
+}
 
 #[derive(Clone)]
 pub struct FsCache {
@@ -71,14 +130,14 @@ pub struct FsCache {
     pub tab_settings: Arc<RwLock<HashMap<File, TabSettings>>>,
     watched_dirs: Arc<RwLock<HashSet<File>>>,
     watcher: Arc<RwLock<RecommendedWatcher>>,
-    pub fs_changes: Arc<RwLock<Vec<(File, Option<File>, Option<File>)>>>,
+    fs_event_dispatcher: FsEventDispatcher
 }
 
 impl FsCache {
     pub fn new(sender: Sender<Events>) -> FsCache {
         let (tx_fs_event, rx_fs_event) = channel();
         let watcher = RecommendedWatcher::new(tx_fs_event,
-                                          Duration::from_secs(2)).unwrap();
+                                              Duration::from_secs(2)).unwrap();
 
 
         let fs_cache = FsCache {
@@ -86,12 +145,11 @@ impl FsCache {
             tab_settings: Arc::new(RwLock::new(HashMap::new())),
             watched_dirs: Arc::new(RwLock::new(HashSet::new())),
             watcher: Arc::new(RwLock::new(watcher)),
-            fs_changes: Arc::new(RwLock::new(vec![])),
+            fs_event_dispatcher: FsEventDispatcher::new()
         };
 
         watch_fs(rx_fs_event,
-                 fs_cache.files.clone(),
-                 fs_cache.fs_changes.clone(),
+                 fs_cache.fs_event_dispatcher.clone(),
                  sender);