use nix's lower level API to read in directories

4 files changed, 83 insertions, 47 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 6884e32..1dffb87 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -608,6 +608,7 @@ dependencies = [
  "lscolors 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "notify 4.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
  "osstrtools 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "parse-ansi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -854,6 +855,18 @@ dependencies = [
 ]
 
 [[package]]
+name = "nix"
+version = "0.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
 name = "nodrop"
 version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1495,6 +1508,11 @@ version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
+name = "void"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
 name = "walkdir"
 version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1649,6 +1667,7 @@ dependencies = [
 "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919"
 "checksum muldiv 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0419348c027fa7be448d2ae7ea0e4e04c2334c31dc4e74ab29f00a2a7ca69204"
 "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
+"checksum nix 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363"
 "checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
 "checksum nom 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf51a729ecf40266a2368ad335a5fdde43471f545a967109cd62146ecf8b66ff"
 "checksum nom 3.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b"
@@ -1728,6 +1747,7 @@ dependencies = [
 "checksum utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a1ca13c08c41c9c3e04224ed9ff80461d97e121589ff27c753a16cb10830ae0f"
 "checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
 "checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
+"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
 "checksum walkdir 2.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
 "checksum wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)" = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
diff --git a/Cargo.toml b/Cargo.toml
index e18845c..1318436 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -43,6 +43,7 @@ strum_macros = "0.15"
 rust-ini = "0.13"
 derivative = "1.0.3"
 itertools = "0.8"
+nix = "0.17"
 
 
 image = { version = "0.21.1", optional = true }
diff --git a/src/files.rs b/src/files.rs
index 43c603f..c02e2a8 100644
--- a/src/files.rs
+++ b/src/files.rs
@@ -9,6 +9,7 @@ use std::sync::mpsc::Sender;
 use std::hash::{Hash, Hasher};
 use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::ffi::OsStr;
 
 use failure;
 use failure::Fail;
@@ -24,6 +25,9 @@ use rayon::{ThreadPool, ThreadPoolBuilder};
 use alphanumeric_sort::compare_str;
 use mime_guess;
 use rayon::prelude::*;
+use nix::{dir::*,
+          fcntl::OFlag,
+          sys::stat::Mode};
 
 use pathbuftools::PathBufTools;
 use async_value::{Async, Stale, StopIter};
@@ -72,7 +76,11 @@ pub fn set_ticking(val: bool) {
 #[derive(Fail, Debug, Clone)]
 pub enum FileError {
     #[fail(display = "Metadata still pending!")]
-    MetaPending
+    MetaPending,
+    #[fail(display = "Couldn't open directory! Error: {}", _0)]
+    OpenDir(#[cause] nix::Error),
+    #[fail(display = "Couldn't read files! Error: {}", _0)]
+    ReadFiles(#[cause] nix::Error),
 }
 
 pub fn get_pool() -> ThreadPool {
@@ -352,23 +360,24 @@ impl Drop for Files {
 
 impl Files {
     pub fn new_from_path_cancellable(path: &Path, stale: Stale) -> HResult<Files> {
-        let direntries: Vec<std::fs::DirEntry> = std::fs::read_dir(&path)?
-            .stop_stale(stale.clone())
-            .collect::<Result<Vec<std::fs::DirEntry>, _>>()?;
-
         let nonhidden = AtomicUsize::default();
 
-        let direntries: Vec<_> = direntries
-            .into_par_iter()
+        let direntries  = Dir::open(path.clone(),
+                                    OFlag::O_DIRECTORY,
+                                    Mode::empty())
+            .map_err(|e| FileError::OpenDir(e))?
+            .iter()
+            .stop_stale(stale.clone())
             .map(|f| {
-                let f = File::new_from_direntry(f);
+                let f = File::new_from_nixentry(f?, path);
                 // Fast check to avoid iterating twice
                 if f.name.as_bytes()[0] != b'.' {
                     nonhidden.fetch_add(1, Ordering::Relaxed);
                 }
-                f
+                Ok(f)
             })
-            .collect();
+            .collect::<Result<_,_>>()
+            .map_err(|e| FileError::ReadFiles(e))?;
 
         if stale.is_stale()? {
             HError::stale()?;
@@ -480,20 +489,20 @@ impl Files {
                         }
 
                         if let Some(dirsize) = dirsize {
-                            std::fs::read_dir(&path)
-                                .map(|dirs| {
-                                    let size = dirs.count();
-                                    dirsize.0.store(true, Ordering::Relaxed);
-                                    dirsize.1.store(size, Ordering::Relaxed);
-                                }).map_err(|e| {
-                                    dirsize.0.store(true, Ordering::Relaxed);
-                                    dirsize.1.store(0, Ordering::Relaxed);
-                                    HError::from(e)
-                                }).log();
+                            let size = Dir::open(&path,
+                                                 OFlag::O_DIRECTORY,
+                                                 Mode::empty())
+                                .map(|mut d| d.iter().count())
+                                .map_err(|e| FileError::OpenDir(e))
+                                .log_and()
+                                .unwrap_or(0);
+
+                            dirsize.0.store(true, Ordering::Relaxed);
+                            dirsize.1.store(size, Ordering::Relaxed);
+
+                            // Ticker will only stop after this reaches 0
+                            jobs_left.fetch_sub(1, Ordering::Relaxed);
                         }
-
-                        // Ticker will only stop after this reaches 0
-                        jobs_left.fetch_sub(1, Ordering::Relaxed);
                     });
 
                     ticker().map(|t| s.spawn_fifo(move |_| t()));
@@ -938,34 +947,39 @@ impl File {
         }
     }
 
-    pub fn new_with_stale(name: &str,
-                          path: PathBuf) -> File {
-        let hidden = name.starts_with(".");
+    pub fn new_from_nixentry(direntry: Entry, path: &Path) -> File {
+        // Scary stuff to avoid some of the overhead in Rusts conversions
+        // Speedup is a solid ~10%
+        let name: &OsStr = unsafe {
+            use std::ffi::CStr;
+            // &CStr -> &[u8]
+            let s: &[u8] = std::mem::transmute::<&CStr, &[u8]>(direntry.file_name());
+            // &Cstr -> &OsStr, minus the NULL byte
+            let len = s.len();
+            let s = &s[..len-1];
+            std::mem::transmute::<&[u8], &OsStr>(s)
+        };
 
-        File {
-            name: name.to_string(),
-            hidden: hidden,
-            kind: if path.is_dir() { Kind::Directory } else { Kind::File },
-            path: path,
-            dirsize: None,
-            target: None,
-            meta: None,
-            selected: false,
-            tag: None,
-        }
-    }
+        // Avoid reallocation on push
+        let mut pathstr = std::ffi::OsString::with_capacity(path.as_os_str().len() +
+                                                            name.len() +
+                                                            2);
+        pathstr.push(path.as_os_str());
+        pathstr.push("/");
+        pathstr.push(name);
+
+        let path = PathBuf::from(pathstr);
+
+        let name = name.to_str()
+                       .map(|n| String::from(n))
+                       .unwrap_or_else(|| name.to_string_lossy().to_string());
 
-    pub fn new_from_direntry(direntry: std::fs::DirEntry) -> File {
-        let path = direntry.path();
-        let name = direntry.file_name();
-        let name = name.to_string_lossy();
-        let name = String::from(name);
-        let hidden = name.chars().nth(0) == Some('.');
+        let hidden = name.as_bytes()[0] == b'.';
 
         let kind = match direntry.file_type() {
-            Ok(ftype) => match ftype.is_dir() {
-                true => Kind::Directory,
-                false => Kind::File
+            Some(ftype) => match ftype {
+                Type::Directory => Kind::Directory,
+                _ => Kind::File
             }
             _ => Kind::Placeholder
         };
diff --git a/src/main.rs b/src/main.rs
index a70e0c1..a0ba464 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -29,6 +29,7 @@ extern crate strum;
 extern crate strum_macros;
 #[macro_use]
 extern crate derivative;
+extern crate nix;
 
 extern crate osstrtools;
 extern crate pathbuftools;