From d2fda42dca410a9319f3f08b24545cbd8b8f1f59 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sun, 15 Mar 2020 20:31:46 +0800 Subject: Revert "Upgrade to jwalk 0.5; stop following symlinks during traversal" This reverts commit 4990fa4202f2b687ee2476efe0a406fdfe23fd96. Performance regression - it only uses a single thread for most of the iteration. --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- README.md | 4 +++- src/aggregate.rs | 20 +++++++++-------- src/common.rs | 8 ++----- src/interactive/app_test/utils.rs | 14 ++++-------- src/traverse.rs | 46 +++++++++++++++++++-------------------- 7 files changed, 45 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e2af8d1..329a9aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,9 +302,9 @@ dependencies = [ [[package]] name = "jwalk" -version = "0.5.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88746778a47f54f83bc0d3d8ba40ce83808024405356b4d521c2bf93c1273cd4" +checksum = "2b3dbf0a8f61baee43a2918ff50ac6a2d3b2c105bc08ed53bc298779f1263409" dependencies = [ "crossbeam", "rayon", diff --git a/Cargo.toml b/Cargo.toml index 6da36b9..9ec68b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ license = "MIT" failure = "0.1.1" failure-tools = "4.0.2" structopt = "0.3" -jwalk = "0.5.0" +jwalk = "0.4.0" byte-unit = "3" termion = "1.5.2" atty = "0.2.11" diff --git a/README.md b/README.md index 37b9038..cbfe5de 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ dua interactive * [ ] Evaluate unit coloring - can we highlight different units better, make them stick out? -#### ✅ v2.3.4 Fast exit from interactive mode for a responsive exit; don't follow symlinks during traversal +#### ✅ v2.3.4 Fast exit from interactive mode for a responsive exit #### ✅ v2.3.3 YANKED - journey tests failed to changed method signature @@ -194,6 +194,8 @@ Thanks to [jwalk][jwalk], all there was left to do is to write a command-line in ### Limitations * Interactive mode only looks good in dark terminals (see [this issue](https://github.com/Byron/dua-cli/issues/13)) +* _Symlinks_ are followed and we obtain the logical size of the file they point to. Ideally, we only + count their actual size. * _easy fix_: file names in main window are not truncated if too large. They are cut off on the right. * There are plenty of examples in `tests/fixtures` which don't render correctly in interactive mode. This can be due to graphemes not interpreted correctly. With Chinese characters for instance, diff --git a/src/aggregate.rs b/src/aggregate.rs index b8d3f6b..0652a03 100644 --- a/src/aggregate.rs +++ b/src/aggregate.rs @@ -29,24 +29,26 @@ pub fn aggregate( stats.entries_traversed += 1; match entry { Ok(entry) => { - let file_size = match entry.metadata() { - Ok(ref m) if !m.is_dir() && (options.count_hard_links || inodes.add(m)) => { + let file_size = match entry.metadata { + Some(Ok(ref m)) if !m.is_dir() && (options.count_hard_links || inodes.add(m)) => { if options.apparent_size { m.len() } else { - filesize::file_real_size_fast(&entry.path(), m).unwrap_or_else( - |_| { + filesize::file_real_size_fast(&entry.path(), m) + .unwrap_or_else(|_| { num_errors += 1; 0 - }, - ) + }) } - } - Ok(_) => 0, - Err(_) => { + }, + Some(Ok(_)) => 0, + Some(Err(_)) => { num_errors += 1; 0 } + None => unreachable!( + "we ask for metadata, so we at least have Some(Err(..))). Issue in jwalk?" + ), }; stats.largest_file_in_bytes = stats.largest_file_in_bytes.max(file_size); stats.smallest_file_in_bytes = stats.smallest_file_in_bytes.min(file_size); diff --git a/src/common.rs b/src/common.rs index c4a71df..a1551b8 100644 --- a/src/common.rs +++ b/src/common.rs @@ -161,17 +161,13 @@ pub struct WalkOptions { impl WalkOptions { pub(crate) fn iter_from_path(&self, path: &Path) -> WalkDir { WalkDir::new(path) - .follow_links(false) + .preload_metadata(true) .sort(match self.sorting { TraversalSorting::None => false, TraversalSorting::AlphabeticalByFileName => true, }) .skip_hidden(false) - .parallelism(if self.threads == 0 { - jwalk::Parallelism::RayonDefaultPool - } else { - jwalk::Parallelism::RayonNewPool(self.threads) - }) + .num_threads(self.threads) } } diff --git a/src/interactive/app_test/utils.rs b/src/interactive/app_test/utils.rs index ca9841c..f95efce 100644 --- a/src/interactive/app_test/utils.rs +++ b/src/interactive/app_test/utils.rs @@ -73,11 +73,8 @@ fn delete_recursive(path: impl AsRef) -> Result<(), Error> { let mut files: Vec<_> = Vec::new(); let mut dirs: Vec<_> = Vec::new(); - for entry in WalkDir::new(&path) - .parallelism(jwalk::Parallelism::Serial) - .into_iter() - { - let entry: DirEntry<_> = entry?; + for entry in WalkDir::new(&path).num_threads(1).into_iter() { + let entry: DirEntry = entry?; let p = entry.path(); match p.is_dir() { true => dirs.push(p), @@ -102,11 +99,8 @@ fn delete_recursive(path: impl AsRef) -> Result<(), Error> { } fn copy_recursive(src: impl AsRef, dst: impl AsRef) -> Result<(), Error> { - for entry in WalkDir::new(&src) - .parallelism(jwalk::Parallelism::Serial) - .into_iter() - { - let entry: DirEntry<_> = entry?; + for entry in WalkDir::new(&src).num_threads(1).into_iter() { + let entry: DirEntry = entry?; let entry_path = entry.path(); entry_path .strip_prefix(&src) diff --git a/src/traverse.rs b/src/traverse.rs index ffbd5d9..a65595e 100644 --- a/src/traverse.rs +++ b/src/traverse.rs @@ -88,36 +88,34 @@ impl Traversal { let mut data = EntryData::default(); match entry { Ok(entry) => { - let metadata = entry.metadata(); data.name = if entry.depth < 1 { path.clone().into() } else { entry.file_name }; - let file_size = match metadata { - Ok(ref m) - if !m.is_dir() - && (walk_options.count_hard_links || inodes.add(m)) => - { - if walk_options.apparent_size { - m.len() - } else { - filesize::file_real_size_fast(&data.name, m).unwrap_or_else( - |_| { - t.io_errors += 1; - data.metadata_io_error = true; - 0 - }, - ) + let file_size = match entry.metadata { + Some(Ok(ref m)) if !m.is_dir() && (walk_options.count_hard_links || inodes.add(m)) => { + if walk_options.apparent_size { + m.len() + } else { + filesize::file_real_size_fast(&data.name, m) + .unwrap_or_else(|_| { + t.io_errors += 1; + data.metadata_io_error = true; + 0 + }) + } + }, + Some(Ok(_)) => 0, + Some(Err(_)) => { + t.io_errors += 1; + data.metadata_io_error = true; + 0 } - } - Ok(_) => 0, - Err(_) => { - t.io_errors += 1; - data.metadata_io_error = true; - 0 - } - }; + None => unreachable!( + "we ask for metadata, so we at least have Some(Err(..))). Issue in jwalk?" + ), + }; match (entry.depth, previous_depth) { (n, p) if n > p => { -- cgit v1.2.3