From 78029853ba687cabd37adbbdf41b2ee480bbcbf8 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Thu, 15 Dec 2022 14:11:51 +0100 Subject: create our own threadpool with minimal stack instead of using the global one. Even though the global one is also ours as we own the process, this way we have control over the stack size, to save memory where we don't need it and scale better to even more cores. --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- src/common.rs | 11 ++++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 274de2f..229475c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,9 +380,9 @@ dependencies = [ [[package]] name = "jwalk" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cb840f933b9fa56c78fde73acc9f2c883a2594fc6c791992a133468106965d" +checksum = "2735847566356cd2179a2a38264839308f7079fa96e6bd5a42d740460e003c56" dependencies = [ "crossbeam", "rayon", diff --git a/Cargo.toml b/Cargo.toml index f991444..1300a2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ trash-move = ["trash"] [dependencies] clap = { version = "4.0.29", features = ["derive"] } -jwalk = "0.8.0" +jwalk = "0.8.1" byte-unit = "4" atty = "0.2.11" petgraph = "0.6" diff --git a/src/common.rs b/src/common.rs index 45e9738..f5f862e 100644 --- a/src/common.rs +++ b/src/common.rs @@ -160,7 +160,16 @@ impl WalkOptions { busy_timeout: std::time::Duration::from_secs(1), }, 1 => jwalk::Parallelism::Serial, - _ => jwalk::Parallelism::RayonNewPool(self.threads), + _ => jwalk::Parallelism::RayonExistingPool { + pool: jwalk::rayon::ThreadPoolBuilder::new() + .stack_size(128 * 1024) + .num_threads(self.threads) + .thread_name(|idx| format!("dua-fs-walk-{idx}")) + .build() + .expect("fields we set cannot fail") + .into(), + busy_timeout: None, + }, }) } } -- cgit v1.2.3