diff options
author | Ben S <ogham@bsago.me> | 2015-09-02 23:19:10 +0100 |
---|---|---|
committer | Ben S <ogham@bsago.me> | 2015-09-02 23:19:10 +0100 |
commit | 4e49b91d2360c23a8ac959222f2d3e87409d8faa (patch) | |
tree | b55fa185bf792ec01383440f08c27015a246eec9 /src/main.rs | |
parent | eee49ece041e96673ce536fc1a4f5afb19d43ead (diff) |
Parallelise the details view!
This commit removes the threadpool in `main.rs` that stats each command-line argument separately, and replaces it with a *scoped* threadpool in `options/details.rs` that builds the table in parallel! Running this on my machine halves the execution time when tree-ing my entire home directory (which isn't exactly a common occurrence, but it's the only way to give exa a large running time)
The statting will be added back in parallel at a later stage. This was facilitated by the previous changes to recursion that made it easier to deal with.
There's a lot of large sweeping architectural changes. Here's a smattering of them:
- In `main.rs`, the files are now passed around as vectors of files rather than array slices of files. This is because `File`s aren't `Clone`, and the `Vec` is necessary to give away ownership of the files at the appropriate point.
- In the details view, files are now sorted *all* the time, rather than obeying the command-line order. As they're run in parallel, they have no guaranteed order anyway, so we *have* to sort them again. (I'm not sure if this should be the intended behaviour or not!) This means that the `Details` struct has to have the filter *all* the time, not only while recursing, so it's been moved out of the `recurse` field.
- We use `scoped_threadpool` over `threadpool`, a recent addition. It's only safely used on Nightly, which we're using anyway, so that's OK!
- Removed a bunch of out-of-date comments.
This also fixes #77, mainly by accident :)
Diffstat (limited to 'src/main.rs')
-rw-r--r-- | src/main.rs | 181 |
1 files changed, 65 insertions, 116 deletions
diff --git a/src/main.rs b/src/main.rs index e64f3fb..fd9e527 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,8 +11,8 @@ extern crate natord; extern crate num_cpus; extern crate number_prefix; extern crate pad; +extern crate scoped_threadpool; extern crate term_grid; -extern crate threadpool; extern crate unicode_width; extern crate users; @@ -21,12 +21,8 @@ extern crate git2; use std::env; -use std::fs; -use std::path::{Component, Path, PathBuf}; +use std::path::{Component, Path}; use std::process; -use std::sync::mpsc::channel; - -use threadpool::ThreadPool; use dir::Dir; use file::File; @@ -44,94 +40,48 @@ mod term; #[cfg(not(test))] -struct Exa<'dir> { - count: usize, +struct Exa { options: Options, - dirs: Vec<PathBuf>, - files: Vec<File<'dir>>, } #[cfg(not(test))] -impl<'dir> Exa<'dir> { - fn new(options: Options) -> Exa<'dir> { - Exa { - count: 0, - options: options, - dirs: Vec::new(), - files: Vec::new(), - } +impl Exa { + fn new(options: Options) -> Exa { + Exa { options: options } } - fn load(&mut self, files: &[String]) { - - // Separate the user-supplied paths into directories and files. - // Files are shown first, and then each directory is expanded - // and listed second. - let is_tree = self.options.dir_action.is_tree() || self.options.dir_action.is_as_file(); - let total_files = files.len(); - - - // Communication between consumer thread and producer threads - enum StatResult<'dir> { - File(File<'dir>), - Dir(PathBuf), - Error - } - - let pool = ThreadPool::new(8 * num_cpus::get()); - let (tx, rx) = channel(); - - for file in files.iter() { - let tx = tx.clone(); - let file = file.clone(); + fn run(&mut self, args_file_names: &[String]) { + let mut files = Vec::new(); + let mut dirs = Vec::new(); - // Spawn producer thread - pool.execute(move || { - let path = Path::new(&*file); - let _ = tx.send(match fs::metadata(&path) { - Ok(metadata) => { - if is_tree || !metadata.is_dir() { - StatResult::File(File::with_metadata(metadata, &path, None)) - } - else { - StatResult::Dir(path.to_path_buf()) + for file_name in args_file_names.iter() { + match File::from_path(Path::new(&file_name), None) { + Err(e) => { + println!("{}: {}", file_name, e); + }, + Ok(f) => { + if f.is_directory() && !self.options.dir_action.treat_dirs_as_files() { + match f.to_dir(self.options.should_scan_for_git()) { + Ok(d) => dirs.push(d), + Err(e) => println!("{}: {}", file_name, e), } } - Err(e) => { - println!("{}: {}", file, e); - StatResult::Error + else { + files.push(f); } - }); - }); - } - - // Spawn consumer thread - for result in rx.iter().take(total_files) { - match result { - StatResult::File(file) => self.files.push(file), - StatResult::Dir(path) => self.dirs.push(path), - StatResult::Error => () + }, } - self.count += 1; } - } - fn print_files(&self) { - if !self.files.is_empty() { - self.print(None, &self.files[..]); - } + let any_files = files.is_empty(); + self.print_files(None, files); + + let is_only_dir = dirs.len() == 1; + self.print_dirs(dirs, any_files, is_only_dir); } - fn print_dirs(&mut self) { - let mut first = self.files.is_empty(); - - // Directories are put on a stack rather than just being iterated through, - // as the vector can change as more directories are added. - loop { - let dir_path = match self.dirs.pop() { - None => break, - Some(f) => f, - }; + fn print_dirs(&self, dir_files: Vec<Dir>, mut first: bool, is_only_dir: bool) { + for dir in dir_files { // Put a gap between directories, or between the list of files and the // first directory. @@ -142,53 +92,54 @@ impl<'dir> Exa<'dir> { print!("\n"); } - match Dir::readdir(&dir_path, self.options.should_scan_for_git()) { - Ok(ref dir) => { - let mut files = Vec::new(); + if !is_only_dir { + println!("{}:", dir.path.display()); + } - for file in dir.files() { - match file { - Ok(file) => files.push(file), - Err((path, e)) => println!("[{}: {}]", path.display(), e), - } - } + let mut children = Vec::new(); + for file in dir.files() { + match file { + Ok(file) => children.push(file), + Err((path, e)) => println!("[{}: {}]", path.display(), e), + } + }; + + self.options.filter_files(&mut children); + self.options.sort_files(&mut children); + + if let Some(recurse_opts) = self.options.dir_action.recurse_options() { + let depth = dir.path.components().filter(|&c| c != Component::CurDir).count() + 1; + if !recurse_opts.tree && !recurse_opts.is_too_deep(depth) { - self.options.transform_files(&mut files); - - // When recursing, add any directories to the dirs stack - // backwards: the *last* element of the stack is used each - // time, so by inserting them backwards, they get displayed in - // the correct sort order. - if let Some(recurse_opts) = self.options.dir_action.recurse_options() { - let depth = dir_path.components().filter(|&c| c != Component::CurDir).count() + 1; - if !recurse_opts.tree && !recurse_opts.is_too_deep(depth) { - for dir in files.iter().filter(|f| f.is_directory()).rev() { - self.dirs.push(dir.path.clone()); - } + let mut child_dirs = Vec::new(); + for child_dir in children.iter().filter(|f| f.is_directory()) { + match child_dir.to_dir(false) { + Ok(d) => child_dirs.push(d), + Err(e) => println!("{}: {}", child_dir.path.display(), e), } } - if self.count > 1 { - println!("{}:", dir_path.display()); + self.print_files(Some(&dir), children); + + if !child_dirs.is_empty() { + self.print_dirs(child_dirs, false, false); } - self.count += 1; - self.print(Some(dir), &files[..]); - } - Err(e) => { - println!("{}: {}", dir_path.display(), e); - return; + continue; } - }; + } + + self.print_files(Some(&dir), children); + } } - fn print(&self, dir: Option<&Dir>, files: &[File]) { + fn print_files(&self, dir: Option<&Dir>, files: Vec<File>) { match self.options.view { - View::Grid(g) => g.view(files), + View::Grid(g) => g.view(&files), View::Details(d) => d.view(dir, files), - View::GridDetails(gd) => gd.view(dir, files), - View::Lines(l) => l.view(files), + View::GridDetails(gd) => gd.view(dir, &files), + View::Lines(l) => l.view(&files), } } } @@ -201,9 +152,7 @@ fn main() { match Options::getopts(&args) { Ok((options, paths)) => { let mut exa = Exa::new(options); - exa.load(&paths); - exa.print_files(); - exa.print_dirs(); + exa.run(&paths); }, Err(e) => { println!("{}", e); |