From 4e49b91d2360c23a8ac959222f2d3e87409d8faa Mon Sep 17 00:00:00 2001 From: Ben S Date: Wed, 2 Sep 2015 23:19:10 +0100 Subject: Parallelise the details view! This commit removes the threadpool in `main.rs` that stats each command-line argument separately, and replaces it with a *scoped* threadpool in `options/details.rs` that builds the table in parallel! Running this on my machine halves the execution time when tree-ing my entire home directory (which isn't exactly a common occurrence, but it's the only way to give exa a large running time) The statting will be added back in parallel at a later stage. This was facilitated by the previous changes to recursion that made it easier to deal with. There's a lot of large sweeping architectural changes. Here's a smattering of them: - In `main.rs`, the files are now passed around as vectors of files rather than array slices of files. This is because `File`s aren't `Clone`, and the `Vec` is necessary to give away ownership of the files at the appropriate point. - In the details view, files are now sorted *all* the time, rather than obeying the command-line order. As they're run in parallel, they have no guaranteed order anyway, so we *have* to sort them again. (I'm not sure if this should be the intended behaviour or not!) This means that the `Details` struct has to have the filter *all* the time, not only while recursing, so it's been moved out of the `recurse` field. - We use `scoped_threadpool` over `threadpool`, a recent addition. It's only safely used on Nightly, which we're using anyway, so that's OK! - Removed a bunch of out-of-date comments. This also fixes #77, mainly by accident :) --- src/output/details.rs | 175 +++++++++++++++++++++++++++++++------------------- 1 file changed, 108 insertions(+), 67 deletions(-) (limited to 'src/output') diff --git a/src/output/details.rs b/src/output/details.rs index 3052448..0757f81 100644 --- a/src/output/details.rs +++ b/src/output/details.rs @@ -49,7 +49,10 @@ pub struct Details { /// Whether to recurse through directories with a tree view, and if so, /// which options to use. This field is only relevant here if the `tree` /// field of the RecurseOptions is `true`. - pub recurse: Option<(RecurseOptions, FileFilter)>, + pub recurse: Option, + + /// How to sort and filter the files after getting their details. + pub filter: FileFilter, /// Whether to show a header line or not. pub header: bool, @@ -63,7 +66,7 @@ pub struct Details { } impl Details { - pub fn view(&self, dir: Option<&Dir>, files: &[File]) { + pub fn view(&self, dir: Option<&Dir>, files: Vec) { // First, transform the Columns object into a vector of columns for // the current directory. @@ -84,76 +87,121 @@ impl Details { /// Adds files to the table - recursively, if the `recurse` option /// is present. - fn add_files_to_table(&self, table: &mut Table, src: &[File], depth: usize) { - for (index, file) in src.iter().enumerate() { - let mut xattrs = Vec::new(); - let mut errors = Vec::new(); - - let has_xattrs = match file.path.attributes() { - Ok(xs) => { - let r = !xs.is_empty(); - if self.xattr { - for xattr in xs { - xattrs.push(xattr); - } - } - r - }, - Err(e) => { - if self.xattr { - errors.push((e, None)); - } - true - }, - }; + fn add_files_to_table<'dir, U: Users+Send+Sync>(&self, mut table: &mut Table, src: Vec>, depth: usize) { + use num_cpus; + use scoped_threadpool::Pool; + use std::sync::{Arc, Mutex}; + + let mut pool = Pool::new(num_cpus::get() as u32); + let mut file_eggs = Vec::new(); + + struct Egg<'_> { + cells: Vec, + name: Cell, + xattrs: Vec, + errors: Vec<(io::Error, Option)>, + dir: Option, + file: Arc>, + } - table.add_file(file, depth, index == src.len() - 1, true, has_xattrs); - - // There are two types of recursion that exa supports: a tree - // view, which is dealt with here, and multiple listings, which is - // dealt with in the main module. So only actually recurse if we - // are in tree mode - the other case will be dealt with elsewhere. - if let Some((r, filter)) = self.recurse { - if file.is_directory() && r.tree && !r.is_too_deep(depth) { - - // Use the filter to remove unwanted files *before* expanding - // them, so we don't examine any directories that wouldn't - // have their contents listed anyway. - match file.to_dir() { - Ok(ref dir) => { - let mut files = Vec::new(); - - for file_to_add in dir.files() { - match file_to_add { - Ok(f) => files.push(f), - Err((path, e)) => errors.push((e, Some(path))) - } - } + pool.scoped(|scoped| { + let file_eggs = Arc::new(Mutex::new(&mut file_eggs)); + let table = Arc::new(Mutex::new(&mut table)); - filter.transform_files(&mut files); + for file in src.into_iter() { + let file: Arc = Arc::new(file); + let file_eggs = file_eggs.clone(); + let table = table.clone(); - if !files.is_empty() { - for xattr in xattrs { - table.add_xattr(xattr, depth + 1, false); - } + scoped.execute(move || { + let mut errors = Vec::new(); - for (error, path) in errors { - table.add_error(&error, depth + 1, false, path); + let mut xattrs = Vec::new(); + match file.path.attributes() { + Ok(xs) => { + if self.xattr { + for xattr in xs { + xattrs.push(xattr); } - - self.add_files_to_table(table, &files, depth + 1); - continue; } }, Err(e) => { - errors.push((e, None)); + if self.xattr { + errors.push((e, None)); + } }, + }; + + let cells = table.lock().unwrap().cells_for_file(&file, !xattrs.is_empty()); + let links = true; + let name = Cell { text: filename(&file, &self.colours, links), length: file.file_name_width() }; + + let mut dir = None; + + if let Some(r) = self.recurse { + if file.is_directory() && r.tree && !r.is_too_deep(depth) { + if let Ok(d) = file.to_dir(false) { + dir = Some(d); + } + } + }; + + let egg = Egg { + cells: cells, + name: name, + xattrs: xattrs, + errors: errors, + dir: dir, + file: file, + }; + + file_eggs.lock().unwrap().push(egg); + }); + } + }); + + file_eggs.sort_by(|a, b| self.filter.compare_files(&*a.file, &*b.file)); + + let num_eggs = file_eggs.len(); + for (index, egg) in file_eggs.into_iter().enumerate() { + let mut files = Vec::new(); + let mut errors = egg.errors; + + let row = Row { + depth: depth, + cells: Some(egg.cells), + name: egg.name, + last: index == num_eggs - 1, + }; + + table.rows.push(row); + + if let Some(ref dir) = egg.dir { + for file_to_add in dir.files() { + match file_to_add { + Ok(f) => files.push(f), + Err((path, e)) => errors.push((e, Some(path))) + } + } + + self.filter.filter_files(&mut files); + + if !files.is_empty() { + for xattr in egg.xattrs { + table.add_xattr(xattr, depth + 1, false); + } + + for (error, path) in errors { + table.add_error(&error, depth + 1, false, path); } + + self.add_files_to_table(table, files, depth + 1); + continue; } } - let count = xattrs.len(); - for (index, xattr) in xattrs.into_iter().enumerate() { + let count = egg.xattrs.len(); + for (index, xattr) in egg.xattrs.into_iter().enumerate() { table.add_xattr(xattr, depth + 1, errors.is_empty() && index == count - 1); } @@ -161,7 +209,6 @@ impl Details { for (index, (error, path)) in errors.into_iter().enumerate() { table.add_error(&error, depth + 1, index == count - 1, path); } - } } } @@ -289,19 +336,13 @@ impl Table where U: Users { let row = Row { depth: depth, cells: None, - name: Cell::paint(self.colours.perms.attribute, &format!("{}\t{}", xattr.name, xattr.size)), + name: Cell::paint(self.colours.perms.attribute, &format!("{} (len {})", xattr.name, xattr.size)), last: last, }; self.rows.push(row); } - /// Get the cells for the given file, and add the result to the table. - fn add_file(&mut self, file: &File, depth: usize, last: bool, links: bool, xattrs: bool) { - let cells = self.cells_for_file(file, xattrs); - self.add_file_with_cells(cells, file, depth, last, links) - } - pub fn add_file_with_cells(&mut self, cells: Vec, file: &File, depth: usize, last: bool, links: bool) { let row = Row { depth: depth, -- cgit v1.2.3