Parallelise the details view!

This commit removes the threadpool in `main.rs` that stats each command-line argument separately, and replaces it with a *scoped* threadpool in `options/details.rs` that builds the table in parallel! Running this on my machine halves the execution time when tree-ing my entire home directory (which isn't exactly a common occurrence, but it's the only way to give exa a large running time)

The statting will be added back in parallel at a later stage. This was facilitated by the previous changes to recursion that made it easier to deal with.

There's a lot of large sweeping architectural changes. Here's a smattering of them:

- In `main.rs`, the files are now passed around as vectors of files rather than array slices of files. This is because `File`s aren't `Clone`, and the `Vec` is necessary to give away ownership of the files at the appropriate point.
- In the details view, files are now sorted *all* the time, rather than obeying the command-line order. As they're run in parallel, they have no guaranteed order anyway, so we *have* to sort them again. (I'm not sure if this should be the intended behaviour or not!) This means that the `Details` struct has to have the filter *all* the time, not only while recursing, so it's been moved out of the `recurse` field.
- We use `scoped_threadpool` over `threadpool`, a recent addition. It's only safely used on Nightly, which we're using anyway, so that's OK!
- Removed a bunch of out-of-date comments.

This also fixes #77, mainly by accident :)

1 files changed, 65 insertions, 116 deletions

diff --git a/src/main.rs b/src/main.rs
index e64f3fb..fd9e527 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -11,8 +11,8 @@ extern crate natord;
 extern crate num_cpus;
 extern crate number_prefix;
 extern crate pad;
+extern crate scoped_threadpool;
 extern crate term_grid;
-extern crate threadpool;
 extern crate unicode_width;
 extern crate users;
 
@@ -21,12 +21,8 @@ extern crate git2;
 
 
 use std::env;
-use std::fs;
-use std::path::{Component, Path, PathBuf};
+use std::path::{Component, Path};
 use std::process;
-use std::sync::mpsc::channel;
-
-use threadpool::ThreadPool;
 
 use dir::Dir;
 use file::File;
@@ -44,94 +40,48 @@ mod term;
 
 
 #[cfg(not(test))]
-struct Exa<'dir> {
-    count:   usize,
+struct Exa {
     options: Options,
-    dirs:    Vec<PathBuf>,
-    files:   Vec<File<'dir>>,
 }
 
 #[cfg(not(test))]
-impl<'dir> Exa<'dir> {
-    fn new(options: Options) -> Exa<'dir> {
-        Exa {
-            count: 0,
-            options: options,
-            dirs: Vec::new(),
-            files: Vec::new(),
-        }
+impl Exa {
+    fn new(options: Options) -> Exa {
+        Exa { options: options }
     }
 
-    fn load(&mut self, files: &[String]) {
-
-        // Separate the user-supplied paths into directories and files.
-        // Files are shown first, and then each directory is expanded
-        // and listed second.
-        let is_tree = self.options.dir_action.is_tree() || self.options.dir_action.is_as_file();
-        let total_files = files.len();
-
-
-        // Communication between consumer thread and producer threads
-        enum StatResult<'dir> {
-            File(File<'dir>),
-            Dir(PathBuf),
-            Error
-        }
-
-        let pool = ThreadPool::new(8 * num_cpus::get());
-        let (tx, rx) = channel();
-
-        for file in files.iter() {
-            let tx = tx.clone();
-            let file = file.clone();
+    fn run(&mut self, args_file_names: &[String]) {
+        let mut files = Vec::new();
+        let mut dirs = Vec::new();
 
-            // Spawn producer thread
-            pool.execute(move || {
-                let path = Path::new(&*file);
-                let _ = tx.send(match fs::metadata(&path) {
-                    Ok(metadata) => {
-                        if is_tree || !metadata.is_dir() {
-                            StatResult::File(File::with_metadata(metadata, &path, None))
-                        }
-                        else {
-                            StatResult::Dir(path.to_path_buf())
+        for file_name in args_file_names.iter() {
+            match File::from_path(Path::new(&file_name), None) {
+                Err(e) => {
+                    println!("{}: {}", file_name, e);
+                },
+                Ok(f) => {
+                    if f.is_directory() && !self.options.dir_action.treat_dirs_as_files() {
+                        match f.to_dir(self.options.should_scan_for_git()) {
+                            Ok(d) => dirs.push(d),
+                            Err(e) => println!("{}: {}", file_name, e),
                         }
                     }
-                    Err(e) => {
-                        println!("{}: {}", file, e);
-                        StatResult::Error
+                    else {
+                        files.push(f);
                     }
-                });
-            });
-        }
-
-        // Spawn consumer thread
-        for result in rx.iter().take(total_files) {
-            match result {
-                StatResult::File(file)  => self.files.push(file),
-                StatResult::Dir(path)   => self.dirs.push(path),
-                StatResult::Error       => ()
+                },
             }
-            self.count += 1;
         }
-    }
 
-    fn print_files(&self) {
-        if !self.files.is_empty() {
-            self.print(None, &self.files[..]);
-        }
+        let any_files = files.is_empty();
+        self.print_files(None, files);
+
+        let is_only_dir = dirs.len() == 1;
+        self.print_dirs(dirs, any_files, is_only_dir);
     }
 
-    fn print_dirs(&mut self) {
-        let mut first = self.files.is_empty();
-
-        // Directories are put on a stack rather than just being iterated through,
-        // as the vector can change as more directories are added.
-        loop {
-            let dir_path = match self.dirs.pop() {
-                None => break,
-                Some(f) => f,
-            };
+    fn print_dirs(&self, dir_files: Vec<Dir>, mut first: bool, is_only_dir: bool) {
+        for dir in dir_files {
 
             // Put a gap between directories, or between the list of files and the
             // first directory.
@@ -142,53 +92,54 @@ impl<'dir> Exa<'dir> {
                 print!("\n");
             }
 
-            match Dir::readdir(&dir_path, self.options.should_scan_for_git()) {
-                Ok(ref dir) => {
-                    let mut files = Vec::new();
+            if !is_only_dir {
+                println!("{}:", dir.path.display());
+            }
 
-                    for file in dir.files() {
-                        match file {
-                            Ok(file) => files.push(file),
-                            Err((path, e))   => println!("[{}: {}]", path.display(), e),
-                        }
-                    }
+            let mut children = Vec::new();
+            for file in dir.files() {
+                match file {
+                    Ok(file)       => children.push(file),
+                    Err((path, e)) => println!("[{}: {}]", path.display(), e),
+                }
+            };
+
+            self.options.filter_files(&mut children);
+            self.options.sort_files(&mut children);
+
+            if let Some(recurse_opts) = self.options.dir_action.recurse_options() {
+                let depth = dir.path.components().filter(|&c| c != Component::CurDir).count() + 1;
+                if !recurse_opts.tree && !recurse_opts.is_too_deep(depth) {
 
-                    self.options.transform_files(&mut files);
-
-                    // When recursing, add any directories to the dirs stack
-                    // backwards: the *last* element of the stack is used each
-                    // time, so by inserting them backwards, they get displayed in
-                    // the correct sort order.
-                    if let Some(recurse_opts) = self.options.dir_action.recurse_options() {
-                        let depth = dir_path.components().filter(|&c| c != Component::CurDir).count() + 1;
-                        if !recurse_opts.tree && !recurse_opts.is_too_deep(depth) {
-                            for dir in files.iter().filter(|f| f.is_directory()).rev() {
-                                self.dirs.push(dir.path.clone());
-                            }
+                    let mut child_dirs = Vec::new();
+                    for child_dir in children.iter().filter(|f| f.is_directory()) {
+                        match child_dir.to_dir(false) {
+                            Ok(d)  => child_dirs.push(d),
+                            Err(e) => println!("{}: {}", child_dir.path.display(), e),
                         }
                     }
 
-                    if self.count > 1 {
-                        println!("{}:", dir_path.display());
+                    self.print_files(Some(&dir), children);
+
+                    if !child_dirs.is_empty() {
+                        self.print_dirs(child_dirs, false, false);
                     }
-                    self.count += 1;
 
-                    self.print(Some(dir), &files[..]);
-                }
-                Err(e) => {
-                    println!("{}: {}", dir_path.display(), e);
-                    return;
+                    continue;
                 }
-            };
+            }
+
+            self.print_files(Some(&dir), children);
+
         }
     }
 
-    fn print(&self, dir: Option<&Dir>, files: &[File]) {
+    fn print_files(&self, dir: Option<&Dir>, files: Vec<File>) {
         match self.options.view {
-            View::Grid(g)         => g.view(files),
+            View::Grid(g)         => g.view(&files),
             View::Details(d)      => d.view(dir, files),
-            View::GridDetails(gd) => gd.view(dir, files),
-            View::Lines(l)        => l.view(files),
+            View::GridDetails(gd) => gd.view(dir, &files),
+            View::Lines(l)        => l.view(&files),
         }
     }
 }
@@ -201,9 +152,7 @@ fn main() {
     match Options::getopts(&args) {
         Ok((options, paths)) => {
             let mut exa = Exa::new(options);
-            exa.load(&paths);
-            exa.print_files();
-            exa.print_dirs();
+            exa.run(&paths);
         },
         Err(e) => {
             println!("{}", e);