From 51b67ff9d009a56272448d1fee1951f30b1de678 Mon Sep 17 00:00:00 2001 From: Piotr Wach Date: Sun, 7 Jan 2024 19:36:43 +0000 Subject: wip --- src/common.rs | 2 +- src/interactive/app/app_state.rs | 70 ++++- src/interactive/app/eventloop.rs | 412 ++++++++++++++++++++-------- src/interactive/app/terminal_app.rs | 190 +++++++------ src/lib.rs | 2 +- src/main.rs | 18 +- src/traverse.rs | 521 +++++++++++++++++------------------- 7 files changed, 745 insertions(+), 470 deletions(-) diff --git a/src/common.rs b/src/common.rs index 47c0b10..254fbbf 100644 --- a/src/common.rs +++ b/src/common.rs @@ -176,7 +176,7 @@ pub struct WalkOptions { type WalkDir = jwalk::WalkDirGeneric<((), Option>)>; impl WalkOptions { - pub(crate) fn iter_from_path(&self, root: &Path, root_device_id: u64) -> WalkDir { + pub fn iter_from_path(&self, root: &Path, root_device_id: u64) -> WalkDir { let ignore_dirs = self.ignore_dirs.clone(); let cwd = std::env::current_dir().unwrap_or_else(|_| root.to_owned()); WalkDir::new(root) diff --git a/src/interactive/app/app_state.rs b/src/interactive/app/app_state.rs index dd0f9a0..8986f09 100644 --- a/src/interactive/app/app_state.rs +++ b/src/interactive/app/app_state.rs @@ -1,4 +1,5 @@ -use dua::WalkResult; +use dua::{WalkResult, traverse::{TreeIndex, Tree}, inodefilter::InodeFilter}; +use petgraph::Direction; use super::{navigation::Navigation, EntryDataBundle, SortMode}; @@ -27,6 +28,73 @@ pub struct AppState { pub message: Option, pub focussed: FocussedPane, pub is_scanning: bool, + pub traversal_state: TraversalState, +} + + +#[derive(Default)] +pub struct TraversalState { + pub previous_node_idx: TreeIndex, + pub parent_node_idx: TreeIndex, + pub directory_info_per_depth_level: Vec, + pub current_directory_at_depth: EntryInfo, + pub previous_depth: usize, + pub inodes: InodeFilter, +} + +impl TraversalState { + pub fn new(root_idx: TreeIndex) -> Self { + Self { + previous_node_idx: root_idx, + parent_node_idx: root_idx, + directory_info_per_depth_level: Vec::new(), + current_directory_at_depth: EntryInfo::default(), + previous_depth: 0, + inodes: InodeFilter::default(), + } + } +} + +#[derive(Default, Copy, Clone)] +pub struct EntryInfo { + pub size: u128, + pub entries_count: Option, +} + +impl EntryInfo { + pub fn add_count(&mut self, other: &Self) { + self.entries_count = match (self.entries_count, other.entries_count) { + (Some(a), Some(b)) => Some(a + b), + (None, Some(b)) => Some(b), + (Some(a), None) => Some(a), + (None, None) => None, + }; + } +} + +pub fn set_entry_info_or_panic( + tree: &mut Tree, + node_idx: TreeIndex, + EntryInfo { + size, + entries_count, + }: EntryInfo, +) { + let node = tree + .node_weight_mut(node_idx) + .expect("node for parent index we just retrieved"); + node.size = size; + node.entry_count = entries_count; +} + +pub fn parent_or_panic(tree: &mut Tree, parent_node_idx: TreeIndex) -> TreeIndex { + tree.neighbors_directed(parent_node_idx, Direction::Incoming) + .next() + .expect("every node in the iteration has a parent") +} + +pub fn pop_or_panic(v: &mut Vec) -> EntryInfo { + v.pop().expect("sizes per level to be in sync with graph") } pub enum ProcessingResult { diff --git a/src/interactive/app/eventloop.rs b/src/interactive/app/eventloop.rs index df297e3..014ce36 100644 --- a/src/interactive/app/eventloop.rs +++ b/src/interactive/app/eventloop.rs @@ -1,24 +1,24 @@ -use crate::interactive::{ +use crate::{interactive::{ app::navigation::Navigation, app_state::FocussedPane, sorted_entries, widgets::{glob_search, MainWindow, MainWindowProps}, ByteVisualization, CursorDirection, CursorMode, DisplayOptions, EntryDataBundle, MarkEntryMode, SortMode, -}; +}, crossdev}; use anyhow::Result; use crossbeam::channel::Receiver; use crosstermion::crossterm::event::{KeyCode, KeyEvent, KeyEventKind, KeyModifiers}; use crosstermion::input::Event; use dua::{ - traverse::{EntryData, Traversal, Tree}, + traverse::{EntryData, Traversal, Tree, size_on_disk}, WalkOptions, WalkResult, }; -use std::path::PathBuf; +use std::{path::PathBuf, time::{SystemTime, UNIX_EPOCH}}; use tui::backend::Backend; use tui_react::Terminal; -use super::tree_view::TreeView; +use super::{tree_view::TreeView, terminal_app::TraversalEvent, app_state::{EntryInfo, set_entry_info_or_panic, pop_or_panic, parent_or_panic}}; use super::{ app_state::{AppState, Cursor, ProcessingResult}, input::input_channel, @@ -74,132 +74,326 @@ impl AppState { traversal: &mut Traversal, display: &mut DisplayOptions, terminal: &mut Terminal, - events: impl Iterator, + walk_options: &WalkOptions, + events: Receiver, + traversal_events: Receiver, ) -> Result where B: Backend, { - use crosstermion::crossterm::event::KeyCode::*; - use FocussedPane::*; - { let tree_view = self.tree_view(traversal); self.draw(window, &tree_view, *display, terminal)?; } - for event in events { - let key = match event { - Event::Key(key) if key.kind != KeyEventKind::Release => key, - Event::Resize(_, _) => refresh_key(), - _ => continue, - }; + loop { + crossbeam::select! { + recv(events) -> event => { + let Ok(event) = event else { + continue; + }; + let result = self.process_event( + window, + traversal, + display, + terminal, + event)?; + if let Some(processing_result) = result { + return Ok(processing_result); + } + }, + recv(traversal_events) -> event => { + let Ok(event) = event else { + continue; + }; + self.process_traversal_event(traversal, walk_options, event); + } + } + } + // TODO: do we need this? + // Ok(ProcessingResult::Finished(WalkResult { + // num_errors: traversal.io_errors, + // })) + } + + // TODO: + // default(Duration::from_millis(250)) => { + // // No events or new entries received, but we still need + // // to keep updating the status message regularly. + // if update(&mut t, None)? { + // return Ok(None); + // } + // } + // } + // } + + fn process_traversal_event<'a>(&mut self, t: &'a mut Traversal, walk_options: &'a WalkOptions, event: TraversalEvent) { + match event { + TraversalEvent::Entry(entry, root_path, device_id) => { + t.entries_traversed += 1; + let mut data = EntryData::default(); + match entry { + Ok(entry) => { + data.name = if entry.depth < 1 { + (*root_path).clone() + } else { + entry.file_name.into() + }; + + let mut file_size = 0u128; + let mut mtime: SystemTime = UNIX_EPOCH; + match &entry.client_state { + Some(Ok(ref m)) => { + if !m.is_dir() + && (walk_options.count_hard_links || self.traversal_state.inodes.add(m)) + && (walk_options.cross_filesystems + || crossdev::is_same_device(device_id, m)) + { + if walk_options.apparent_size { + file_size = m.len() as u128; + } else { + file_size = size_on_disk(&entry.parent_path, &data.name, m) + .unwrap_or_else(|_| { + t.io_errors += 1; + data.metadata_io_error = true; + 0 + }) + as u128; + } + } else { + data.entry_count = Some(0); + data.is_dir = true; + } + + match m.modified() { + Ok(modified) => { + mtime = modified; + } + Err(_) => { + t.io_errors += 1; + data.metadata_io_error = true; + } + } + } + Some(Err(_)) => { + t.io_errors += 1; + data.metadata_io_error = true; + } + None => {} + } + + match (entry.depth, self.traversal_state.previous_depth) { + (n, p) if n > p => { + self.traversal_state.directory_info_per_depth_level.push(self.traversal_state.current_directory_at_depth); + self.traversal_state.current_directory_at_depth = EntryInfo { + size: file_size, + entries_count: Some(1), + }; + self.traversal_state.parent_node_idx = self.traversal_state.previous_node_idx; + } + (n, p) if n < p => { + for _ in n..p { + set_entry_info_or_panic( + &mut t.tree, + self.traversal_state.parent_node_idx, + self.traversal_state.current_directory_at_depth, + ); + let dir_info = + pop_or_panic(&mut self.traversal_state.directory_info_per_depth_level); + + self.traversal_state.current_directory_at_depth.size += dir_info.size; + self.traversal_state.current_directory_at_depth.add_count(&dir_info); + + self.traversal_state.parent_node_idx = parent_or_panic(&mut t.tree, self.traversal_state.parent_node_idx); + } + self.traversal_state.current_directory_at_depth.size += file_size; + *self.traversal_state.current_directory_at_depth.entries_count.get_or_insert(0) += 1; + set_entry_info_or_panic( + &mut t.tree, + self.traversal_state.parent_node_idx, + self.traversal_state.current_directory_at_depth, + ); + } + _ => { + self.traversal_state.current_directory_at_depth.size += file_size; + *self.traversal_state.current_directory_at_depth.entries_count.get_or_insert(0) += 1; + } + }; - self.reset_message(); + data.mtime = mtime; + data.size = file_size; + let entry_index = t.tree.add_node(data); - let glob_focussed = self.focussed == Glob; - let mut tree_view = self.tree_view(traversal); - let mut handled = true; - match key.code { - Esc => { - if let Some(value) = self.handle_quit(&mut tree_view, window) { - return value; + t.tree.add_edge(self.traversal_state.parent_node_idx, entry_index, ()); + self.traversal_state.previous_node_idx = entry_index; + self.traversal_state.previous_depth = entry.depth; + } + Err(_) => { + if self.traversal_state.previous_depth == 0 { + data.name = (*root_path).clone(); + let entry_index = t.tree.add_node(data); + t.tree.add_edge(self.traversal_state.parent_node_idx, entry_index, ()); + } + + t.io_errors += 1 } } - Tab => { - self.cycle_focus(window); - } - Char('/') if !glob_focussed => { - self.toggle_glob_search(window); - } - Char('?') if !glob_focussed => self.toggle_help_pane(window), - Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) && !glob_focussed => { - return Ok(ProcessingResult::ExitRequested(WalkResult { - num_errors: tree_view.traversal.io_errors, - })) + + // TODO: + // if throttle.can_update() && update(&mut t, None)? { + // return Ok(None); + // } + }, + TraversalEvent::Finished(io_errors) => { + self.traversal_state.directory_info_per_depth_level.push(self.traversal_state.current_directory_at_depth); + self.traversal_state.current_directory_at_depth = EntryInfo::default(); + for _ in 0..self.traversal_state.previous_depth { + let dir_info = pop_or_panic(&mut self.traversal_state.directory_info_per_depth_level); + self.traversal_state.current_directory_at_depth.size += dir_info.size; + self.traversal_state.current_directory_at_depth.add_count(&dir_info); + + set_entry_info_or_panic(&mut t.tree, self.traversal_state.parent_node_idx, self.traversal_state.current_directory_at_depth); + self.traversal_state.parent_node_idx = parent_or_panic(&mut t.tree, self.traversal_state.parent_node_idx); } - Char('q') if !glob_focussed => { - if let Some(value) = self.handle_quit(&mut tree_view, window) { - return value; - } + let root_size = t.recompute_root_size(); + set_entry_info_or_panic( + &mut t.tree, + t.root_index, + EntryInfo { + size: root_size, + entries_count: (t.entries_traversed > 0).then_some(t.entries_traversed), + }, + ); + t.total_bytes = Some(root_size); + t.elapsed = Some(t.start.elapsed()); + // Ok(Some(t)) + } + } + } + + fn process_event(&mut self, + window: &mut MainWindow, + traversal: &mut Traversal, + display: &mut DisplayOptions, + terminal: &mut Terminal, + event: Event + ) -> Result> + where + B: Backend, + { + use crosstermion::crossterm::event::KeyCode::*; + use FocussedPane::*; + + let key = match event { + Event::Key(key) if key.kind != KeyEventKind::Release => key, + Event::Resize(_, _) => refresh_key(), + _ => return Ok(None), + }; + + self.reset_message(); + + let glob_focussed = self.focussed == Glob; + let mut tree_view = self.tree_view(traversal); + let mut handled = true; + match key.code { + Esc => { + if let Some(value) = self.handle_quit(&mut tree_view, window) { + return Ok(Some(value?)); } - _ => { - handled = false; + } + Tab => { + self.cycle_focus(window); + } + Char('/') if !glob_focussed => { + self.toggle_glob_search(window); + } + Char('?') if !glob_focussed => self.toggle_help_pane(window), + Char('c') if key.modifiers.contains(KeyModifiers::CONTROL) && !glob_focussed => { + return Ok(Some(ProcessingResult::ExitRequested(WalkResult { + num_errors: tree_view.traversal.io_errors, + }))) + } + Char('q') if !glob_focussed => { + if let Some(result) = self.handle_quit(&mut tree_view, window) { + return Ok(Some(result?)); } } + _ => { + handled = false; + } + } - if !handled { - match self.focussed { - Mark => { - self.dispatch_to_mark_pane(key, window, &mut tree_view, *display, terminal) + if !handled { + match self.focussed { + Mark => { + self.dispatch_to_mark_pane(key, window, &mut tree_view, *display, terminal) + } + Help => { + window + .help_pane + .as_mut() + .expect("help pane") + .process_events(key); + } + Glob => { + let glob_pane = window.glob_pane.as_mut().expect("glob pane"); + match key.code { + Enter => self.search_glob_pattern(&mut tree_view, &glob_pane.input), + _ => glob_pane.process_events(key), } - Help => { - window - .help_pane - .as_mut() - .expect("help pane") - .process_events(key); + } + Main => match key.code { + Char('O') => self.open_that(&tree_view), + Char(' ') => self.mark_entry( + CursorMode::KeepPosition, + MarkEntryMode::Toggle, + window, + &tree_view, + ), + Char('x') => self.mark_entry( + CursorMode::Advance, + MarkEntryMode::MarkForDeletion, + window, + &tree_view, + ), + Char('a') => { + self.mark_all_entries(MarkEntryMode::Toggle, window, &tree_view) } - Glob => { - let glob_pane = window.glob_pane.as_mut().expect("glob pane"); - match key.code { - Enter => self.search_glob_pattern(&mut tree_view, &glob_pane.input), - _ => glob_pane.process_events(key), - } + Char('o') | Char('l') | Enter | Right => { + self.enter_node_with_traversal(&tree_view) } - Main => match key.code { - Char('O') => self.open_that(&tree_view), - Char(' ') => self.mark_entry( - CursorMode::KeepPosition, - MarkEntryMode::Toggle, - window, - &tree_view, - ), - Char('x') => self.mark_entry( - CursorMode::Advance, - MarkEntryMode::MarkForDeletion, - window, - &tree_view, - ), - Char('a') => { - self.mark_all_entries(MarkEntryMode::Toggle, window, &tree_view) - } - Char('o') | Char('l') | Enter | Right => { - self.enter_node_with_traversal(&tree_view) - } - Char('H') | Home => self.change_entry_selection(CursorDirection::ToTop), - Char('G') | End => self.change_entry_selection(CursorDirection::ToBottom), - PageUp => self.change_entry_selection(CursorDirection::PageUp), - Char('u') if key.modifiers.contains(KeyModifiers::CONTROL) => { - self.change_entry_selection(CursorDirection::PageUp) - } - Char('k') | Up => self.change_entry_selection(CursorDirection::Up), - Char('j') | Down => self.change_entry_selection(CursorDirection::Down), - PageDown => self.change_entry_selection(CursorDirection::PageDown), - Char('d') if key.modifiers.contains(KeyModifiers::CONTROL) => { - self.change_entry_selection(CursorDirection::PageDown) - } - Char('s') => self.cycle_sorting(&tree_view), - Char('m') => self.cycle_mtime_sorting(&tree_view), - Char('c') => self.cycle_count_sorting(&tree_view), - Char('g') => display.byte_vis.cycle(), - Char('d') => self.mark_entry( - CursorMode::Advance, - MarkEntryMode::Toggle, - window, - &tree_view, - ), - Char('u') | Char('h') | Backspace | Left => { - self.exit_node_with_traversal(&tree_view) - } - _ => {} - }, - }; - } - self.draw(window, &tree_view, *display, terminal)?; + Char('H') | Home => self.change_entry_selection(CursorDirection::ToTop), + Char('G') | End => self.change_entry_selection(CursorDirection::ToBottom), + PageUp => self.change_entry_selection(CursorDirection::PageUp), + Char('u') if key.modifiers.contains(KeyModifiers::CONTROL) => { + self.change_entry_selection(CursorDirection::PageUp) + } + Char('k') | Up => self.change_entry_selection(CursorDirection::Up), + Char('j') | Down => self.change_entry_selection(CursorDirection::Down), + PageDown => self.change_entry_selection(CursorDirection::PageDown), + Char('d') if key.modifiers.contains(KeyModifiers::CONTROL) => { + self.change_entry_selection(CursorDirection::PageDown) + } + Char('s') => self.cycle_sorting(&tree_view), + Char('m') => self.cycle_mtime_sorting(&tree_view), + Char('c') => self.cycle_count_sorting(&tree_view), + Char('g') => display.byte_vis.cycle(), + Char('d') => self.mark_entry( + CursorMode::Advance, + MarkEntryMode::Toggle, + window, + &tree_view, + ), + Char('u') | Char('h') | Backspace | Left => { + self.exit_node_with_traversal(&tree_view) + } + _ => {} + }, + }; } - Ok(ProcessingResult::Finished(WalkResult { - num_errors: traversal.io_errors, - })) + self.draw(window, &tree_view, *display, terminal)?; + + Ok(None) } fn tree_view<'a>(&mut self, traversal: &'a mut Traversal) -> TreeView<'a> { diff --git a/src/interactive/app/terminal_app.rs b/src/interactive/app/terminal_app.rs index 0c96140..01398df 100644 --- a/src/interactive/app/terminal_app.rs +++ b/src/interactive/app/terminal_app.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; use anyhow::Result; use crossbeam::channel::Receiver; @@ -10,10 +10,10 @@ use dua::{ use tui::prelude::Backend; use tui_react::Terminal; -use crate::interactive::widgets::MainWindow; +use crate::{interactive::widgets::MainWindow, crossdev}; use super::{ - app_state::{AppState, ProcessingResult}, + app_state::{AppState, ProcessingResult, TraversalState}, refresh_key, sorted_entries, ByteVisualization, DisplayOptions, }; @@ -23,73 +23,108 @@ pub struct TerminalApp { pub display: DisplayOptions, pub state: AppState, pub window: MainWindow, + pub walk_options: WalkOptions, } -type KeyboardInputAndApp = (crossbeam::channel::Receiver, TerminalApp); +pub type TraversalEntry = Result>)>, jwalk::Error>; -impl TerminalApp { - pub fn refresh_view(&mut self, terminal: &mut Terminal) - where - B: Backend, - { - // Use an event that does nothing to trigger a refresh - self.state - .process_events( - &mut self.window, - &mut self.traversal, - &mut self.display, - terminal, - std::iter::once(Event::Key(refresh_key())), - ) - .ok(); - } - - pub fn process_events( - &mut self, - terminal: &mut Terminal, - events: impl Iterator, - ) -> Result - where - B: Backend, - { - match self.state.process_events( - &mut self.window, - &mut self.traversal, - &mut self.display, - terminal, - events, - )? { - ProcessingResult::Finished(res) | ProcessingResult::ExitRequested(res) => Ok(res), - } - } +pub enum TraversalEvent { + Entry(TraversalEntry, Arc, u64), + Finished(u64), +} - pub fn initialize(terminal: &mut Terminal, byte_format: ByteFormat) -> Result +impl TerminalApp { + pub fn initialize(terminal: &mut Terminal, walk_options: WalkOptions, byte_format: ByteFormat) -> Result where B: Backend, { terminal.hide_cursor()?; terminal.clear()?; - let mut display = DisplayOptions::new(byte_format); - let mut window = MainWindow::default(); - - // #[inline] - // fn fetch_buffered_key_events(keys_rx: &Receiver) -> Vec { - // let mut keys = Vec::new(); - // while let Ok(key) = keys_rx.try_recv() { - // keys.push(key); - // } - // keys - // } + let display = DisplayOptions::new(byte_format); + let window = MainWindow::default(); let mut state = AppState { is_scanning: false, ..Default::default() }; + let traversal = { + let mut tree = Tree::new(); + let root_index = tree.add_node(EntryData::default()); + Traversal { + tree, + root_index, + entries_traversed: 0, + start: std::time::Instant::now(), + elapsed: None, + io_errors: 0, + total_bytes: None, + } + }; + + state.navigation_mut().view_root = traversal.root_index; + state.entries = sorted_entries( + &traversal.tree, + state.navigation().view_root, + state.sorting, + state.glob_root(), + ); + state.navigation_mut().selected = state.entries.first().map(|b| b.index); + + let mut app = TerminalApp { + state, + display, + traversal, + window, + walk_options, + }; + Ok(app) + } + + pub fn scan<'a>(&mut self, input: Vec) -> Result> { + self.state.traversal_state = TraversalState::new(self.traversal.root_index); + + let (entry_tx, entry_rx) = crossbeam::channel::bounded(100); + std::thread::Builder::new() + .name("dua-fs-walk-dispatcher".to_string()) + .spawn({ + let walk_options = self.walk_options.clone(); + let mut io_errors: u64 = 0; + move || { + for root_path in input.into_iter() { + let device_id = match crossdev::init(root_path.as_ref()) { + Ok(id) => id, + Err(_) => { + io_errors += 1; + continue; + } + }; + + let root_path = Arc::new(root_path); + for entry in walk_options + .iter_from_path(root_path.as_ref(), device_id) + .into_iter() + { + if entry_tx + .send(TraversalEvent::Entry(entry, Arc::clone(&root_path), device_id)) + .is_err() + { + // The channel is closed, this means the user has + // requested to quit the app. Abort the walking. + return; + } + } + } + if entry_tx.send(TraversalEvent::Finished(io_errors)).is_err() { + log::error!("Failed to send TraversalEvents::Finished event"); + } + } + })?; + // let mut received_events = false; // let traversal = - // Traversal::from_walk(options, input_paths, &keys_rx, |traversal, event| { + // Traversal::from_walk(options, input_paths, |traversal, event| { // if !received_events { // state.navigation_mut().view_root = traversal.root_index; // } @@ -135,37 +170,28 @@ impl TerminalApp { // if !received_events { // } - let traversal = { - let mut tree = Tree::new(); - let root_index = tree.add_node(EntryData::default()); - Traversal { - tree, - root_index, - entries_traversed: 0, - start: std::time::Instant::now(), - elapsed: None, - io_errors: 0, - total_bytes: None, - } - }; - - state.navigation_mut().view_root = traversal.root_index; - state.entries = sorted_entries( - &traversal.tree, - state.navigation().view_root, - state.sorting, - state.glob_root(), - ); - state.navigation_mut().selected = state.entries.first().map(|b| b.index); + Ok(entry_rx) + } - let mut app = TerminalApp { - state, - display, + pub fn process_events( + &mut self, + terminal: &mut Terminal, + events: Receiver, + traversal: Receiver, + ) -> Result + where + B: Backend, + { + match self.state.process_events( + &mut self.window, + &mut self.traversal, + &mut self.display, + terminal, + &self.walk_options, + events, traversal, - window, - }; - app.refresh_view(terminal); - - Ok(app) + )? { + ProcessingResult::Finished(res) | ProcessingResult::ExitRequested(res) => Ok(res), + } } } diff --git a/src/lib.rs b/src/lib.rs index bd46b4a..6b08772 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ mod aggregate; mod common; mod crossdev; -mod inodefilter; +pub mod inodefilter; pub mod traverse; diff --git a/src/main.rs b/src/main.rs index e579aad..a3351bc 100644 --- a/src/main.rs +++ b/src/main.rs @@ -44,7 +44,7 @@ fn main() -> Result<()> { } let byte_format: dua::ByteFormat = opt.format.into(); - let walk_options = dua::WalkOptions { + let mut walk_options = dua::WalkOptions { threads: opt.threads, apparent_size: opt.apparent_size, count_hard_links: opt.count_hard_links, @@ -52,6 +52,13 @@ fn main() -> Result<()> { cross_filesystems: !opt.stay_on_filesystem, ignore_dirs: canonicalize_ignore_dirs(&opt.ignore_dirs), }; + + if walk_options.threads == 0 { + // avoid using the global rayon pool, as it will keep a lot of threads alive after we are done. + // Also means that we will spin up a bunch of threads per root path, instead of reusing them. + walk_options.threads = num_cpus::get(); + } + let res = match opt.command { #[cfg(feature = "tui-crossplatform")] Some(Interactive { input }) => { @@ -68,13 +75,12 @@ fn main() -> Result<()> { ) .with_context(|| "Could not instantiate terminal")?; - // TODO: use - // extract_paths_maybe_set_cwd(input, !opt.stay_on_filesystem)?, - let keys_rx = input_channel(); - let mut app = TerminalApp::initialize(&mut terminal, byte_format)?; + let mut app = TerminalApp::initialize(&mut terminal, walk_options, byte_format)?; - let res = app.process_events(&mut terminal, keys_rx.into_iter()); + let traversal_rx = app.scan(extract_paths_maybe_set_cwd(input, !opt.stay_on_filesystem)?)?; + + let res = app.process_events(&mut terminal, keys_rx, traversal_rx); let res = res.map(|r| { ( diff --git a/src/traverse.rs b/src/traverse.rs index 69c1639..8838438 100644 --- a/src/traverse.rs +++ b/src/traverse.rs @@ -71,283 +71,254 @@ pub struct Traversal { } impl Traversal { - pub fn from_walk( - mut walk_options: WalkOptions, - input: Vec, - waker_rx: &crossbeam::channel::Receiver, - mut update: impl FnMut(&mut Traversal, Option) -> Result, - ) -> Result> { - #[derive(Default, Copy, Clone)] - struct EntryInfo { - size: u128, - entries_count: Option, - } - impl EntryInfo { - fn add_count(&mut self, other: &Self) { - self.entries_count = match (self.entries_count, other.entries_count) { - (Some(a), Some(b)) => Some(a + b), - (None, Some(b)) => Some(b), - (Some(a), None) => Some(a), - (None, None) => None, - }; - } - } - fn set_entry_info_or_panic( - tree: &mut Tree, - node_idx: TreeIndex, - EntryInfo { - size, - entries_count, - }: EntryInfo, - ) { - let node = tree - .node_weight_mut(node_idx) - .expect("node for parent index we just retrieved"); - node.size = size; - node.entry_count = entries_count; - } - fn parent_or_panic(tree: &mut Tree, parent_node_idx: TreeIndex) -> TreeIndex { - tree.neighbors_directed(parent_node_idx, Direction::Incoming) - .next() - .expect("every node in the iteration has a parent") - } - fn pop_or_panic(v: &mut Vec) -> EntryInfo { - v.pop().expect("sizes per level to be in sync with graph") - } + // pub fn from_walk( + // mut walk_options: WalkOptions, + // input: Vec, + // mut update: impl FnMut(&mut Traversal, Option) -> Result, + // ) -> Result> { + - let mut t = { - let mut tree = Tree::new(); - let root_index = tree.add_node(EntryData::default()); - Traversal { - tree, - root_index, - entries_traversed: 0, - start: std::time::Instant::now(), - elapsed: None, - io_errors: 0, - total_bytes: None, - } - }; - - let (mut previous_node_idx, mut parent_node_idx) = (t.root_index, t.root_index); - let mut directory_info_per_depth_level = Vec::new(); - let mut current_directory_at_depth = EntryInfo::default(); - let mut previous_depth = 0; - let mut inodes = InodeFilter::default(); - - let throttle = Throttle::new(Duration::from_millis(250), None); - if walk_options.threads == 0 { - // avoid using the global rayon pool, as it will keep a lot of threads alive after we are done. - // Also means that we will spin up a bunch of threads per root path, instead of reusing them. - walk_options.threads = num_cpus::get(); - } + // let mut t = { + // let mut tree = Tree::new(); + // let root_index = tree.add_node(EntryData::default()); + // Traversal { + // tree, + // root_index, + // entries_traversed: 0, + // start: std::time::Instant::now(), + // elapsed: None, + // io_errors: 0, + // total_bytes: None, + // } + // }; - #[cfg(not(windows))] - fn size_on_disk(_parent: &Path, name: &Path, meta: &Metadata) -> io::Result { - name.size_on_disk_fast(meta) - } - #[cfg(windows)] - fn size_on_disk(parent: &Path, name: &Path, meta: &Metadata) -> io::Result { - parent.join(name).size_on_disk_fast(meta) - } + // let (mut previous_node_idx, mut parent_node_idx) = (t.root_index, t.root_index); + // let mut directory_info_per_depth_level = Vec::new(); + // let mut current_directory_at_depth = EntryInfo::default(); + // let mut previous_depth = 0; + // let mut inodes = InodeFilter::default(); - let (entry_tx, entry_rx) = crossbeam::channel::bounded(100); - std::thread::Builder::new() - .name("dua-fs-walk-dispatcher".to_string()) - .spawn({ - let walk_options = walk_options.clone(); - move || { - for root_path in input.into_iter() { - let device_id = match crossdev::init(root_path.as_ref()) { - Ok(id) => id, - Err(_) => { - t.io_errors += 1; - continue; - } - }; - - let root_path = Arc::new(root_path); - for entry in walk_options - .iter_from_path(root_path.as_ref(), device_id) - .into_iter() - { - if entry_tx - .send((entry, Arc::clone(&root_path), device_id)) - .is_err() - { - // The channel is closed, this means the user has - // requested to quit the app. Abort the walking. - return; - } - } - } - } - })?; - - loop { - crossbeam::select! { - recv(entry_rx) -> entry => { - let Ok((entry, root_path, device_id)) = entry else { - break; - }; - - t.entries_traversed += 1; - let mut data = EntryData::default(); - match entry { - Ok(entry) => { - data.name = if entry.depth < 1 { - (*root_path).clone() - } else { - entry.file_name.into() - }; - - let mut file_size = 0u128; - let mut mtime: SystemTime = UNIX_EPOCH; - match &entry.client_state { - Some(Ok(ref m)) => { - if !m.is_dir() - && (walk_options.count_hard_links || inodes.add(m)) - && (walk_options.cross_filesystems - || crossdev::is_same_device(device_id, m)) - { - if walk_options.apparent_size { - file_size = m.len() as u128; - } else { - file_size = size_on_disk(&entry.parent_path, &data.name, m) - .unwrap_or_else(|_| { - t.io_errors += 1; - data.metadata_io_error = true; - 0 - }) - as u128; - } - } else { - data.entry_count = Some(0); - data.is_dir = true; - } - - match m.modified() { - Ok(modified) => { - mtime = modified; - } - Err(_) => { - t.io_errors += 1; - data.metadata_io_error = true; - } - } - } - Some(Err(_)) => { - t.io_errors += 1; - data.metadata_io_error = true; - } - None => {} - } - - match (entry.depth, previous_depth) { - (n, p) if n > p => { - directory_info_per_depth_level.push(current_directory_at_depth); - current_directory_at_depth = EntryInfo { - size: file_size, - entries_count: Some(1), - }; - parent_node_idx = previous_node_idx; - } - (n, p) if n < p => { - for _ in n..p { - set_entry_info_or_panic( - &mut t.tree, - parent_node_idx, - current_directory_at_depth, - ); - let dir_info = - pop_or_panic(&mut directory_info_per_depth_level); - - current_directory_at_depth.size += dir_info.size; - current_directory_at_depth.add_count(&dir_info); - - parent_node_idx = parent_or_panic(&mut t.tree, parent_node_idx); - } - current_directory_at_depth.size += file_size; - *current_directory_at_depth.entries_count.get_or_insert(0) += 1; - set_entry_info_or_panic( - &mut t.tree, - parent_node_idx, - current_directory_at_depth, - ); - } - _ => { - current_directory_at_depth.size += file_size; - *current_directory_at_depth.entries_count.get_or_insert(0) += 1; - } - }; - - data.mtime = mtime; - data.size = file_size; - let entry_index = t.tree.add_node(data); - - t.tree.add_edge(parent_node_idx, entry_index, ()); - previous_node_idx = entry_index; - previous_depth = entry.depth; - } - Err(_) => { - if previous_depth == 0 { - data.name = (*root_path).clone(); - let entry_index = t.tree.add_node(data); - t.tree.add_edge(parent_node_idx, entry_index, ()); - } - - t.io_errors += 1 - } - } - - if throttle.can_update() && update(&mut t, None)? { - return Ok(None); - } - }, - recv(waker_rx) -> waker_value => { - let Ok(waker_value) = waker_value else { - continue; - }; - if update(&mut t, Some(waker_value))? { - return Ok(None); - } - }, - default(Duration::from_millis(250)) => { - // No events or new entries received, but we still need - // to keep updating the status message regularly. - if update(&mut t, None)? { - return Ok(None); - } - } - } - } + // let throttle = Throttle::new(Duration::from_millis(250), None); + // // if walk_options.threads == 0 { + // // // avoid using the global rayon pool, as it will keep a lot of threads alive after we are done. + // // // Also means that we will spin up a bunch of threads per root path, instead of reusing them. + // // walk_options.threads = num_cpus::get(); + // // } - directory_info_per_depth_level.push(current_directory_at_depth); - current_directory_at_depth = EntryInfo::default(); - for _ in 0..previous_depth { - let dir_info = pop_or_panic(&mut directory_info_per_depth_level); - current_directory_at_depth.size += dir_info.size; - current_directory_at_depth.add_count(&dir_info); + // // #[cfg(not(windows))] + // // fn size_on_disk(_parent: &Path, name: &Path, meta: &Metadata) -> io::Result { + // // name.size_on_disk_fast(meta) + // // } + // // #[cfg(windows)] + // // fn size_on_disk(parent: &Path, name: &Path, meta: &Metadata) -> io::Result { + // // parent.join(name).size_on_disk_fast(meta) + // // } - set_entry_info_or_panic(&mut t.tree, parent_node_idx, current_directory_at_depth); - parent_node_idx = parent_or_panic(&mut t.tree, parent_node_idx); - } - let root_size = t.recompute_root_size(); - set_entry_info_or_panic( - &mut t.tree, - t.root_index, - EntryInfo { - size: root_size, - entries_count: (t.entries_traversed > 0).then_some(t.entries_traversed), - }, - ); - t.total_bytes = Some(root_size); + // // enum TraversalEvents { + // // Entry(Result>)>, jwalk::Error>, Arc, u64), + // // Finished, + // // } - t.elapsed = Some(t.start.elapsed()); - Ok(Some(t)) - } + // let (entry_tx, entry_rx) = crossbeam::channel::bounded(100); + // std::thread::Builder::new() + // .name("dua-fs-walk-dispatcher".to_string()) + // .spawn({ + // let walk_options = walk_options.clone(); + // move || { + // for root_path in input.into_iter() { + // let device_id = match crossdev::init(root_path.as_ref()) { + // Ok(id) => id, + // Err(_) => { + // t.io_errors += 1; + // continue; + // } + // }; + + // let root_path = Arc::new(root_path); + // for entry in walk_options + // .iter_from_path(root_path.as_ref(), device_id) + // .into_iter() + // { + // if entry_tx + // .send(TraversalEvents::Entry(entry, Arc::clone(&root_path), device_id)) + // .is_err() + // { + // // The channel is closed, this means the user has + // // requested to quit the app. Abort the walking. + // return; + // } + // } + // } + // if entry_tx.send(TraversalEvents::Finished).is_err() { + // log::error!("Failed to send TraversalEvents::Finished event"); + // } + // } + // })?; + + // // loop { + // // crossbeam::select! { + // // recv(entry_rx) -> entry => { + // // let Ok((entry, root_path, device_id)) = entry else { + // // break; + // // }; + + // // t.entries_traversed += 1; + // // let mut data = EntryData::default(); + // // match entry { + // // Ok(entry) => { + // // data.name = if entry.depth < 1 { + // // (*root_path).clone() + // // } else { + // // entry.file_name.into() + // // }; + + // // let mut file_size = 0u128; + // // let mut mtime: SystemTime = UNIX_EPOCH; + // // match &entry.client_state { + // // Some(Ok(ref m)) => { + // // if !m.is_dir() + // // && (walk_options.count_hard_links || inodes.add(m)) + // // && (walk_options.cross_filesystems + // // || crossdev::is_same_device(device_id, m)) + // // { + // // if walk_options.apparent_size { + // // file_size = m.len() as u128; + // // } else { + // // file_size = size_on_disk(&entry.parent_path, &data.name, m) + // // .unwrap_or_else(|_| { + // // t.io_errors += 1; + // // data.metadata_io_error = true; + // // 0 + // // }) + // // as u128; + // // } + // // } else { + // // data.entry_count = Some(0); + // // data.is_dir = true; + // // } - fn recompute_root_size(&self) -> u128 { + // // match m.modified() { + // // Ok(modified) => { + // // mtime = modified; + // // } + // // Err(_) => { + // // t.io_errors += 1; + // // data.metadata_io_error = true; + // // } + // // } + // // } + // // Some(Err(_)) => { + // // t.io_errors += 1; + // // data.metadata_io_error = true; + // // } + // // None => {} + // // } + + // // match (entry.depth, previous_depth) { + // // (n, p) if n > p => { + // // directory_info_per_depth_level.push(current_directory_at_depth); + // // current_directory_at_depth = EntryInfo { + // // size: file_size, + // // entries_count: Some(1), + // // }; + // // parent_node_idx = previous_node_idx; + // // } + // // (n, p) if n < p => { + // // for _ in n..p { + // // set_entry_info_or_panic( + // // &mut t.tree, + // // parent_node_idx, + // // current_directory_at_depth, + // // ); + // // let dir_info = + // // pop_or_panic(&mut directory_info_per_depth_level); + + // // current_directory_at_depth.size += dir_info.size; + // // current_directory_at_depth.add_count(&dir_info); + + // // parent_node_idx = parent_or_panic(&mut t.tree, parent_node_idx); + // // } + // // current_directory_at_depth.size += file_size; + // // *current_directory_at_depth.entries_count.get_or_insert(0) += 1; + // // set_entry_info_or_panic( + // // &mut t.tree, + // // parent_node_idx, + // // current_directory_at_depth, + // // ); + // // } + // // _ => { + // // current_directory_at_depth.size += file_size; + // // *current_directory_at_depth.entries_count.get_or_insert(0) += 1; + // // } + // // }; + + // // data.mtime = mtime; + // // data.size = file_size; + // // let entry_index = t.tree.add_node(data); + + // // t.tree.add_edge(parent_node_idx, entry_index, ()); + // // previous_node_idx = entry_index; + // // previous_depth = entry.depth; + // // } + // // Err(_) => { + // // if previous_depth == 0 { + // // data.name = (*root_path).clone(); + // // let entry_index = t.tree.add_node(data); + // // t.tree.add_edge(parent_node_idx, entry_index, ()); + // // } + + // // t.io_errors += 1 + // // } + // // } + + // // if throttle.can_update() && update(&mut t, None)? { + // // return Ok(None); + // // } + // // }, + // // recv(waker_rx) -> waker_value => { + // // let Ok(waker_value) = waker_value else { + // // continue; + // // }; + // // if update(&mut t, Some(waker_value))? { + // // return Ok(None); + // // } + // // }, + // // default(Duration::from_millis(250)) => { + // // // No events or new entries received, but we still need + // // // to keep updating the status message regularly. + // // if update(&mut t, None)? { + // // return Ok(None); + // // } + // // } + // // } + // // } + + // directory_info_per_depth_level.push(current_directory_at_depth); + // current_directory_at_depth = EntryInfo::default(); + // for _ in 0..previous_depth { + // let dir_info = pop_or_panic(&mut directory_info_per_depth_level); + // current_directory_at_depth.size += dir_info.size; + // current_directory_at_depth.add_count(&dir_info); + + // set_entry_info_or_panic(&mut t.tree, parent_node_idx, current_directory_at_depth); + // parent_node_idx = parent_or_panic(&mut t.tree, parent_node_idx); + // } + // let root_size = t.recompute_root_size(); + // set_entry_info_or_panic( + // &mut t.tree, + // t.root_index, + // EntryInfo { + // size: root_size, + // entries_count: (t.entries_traversed > 0).then_some(t.entries_traversed), + // }, + // ); + // t.total_bytes = Some(root_size); + + // t.elapsed = Some(t.start.elapsed()); + // Ok(Some(t)) + // } + + pub fn recompute_root_size(&self) -> u128 { self.tree .neighbors_directed(self.root_index, Direction::Outgoing) .map(|idx| get_size_or_panic(&self.tree, idx)) @@ -355,6 +326,16 @@ impl Traversal { } } +#[cfg(not(windows))] +pub fn size_on_disk(_parent: &Path, name: &Path, meta: &Metadata) -> io::Result { + name.size_on_disk_fast(meta) +} + +#[cfg(windows)] +pub fn size_on_disk(parent: &Path, name: &Path, meta: &Metadata) -> io::Result { + parent.join(name).size_on_disk_fast(meta) +} + #[cfg(test)] mod tests { use super::*; -- cgit v1.2.3