From 1d8ba524ac83a0c3b5e4146cf937ed75650f1e97 Mon Sep 17 00:00:00 2001 From: Thomas Hurst Date: Wed, 1 Jul 2020 20:28:38 +0000 Subject: Use u128 for byte sizes Per issue #58, u64 is insufficient for use with very large sparse files. Enormous file sizes are also a common filesystem error trope, either from disk corruption or software bugs, and they're also conceivable with virtual filesystems. Handle this as gracefully as can be reasonably expected using 128-bit integers, which should be sufficient for most uses. --- src/aggregate.rs | 12 ++++++------ src/common.rs | 8 ++++---- src/interactive/app_test/utils.rs | 4 ++-- src/interactive/widgets/entries.rs | 2 +- src/interactive/widgets/footer.rs | 2 +- src/interactive/widgets/mark.rs | 4 ++-- src/traverse.rs | 14 +++++++------- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/aggregate.rs b/src/aggregate.rs index b7ef995..fdc6694 100644 --- a/src/aggregate.rs +++ b/src/aggregate.rs @@ -17,7 +17,7 @@ pub fn aggregate( ) -> Result<(WalkResult, Statistics), Error> { let mut res = WalkResult::default(); let mut stats = Statistics::default(); - stats.smallest_file_in_bytes = u64::max_value(); + stats.smallest_file_in_bytes = u128::max_value(); let mut total = 0; let mut num_roots = 0; let mut aggregates = Vec::new(); @@ -25,7 +25,7 @@ pub fn aggregate( let paths: Vec<_> = paths.into_iter().collect(); for path in paths.into_iter() { num_roots += 1; - let mut num_bytes = 0u64; + let mut num_bytes = 0u128; let mut num_errors = 0u64; let device_id = crossdev::init(path.as_ref())?; for entry in walk_options.iter_from_path(path.as_ref()) { @@ -54,7 +54,7 @@ pub fn aggregate( 0 } None => unreachable!("must have populated client state for metadata"), - }; + } as u128; stats.largest_file_in_bytes = stats.largest_file_in_bytes.max(file_size); stats.smallest_file_in_bytes = stats.smallest_file_in_bytes.min(file_size); num_bytes += file_size; @@ -122,7 +122,7 @@ fn write_path( out: &mut impl io::Write, options: &WalkOptions, path: impl AsRef, - num_bytes: u64, + num_bytes: u128, num_errors: u64, path_color: C, ) -> Result<(), io::Error> { @@ -154,7 +154,7 @@ pub struct Statistics { /// The amount of entries we have seen during filesystem traversal pub entries_traversed: u64, /// The size of the smallest file encountered in bytes - pub smallest_file_in_bytes: u64, + pub smallest_file_in_bytes: u128, /// The size of the largest file encountered in bytes - pub largest_file_in_bytes: u64, + pub largest_file_in_bytes: u128, } diff --git a/src/common.rs b/src/common.rs index e5cf697..d297c45 100644 --- a/src/common.rs +++ b/src/common.rs @@ -7,7 +7,7 @@ pub fn get_entry_or_panic(tree: &Tree, node_idx: TreeIndex) -> &EntryData { .expect("node should always be retrievable with valid index") } -pub(crate) fn get_size_or_panic(tree: &Tree, node_idx: TreeIndex) -> u64 { +pub(crate) fn get_size_or_panic(tree: &Tree, node_idx: TreeIndex) -> u128 { get_entry_or_panic(tree, node_idx).size } @@ -52,7 +52,7 @@ impl ByteFormat { } + THE_SPACE_BETWEEN_UNIT_AND_NUMBER } - pub fn display(self, bytes: u64) -> ByteFormatDisplay { + pub fn display(self, bytes: u128) -> ByteFormatDisplay { ByteFormatDisplay { format: self, bytes, @@ -62,7 +62,7 @@ impl ByteFormat { pub struct ByteFormatDisplay { format: ByteFormat, - bytes: u64, + bytes: u128, } impl fmt::Display for ByteFormatDisplay { @@ -84,7 +84,7 @@ impl fmt::Display for ByteFormatDisplay { (_, Some((divisor, unit))) => Byte::from_unit(self.bytes as f64 / divisor as f64, unit) .expect("byte count > 0") .get_adjusted_unit(unit), - (binary, None) => Byte::from_bytes(u128::from(self.bytes)).get_appropriate_unit(binary), + (binary, None) => Byte::from_bytes(self.bytes).get_appropriate_unit(binary), } .format(2); let mut splits = b.split(' '); diff --git a/src/interactive/app_test/utils.rs b/src/interactive/app_test/utils.rs index 57854d9..ed94720 100644 --- a/src/interactive/app_test/utils.rs +++ b/src/interactive/app_test/utils.rs @@ -33,7 +33,7 @@ pub fn node_by_name(app: &TerminalApp, name: impl AsRef) -> &EntryData { pub fn index_by_name_and_size( app: &TerminalApp, name: impl AsRef, - size: Option, + size: Option, ) -> TreeIndex { let name = name.as_ref(); let t: Vec<_> = app @@ -276,7 +276,7 @@ pub fn sample_02_tree() -> Tree { pub fn make_add_node<'a>( t: &'a mut Tree, -) -> impl FnMut(&str, u64, Option) -> NodeIndex + 'a { +) -> impl FnMut(&str, u128, Option) -> NodeIndex + 'a { move |name, size, maybe_from_idx| { let n = t.add_node(EntryData { name: PathBuf::from(name), diff --git a/src/interactive/widgets/entries.rs b/src/interactive/widgets/entries.rs index fc68f44..279a7fa 100644 --- a/src/interactive/widgets/entries.rs +++ b/src/interactive/widgets/entries.rs @@ -60,7 +60,7 @@ impl Entries { .is_none() }; - let total: u64 = entries.iter().map(|b| b.data.size).sum(); + let total: u128 = entries.iter().map(|b| b.data.size).sum(); let title = match path_of(tree, *root).to_string_lossy().to_string() { ref p if p.is_empty() => Path::new(".") .canonicalize() diff --git a/src/interactive/widgets/footer.rs b/src/interactive/widgets/footer.rs index 461580c..190c6c9 100644 --- a/src/interactive/widgets/footer.rs +++ b/src/interactive/widgets/footer.rs @@ -11,7 +11,7 @@ use tui::{ pub struct Footer; pub struct FooterProps { - pub total_bytes: Option, + pub total_bytes: Option, pub entries_traversed: u64, pub format: ByteFormat, pub message: Option, diff --git a/src/interactive/widgets/mark.rs b/src/interactive/widgets/mark.rs index 1f59738..3c26a3a 100644 --- a/src/interactive/widgets/mark.rs +++ b/src/interactive/widgets/mark.rs @@ -31,7 +31,7 @@ pub enum MarkMode { pub type EntryMarkMap = BTreeMap; pub struct EntryMark { - pub size: u64, + pub size: u128, pub path: PathBuf, pub index: usize, pub num_errors_during_deletion: usize, @@ -230,7 +230,7 @@ impl MarkPane { let title = format!( "Marked {} items ({}) ", marked.len(), - format.display(marked.iter().map(|(_k, v)| v.size).sum::()) + format.display(marked.iter().map(|(_k, v)| v.size).sum::()) ); let selected = self.selected; let has_focus = self.has_focus; diff --git a/src/traverse.rs b/src/traverse.rs index 5027b69..d19dd9e 100644 --- a/src/traverse.rs +++ b/src/traverse.rs @@ -11,7 +11,7 @@ pub type Tree = StableGraph; pub struct EntryData { pub name: PathBuf, /// The entry's size in bytes. If it's a directory, the size is the aggregated file size of all children - pub size: u64, + pub size: u128, /// If set, the item meta-data could not be obtained pub metadata_io_error: bool, } @@ -30,7 +30,7 @@ pub struct Traversal { /// Total amount of IO errors encountered when traversing the filesystem pub io_errors: u64, /// Total amount of bytes seen during the traversal - pub total_bytes: Option, + pub total_bytes: Option, } impl Traversal { @@ -39,7 +39,7 @@ impl Traversal { input: Vec, mut update: impl FnMut(&mut Traversal) -> Result, ) -> Result, Error> { - fn set_size_or_panic(tree: &mut Tree, node_idx: TreeIndex, current_size_at_depth: u64) { + fn set_size_or_panic(tree: &mut Tree, node_idx: TreeIndex, current_size_at_depth: u128) { tree.node_weight_mut(node_idx) .expect("node for parent index we just retrieved") .size = current_size_at_depth; @@ -49,7 +49,7 @@ impl Traversal { .next() .expect("every node in the iteration has a parent") } - fn pop_or_panic(v: &mut Vec) -> u64 { + fn pop_or_panic(v: &mut Vec) -> u128 { v.pop().expect("sizes per level to be in sync with graph") } @@ -65,7 +65,7 @@ impl Traversal { let (mut previous_node_idx, mut parent_node_idx) = (t.root_index, t.root_index); let mut sizes_per_depth_level = Vec::new(); - let mut current_size_at_depth = 0; + let mut current_size_at_depth: u128 = 0; let mut previous_depth = 0; let mut inodes = InodeFilter::default(); @@ -119,7 +119,7 @@ impl Traversal { 0 } None => unreachable!("must have populated client state for metadata"), - }; + } as u128; match (entry.depth, previous_depth) { (n, p) if n > p => { @@ -202,7 +202,7 @@ impl Traversal { Ok(Some(t)) } - fn recompute_root_size(&self) -> u64 { + fn recompute_root_size(&self) -> u128 { self.tree .neighbors_directed(self.root_index, Direction::Outgoing) .map(|idx| get_size_or_panic(&self.tree, idx)) -- cgit v1.2.3