diff --git a/src/fs_tree_builder.rs b/src/fs_tree_builder.rs index e942a630..d414af7e 100644 --- a/src/fs_tree_builder.rs +++ b/src/fs_tree_builder.rs @@ -66,12 +66,19 @@ where max_depth, } = builder; - TreeBuilder:: { + TreeBuilder:: { name: OsStringDisplay::os_string_from(&root), path: root, get_info: |path| { + let make_info = |size: Size, children: Option<_>| Info { + size, + children: children.into_iter().flatten(), + }; + + let empty_info = || make_info(Size::default(), None); + let stats = match symlink_metadata(path) { Err(error) => { reporter.report(Event::EncounterError(ErrorReport { @@ -79,15 +86,12 @@ where path, error, })); - return Info { - size: Size::default(), - children: Vec::new(), - }; + return empty_info(); } Ok(stats) => stats, }; - let children: Vec<_> = if stats.is_dir() { + let children = if stats.is_dir() { match read_dir(path) { Err(error) => { reporter.report(Event::EncounterError(ErrorReport { @@ -95,30 +99,22 @@ where path, error, })); - return Info::default(); + return empty_info(); } Ok(entries) => entries, } - .filter_map(|entry| match entry { - Err(error) => { - reporter.report(Event::EncounterError(ErrorReport { - operation: AccessEntry, - path, - error, - })); - None - } - Ok(entry) => entry.file_name().pipe(OsStringDisplay::from).pipe(Some), - }) - .collect() + .flatten() + .map(|entry| entry.file_name()) + .map(OsStringDisplay::from) + .pipe(Some) } else { - Vec::new() + None }; let size = size_getter.get_size(&stats); reporter.report(Event::ReceiveData(size)); - Info { size, children } + make_info(size, children) }, join_path: |prefix, name| prefix.join(&name.0), diff --git a/src/reporter/error_report/operation.rs b/src/reporter/error_report/operation.rs index 55850632..1d16defc 100644 --- a/src/reporter/error_report/operation.rs +++ b/src/reporter/error_report/operation.rs @@ -5,8 +5,6 @@ pub enum Operation { SymlinkMetadata, /// Error is caused by calling [`std::fs::read_dir`]. ReadDirectory, - /// Error when trying to access [`std::fs::DirEntry`] of one of the element of [`std::fs::read_dir`]. - AccessEntry, } impl Operation { @@ -16,7 +14,6 @@ impl Operation { match self { SymlinkMetadata => "symlink_metadata", ReadDirectory => "read_dir", - AccessEntry => "access entry", } } } @@ -36,5 +33,4 @@ mod test_operation { name_display!(symlink_metadata, SymlinkMetadata, "symlink_metadata"); name_display!(read_directory, ReadDirectory, "read_dir"); - name_display!(access_entry, AccessEntry, "access entry"); } diff --git a/src/tree_builder.rs b/src/tree_builder.rs index c6e1494d..c0982bbc 100644 --- a/src/tree_builder.rs +++ b/src/tree_builder.rs @@ -3,22 +3,25 @@ pub mod info; pub use info::Info; use super::{data_tree::DataTree, size}; +use itertools::Itertools; use rayon::prelude::*; /// Collection of functions and starting points in order to build a [`DataTree`] with [`From`] or [`Into`]. #[derive(Debug)] -pub struct TreeBuilder +pub struct TreeBuilder where Path: Send + Sync, - Name: Send + Sync, - GetInfo: Fn(&Path) -> Info + Copy + Send + Sync, - JoinPath: Fn(&Path, &Name) -> Path + Copy + Send + Sync, + NameIter: IntoIterator, + NameIter::IntoIter: Send, + NameIter::Item: Send, + GetInfo: Fn(&Path) -> Info + Copy + Send + Sync, + JoinPath: Fn(&Path, &NameIter::Item) -> Path + Copy + Send + Sync, Size: size::Size + Send, { /// Path to the root. pub path: Path, /// Name of the root. - pub name: Name, + pub name: NameIter::Item, /// Function to extract necessary information from `path` (`size` and `children`). pub get_info: GetInfo, /// Function to join parent's `path` with a child's name to make the child's `name`. @@ -27,17 +30,19 @@ where pub max_depth: u64, } -impl From> - for DataTree +impl + From> for DataTree where Path: Send + Sync, - Name: Send + Sync, - GetInfo: Fn(&Path) -> Info + Copy + Send + Sync, - JoinPath: Fn(&Path, &Name) -> Path + Copy + Send + Sync, + NameIter: IntoIterator, + NameIter::IntoIter: Send, + NameIter::Item: Send, + GetInfo: Fn(&Path) -> Info + Copy + Send + Sync, + JoinPath: Fn(&Path, &NameIter::Item) -> Path + Copy + Send + Sync, Size: size::Size + Send, { /// Create a [`DataTree`] from a [`TreeBuilder`]. - fn from(builder: TreeBuilder) -> Self { + fn from(builder: TreeBuilder) -> Self { let TreeBuilder { path, name, @@ -50,20 +55,32 @@ where let max_depth = max_depth.saturating_sub(1); let children = children - .into_par_iter() - .map(|name| TreeBuilder { - path: join_path(&path, &name), - name, - get_info, - join_path, - max_depth, + .into_iter() + .chunks(64) + .into_iter() + .map(Vec::::from_iter) + .map(|names| -> Vec<_> { + names + .into_par_iter() + .map(|name| TreeBuilder { + path: join_path(&path, &name), + name, + get_info, + join_path, + max_depth, + }) + .map(Self::from) + .collect() }) - .map(Self::from); + .fold(Vec::new(), |mut a, b| { + a.extend(b); + a + }); if max_depth > 0 { - DataTree::dir(name, size, children.collect()) + DataTree::dir(name, size, children) } else { - let size = size + children.map(|child| child.size()).sum(); + let size = size + children.into_iter().map(|child| child.size()).sum(); DataTree::dir(name, size, Vec::new()) } } diff --git a/src/tree_builder/info.rs b/src/tree_builder/info.rs index c6a72d28..898052d3 100644 --- a/src/tree_builder/info.rs +++ b/src/tree_builder/info.rs @@ -1,12 +1,11 @@ use crate::size; use derive_more::From; -use smart_default::SmartDefault; /// Information to return from `get_info` of [`super::TreeBuilder`]. -#[derive(Debug, SmartDefault, From)] -pub struct Info { +#[derive(Debug, Default, From)] +pub struct Info { /// Size associated with given `path`. pub size: Size, /// Direct descendants of given `path`. - pub children: Vec, + pub children: NameIter, }