perf!: eagerly process each 64 tasks

src/fs_tree_builder.rs

@@ -66,59 +66,55 @@ where
             max_depth,
         } = builder;
 
-        TreeBuilder::<PathBuf, OsStringDisplay, Size, _, _> {
+        TreeBuilder::<PathBuf, _, Size, _, _> {
             name: OsStringDisplay::os_string_from(&root),
 
             path: root,
 
             get_info: |path| {
+                let make_info = |size: Size, children: Option<_>| Info {
+                    size,
+                    children: children.into_iter().flatten(),
+                };
+
+                let empty_info = || make_info(Size::default(), None);
+
                 let stats = match symlink_metadata(path) {
                     Err(error) => {
                         reporter.report(Event::EncounterError(ErrorReport {
                             operation: SymlinkMetadata,
                             path,
                             error,
                         }));
-                        return Info {
-                            size: Size::default(),
-                            children: Vec::new(),
-                        };
+                        return empty_info();
                     }
                     Ok(stats) => stats,
                 };
 
-                let children: Vec<_> = if stats.is_dir() {
+                let children = if stats.is_dir() {
                     match read_dir(path) {
                         Err(error) => {
                             reporter.report(Event::EncounterError(ErrorReport {
                                 operation: ReadDirectory,
                                 path,
                                 error,
                             }));
-                            return Info::default();
+                            return empty_info();
                         }
                         Ok(entries) => entries,
                     }
-                    .filter_map(|entry| match entry {
-                        Err(error) => {
-                            reporter.report(Event::EncounterError(ErrorReport {
-                                operation: AccessEntry,
-                                path,
-                                error,
-                            }));
-                            None
-                        }
-                        Ok(entry) => entry.file_name().pipe(OsStringDisplay::from).pipe(Some),
-                    })
-                    .collect()
+                    .flatten()
+                    .map(|entry| entry.file_name())
+                    .map(OsStringDisplay::from)
+                    .pipe(Some)
                 } else {
-                    Vec::new()
+                    None
                 };
 
                 let size = size_getter.get_size(&stats);
                 reporter.report(Event::ReceiveData(size));
 
-                Info { size, children }
+                make_info(size, children)
             },
 
             join_path: |prefix, name| prefix.join(&name.0),

src/reporter/error_report/operation.rs

@@ -5,8 +5,6 @@ pub enum Operation {
     SymlinkMetadata,
     /// Error is caused by calling [`std::fs::read_dir`].
     ReadDirectory,
-    /// Error when trying to access [`std::fs::DirEntry`] of one of the element of [`std::fs::read_dir`].
-    AccessEntry,
 }
 
 impl Operation {
@@ -16,7 +14,6 @@ impl Operation {
         match self {
             SymlinkMetadata => "symlink_metadata",
             ReadDirectory => "read_dir",
-            AccessEntry => "access entry",
         }
     }
 }
@@ -36,5 +33,4 @@ mod test_operation {
 
     name_display!(symlink_metadata, SymlinkMetadata, "symlink_metadata");
     name_display!(read_directory, ReadDirectory, "read_dir");
-    name_display!(access_entry, AccessEntry, "access entry");
 }

src/tree_builder.rs

@@ -3,22 +3,25 @@ pub mod info;
 pub use info::Info;
 
 use super::{data_tree::DataTree, size};
+use itertools::Itertools;
 use rayon::prelude::*;
 
 /// Collection of functions and starting points in order to build a [`DataTree`] with [`From`] or [`Into`].
 #[derive(Debug)]
-pub struct TreeBuilder<Path, Name, Size, GetInfo, JoinPath>
+pub struct TreeBuilder<Path, NameIter, Size, GetInfo, JoinPath>
 where
     Path: Send + Sync,
-    Name: Send + Sync,
-    GetInfo: Fn(&Path) -> Info<Name, Size> + Copy + Send + Sync,
-    JoinPath: Fn(&Path, &Name) -> Path + Copy + Send + Sync,
+    NameIter: IntoIterator,
+    NameIter::IntoIter: Send,
+    NameIter::Item: Send,
+    GetInfo: Fn(&Path) -> Info<NameIter, Size> + Copy + Send + Sync,
+    JoinPath: Fn(&Path, &NameIter::Item) -> Path + Copy + Send + Sync,
     Size: size::Size + Send,
 {
     /// Path to the root.
     pub path: Path,
     /// Name of the root.
-    pub name: Name,
+    pub name: NameIter::Item,
     /// Function to extract necessary information from `path` (`size` and `children`).
     pub get_info: GetInfo,
     /// Function to join parent's `path` with a child's name to make the child's `name`.
@@ -27,17 +30,19 @@ where
     pub max_depth: u64,
 }
 
-impl<Path, Name, Size, GetInfo, JoinPath> From<TreeBuilder<Path, Name, Size, GetInfo, JoinPath>>
-    for DataTree<Name, Size>
+impl<Path, NameIter, Size, GetInfo, JoinPath>
+    From<TreeBuilder<Path, NameIter, Size, GetInfo, JoinPath>> for DataTree<NameIter::Item, Size>
 where
     Path: Send + Sync,
-    Name: Send + Sync,
-    GetInfo: Fn(&Path) -> Info<Name, Size> + Copy + Send + Sync,
-    JoinPath: Fn(&Path, &Name) -> Path + Copy + Send + Sync,
+    NameIter: IntoIterator,
+    NameIter::IntoIter: Send,
+    NameIter::Item: Send,
+    GetInfo: Fn(&Path) -> Info<NameIter, Size> + Copy + Send + Sync,
+    JoinPath: Fn(&Path, &NameIter::Item) -> Path + Copy + Send + Sync,
     Size: size::Size + Send,
 {
     /// Create a [`DataTree`] from a [`TreeBuilder`].
-    fn from(builder: TreeBuilder<Path, Name, Size, GetInfo, JoinPath>) -> Self {
+    fn from(builder: TreeBuilder<Path, NameIter, Size, GetInfo, JoinPath>) -> Self {
         let TreeBuilder {
             path,
             name,
@@ -50,20 +55,32 @@ where
         let max_depth = max_depth.saturating_sub(1);
 
         let children = children
-            .into_par_iter()
-            .map(|name| TreeBuilder {
-                path: join_path(&path, &name),
-                name,
-                get_info,
-                join_path,
-                max_depth,
+            .into_iter()
+            .chunks(64)
+            .into_iter()
+            .map(Vec::<NameIter::Item>::from_iter)
+            .map(|names| -> Vec<_> {
+                names
+                    .into_par_iter()
+                    .map(|name| TreeBuilder {
+                        path: join_path(&path, &name),
+                        name,
+                        get_info,
+                        join_path,
+                        max_depth,
+                    })
+                    .map(Self::from)
+                    .collect()
             })
-            .map(Self::from);
+            .fold(Vec::new(), |mut a, b| {
+                a.extend(b);
+                a
+            });
 
         if max_depth > 0 {
-            DataTree::dir(name, size, children.collect())
+            DataTree::dir(name, size, children)
         } else {
-            let size = size + children.map(|child| child.size()).sum();
+            let size = size + children.into_iter().map(|child| child.size()).sum();
             DataTree::dir(name, size, Vec::new())
         }
     }

src/tree_builder/info.rs

@@ -1,12 +1,11 @@
 use crate::size;
 use derive_more::From;
-use smart_default::SmartDefault;
 
 /// Information to return from `get_info` of [`super::TreeBuilder`].
-#[derive(Debug, SmartDefault, From)]
-pub struct Info<Name, Size: size::Size> {
+#[derive(Debug, Default, From)]
+pub struct Info<NameIter, Size: size::Size> {
     /// Size associated with given `path`.
     pub size: Size,
     /// Direct descendants of given `path`.
-    pub children: Vec<Name>,
+    pub children: NameIter,
 }