Merge 93bbea3 into f128a9e

Author: KSXGitHub

src/fs_tree_builder.rs

@@ -66,59 +66,55 @@ where
             max_depth,
         } = builder;
 
-        TreeBuilder::<PathBuf, OsStringDisplay, Size, _, _> {
+        TreeBuilder::<PathBuf, _, Size, _, _> {
             name: OsStringDisplay::os_string_from(&root),
 
             path: root,
 
             get_info: |path| {
+                let make_info = |size: Size, children: Option<_>| Info {
+                    size,
+                    children: children.into_iter().flatten(),
+                };
+
+                let empty_info = || make_info(Size::default(), None);
+
                 let stats = match symlink_metadata(path) {
                     Err(error) => {
                         reporter.report(Event::EncounterError(ErrorReport {
                             operation: SymlinkMetadata,
                             path,
                             error,
                         }));
-                        return Info {
-                            size: Size::default(),
-                            children: Vec::new(),
-                        };
+                        return empty_info();
                     }
                     Ok(stats) => stats,
                 };
 
-                let children: Vec<_> = if stats.is_dir() {
+                let children = if stats.is_dir() {
                     match read_dir(path) {
                         Err(error) => {
                             reporter.report(Event::EncounterError(ErrorReport {
                                 operation: ReadDirectory,
                                 path,
                                 error,
                             }));
-                            return Info::default();
+                            return empty_info();
                         }
                         Ok(entries) => entries,
                     }
-                    .filter_map(|entry| match entry {
-                        Err(error) => {
-                            reporter.report(Event::EncounterError(ErrorReport {
-                                operation: AccessEntry,
-                                path,
-                                error,
-                            }));
-                            None
-                        }
-                        Ok(entry) => entry.file_name().pipe(OsStringDisplay::from).pipe(Some),
-                    })
-                    .collect()
+                    .flatten()
+                    .map(|entry| entry.file_name())
+                    .map(OsStringDisplay::from)
+                    .pipe(Some)
                 } else {
-                    Vec::new()
+                    None
                 };
 
                 let size = size_getter.get_size(&stats);
                 reporter.report(Event::ReceiveData(size));
 
-                Info { size, children }
+                make_info(size, children)
             },
 
             join_path: |prefix, name| prefix.join(&name.0),

src/reporter/error_report/operation.rs

@@ -5,8 +5,6 @@ pub enum Operation {
     SymlinkMetadata,
     /// Error is caused by calling [`std::fs::read_dir`].
     ReadDirectory,
-    /// Error when trying to access [`std::fs::DirEntry`] of one of the element of [`std::fs::read_dir`].
-    AccessEntry,
 }
 
 impl Operation {
@@ -16,7 +14,6 @@ impl Operation {
         match self {
             SymlinkMetadata => "symlink_metadata",
             ReadDirectory => "read_dir",
-            AccessEntry => "access entry",
         }
     }
 }
@@ -36,5 +33,4 @@ mod test_operation {
 
     name_display!(symlink_metadata, SymlinkMetadata, "symlink_metadata");
     name_display!(read_directory, ReadDirectory, "read_dir");
-    name_display!(access_entry, AccessEntry, "access entry");
 }

src/tree_builder.rs

@@ -3,22 +3,26 @@ pub mod info;
 pub use info::Info;
 
 use super::{data_tree::DataTree, size};
+use pipe_trait::Pipe;
 use rayon::prelude::*;
+use std::{marker::PhantomData, ops::Not};
 
 /// Collection of functions and starting points in order to build a [`DataTree`] with [`From`] or [`Into`].
 #[derive(Debug)]
-pub struct TreeBuilder<Path, Name, Size, GetInfo, JoinPath>
+pub struct TreeBuilder<Path, NameIter, Size, GetInfo, JoinPath>
 where
     Path: Send + Sync,
-    Name: Send + Sync,
-    GetInfo: Fn(&Path) -> Info<Name, Size> + Copy + Send + Sync,
-    JoinPath: Fn(&Path, &Name) -> Path + Copy + Send + Sync,
+    NameIter: IntoIterator,
+    NameIter::IntoIter: Send,
+    NameIter::Item: Send,
+    GetInfo: Fn(&Path) -> Info<NameIter, Size> + Copy + Send + Sync,
+    JoinPath: Fn(&Path, &NameIter::Item) -> Path + Copy + Send + Sync,
     Size: size::Size + Send,
 {
     /// Path to the root.
     pub path: Path,
     /// Name of the root.
-    pub name: Name,
+    pub name: NameIter::Item,
     /// Function to extract necessary information from `path` (`size` and `children`).
     pub get_info: GetInfo,
     /// Function to join parent's `path` with a child's name to make the child's `name`.
@@ -27,17 +31,19 @@ where
     pub max_depth: u64,
 }
 
-impl<Path, Name, Size, GetInfo, JoinPath> From<TreeBuilder<Path, Name, Size, GetInfo, JoinPath>>
-    for DataTree<Name, Size>
+impl<Path, NameIter, Size, GetInfo, JoinPath>
+    From<TreeBuilder<Path, NameIter, Size, GetInfo, JoinPath>> for DataTree<NameIter::Item, Size>
 where
     Path: Send + Sync,
-    Name: Send + Sync,
-    GetInfo: Fn(&Path) -> Info<Name, Size> + Copy + Send + Sync,
-    JoinPath: Fn(&Path, &Name) -> Path + Copy + Send + Sync,
+    NameIter: IntoIterator,
+    NameIter::IntoIter: Send,
+    NameIter::Item: Send,
+    GetInfo: Fn(&Path) -> Info<NameIter, Size> + Copy + Send + Sync,
+    JoinPath: Fn(&Path, &NameIter::Item) -> Path + Copy + Send + Sync,
     Size: size::Size + Send,
 {
     /// Create a [`DataTree`] from a [`TreeBuilder`].
-    fn from(builder: TreeBuilder<Path, Name, Size, GetInfo, JoinPath>) -> Self {
+    fn from(builder: TreeBuilder<Path, NameIter, Size, GetInfo, JoinPath>) -> Self {
         let TreeBuilder {
             path,
             name,
@@ -50,21 +56,68 @@ where
         let max_depth = max_depth.saturating_sub(1);
 
         let children = children
-            .into_par_iter()
-            .map(|name| TreeBuilder {
-                path: join_path(&path, &name),
-                name,
-                get_info,
-                join_path,
-                max_depth,
+            .into_iter()
+            .pipe(Chunks::<64, _>::new)
+            .par_bridge()
+            .map(|names: Vec<_>| -> Vec<_> {
+                names
+                    .into_par_iter()
+                    .map(|name| TreeBuilder {
+                        path: join_path(&path, &name),
+                        name,
+                        get_info,
+                        join_path,
+                        max_depth,
+                    })
+                    .map(Self::from)
+                    .collect()
             })
-            .map(Self::from);
+            .reduce(Vec::new, add_short_vec_to_long);
 
         if max_depth > 0 {
-            DataTree::dir(name, size, children.collect())
+            DataTree::dir(name, size, children)
         } else {
-            let size = size + children.map(|child| child.size()).sum();
+            let size = size + children.into_iter().map(|child| child.size()).sum();
             DataTree::dir(name, size, Vec::new())
         }
     }
 }
+
+/// Concat 2 `Vec`s in any order with minimal copying.
+fn add_short_vec_to_long<Item>(mut a: Vec<Item>, mut b: Vec<Item>) -> Vec<Item> {
+    if a.len() > b.len() {
+        a.extend(b);
+        a
+    } else {
+        b.extend(a);
+        b
+    }
+}
+
+/// Utility type to iterate over each `Vec` of at most `LEN` items.
+#[derive(Debug, Clone, Copy)]
+struct Chunks<const LEN: usize, Iter> {
+    iter: Iter,
+    _phantom: PhantomData<[(); LEN]>,
+}
+
+impl<const LEN: usize, Iter> Chunks<LEN, Iter> {
+    /// Start iterating chunks.
+    fn new(iter: Iter) -> Self {
+        Chunks {
+            iter,
+            _phantom: PhantomData,
+        }
+    }
+}
+
+impl<const LEN: usize, Iter> Iterator for Chunks<LEN, Iter>
+where
+    Iter: Iterator,
+{
+    type Item = Vec<Iter::Item>;
+    fn next(&mut self) -> Option<Self::Item> {
+        let chunk: Vec<_> = self.iter.by_ref().take(LEN).collect();
+        chunk.is_empty().not().then_some(chunk)
+    }
+}

src/tree_builder/info.rs

@@ -1,12 +1,11 @@
 use crate::size;
 use derive_more::From;
-use smart_default::SmartDefault;
 
 /// Information to return from `get_info` of [`super::TreeBuilder`].
-#[derive(Debug, SmartDefault, From)]
-pub struct Info<Name, Size: size::Size> {
+#[derive(Debug, Default, From)]
+pub struct Info<NameIter, Size: size::Size> {
     /// Size associated with given `path`.
     pub size: Size,
     /// Direct descendants of given `path`.
-    pub children: Vec<Name>,
+    pub children: NameIter,
 }