perf: eagerly process each 100 tasks

Author: KSXGitHub

src/tree_builder.rs

@@ -3,6 +3,7 @@ pub mod info;
 pub use info::Info;
 
 use super::{data_tree::DataTree, size};
+use itertools::Itertools;
 use rayon::prelude::*;
 
 /// Collection of functions and starting points in order to build a [`DataTree`] with [`From`] or [`Into`].
@@ -55,21 +56,39 @@ where
 
         let children = children
             .into_iter()
-            .par_bridge()
-            .map(|name| TreeBuilder {
-                path: join_path(&path, &name),
-                name,
-                get_info,
-                join_path,
-                max_depth,
+            .chunks(100)
+            .into_iter()
+            .map(Vec::<NameIter::Item>::from_iter)
+            .map(|names| -> Vec<_> {
+                names
+                    .into_par_iter()
+                    .map(|name| TreeBuilder {
+                        path: join_path(&path, &name),
+                        name,
+                        get_info,
+                        join_path,
+                        max_depth,
+                    })
+                    .map(Self::from)
+                    .collect()
             })
-            .map(Self::from);
+            .fold(Vec::new(), add_short_vec_to_long);
 
         if max_depth > 0 {
-            DataTree::dir(name, size, children.collect())
+            DataTree::dir(name, size, children)
         } else {
-            let size = size + children.map(|child| child.size()).sum();
+            let size = size + children.into_iter().map(|child| child.size()).sum();
             DataTree::dir(name, size, Vec::new())
         }
     }
 }
+
+fn add_short_vec_to_long<Item>(mut a: Vec<Item>, mut b: Vec<Item>) -> Vec<Item> {
+    if a.len() > b.len() {
+        a.extend(b);
+        a
+    } else {
+        b.extend(a);
+        b
+    }
+}