feat!: hardlink deduplications (wip)

Author: KSXGitHub

Cargo.lock

@@ -67,6 +67,9 @@ serde = { version = "^1.0.219", optional = true }
 serde_json = { version = "^1.0.140", optional = true }
 sysinfo = "^0.35.2"
 
+[target.'cfg(unix)'.dependencies]
+dashmap = "^6.1.0"
+
 [dev-dependencies]
 build-fs-tree = "^0.7.1"
 command-extra = "^1.0.0"

Cargo.toml

@@ -23,7 +23,7 @@ _pdu() {
 
     case "${cmd}" in
         pdu)
-            opts="-h -V --json-input --json-output --bytes-format --top-down --align-right --quantity --max-depth --total-width --column-width --min-ratio --no-sort --silent-errors --progress --threads --help --version [FILES]..."
+            opts="-h -V --json-input --json-output --bytes-format --deduplicate-hardlinks --top-down --align-right --quantity --max-depth --total-width --column-width --min-ratio --no-sort --silent-errors --progress --threads --help --version [FILES]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0

exports/completion.bash

@@ -27,6 +27,7 @@ set edit:completion:arg-completer[pdu] = {|@words|
             cand --threads 'Set the maximum number of threads to spawn. Could be either "auto", "max", or a number'
             cand --json-input 'Read JSON data from stdin'
             cand --json-output 'Print JSON data instead of an ASCII chart'
+            cand --deduplicate-hardlinks 'Detect duplicated hardlinks and remove their sizes from total'
             cand --top-down 'Print the tree top-down instead of bottom-up'
             cand --align-right 'Set the root of the bars to the right'
             cand --no-sort 'Preserve order of entries'

exports/completion.elv

@@ -11,6 +11,7 @@ complete -c pdu -l min-ratio -d 'Minimal size proportion required to appear' -r
 complete -c pdu -l threads -d 'Set the maximum number of threads to spawn. Could be either "auto", "max", or a number' -r
 complete -c pdu -l json-input -d 'Read JSON data from stdin'
 complete -c pdu -l json-output -d 'Print JSON data instead of an ASCII chart'
+complete -c pdu -l deduplicate-hardlinks -d 'Detect duplicated hardlinks and remove their sizes from total'
 complete -c pdu -l top-down -d 'Print the tree top-down instead of bottom-up'
 complete -c pdu -l align-right -d 'Set the root of the bars to the right'
 complete -c pdu -l no-sort -d 'Preserve order of entries'

exports/completion.fish

@@ -30,6 +30,7 @@ Register-ArgumentCompleter -Native -CommandName 'pdu' -ScriptBlock {
             [CompletionResult]::new('--threads', '--threads', [CompletionResultType]::ParameterName, 'Set the maximum number of threads to spawn. Could be either "auto", "max", or a number')
             [CompletionResult]::new('--json-input', '--json-input', [CompletionResultType]::ParameterName, 'Read JSON data from stdin')
             [CompletionResult]::new('--json-output', '--json-output', [CompletionResultType]::ParameterName, 'Print JSON data instead of an ASCII chart')
+            [CompletionResult]::new('--deduplicate-hardlinks', '--deduplicate-hardlinks', [CompletionResultType]::ParameterName, 'Detect duplicated hardlinks and remove their sizes from total')
             [CompletionResult]::new('--top-down', '--top-down', [CompletionResultType]::ParameterName, 'Print the tree top-down instead of bottom-up')
             [CompletionResult]::new('--align-right', '--align-right', [CompletionResultType]::ParameterName, 'Set the root of the bars to the right')
             [CompletionResult]::new('--no-sort', '--no-sort', [CompletionResultType]::ParameterName, 'Preserve order of entries')

exports/completion.ps1

@@ -28,6 +28,7 @@ block-count\:"Count numbers of blocks"))' \
 '--threads=[Set the maximum number of threads to spawn. Could be either "auto", "max", or a number]:THREADS:_default' \
 '(--quantity)--json-input[Read JSON data from stdin]' \
 '--json-output[Print JSON data instead of an ASCII chart]' \
+'--deduplicate-hardlinks[Detect duplicated hardlinks and remove their sizes from total]' \
 '--top-down[Print the tree top-down instead of bottom-up]' \
 '--align-right[Set the root of the bars to the right]' \
 '--no-sort[Preserve order of entries]' \

exports/completion.zsh

@@ -6,6 +6,7 @@ use crate::{
     args::{Args, Quantity, Threads},
     bytes_format::BytesFormat,
     get_size::{GetApparentSize, GetSize},
+    hook,
     json_data::{JsonData, UnitAndTree},
     reporter::{ErrorOnlyReporter, ErrorReport, ProgressAndErrorReporter, ProgressReport},
     runtime_error::RuntimeError,
@@ -179,15 +180,54 @@ impl App {
             }
         }
 
+        trait HardlinkDeduplicationSystem<const DEDUPLICATE_HARDLINKS: bool>: GetSizeUtils {
+            type Hook: hook::Hook<Self::Size> + sub::DeduplicateHardlinkSizes<Self::Size>;
+            fn create_hook(
+                record: <Self::Hook as sub::DeduplicateHardlinkSizes<Self::Size>>::HardlinkRecord,
+            ) -> Self::Hook;
+            fn init_hardlink_record(
+            ) -> <Self::Hook as sub::DeduplicateHardlinkSizes<Self::Size>>::HardlinkRecord;
+        }
+
+        impl<SizeGetter> HardlinkDeduplicationSystem<false> for SizeGetter
+        where
+            SizeGetter: GetSizeUtils,
+            SizeGetter::Size: Send + Sync,
+        {
+            type Hook = hook::DoNothing;
+            fn create_hook((): ()) -> Self::Hook {
+                hook::DoNothing
+            }
+            fn init_hardlink_record() {}
+        }
+
+        #[cfg(unix)]
+        impl<SizeGetter> HardlinkDeduplicationSystem<true> for SizeGetter
+        where
+            SizeGetter: GetSizeUtils,
+            SizeGetter::Size: Send + Sync + 'static,
+        {
+            type Hook = hook::RecordHardLink<'static, Self::Size>;
+            fn create_hook(record: &'static hook::RecordHardLinkStorage<Self::Size>) -> Self::Hook {
+                hook::RecordHardLink::new(record)
+            }
+            fn init_hardlink_record() -> &'static hook::RecordHardLinkStorage<Self::Size> {
+                hook::RecordHardLinkStorage::new()
+                    .pipe(Box::new)
+                    .pipe(Box::leak)
+            }
+        }
+
         macro_rules! run {
             ($(
                 $(#[$variant_attrs:meta])*
-                $size_getter:ident, $progress:literal;
+                $size_getter:ident, $progress:literal, $deduplicate_hardlinks:ident;
             )*) => { match self.args {$(
                 $(#[$variant_attrs])*
                 Args {
                     quantity: <$size_getter as GetSizeUtils>::QUANTITY,
                     progress: $progress,
+                    #[cfg(unix)] deduplicate_hardlinks: $deduplicate_hardlinks,
                     files,
                     json_output,
                     bytes_format,
@@ -197,30 +237,44 @@ impl App {
                     min_ratio,
                     no_sort,
                     ..
-                } => Sub {
-                    direction: Direction::from_top_down(top_down),
-                    bar_alignment: BarAlignment::from_align_right(align_right),
-                    size_getter: <$size_getter as GetSizeUtils>::INSTANCE,
-                    reporter: <$size_getter as CreateReporter<$progress>>::create_reporter(report_error),
-                    bytes_format: <$size_getter as GetSizeUtils>::formatter(bytes_format),
-                    files,
-                    json_output,
-                    column_width_distribution,
-                    max_depth,
-                    min_ratio,
-                    no_sort,
-                }
-                .run(),
+                } => {
+                    const DEDUPLICATE_HARDLINKS: bool = cfg!(unix) && $deduplicate_hardlinks;
+                    let hardlink_record = <$size_getter as HardlinkDeduplicationSystem<DEDUPLICATE_HARDLINKS>>::init_hardlink_record();
+                    let hook = <$size_getter as HardlinkDeduplicationSystem<DEDUPLICATE_HARDLINKS>>::create_hook(hardlink_record);
+
+                    Sub {
+                        direction: Direction::from_top_down(top_down),
+                        bar_alignment: BarAlignment::from_align_right(align_right),
+                        size_getter: <$size_getter as GetSizeUtils>::INSTANCE,
+                        hook,
+                        hardlink_record,
+                        reporter: <$size_getter as CreateReporter<$progress>>::create_reporter(report_error),
+                        bytes_format: <$size_getter as GetSizeUtils>::formatter(bytes_format),
+                        files,
+                        json_output,
+                        column_width_distribution,
+                        max_depth,
+                        min_ratio,
+                        no_sort,
+                    }
+                    .run()
+                },
             )*} };
         }
 
         run! {
-            GetApparentSize, false;
-            GetApparentSize, true;
-            #[cfg(unix)] GetBlockSize, false;
-            #[cfg(unix)] GetBlockSize, true;
-            #[cfg(unix)] GetBlockCount, false;
-            #[cfg(unix)] GetBlockCount, true;
+            GetApparentSize, false, false;
+            GetApparentSize, true, false;
+            #[cfg(unix)] GetBlockSize, false, false;
+            #[cfg(unix)] GetBlockSize, true, false;
+            #[cfg(unix)] GetBlockCount, false, false;
+            #[cfg(unix)] GetBlockCount, true, false;
+            #[cfg(unix)] GetApparentSize, false, true;
+            #[cfg(unix)] GetApparentSize, true, true;
+            #[cfg(unix)] GetBlockSize, false, true;
+            #[cfg(unix)] GetBlockSize, true, true;
+            #[cfg(unix)] GetBlockCount, false, true;
+            #[cfg(unix)] GetBlockCount, true, true;
         }
     }
 }