feat: Custom stack size

If a user reports a crash from stacksize - we can try different stack
memory sizes to try and figure out the cause.

Do not try and initialise rayon on 32 bit builds

README.md

@@ -83,6 +83,7 @@ Usage: dust -v regex (Exclude files matching this regex (eg dust -v "\.png$" wou
 Usage: dust -L (dereference-links - Treat sym links as directories and go into them)
 Usage: dust -P (Disable the progress indicator)
 Usage: dust -R (For screen readers. Removes bars/symbols. Adds new column: depth level. (May want to use -p for full path too))
+Usage: dust -S (Custom Stack size - Use if you see: 'fatal runtime error: stack overflow' (default allocation: low memory=1048576, high memory=1073741824)"),
 Usage: dust --skip-total (No total row will be displayed)
 Usage: dust -z 4000000 (Exclude files below size 4MB)
 ```

completions/_dust

@@ -29,6 +29,8 @@ _dust() {
 '(-t --file_types)*--filter=[Only include filepaths matching this regex. For png files type\: -e "\\.png\$" ]: : ' \
 '-w+[Specify width of output overriding the auto detection of terminal width]: : ' \
 '--terminal_width=[Specify width of output overriding the auto detection of terminal width]: : ' \
+'-S+[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]: : ' \
+'--stack-size=[Specify memory to use as stack size - use if you see\: '\''fatal runtime error\: stack overflow'\'' (default low memory=1048576, high memory=1073741824)]: : ' \
 '-p[Subdirectories will not have their path shortened]' \
 '--full-paths[Subdirectories will not have their path shortened]' \
 '-L[dereference sym links - Treat sym links as directories and go into them]' \

completions/_dust.ps1

@@ -35,6 +35,8 @@ Register-ArgumentCompleter -Native -CommandName 'dust' -ScriptBlock {
             [CompletionResult]::new('--filter', 'filter', [CompletionResultType]::ParameterName, 'Only include filepaths matching this regex. For png files type: -e "\.png$" ')
             [CompletionResult]::new('-w', 'w', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width')
             [CompletionResult]::new('--terminal_width', 'terminal_width', [CompletionResultType]::ParameterName, 'Specify width of output overriding the auto detection of terminal width')
+            [CompletionResult]::new('-S', 'S ', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)')
+            [CompletionResult]::new('--stack-size', 'stack-size', [CompletionResultType]::ParameterName, 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)')
             [CompletionResult]::new('-p', 'p', [CompletionResultType]::ParameterName, 'Subdirectories will not have their path shortened')
             [CompletionResult]::new('--full-paths', 'full-paths', [CompletionResultType]::ParameterName, 'Subdirectories will not have their path shortened')
             [CompletionResult]::new('-L', 'L ', [CompletionResultType]::ParameterName, 'dereference sym links - Treat sym links as directories and go into them')

completions/dust.bash

@@ -19,7 +19,7 @@ _dust() {
 
     case "${cmd}" in
         dust)
-            opts="-d -n -p -X -L -x -s -r -c -b -B -z -R -f -i -v -e -t -w -H -P -D -F -h -V --depth --number-of-lines --full-paths --ignore-directory --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore_hidden --invert-filter --filter --file_types --terminal_width --si --no-progress --only-dir --only-file --help --version [params]..."
+            opts="-d -n -p -X -L -x -s -r -c -b -B -z -R -f -i -v -e -t -w -H -P -D -F -S -h -V --depth --number-of-lines --full-paths --ignore-directory --dereference-links --limit-filesystem --apparent-size --reverse --no-colors --no-percent-bars --bars-on-right --min-size --screen-reader --skip-total --filecount --ignore_hidden --invert-filter --filter --file_types --terminal_width --si --no-progress --only-dir --only-file --stack-size --help --version [params]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -81,6 +81,14 @@ _dust() {
                     COMPREPLY=($(compgen -f "${cur}"))
                     return 0
                     ;;
+                --stack-size)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
+                -S)
+                    COMPREPLY=($(compgen -f "${cur}"))
+                    return 0
+                    ;;
                 *)
                     COMPREPLY=()
                     ;;

completions/dust.elv

@@ -32,6 +32,8 @@ set edit:completion:arg-completer[dust] = {|@words|
             cand --filter 'Only include filepaths matching this regex. For png files type: -e "\.png$" '
             cand -w 'Specify width of output overriding the auto detection of terminal width'
             cand --terminal_width 'Specify width of output overriding the auto detection of terminal width'
+            cand -S 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)'
+            cand --stack-size 'Specify memory to use as stack size - use if you see: ''fatal runtime error: stack overflow'' (default low memory=1048576, high memory=1073741824)'
             cand -p 'Subdirectories will not have their path shortened'
             cand --full-paths 'Subdirectories will not have their path shortened'
             cand -L 'dereference sym links - Treat sym links as directories and go into them'

completions/dust.fish

@@ -5,6 +5,7 @@ complete -c dust -s z -l min-size -d 'Minimum size file to include in output' -r
 complete -c dust -s v -l invert-filter -d 'Exclude filepaths matching this regex. To ignore png files type: -v "\\.png$" ' -r
 complete -c dust -s e -l filter -d 'Only include filepaths matching this regex. For png files type: -e "\\.png$" ' -r
 complete -c dust -s w -l terminal_width -d 'Specify width of output overriding the auto detection of terminal width' -r
+complete -c dust -s S -l stack-size -d 'Specify memory to use as stack size - use if you see: \'fatal runtime error: stack overflow\' (default low memory=1048576, high memory=1073741824)' -r
 complete -c dust -s p -l full-paths -d 'Subdirectories will not have their path shortened'
 complete -c dust -s L -l dereference-links -d 'dereference sym links - Treat sym links as directories and go into them'
 complete -c dust -s x -l limit-filesystem -d 'Only count the files and directories on the same filesystem as the supplied directory'

man-page/dust.1

@@ -4,7 +4,7 @@
 .SH NAME
 Dust \- Like du but more intuitive
 .SH SYNOPSIS
-\fBDust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore_hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file_types\fR] [\fB\-w\fR|\fB\-\-terminal_width\fR] [\fB\-H\fR|\fB\-\-si\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIparams\fR] 
+\fBDust\fR [\fB\-d\fR|\fB\-\-depth\fR] [\fB\-n\fR|\fB\-\-number\-of\-lines\fR] [\fB\-p\fR|\fB\-\-full\-paths\fR] [\fB\-X\fR|\fB\-\-ignore\-directory\fR] [\fB\-L\fR|\fB\-\-dereference\-links\fR] [\fB\-x\fR|\fB\-\-limit\-filesystem\fR] [\fB\-s\fR|\fB\-\-apparent\-size\fR] [\fB\-r\fR|\fB\-\-reverse\fR] [\fB\-c\fR|\fB\-\-no\-colors\fR] [\fB\-b\fR|\fB\-\-no\-percent\-bars\fR] [\fB\-B\fR|\fB\-\-bars\-on\-right\fR] [\fB\-z\fR|\fB\-\-min\-size\fR] [\fB\-R\fR|\fB\-\-screen\-reader\fR] [\fB\-\-skip\-total\fR] [\fB\-f\fR|\fB\-\-filecount\fR] [\fB\-i\fR|\fB\-\-ignore_hidden\fR] [\fB\-v\fR|\fB\-\-invert\-filter\fR] [\fB\-e\fR|\fB\-\-filter\fR] [\fB\-t\fR|\fB\-\-file_types\fR] [\fB\-w\fR|\fB\-\-terminal_width\fR] [\fB\-H\fR|\fB\-\-si\fR] [\fB\-P\fR|\fB\-\-no\-progress\fR] [\fB\-D\fR|\fB\-\-only\-dir\fR] [\fB\-F\fR|\fB\-\-only\-file\fR] [\fB\-S\fR|\fB\-\-stack\-size\fR] [\fB\-h\fR|\fB\-\-help\fR] [\fB\-V\fR|\fB\-\-version\fR] [\fIparams\fR] 
 .SH DESCRIPTION
 Like du but more intuitive
 .SH OPTIONS
@@ -81,6 +81,9 @@ Only directories will be displayed.
 \fB\-F\fR, \fB\-\-only\-file\fR
 Only files will be displayed. (Finds your largest files)
 .TP
+\fB\-S\fR, \fB\-\-stack\-size\fR
+Specify memory to use as stack size \- use if you see: \*(Aqfatal runtime error: stack overflow\*(Aq (default low memory=1048576, high memory=1073741824)
+.TP
 \fB\-h\fR, \fB\-\-help\fR
 Print help
 .TP

src/cli.rs

@@ -186,6 +186,14 @@ pub fn build_cli() -> Command {
                 .action(clap::ArgAction::SetTrue)
                 .help("Only files will be displayed. (Finds your largest files)"),
         )
+        .arg(
+            Arg::new("stack_size")
+                .short('S')
+                .long("stack-size")
+                .num_args(1)
+                .value_parser(value_parser!(usize))
+                .help("Specify memory to use as stack size - use if you see: 'fatal runtime error: stack overflow' (default low memory=1048576, high memory=1073741824)"),
+        )
         .arg(Arg::new("params").num_args(1..)
             .value_parser(value_parser!(String)))
 }

src/config.rs

@@ -26,6 +26,7 @@ pub struct Config {
     pub disable_progress: Option<bool>,
     pub depth: Option<usize>,
     pub bars_on_right: Option<bool>,
+    pub stack_size: Option<usize>,
 }
 
 impl Config {
@@ -95,6 +96,14 @@ impl Config {
     pub fn get_bars_on_right(&self, options: &ArgMatches) -> bool {
         Some(true) == self.bars_on_right || options.get_flag("bars_on_right")
     }
+    pub fn get_custom_stack_size(&self, options: &ArgMatches) -> Option<usize> {
+        let from_cmd_line = options.get_one::<usize>("stack_size");
+        if from_cmd_line.is_none() {
+            self.stack_size
+        } else {
+            from_cmd_line.copied()
+        }
+    }
 }
 
 fn convert_min_size(input: &str, iso: bool) -> Option<usize> {

src/main.rs

@@ -27,7 +27,6 @@ use config::get_config;
 use dir_walker::walk_it;
 use filter::get_biggest;
 use filter_type::get_all_file_types;
-use rayon::ThreadPoolBuildError;
 use regex::Regex;
 use std::cmp::max;
 use std::path::PathBuf;
@@ -177,11 +176,8 @@ fn main() {
         follow_links,
         progress_data: indicator.data.clone(),
     };
-
-    let result = panic::catch_unwind(|| init_rayon);
-    if result.is_err() {
-        eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1")
-    }
+    let stack_size = config.get_custom_stack_size(&options);
+    init_rayon(&stack_size);
 
     let top_level_nodes = walk_it(simplified_dirs, walk_data);
 
@@ -227,18 +223,34 @@ fn main() {
     }
 }
 
-fn init_rayon() -> Result<(), ThreadPoolBuildError> {
-    let large_stack = usize::pow(1024, 3);
-    let mut s = System::new();
-    s.refresh_memory();
-    let available = s.available_memory();
-
-    if available > large_stack.try_into().unwrap() {
-        // Larger stack size to handle cases with lots of nested directories
-        rayon::ThreadPoolBuilder::new()
-            .stack_size(large_stack)
-            .build_global()
-    } else {
-        rayon::ThreadPoolBuilder::new().build_global()
+fn init_rayon(stack_size: &Option<usize>) {
+    // Rayon seems to raise this error on 32-bit builds
+    // The global thread pool has not been initialized.: ThreadPoolBuildError { kind: GlobalPoolAlreadyInitialized }
+    if cfg!(target_pointer_width = "64") {
+        let result = panic::catch_unwind(|| {
+            match stack_size {
+                Some(n) => rayon::ThreadPoolBuilder::new()
+                    .stack_size(*n)
+                    .build_global(),
+                None => {
+                    let large_stack = usize::pow(1024, 3);
+                    let mut s = System::new();
+                    s.refresh_memory();
+                    let available = s.available_memory();
+
+                    if available > large_stack.try_into().unwrap() {
+                        // Larger stack size to handle cases with lots of nested directories
+                        rayon::ThreadPoolBuilder::new()
+                            .stack_size(large_stack)
+                            .build_global()
+                    } else {
+                        rayon::ThreadPoolBuilder::new().build_global()
+                    }
+                }
+            }
+        });
+        if result.is_err() {
+            eprintln!("Problem initializing rayon, try: export RAYON_NUM_THREADS=1")
+        }
     }
 }

tests/test_exact_output.rs

@@ -51,6 +51,7 @@ fn exact_output_test<T: AsRef<OsStr>>(valid_outputs: Vec<String>, command_args:
     }
 
     let output = str::from_utf8(&a.unwrap().stdout).unwrap().to_owned();
+    println!("{}", output);
 
     assert!(valid_outputs.iter().any(|i| output.contains(i)));
 }