modulised some data structures

.gitignore

@@ -1 +1,2 @@
-target/
+target/
+test/

Cargo.lock

@@ -50,6 +50,12 @@ dependencies = [
  "windows-sys",
 ]
 
+[[package]]
+name = "anyhow"
+version = "1.0.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -157,11 +163,19 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 
+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
 [[package]]
 name = "hb"
 version = "0.1.0"
 dependencies = [
+ "anyhow",
  "clap",
+ "glob",
  "rayon",
 ]
 

Cargo.toml

@@ -6,7 +6,9 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = "1.0.81"
 clap = { version = "4.4.3", features = ["derive"] }
+glob = "0.3.1"
 rayon = "1.7.0"
 
 # lints from https://github.com/0atman/noboilerplate/blob/main/scripts/37-functional-rust.md
@@ -24,4 +26,4 @@ opt-level = 'z' # size optimisation
 lto = true # link time optimisation
 codegen-units = 1 # fewer -> more optimisation
 panic = 'abort' # abort on panic
-strip = 'symbols' # strip symbols
+strip = 'symbols' # strip symbols

src/args.rs

@@ -0,0 +1,204 @@
+use clap::{Parser, ArgAction};
+use glob::Pattern;
+
+use crate::unit::Unit;
+
+use std::fs::Metadata;
+use std::path::{Component, Path};
+use std::slice::Iter;
+use std::iter::once_with;
+
+#[allow(clippy::struct_excessive_bools)]
+#[derive(Parser, Debug, Clone)]
+pub struct Args {
+    #[arg(
+        short, long, default_value_t = false,
+        help = "keep going if an error occurs",
+        long_help = "keep going if an error occurs (ex. unreadable subdirectories in a readable directory)"
+    )]
+    persistant: bool,
+
+    #[arg(
+        short, long,
+        help = "minimize output",
+        long_help = "print nothing but the total size for all directories, without a newline. Also supresses all error messages",
+        conflicts_with = "total",
+        default_value_t = false,
+    )]
+    minimal: bool,
+
+    #[arg(
+        short='T', long,
+        help = "display in tree",
+        default_value_t = false,
+        conflicts_with = "minimal",
+    )]
+    tree: bool,
+
+    #[arg(
+        short, long,
+        help = "display the total size",
+        conflicts_with = "minimal",
+        default_value_t = false,
+    )]
+    total: bool,
+
+    #[arg(
+        short='2', long,
+        help = "alias for --unit 1024",
+        default_value_t = false,
+        conflicts_with_all = ["si","unit"],
+    )]
+    base_two: bool,
+
+    #[arg(
+        short='k', long,
+        help = "alias for --unit 1000",
+        default_value_t = false,
+        conflicts_with_all = ["base_two","unit"],
+    )]
+    si: bool,
+
+    #[arg(
+        short, long,
+        help = "unit to print in",
+        long_help = "printing unit (case insensitive): b = bytes, kb = kilobytes, ki = kibibytes, gb = gigabytes, gi = gibibytes, tb = terabytes, ti = tibibytes",
+        value_parser = Unit::parse,
+        default_value_t = Unit::Byte,
+        conflicts_with_all = ["base_two","si"],
+    )]
+    unit: Unit,
+
+    #[arg(
+        short='s', long,
+        help = "follow symlinks",
+        default_value_t = false,
+    )]
+    follow_links: bool,
+
+    #[arg(
+        short='x', long = "exclude",
+        help = "include in search, but exclude from printing",
+        long_help = "include in search, but exclude from printing. accepts glob syntax",
+        default_values_t = once_with(|| Pattern::new(".*").unwrap()),
+        value_parser = parse_glob,
+        value_delimiter = ',',
+        action = ArgAction::Append,
+    )]
+    exclude_print: Vec<Pattern>,
+
+    #[arg(
+        short='X', long,
+        help = "exclude from search and printing",
+        default_values_t = once_with(|| Pattern::new("").unwrap()),
+        value_parser = parse_glob,
+        value_delimiter = ',',
+        action = ArgAction::Append,
+    )]
+    exclude_search: Vec<Pattern>,
+
+    #[arg(
+        short='H', long,
+        help = "disable implicit hiding of results",
+        long_help = "don't implicitly hide dotfiles and dot directories",
+        conflicts_with = "exclude_print",
+        default_value_t = false,
+    )]
+    show_hidden: bool,
+
+    #[arg(
+        value_parser = validate_path,
+        help = "items to summate",
+        action = ArgAction::Append,
+        num_args = 1..
+    )]
+    path: Vec<String>,
+}
+impl Args {
+    pub fn post_process(mut self) -> Self {
+        if self.base_two {
+            self.unit = Unit::Kibi;
+        } else if self.si {
+            self.unit = Unit::Kilo;
+        }
+
+        if self.show_hidden {
+            self.exclude_print = Vec::new();
+        }
+
+        if self.path.is_empty() {
+            self.path = vec![ ".".to_owned() ];
+        }
+        
+        self
+    }
+
+    pub fn should_exclude(&self, path: &Path, file: &Metadata) -> bool {
+        if !self.follow_links && file.is_symlink() {
+            return true
+        }
+
+        any_pattern_matches_any_component(&self.exclude_search, path)
+    }
+    
+    pub fn should_print(&self, path: &Path) -> bool {
+        ! any_pattern_matches_any_component(&self.exclude_print, path)
+        // TODO: this exists because when a file matches an exclude pattern
+        // is it still returned, just with no size or children, so in order
+        // to not accidentally print things that we said we were excluding,
+        // we also have to check that it's not excluded by search.
+        // `self.exclude_print.extend(&self.exclude_search)` is wasteful,
+        // but until I find a better way this is what it's gotta be`
+        && ! any_pattern_matches_any_component(&self.exclude_search, path)
+    }
+
+    pub const fn persistant(&self) -> bool {
+        self.persistant
+    }
+
+    pub const fn minimal(&self) -> bool {
+        self.minimal
+    }
+    
+    pub const fn tree(&self) -> bool {
+        self.tree
+    }
+    
+    pub const fn total(&self) -> bool {
+        self.total
+    }
+    
+    pub const fn unit(&self) -> Unit {
+        self.unit
+    }
+
+    pub fn iter(&self) -> Iter<'_, String> {
+        self.path.iter()
+    }
+}
+
+fn validate_path(s: &str) -> Result<String, String> {
+    // try to access it's metadata, since that is what is used
+    // to get its length
+    std::fs::metadata(s)
+        .map(|_| s.to_string())
+        .map_err(|e| e.to_string())
+}
+
+fn parse_glob(s: &str) -> Result<Pattern, String> {
+    Pattern::new(s).map_err(|_| format!("invalid glob: {s}"))
+}
+
+fn any_pattern_matches_any_component(patterns: &[Pattern], path: &Path) -> bool {
+    for pat in patterns {
+        for cmp in path.components() {
+            let Component::Normal(cmp) = cmp else { continue };
+            let Some(s) = cmp.to_str() else { continue };
+            if pat.matches(s) {
+                return true
+            }
+        }
+    }
+
+    false
+}

src/directory.rs

@@ -1,10 +1,14 @@
+use std::ffi::OsString;
 use std::fs::read_dir;
-use std::hint::unreachable_unchecked;
 use std::path::{Path, PathBuf};
-use std::io::{Result, ErrorKind};
+use std::io::ErrorKind;
+use anyhow::{Context, Result};
 
+use crate::args::Args;
 use crate::unit::Unit;
 
+use rayon::prelude::*;
+
 #[derive(Debug, Clone)]
 pub struct Directory {
     name: PathBuf,
@@ -17,82 +21,142 @@ impl Directory {
         self.size
     }
 
-    #[inline]
-    pub fn scale(&self, unit: Unit) -> String {
-        unit.convert(self.size)
-    }
-
     #[inline]
     pub fn path(&self) -> &Path {
         self.name.as_ref()
     }
 
-    pub fn new<P>(path: P) -> Result<Self>
-    where
-        P: AsRef<Path>
-    {
+    pub fn new< P: AsRef<Path> >(path: P, args: &Args) -> Result<Option<Self>> {
         let path = path.as_ref();
-        let name = path.canonicalize()?
-            .file_name()
-            // file_name() returns None if and only if the path ends in "..".
-            // due to the call to canonicalize(), this can never be the case,
-            // so this can be safely unwrapped
-            .unwrap_or_else(|| unsafe { unreachable_unchecked() })
+        let name = path.file_name()
+            .map_or_else(|| OsString::from("/"), ToOwned::to_owned)
             .into();
 
+        // symlink_metadata() is the same as metadata() but it doesn't
+        // traverse symlinks, so that we can exclude them if necessary
+        let meta = match (path.symlink_metadata(), args.persistant()) {
+            (Ok(md), _) => md,
+            (Err(_), true) => return Ok(None),
+            (Err(e), false) => return Err(e.into()),
+        };
+
+        if args.should_exclude(path, &meta) {
+            // Ok(None) is only meant to arise from an error
+            // while persistant. When that happens, a no-op
+            // entry is substituted, which is precisely 
+            // we want to happen when we hit an excluded file.
+
+            // NOTE: return Ok(None) is *not* equivalent, because
+            // that's only produced when an error occurs but
+            // the program is running in persistant mode. I used
+            // to return that, but that causes incredibly
+            // bizarre and wrong behaviour.
+            return Ok(Some( Self { name, size: 0, children: Vec::new() } ))
+        }
+
         let dir = match read_dir(path) {
             Ok(dir) => dir,
             Err(io_error) => match io_error.kind() {
-                ErrorKind::NotADirectory => return Ok(Self {
-                    name,
-                    size: path.metadata()?.len(),
-                    children: Vec::new()
-                }),
-                _ => return Err(io_error),
+                ErrorKind::NotADirectory => {
+                    return Ok(Some(
+                        Self {
+                            name,
+                            size: meta.len(),
+                            children: Vec::new()
+                        }
+                    ))
+                },
+                other => return Result::context(
+                    Err(io_error),
+                    format!("{}: {}", path.display(), other)
+                ),
             }
         };
 
-        let mut size = 0;
-        let mut children = Vec::new();
-        for entry in dir {
-            let child = Self::new(entry?.path())?;
-            size += child.size;
-            children.push(child);
-        }
+        // this is a compicated iterator pattern. I'll do my best to explain.
+        // 1. the end result is that we `reduce()` the iterator to a single
+        //    (u64, Vec<Directory>) tuple to return. this is done by...
+        let (size, children) =
+            // 2. taking the iterator over the directory and parallelising it...
+            dir.par_bridge()
+            // 3, this is the recursive step: try to create new Directory
+            //    objects from each item in the iterator
+            .map(|entry| Self::new(entry?.path(), args))
+            // 4. the fold (this is try_fold because we're iterating over Result.).
+            //    each fold adds a directory as a child and increases the total size
+            .try_fold(
+                || (0, Vec::new()),
+                |(mut size, mut children), dir| -> Result<(u64, Vec<Self>)> {
+                    let Some(dir) = Result::from(dir)?
+                    else {
+                        // some intermediate operation failed, but we
+                        // are persistant, so just skip
+                        return Result::Ok((0, Vec::new()))
+                    };
+                    size += dir.size;
+                    children.push(dir);
+                    // have to specify anyhow::Result::Ok otherwise it complains
+                    // that it can't infer the E in Result<T, E>
+                    Result::Ok((size, children))
+                }
+            )
+            // 5. the final step is to reduce, which is as simple as concatenating
+            //    every vector and summing up their sizes.
+            .try_reduce(
+                || (0, Vec::new()),
+                |(asize, mut avec), (bsize, bvec)| {
+                    avec.extend(bvec);
+                    Result::Ok((asize + bsize, avec))
+                }
+            )?;
+        //   ^ note the Try, because of course any of these operations could
+        //     fail
 
-        Ok(Self{
-            name,
-            size,
-            children,
-        })
+        // final notes:
+        // 1. I am unsure if it is better to do a bunch of partial sums
+        //    during the fold() and reduce() steps, or if it is best to
+        //    have them only do data collection and sum the lengths
+        //    later. intuitively we would want to do everything in
+        //    parallel but I have no data to support this.
+        // 2. this is a super complicated iterator pattern, If anyone
+        //    knows how to simplify it I'm all ears, but being
+        //    parallel is the main advantage it has over du so I don't
+        //    want to abandon that, even though a serial for loop is
+        //    *incredibly* clearer.
+
+        Ok(Some(
+            Self {
+                name,
+                size,
+                children,
+            }
+        ))
     }
 
-    pub fn display(self, unit: Unit) -> String {
+    pub fn tree(self, args: &Args) -> String {
         // since self.size is definitionally the greatest value, the tab length
         // is just the length of self.len, plus two for a tab width
         let tab_size = self.size.to_string().len() + 2;
-        let mut result = self.vectorise(unit).iter().map(|e| e.stringify_tabbed(tab_size) + "\n").collect::<String>();
-
-        if ! result.is_empty() {
-            let final_newline_char_range = result.len()-2 .. result.len();
-            result.drain(final_newline_char_range);
-        }
-
-        result
+        self.vectorise(args)
+            .iter()
+            .map(|e| e.stringify_tabbed(tab_size))
+            .reduce(|s1, s2| s1 + "\n" + &s2)
+            .unwrap_or_default()
     }
 
     /// TODO: make not recursive, take &self if possible,
     /// and maybe write directly to stdout to not use so much mem
-    fn vectorise(self, unit: Unit) -> Vec<TreeEntry> {
+    fn vectorise(mut self, args: &Args) -> Vec<TreeEntry> {        
         let mut result = Vec::new();
 
         result.push(TreeEntry::new(
-            Vec::new(), self.name.display().to_string(), self.size, unit
+            self.name.display().to_string(), self.size, args.unit()
         ));
 
         let mut new_entry_part = TreePart::First;
         let mut continue_part = TreePart::Wait;
-
+        
+        self.children.retain(|dir| args.should_print(dir.path()));
         let len = self.children.len();
 
         for (idx, child) in self.children.into_iter().enumerate() {
@@ -101,7 +165,7 @@ impl Directory {
                 continue_part = TreePart::Blank;
             }
 
-            let subtree = child.vectorise(unit);
+            let subtree = child.vectorise(args);
 
             for mut item in subtree {
                 if item.parts.is_empty() {
@@ -125,9 +189,9 @@ struct TreeEntry {
     unit: Unit
 }
 impl TreeEntry {
-    fn new(parts: Vec<TreePart>, path: String, size: u64, unit: Unit) -> Self {
+    fn new(path: String, size: u64, unit: Unit) -> Self {
         Self {
-            parts, path, size, unit
+            parts: Vec::new(), path, size, unit
         }
     }
     

src/main.rs

@@ -1,107 +1,32 @@
 #![feature(io_error_more, fs_try_exists)]
 
-use clap::{Parser, ArgAction};
-
+mod args;
 mod directory;
 mod unit;
+
+use args::Args;
 use directory::Directory;
-use unit::Unit;
+
+use clap::Parser;
 
 use std::process::ExitCode;
 
-#[allow(clippy::struct_excessive_bools)]
-#[derive(Parser, Debug, Clone)]
-pub struct Args {
-    #[arg(
-        short, long, default_value_t = false,
-        help = "keep going if an error occurs",
-        long_help = "keep going if an error occurs (ex. unreadable subdirectories in a readable directory)"
-    )]
-    persistant: bool,
-
-    #[arg(
-        short, long,
-        help = "minimize output",
-        long_help = "print nothing but the total size for all directories, without a newline. Also supresses all error messages",
-        conflicts_with = "total",
-        default_value_t = false,
-    )]
-    minimal: bool,
-
-    #[arg(
-        short='T', long,
-        help = "display in tree",
-        default_value_t = false,
-        conflicts_with = "minimal",
-    )]
-    tree: bool,
-
-    #[arg(
-        short, long,
-        help = "display the total size",
-        conflicts_with = "minimal",
-        default_value_t = false,
-    )]
-    total: bool,
-
-    #[arg(
-        short='2', long,
-        help = "alias for --unit 1024",
-        default_value_t = false,
-        conflicts_with_all = ["si","unit"],
-    )]
-    base_two: bool,
-
-    #[arg(
-        short='0', long,
-        help = "alias for --unit 1000",
-        default_value_t = false,
-        conflicts_with_all = ["base_two","unit"],
-    )]
-    si: bool,
-
-    #[arg(
-        short, long,
-        help = "unit to print in",
-        long_help = "printing unit (case insensitive): b = bytes, kb = kilobytes, ki = kibibytes, gb = gigabytes, gi = gibibytes, tb = terabytes, ti = tibibytes",
-        value_parser = Unit::parse,
-        default_value_t = Unit::Byte,
-        conflicts_with_all = ["base_two","si"],
-    )]
-    unit: Unit,
-
-    #[arg(
-        value_parser = validate_path,
-        help = "items to summate",
-        action = ArgAction::Append,
-        num_args = 1..
-    )]
-    path: Vec<String>,
-}
-
-fn validate_path(s: &str) -> Result<String, String> {
-    // try to access it's metadata, since that is what is used
-    // to get its length
-    std::fs::metadata(s)
-        .map(|_| s.to_string())
-        .map_err(|e| e.to_string())
-}
-
 fn main() -> ExitCode {
-    let mut args = Args::parse();
+    let args = Args::parse().post_process();
+    // dbg!(&args);
 
-    if args.base_two {
-        args.unit = Unit::Kibi;
-    } else if args.si {
-        args.unit = Unit::Kilo;
-    }
-    
     let mut total = 0;
-    for path in args.path {
-        let dir_structure = match Directory::new(path) {
-            Ok(ds) => ds,
+    for path in args.iter() {
+        let dir_structure = match Directory::new(path, &args) {
+            Ok(Some(ds)) => ds,
+            // this only ever returns None when persistant,
+            // so we don't need a match guard
+            Ok(None) => continue,
             Err(e) => {
-                if !args.minimal {
+                if args.persistant() {
+                    continue;
+                }
+                if !args.minimal() {
                     eprintln!("hb: {e}");
                 }
                 return ExitCode::FAILURE;
@@ -110,28 +35,28 @@ fn main() -> ExitCode {
 
         total += dir_structure.size();
 
-        if args.minimal {
+        if args.minimal() {
             // skip printing (this is a matter of indentation)
             continue;
         }
 
-        if args.tree {
-            println!("{}", dir_structure.display(args.unit));
+        if args.tree() {
+            println!("{}", dir_structure.tree(&args));
         } else {
             println!(
                 "{}: {}",
                 dir_structure.path().display(),
-                dir_structure.scale(args.unit),
+                args.unit().convert(dir_structure.size())
             );
         }
     }
 
-    let total = args.unit.convert(total);
+    let total = args.unit().convert(total);
 
-    if args.total {
+    if args.total() {
         println!("total: {total}");
     }
-    else if args.minimal {
+    else if args.minimal() {
         print!("{total}");
     }
 

src/unit.rs

@@ -15,12 +15,14 @@ pub enum Unit {
 
     Tera,
     Tibi,
+
+    Blocks,
 }
 impl Unit {
     pub fn parse(s: &str) -> Result<Self, String> {
         let s = s.to_lowercase();
         match s.as_str() {
-            "b" => Ok(Self::Byte),
+            "b" | "bytes" => Ok(Self::Byte),
             "k" | "kb" => Ok(Self::Kilo),
             "ki" => Ok(Self::Kibi),
             "m" | "mb" => Ok(Self::Mega),
@@ -29,11 +31,19 @@ impl Unit {
             "gi" => Ok(Self::Gibi),
             "t" | "tb" => Ok(Self::Tera),
             "ti" => Ok(Self::Tibi),
+            "blk" | "blks"
+            | "blck" |"blcks"
+            | "block" | "blocks" => Ok(Self::Blocks),
             _ => Err(s),
         }
     }
 
     pub fn convert(self, n: u64) -> String {
+        let n = if self == Self::Blocks {
+            n.next_multiple_of(self.integer_value())
+        } else {
+            n
+        };
         format!("{}{}", n/self.integer_value(), self.units_pretty())
     }
 
@@ -48,6 +58,7 @@ impl Unit {
             Self::Gibi => " Gi",
             Self::Tera => " T",
             Self::Tibi => " Ti",
+            Self::Blocks => " blocks"
         }
     }
 
@@ -62,13 +73,14 @@ impl Unit {
             Self::Gibi => 1_073_741_824,
             Self::Tera => 1_000_000_000_000,
             Self::Tibi => 1_099_511_627_776,
+            Self::Blocks => 512,
         }
     }
 }
 impl Display for Unit {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let s = match self {
-            Self::Byte => "b",
+            Self::Byte => "bytes",
             Self::Kilo => "K",
             Self::Kibi => "Ki",
             Self::Mega => "M",
@@ -77,6 +89,7 @@ impl Display for Unit {
             Self::Gibi => "Gi",
             Self::Tera => "T",
             Self::Tibi => "Ti",
+            Self::Blocks => "blk"
         };
 
         f.write_str(s)