diff --git a/Cargo.lock b/Cargo.lock index 0798c16..d2104b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -330,7 +330,7 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "diskghost-cli" -version = "0.11.1" +version = "0.12.0" dependencies = [ "clap", "diskghost-core", @@ -340,7 +340,7 @@ dependencies = [ [[package]] name = "diskghost-core" -version = "0.11.1" +version = "0.12.0" dependencies = [ "blake3", "criterion", diff --git a/Cargo.toml b/Cargo.toml index 10197b8..d65dffc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ exclude = ["gui"] resolver = "2" [workspace.package] -version = "0.11.1" +version = "0.12.0" edition = "2021" license = "MIT" authors = ["kingchenc"] diff --git a/README.md b/README.md index 0bab546..706a4d5 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,10 @@ cargo install --path crates/diskghost-cli diskghost scan "C:\Users\me\Desktop" --top 20 # Scan options (an invalid exclude glob is an error, never silently skipped) -diskghost scan . --exclude node_modules --exclude "*.tmp" --max-depth 3 --follow-symlinks +diskghost scan . --exclude node_modules --exclude "*.tmp" --follow-symlinks + +# Read only 2 levels deep: faster, but folders below are not measured +diskghost scan . --max-depth 2 # Duplicates (ignore anything under 5 MB) diskghost dupes "D:\Media" --min-mb 5 @@ -110,6 +113,13 @@ diskghost diff before.dgs after.dgs # compare two saved sna diskghost diff before.dgs after.dgs --at "C:\Users\me\Downloads" # ...at a sub-folder ``` +`--max-depth N` stops the walk N levels below the root (1 = only the +root's own entries). Folders it cuts off are never shown as empty: one that +was not read at all reads `? … (not measured)`, one read only partly +`≥ 1.2 GB` (a lower bound), and the totals carry `≥` too. Such a scan cannot +be saved or compared (`--save` / `--since`), since every folder it skipped +would look emptied. For exact numbers, scan without it. + A scheduled task (cron, Task Scheduler) that runs `diskghost scan --since last.dgs --save last.dgs --json` gives you a daily "what grew" report with nothing else to set up. Snapshots saved by @@ -138,6 +148,11 @@ stderr with a non-zero exit code and leave stdout empty; only `rm` and `dupes --reclaim` still print their report when some files could not be removed, with the failures in its `errors`. +A scan cut short by `--max-depth` (or cancelled) has `"complete": false` at +the top and on every folder in `children` that was not fully read; its +`size` and `file_count` are then lower bounds, 0 for a folder that was not +read at all. Complete entries carry no `complete` field. + Perfect for wiring into an automation/agent that decides what to clean. ## Desktop app (GUI) @@ -174,7 +189,9 @@ A modern Tauri UI lives in `gui/`: list length, max depth, symlinks, parallel scans, snapshots kept per folder, duplicate minimum size, Recycle Bin vs. permanent delete, window size at start (1440p if it fits the screen, otherwise maximized; or 1080p / 4K / maximized), - language. The version is shown next to the name. + language. The version is shown next to the name. With a max depth set, + folders below it show "not measured" (or `≥` when partly read) instead of a + size, and such a scan is neither kept as a snapshot nor compared. - **Find duplicates** — every group is byte-identical content (BLAKE3 over the whole file), whatever the names. Copies inside installed programs and games (Steam, Epic, Battle.net, Program Files, …; DLLs, game archives) are there on diff --git a/crates/diskghost-cli/src/main.rs b/crates/diskghost-cli/src/main.rs index c517c2e..4ef9697 100644 --- a/crates/diskghost-cli/src/main.rs +++ b/crates/diskghost-cli/src/main.rs @@ -96,12 +96,12 @@ enum Command { #[command(flatten)] walk: WalkArgs, /// Save the scan as a snapshot (every folder and file, compressed) for a - /// later `--since` or `diff`. - #[arg(long, value_name = "FILE")] + /// later `--since` or `diff`. Needs the whole tree: not with --max-depth. + #[arg(long, value_name = "FILE", conflicts_with = "max_depth")] save: Option, /// Compare against a snapshot saved earlier: what grew, shrank, - /// appeared or vanished since then. - #[arg(long, value_name = "FILE")] + /// appeared or vanished since then. Not with --max-depth. + #[arg(long, value_name = "FILE", conflicts_with = "max_depth")] since: Option, }, /// Compare two snapshots saved with `scan --save`. @@ -158,8 +158,10 @@ struct WalkArgs { /// e.g. --exclude "*.tmp" --exclude node_modules #[arg(long)] exclude: Vec, - /// Limit recursion depth below the root (1 = the root's direct children). - #[arg(long)] + /// Read only this many levels below the root (1 = the root's own entries). + /// Folders the limit cuts off are not measured: their sizes are reported + /// as lower bounds, or as not measured at all. + #[arg(long, value_parser = clap::builder::RangedU64ValueParser::::new().range(1..))] max_depth: Option, /// Follow symbolic links / junctions. #[arg(long)] @@ -274,7 +276,7 @@ fn main() -> ExitCode { }; println!("{}", serde_json::to_string_pretty(&out).unwrap()); } else { - print_scan(&report); + print_scan(&report, walk.max_depth); if let Some(d) = &changes { println!(); print_diff(d); @@ -527,10 +529,21 @@ fn fmt_duration(d: std::time::Duration) -> String { } } -fn print_scan(r: &ScanReport) { +/// A folder's size as far as it is known: `≥` when part of it was not read, +/// `?` when none of it was. +fn dir_size_text(size: u64, complete: bool) -> String { + match (complete, size) { + (true, _) => human_size(size), + (false, 0) => "?".to_string(), + (false, _) => format!("≥ {}", human_size(size)), + } +} + +fn print_scan(r: &ScanReport, max_depth: Option) { println!("Scan of {}", r.root.display()); + let at_least = if r.complete { "" } else { "≥ " }; println!( - " total: {} across {} files in {} dirs", + " total: {at_least}{} across {at_least}{} files in {at_least}{} dirs", human_size(r.total_size), r.total_files, r.total_dirs @@ -550,12 +563,26 @@ fn print_scan(r: &ScanReport) { } if r.cancelled { println!(" CANCELLED: partial result, the numbers cover only what was scanned"); + } else if !r.complete { + let limit = max_depth.map_or(String::new(), |m| format!(" (--max-depth {m})")); + println!( + " DEPTH LIMIT{limit}: deeper folders were not read; ≥ marks a lower bound, ? a folder not measured" + ); } println!(); println!("Biggest sub-folders:"); for d in &r.children { - println!(" {:>10} {}", human_size(d.size), d.path.display()); + let note = if !d.complete && d.size == 0 { + " (not measured)" + } else { + "" + }; + println!( + " {:>10} {}{note}", + dir_size_text(d.size, d.complete), + d.path.display() + ); } if r.root_files_count > 0 { println!( diff --git a/crates/diskghost-cli/tests/cli.rs b/crates/diskghost-cli/tests/cli.rs index fc86212..d6d039b 100644 --- a/crates/diskghost-cli/tests/cli.rs +++ b/crates/diskghost-cli/tests/cli.rs @@ -437,3 +437,145 @@ fn snapshot_errors_are_reported() { std::fs::remove_dir_all(&d).ok(); } + +/// root/top.bin, root/a/1.bin, root/a/x/2.bin and an empty root/c. +fn depth_fixture(label: &str) -> std::path::PathBuf { + let d = scratch(label); + std::fs::create_dir_all(d.join("a/x")).unwrap(); + std::fs::create_dir_all(d.join("c")).unwrap(); + std::fs::write(d.join("top.bin"), [0u8; 100]).unwrap(); + std::fs::write(d.join("a/1.bin"), [0u8; 100]).unwrap(); + std::fs::write(d.join("a/x/2.bin"), [0u8; 100]).unwrap(); + d +} + +#[test] +fn max_depth_never_shows_unread_folders_as_empty() { + let d = depth_fixture("depth"); + let dir = d.to_str().unwrap(); + let run = |args: &[&str]| { + Command::new(bin()) + .args(["scan", dir]) + .args(args) + .output() + .expect("run diskghost") + }; + let child = |v: &serde_json::Value, name: &str| { + v["children"] + .as_array() + .unwrap() + .iter() + .find(|c| c["path"].as_str().unwrap().ends_with(name)) + .cloned() + .unwrap() + }; + + // A full scan is complete: no flag anywhere. + let v = json_only(&run(&["--json"])); + assert!(v.get("complete").is_none(), "{v}"); + assert!(v["children"] + .as_array() + .unwrap() + .iter() + .all(|c| c.get("complete").is_none())); + assert_eq!(v["total_files"], 3); + + // Depth 1: a and c were not read — not measured, never "empty". + let v = json_only(&run(&["--max-depth", "1", "--json"])); + assert_eq!(v["complete"], false); + assert_eq!(v["total_files"], 1); + for name in ["a", "c"] { + let c = child(&v, name); + assert_eq!(c["complete"], false, "{c}"); + assert_eq!(c["size"], 0); + } + let out = run(&["--max-depth", "1"]); + assert!(out.status.success()); + let s = String::from_utf8_lossy(&out.stdout); + assert!(s.contains("total: ≥ "), "{s}"); + assert!(s.contains("DEPTH LIMIT (--max-depth 1)"), "{s}"); + for name in ["a", "c"] { + let line = s + .lines() + .find(|l| l.trim_end().ends_with(&format!("{name} (not measured)"))) + .unwrap_or_else(|| panic!("no unmeasured line for {name}:\n{s}")); + assert!(line.trim_start().starts_with("? "), "{line}"); + assert!(!line.contains(" B "), "{line}"); + } + + // Depth 2: a's own file is counted, a/x is not — a is a lower bound; + // c was read and really is empty. + let v = json_only(&run(&["--max-depth", "2", "--json"])); + let a = child(&v, "a"); + assert_eq!(a["complete"], false); + assert_eq!(a["file_count"], 1); + assert!(a["size"].as_u64().unwrap() > 0); + assert!(child(&v, "c").get("complete").is_none()); + let s = String::from_utf8_lossy(&run(&["--max-depth", "2"]).stdout).into_owned(); + let line = s + .lines() + .find(|l| { + l.trim_end() + .ends_with(&format!("{}a", std::path::MAIN_SEPARATOR)) + }) + .unwrap_or_else(|| panic!("no line for a:\n{s}")); + assert!(line.trim_start().starts_with("≥ "), "{line}"); + + // A limit deeper than the tree reads everything. + let v = json_only(&run(&["--max-depth", "3", "--json"])); + assert!(v.get("complete").is_none(), "{v}"); + assert_eq!(v["total_files"], 3); + + std::fs::remove_dir_all(&d).ok(); +} + +#[test] +fn max_depth_is_refused_where_it_would_mislead() { + let d = depth_fixture("depthbad"); + let dir = d.to_str().unwrap(); + // A valid snapshot to compare against, from a full scan. + let full = d.join("full.dgs"); + let out = Command::new(bin()) + .args(["scan", dir, "--save", full.to_str().unwrap()]) + .output() + .expect("run diskghost"); + assert!(out.status.success(), "{out:?}"); + let snap = d.join("s.dgs"); + for (args, why) in [ + ( + vec![ + "scan", + dir, + "--max-depth", + "1", + "--save", + snap.to_str().unwrap(), + ], + "cannot be used with", + ), + ( + vec![ + "scan", + dir, + "--max-depth", + "1", + "--since", + full.to_str().unwrap(), + ], + "cannot be used with", + ), + (vec!["scan", dir, "--max-depth", "0"], "invalid value '0'"), + (vec!["dupes", dir, "--max-depth", "0"], "invalid value '0'"), + ] { + let out = Command::new(bin()) + .args(&args) + .output() + .expect("run diskghost"); + assert!(!out.status.success(), "{args:?} should fail"); + assert!(out.stdout.is_empty(), "{args:?}: {out:?}"); + let err = String::from_utf8_lossy(&out.stderr); + assert!(err.contains(why), "{args:?}: {err}"); + } + assert!(!snap.exists(), "no snapshot of a partial scan"); + std::fs::remove_dir_all(&d).ok(); +} diff --git a/crates/diskghost-core/src/lib.rs b/crates/diskghost-core/src/lib.rs index b6864a1..a7dee3c 100644 --- a/crates/diskghost-core/src/lib.rs +++ b/crates/diskghost-core/src/lib.rs @@ -148,6 +148,20 @@ pub struct DirSize { pub path: PathBuf, pub size: u64, pub file_count: u64, + /// False if part of the folder was never read (it lies at or below the + /// depth limit, or the walk was cancelled): `size` and `file_count` are + /// then lower bounds, and 0 when the folder itself was not read. Written + /// only when false. + #[serde(default = "yes", skip_serializing_if = "is_true")] + pub complete: bool, +} + +fn yes() -> bool { + true +} + +fn is_true(b: &bool) -> bool { + *b } /// Bytes and file count of one file extension. @@ -212,6 +226,11 @@ pub struct ScanReport { /// above then covers only the part of the tree seen before the stop. #[serde(default)] pub cancelled: bool, + /// False if some folder below was never read (depth limit, or a + /// cancelled walk): every number above is then a lower bound. Written + /// only when false. + #[serde(default = "yes", skip_serializing_if = "is_true")] + pub complete: bool, } /// A group of byte-identical files (hard links to the same physical file are diff --git a/crates/diskghost-core/src/snapshot.rs b/crates/diskghost-core/src/snapshot.rs index 6f8097d..19140e2 100644 --- a/crates/diskghost-core/src/snapshot.rs +++ b/crates/diskghost-core/src/snapshot.rs @@ -33,6 +33,18 @@ pub const DIFF_FILE_LIMIT: usize = 1000; const MAGIC: &[u8; 8] = b"DGSNAP02"; +/// What an entry of the stream is (its first byte). +pub(crate) const ENTRY_FILE: u8 = 0; +pub(crate) const ENTRY_DIR: u8 = 1; +/// A directory whose contents were never read (the depth limit, or a +/// cancelled walk). Readers before 0.12 take it for an ordinary directory. +pub(crate) const ENTRY_UNREAD_DIR: u8 = 2; + +/// Header flags. +const FLAG_HAS_FILES: u64 = 1; +/// Some directory was not read, or the walk was cancelled. +const FLAG_PARTIAL: u64 = 2; + /// Why a snapshot could not be read, written or compared. #[derive(Debug)] pub enum SnapshotError { @@ -49,6 +61,9 @@ pub enum SnapshotError { }, /// The directory to compare at is not part of the newer snapshot. NotInSnapshot(PathBuf), + /// A snapshot of a scan that did not read every folder: compared, the + /// folders it skipped would look emptied. + Partial, } impl fmt::Display for SnapshotError { @@ -70,6 +85,10 @@ impl fmt::Display for SnapshotError { SnapshotError::NotInSnapshot(p) => { write!(f, "not a directory in the snapshot: {}", p.display()) } + SnapshotError::Partial => write!( + f, + "a scan that did not read every folder (depth limit, or cancelled) cannot be compared" + ), } } } @@ -101,6 +120,9 @@ pub struct Snapshot { pub total_dirs: u64, /// Whether every file is recorded. Format 1 snapshots only know folders. pub has_files: bool, + /// Whether the scan read every folder (see [`ScanTree::complete`]). Only + /// complete snapshots can be compared. + pub complete: bool, /// Number of recorded entries (directories plus files). pub entries: u64, /// The zstd-compressed entry stream. @@ -220,19 +242,19 @@ impl Encoder { } } - fn entry(&mut self, key: &[u8], dir: bool, size: u64, files: u64) { + fn entry(&mut self, key: &[u8], kind: u8, size: u64, files: u64) { let shared = self .prev .iter() .zip(key) .take_while(|(a, b)| a == b) .count(); - self.buf.push(u8::from(dir)); + self.buf.push(kind); put_varint(&mut self.buf, shared as u64); put_varint(&mut self.buf, (key.len() - shared) as u64); self.buf.extend_from_slice(&key[shared..]); put_varint(&mut self.buf, size); - if dir { + if kind != ENTRY_FILE { put_varint(&mut self.buf, files); } self.prev.truncate(shared); @@ -261,6 +283,7 @@ struct Cursor<'a> { r: BufReader>, left: u64, key: Vec, + kind: u8, dir: bool, size: u64, files: u64, @@ -276,6 +299,7 @@ impl<'a> Cursor<'a> { ), left: s.entries, key: Vec::new(), + kind: ENTRY_FILE, dir: false, size: 0, files: 0, @@ -290,7 +314,8 @@ impl<'a> Cursor<'a> { return Ok(()); } self.left -= 1; - self.dir = get_u8(&mut self.r)? != 0; + self.kind = get_u8(&mut self.r)?; + self.dir = self.kind != ENTRY_FILE; let shared = get_varint(&mut self.r)? as usize; let len = get_varint(&mut self.r)? as usize; if shared > self.key.len() || len > 1 << 20 { @@ -328,6 +353,7 @@ impl Snapshot { total_files: root.file_count, total_dirs: r.total_dirs, has_files: true, + complete: tree.complete(), entries, data, } @@ -389,7 +415,8 @@ impl Snapshot { total_size, total_files, total_dirs, - has_files: flags & 1 == 1, + has_files: flags & FLAG_HAS_FILES != 0, + complete: flags & FLAG_PARTIAL == 0, entries, data: b[root_len..].to_vec(), }) @@ -408,7 +435,7 @@ impl Snapshot { dirs.sort_by(|a, b| cmp_key(&a.0, &b.0)); let mut enc = Encoder::new(); for (k, d) in &dirs { - enc.entry(k, true, d.size, d.file_count); + enc.entry(k, ENTRY_DIR, d.size, d.file_count); } let (data, entries) = enc.finish(); Ok(Snapshot { @@ -419,6 +446,7 @@ impl Snapshot { total_files: old.total_files, total_dirs: old.total_dirs, has_files: false, + complete: true, entries, data, }) @@ -432,6 +460,13 @@ impl Snapshot { std::fs::create_dir_all(parent)?; } let root = self.root.to_string_lossy(); + let mut flags = 0; + if self.has_files { + flags |= FLAG_HAS_FILES; + } + if !self.complete { + flags |= FLAG_PARTIAL; + } let mut out = Vec::with_capacity(self.data.len() + 80 + root.len()); out.extend_from_slice(MAGIC); for v in [ @@ -440,7 +475,7 @@ impl Snapshot { self.total_files, self.total_dirs, self.entries, - u64::from(self.has_files), + flags, root.len() as u64, self.data.len() as u64, ] { @@ -472,7 +507,7 @@ impl Snapshot { let mut c = Cursor::new(self)?; c.advance()?; while !c.done { - entries.push((c.key.clone(), c.dir, c.size)); + entries.push((c.key.clone(), c.kind, c.size)); c.advance()?; } Ok(ScanTree::from_entries(&self.root, entries)) @@ -489,6 +524,9 @@ impl Snapshot { path: path_of_key(&self.root, &c.key), size: c.size, file_count: c.files, + // Which folders a partial scan read fully is not + // recorded: all of them are lower bounds. + complete: self.complete, }); } c.advance()?; @@ -516,13 +554,18 @@ fn capture_node(tree: &ScanTree, id: usize, key: &mut Vec, enc: &mut Encoder let c = node.children[d]; let child = &tree.nodes[c]; key.extend_from_slice(child.name.as_encoded_bytes()); - enc.entry(key, true, child.size, child.file_count); + let kind = if child.listed { + ENTRY_DIR + } else { + ENTRY_UNREAD_DIR + }; + enc.entry(key, kind, child.size, child.file_count); capture_node(tree, c, key, enc); d += 1; } else { let file = &node.files[f]; key.extend_from_slice(file.name.as_encoded_bytes()); - enc.entry(key, false, file.size, 0); + enc.entry(key, ENTRY_FILE, file.size, 0); f += 1; } key.truncate(len); @@ -673,6 +716,9 @@ impl TopFiles { /// — every file below `at` that is new, gone or changed in size. A folder /// missing from the older snapshot counts as having been empty. pub fn diff(old: &Snapshot, new: &Snapshot, at: &Path) -> Result { + if !old.complete || !new.complete { + return Err(SnapshotError::Partial); + } if old.root != new.root { return Err(SnapshotError::RootMismatch { old: old.root.clone(), @@ -801,6 +847,7 @@ impl Acc<'_> { path: path_of_key(self.root, &c.key), size: c.size, file_count: c.files, + complete: true, }; if is_new { self.added.push(d); diff --git a/crates/diskghost-core/src/tree.rs b/crates/diskghost-core/src/tree.rs index 62db952..b629746 100644 --- a/crates/diskghost-core/src/tree.rs +++ b/crates/diskghost-core/src/tree.rs @@ -15,6 +15,7 @@ use std::collections::{BinaryHeap, HashMap}; use std::ffi::OsStr; use std::path::{Component, Path, PathBuf}; +use crate::snapshot::{ENTRY_DIR, ENTRY_FILE}; use crate::walk::{walk, Batch, Engine}; use crate::{ disk_space, DirSize, ExtStat, FileEntry, FileStats, Options, Progress, ScanReport, SizeBucket, @@ -44,6 +45,12 @@ pub(crate) struct Node { pub(crate) children: Vec, /// Files directly in this directory, sorted by name once the walk is done. pub(crate) files: Vec, + /// Whether this directory's own listing was read (it was not cut off by + /// the depth limit or a cancelled walk). + pub(crate) listed: bool, + /// Whether this directory and everything below it was read, so its totals + /// are exact — set once the walk is done. + complete: bool, } /// Byte order of two names — the order children and files are kept in. @@ -164,6 +171,7 @@ impl Builder { }); } let node = &mut self.tree.nodes[id]; + node.listed = true; node.size += size; node.file_count += files.len() as u64; node.files.extend(files); @@ -172,17 +180,28 @@ impl Builder { fn finish(mut self, cancelled: bool) -> ScanTree { let tree = &mut self.tree; tree.cancelled = cancelled; + // A cancelled walk may have stopped inside any listing: nothing in + // it is exact. + for node in &mut tree.nodes { + node.complete = node.listed && !cancelled; + } // Roll the totals up. A child is always created after its parent, so // walking the arena backwards visits every node before its parent. for id in (1..tree.nodes.len()).rev() { let node = &tree.nodes[id]; - let (size, files, dirs, parent) = - (node.size, node.file_count, node.dir_count + 1, node.parent); + let (size, files, dirs, complete, parent) = ( + node.size, + node.file_count, + node.dir_count + 1, + node.complete, + node.parent, + ); if let Some(p) = parent { let pn = &mut tree.nodes[p]; pn.size += size; pn.file_count += files; pn.dir_count += dirs; + pn.complete &= complete; } } // Sort children and files by name: lookups binary-search them, and @@ -231,6 +250,7 @@ impl Builder { continue; } let descend = opts.max_depth.map_or(true, |m| child_depth < m); + self.tree.nodes[id].listed = true; let (mut size, mut count) = (0u64, 0u64); for e in table.children(dir) { if e.is_dir { @@ -337,6 +357,8 @@ impl Node { dir_count: 0, children: Vec::new(), files: Vec::new(), + listed: false, + complete: false, } } } @@ -374,10 +396,11 @@ impl ScanTree { tree } - /// A tree from snapshot entries: (key, is directory, size) in snapshot - /// order, where a folder comes before everything inside it. Keys are - /// paths relative to `root` with `/` between components. - pub(crate) fn from_entries(root: &Path, entries: Vec<(Vec, bool, u64)>) -> ScanTree { + /// A tree from snapshot entries: (key, kind, size) in snapshot order, + /// where a folder comes before everything inside it. Keys are paths + /// relative to `root` with `/` between components; kinds are the + /// snapshot's `ENTRY_*` values. + pub(crate) fn from_entries(root: &Path, entries: Vec<(Vec, u8, u64)>) -> ScanTree { let mut b = Builder { tree: ScanTree { root: root.to_path_buf(), @@ -393,9 +416,10 @@ impl ScanTree { pending: HashMap::new(), ext_ids: HashMap::new(), }; + b.tree.nodes[0].listed = true; // Node ids of the folders on the path to the current entry. let mut stack: Vec = vec![0]; - for (key, is_dir, size) in entries { + for (key, kind, size) in entries { let depth = key.iter().filter(|&&c| c == b'/').count() + 1; let name_bytes = key.rsplit(|&c| c == b'/').next().unwrap_or(&[]); let name = std::ffi::OsString::from(String::from_utf8_lossy(name_bytes).into_owned()); @@ -403,8 +427,9 @@ impl ScanTree { let Some(&parent) = stack.last() else { continue; }; - if is_dir { + if kind != ENTRY_FILE { let id = b.push(parent, &name); + b.tree.nodes[id].listed = kind == ENTRY_DIR; stack.push(id); } else { let ext = b.ext_id(&name); @@ -488,6 +513,16 @@ impl ScanTree { /// Flag a tree rebuilt elsewhere as the result of a cancelled walk. pub fn mark_cancelled(&mut self) { self.cancelled = true; + for node in &mut self.nodes { + node.complete = false; + } + } + + /// True if every directory was read: no depth limit cut the walk short + /// and it was not cancelled. Only then are the totals exact — and only + /// then is the tree worth a snapshot to compare against. + pub fn complete(&self) -> bool { + self.nodes[0].complete } /// Why the MFT engine was requested but not used (e.g. no administrator @@ -533,6 +568,7 @@ impl ScanTree { path: base.join(&*n.name), size: n.size, file_count: n.file_count, + complete: n.complete, } }) .collect() @@ -642,6 +678,7 @@ impl ScanTree { disk_total: self.disk_total, disk_free: self.disk_free, cancelled: self.cancelled, + complete: node.complete, } } @@ -664,6 +701,7 @@ impl ScanTree { path, size: n.size, file_count: n.file_count, + complete: n.complete, }); } out @@ -682,6 +720,7 @@ impl ScanTree { let (size, files, dirs) = (n.size, n.file_count, n.dir_count + 1); self.nodes[parent].children.retain(|&c| c != id); self.subtract(parent, size, files, dirs); + self.recheck_complete(parent); // The subtree is unreachable now; give its memory back. let mut stack = vec![id]; while let Some(i) = stack.pop() { @@ -705,6 +744,21 @@ impl ScanTree { true } + /// Recompute `complete` for `id` and its ancestors after a child went + /// away: removing the only unread folder makes the rest exact again. + fn recheck_complete(&mut self, mut id: usize) { + loop { + let n = &self.nodes[id]; + let complete = + n.listed && !self.cancelled && n.children.iter().all(|&c| self.nodes[c].complete); + self.nodes[id].complete = complete; + match self.nodes[id].parent { + Some(p) => id = p, + None => break, + } + } + } + /// Subtract totals from `id` and every ancestor up to the root. fn subtract(&mut self, mut id: usize, size: u64, files: u64, dirs: u64) { loop { @@ -901,6 +955,106 @@ mod tests { std::fs::remove_dir_all(&d).ok(); } + #[test] + fn depth_limit_marks_what_was_not_read() { + // root (depth 0) > a, b, c (1) > a/x (2) > a/x/y (3) + let d = tmpdir("depth"); + write(&d.join("top.bin"), &[0u8; 100_000]); + write(&d.join("a/1.bin"), &[0u8; 100_000]); + write(&d.join("a/x/2.bin"), &[0u8; 40_000]); + write(&d.join("a/x/y/3.bin"), &[0u8; 20_000]); + write(&d.join("b/4.bin"), &[0u8; 10_000]); + std::fs::create_dir_all(d.join("c")).unwrap(); + let build = |depth| { + let opts = Options { + max_depth: depth, + ..Default::default() + }; + ScanTree::build(&d, &opts, &Progress::default()) + }; + let by_name = |r: &ScanReport, name: &str| { + r.children + .iter() + .find(|c| c.path.file_name() == Some(OsStr::new(name))) + .cloned() + .unwrap() + }; + + let full = build(None); + assert!(full.complete()); + let r = full.root_report(10); + assert!(r.complete && r.children.iter().all(|c| c.complete)); + + // Depth 1: the root is listed, its folders are not. + let one = build(Some(1)); + assert!(!one.complete()); + let r = one.root_report(10); + assert!(!r.complete); + assert_eq!(r.total_files, 1); + assert_eq!(r.children.len(), 3); + for c in &r.children { + assert!(!c.complete, "{} was not read", c.path.display()); + assert_eq!((c.size, c.file_count), (0, 0)); + } + + // Depth 2: a, b and c are listed; only a has a folder below. + let two = build(Some(2)); + let r = two.root_report(10); + assert!(!r.complete); + let a = by_name(&r, "a"); + assert!(!a.complete); + assert_eq!(a.file_count, 1, "a's own file is counted"); + assert!(a.size > 0); + assert!(by_name(&r, "b").complete); + assert!(by_name(&r, "c").complete); + let x = &two.report(&d.join("a"), 10).unwrap().children[0]; + assert!(!x.complete); + assert_eq!((x.size, x.file_count), (0, 0)); + assert!(two.report(&d.join("b"), 10).unwrap().complete); + + // Depth 3 still stops above y; depth 4 reads everything. + assert!(!build(Some(3)).complete()); + let four = build(Some(4)); + assert!(four.complete()); + assert_eq!( + four.root_report(10).total_size, + full.root_report(10).total_size + ); + + // What a snapshot of a partial scan keeps, and refuses. + let snap = crate::Snapshot::capture(&two); + assert!(!snap.complete); + let file = d.join("partial.dgs"); + snap.save(&file).unwrap(); + let loaded = crate::Snapshot::load(&file).unwrap(); + std::fs::remove_file(&file).unwrap(); + assert!(!loaded.complete && loaded.has_files); + let back = loaded.to_tree().unwrap(); + assert!(!back.complete()); + assert_eq!(back.root_report(10).children, r.children); + let full_snap = crate::Snapshot::capture(&full); + assert!(full_snap.complete); + for (old, new) in [(&full_snap, &snap), (&snap, &full_snap)] { + assert!(matches!( + crate::diff(old, new, &d), + Err(crate::SnapshotError::Partial) + )); + } + assert!(full_snap.to_tree().unwrap().complete()); + + // Removing the only unread folders makes the rest exact again. + let mut one = one; + for name in ["a", "b"] { + assert!(one.remove(&d.join(name))); + assert!(!one.complete()); + } + assert!(one.remove(&d.join("c"))); + assert!(one.complete()); + assert!(one.root_report(10).complete); + + std::fs::remove_dir_all(&d).ok(); + } + #[test] fn cancelled_walk_is_flagged() { let d = tmpdir("cancel"); @@ -911,9 +1065,11 @@ mod tests { assert!(tree.cancelled()); let r = tree.root_report(10); assert!(r.cancelled); + assert!(!r.complete && !tree.complete()); assert_eq!(r.total_files, 0); let ok = ScanTree::build(&d, &Options::default(), &Progress::default()); assert!(!ok.root_report(10).cancelled); + assert!(ok.complete() && ok.root_report(10).complete); assert_eq!(ok.root_report(10).total_files, 1); assert_eq!(logical_total(&d), 10); std::fs::remove_dir_all(&d).ok(); @@ -962,6 +1118,7 @@ mod tests { let tree = b.finish(false); assert_eq!(tree.nodes.len(), 4, "no node created twice"); + assert!(tree.complete(), "every folder got its batch"); let r = tree.root_report(10); assert_eq!(r.total_size, 8); assert_eq!(r.total_files, 3); diff --git a/gui/src-tauri/Cargo.lock b/gui/src-tauri/Cargo.lock index fdd31b0..bdf9da1 100644 --- a/gui/src-tauri/Cargo.lock +++ b/gui/src-tauri/Cargo.lock @@ -645,7 +645,7 @@ dependencies = [ [[package]] name = "diskghost-core" -version = "0.11.1" +version = "0.12.0" dependencies = [ "blake3", "fs4", @@ -659,7 +659,7 @@ dependencies = [ [[package]] name = "diskghost-gui" -version = "0.11.1" +version = "0.12.0" dependencies = [ "diskghost-core", "serde", diff --git a/gui/src-tauri/Cargo.toml b/gui/src-tauri/Cargo.toml index d3457b5..5fac055 100644 --- a/gui/src-tauri/Cargo.toml +++ b/gui/src-tauri/Cargo.toml @@ -4,7 +4,7 @@ [package] name = "diskghost-gui" -version = "0.11.1" +version = "0.12.0" edition = "2021" license = "MIT" description = "Diskghost desktop GUI (Tauri) — modern disk usage & duplicate finder." diff --git a/gui/src-tauri/src/main.rs b/gui/src-tauri/src/main.rs index e7bcdd6..8bbecf2 100644 --- a/gui/src-tauri/src/main.rs +++ b/gui/src-tauri/src/main.rs @@ -295,7 +295,7 @@ async fn scan_dir( let (tree, snap) = build_tree(&app, job, &requested, &opts, &running.progress); drop(running); let view = view_of(&tree, tree.root(), top).ok_or("scan produced no tree")?; - let complete = !tree.cancelled(); + let complete = tree.complete(); let root = tree.root().to_path_buf(); let shared: Shared = Arc::new(RwLock::new(Cached { tree, @@ -305,8 +305,8 @@ async fn scan_dir( if let Ok(mut trees) = app.state::().trees.lock() { trees.insert(job, shared.clone()); } - // A cancelled walk is a partial view: browsable, but never recorded - // as a point in the folder's history. + // A cancelled or depth-limited walk is a partial view: browsable, but + // never recorded as a point in the folder's history. if complete { let app = app.clone(); std::thread::spawn(move || { diff --git a/gui/src-tauri/tauri.conf.json b/gui/src-tauri/tauri.conf.json index c7d02f9..dfbaf53 100644 --- a/gui/src-tauri/tauri.conf.json +++ b/gui/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Diskghost", - "version": "0.11.1", + "version": "0.12.0", "identifier": "org.diskghost.app", "build": { "frontendDist": "../ui", diff --git a/gui/ui/app.js b/gui/ui/app.js index d310df5..d0b981a 100644 --- a/gui/ui/app.js +++ b/gui/ui/app.js @@ -1,6 +1,7 @@ "use strict"; -// Pure helpers (human, humanDelta, fmtMs, baseName, fmtDate, crumbs, squarify, +// Pure helpers (human, humanDelta, unmeasured, sizeText, countText, fmtMs, +// baseName, fmtDate, crumbs, squarify, // deltaTint, changesByPath, pct, EXCLUDE_PRESETS, mergeSettings, // activeExcludes, categorize, bucketLabel, jobsToStart, mapItems, parentPath, // isUnder) come from lib.js as globals. @@ -44,6 +45,8 @@ const I18N = { backHint: "Previous folder (mouse back button, Alt+←)", upHint: "Parent folder (right-click the map, Backspace, Alt+↑)", cancelling: "Cancelling…", cancelledNotice: "This scan was cancelled — every number covers only the part that was walked.", + depthNotice: "Max depth: deeper folders were not read. ≥ marks a lower bound, “not measured” a folder that was not read at all — and only a complete scan can be compared.", + notMeasured: "not measured", kSize: "Size", kSizeOfUsed: "of the used space on", kSizeOfDisk: "of the disk", kFiles: "Files", kAvg: "avg", kPerFile: "per file", kFolders: "Folders", kDirect: "files directly in this folder", @@ -153,6 +156,8 @@ const I18N = { backHint: "Vorheriger Ordner (Maus-Zurück-Taste, Alt+←)", upHint: "Übergeordneter Ordner (Rechtsklick auf die Karte, Rücktaste, Alt+↑)", cancelling: "Breche ab…", cancelledNotice: "Dieser Scan wurde abgebrochen — alle Zahlen decken nur den gescannten Teil ab.", + depthNotice: "Max. Tiefe: Tiefere Ordner wurden nicht gelesen. ≥ markiert eine Untergrenze, „nicht gemessen“ einen Ordner, der gar nicht gelesen wurde — und vergleichen lässt sich nur ein vollständiger Scan.", + notMeasured: "nicht gemessen", kSize: "Größe", kSizeOfUsed: "des belegten Platzes auf", kSizeOfDisk: "der Platte", kFiles: "Dateien", kAvg: "Ø", kPerFile: "pro Datei", kFolders: "Ordner", kDirect: "Dateien direkt in diesem Ordner", @@ -601,10 +606,12 @@ async function runJob(j) { j.diff = null; j.history = null; j.state = view.cancelled ? "cancelled" : "done"; + // Cancelled or cut off by the depth limit: browsable, not comparable. + j.partial = view.complete === false; const back = j.returnTo; j.returnTo = null; if (back && back !== j.root) navigate(j, back, { keepTab: true, record: false }); - if (!view.cancelled) autoCompare(j); + if (!j.partial) autoCompare(j); if (view.mft_fallback) status(`${t("mftFallback")}: ${view.mft_fallback}`, true); } else { j.groups = await invoke("find_dupes", { job: j.id, path: j.root, mb: j.mb, opts: j.opts }); @@ -968,7 +975,8 @@ function scanView(j) { el("button", { type: "button", class: "small", onclick: () => enqueue("dupes", j.path) }, `👯 ${t("dupesHere")}`), el("button", { type: "button", class: "small", onclick: () => rescan(j) }, `↻ ${t("rescan")}`)); view.append(el("div", { class: "full", style: { display: "flex", flexDirection: "column", gap: "12px" } }, - head, v.cancelled ? el("div", { class: "notice", text: `⚠ ${t("cancelledNotice")}` }) : null)); + head, v.cancelled ? el("div", { class: "notice", text: `⚠ ${t("cancelledNotice")}` }) + : v.complete === false ? el("div", { class: "notice", text: `⚠ ${t("depthNotice")}` }) : null)); view.append(kpiRow(j, v)); @@ -1002,14 +1010,15 @@ function kpiRow(j, v) { const row = el("div", { class: "kpis full" }); const d = driveOf(v.root); const used = v.disk_total - v.disk_free; - row.append(kpi(t("kSize"), human(v.total_size), + const atLeast = v.complete === false ? "≥ " : ""; + row.append(kpi(t("kSize"), atLeast + human(v.total_size), used > 0 ? `${pct(v.total_size, used)} ${t("kSizeOfUsed")} ${d ? d.path : t("kSizeOfDisk")}` : null)); - row.append(kpi(t("kFiles"), num(v.total_files), + row.append(kpi(t("kFiles"), atLeast + num(v.total_files), v.total_files ? `${t("kAvg")} ${human(Math.round(v.total_size / v.total_files))} ${t("kPerFile")}` : null)); - row.append(kpi(t("kFolders"), num(v.total_dirs), `${num(v.root_files_count)} ${t("kDirect")}`)); + row.append(kpi(t("kFolders"), atLeast + num(v.total_dirs), `${num(v.root_files_count)} ${t("kDirect")}`)); const bigDir = v.children[0]; if (bigDir) { - row.append(kpi(t("kBiggestFolder"), baseName(bigDir.path), `${human(bigDir.size)} · ${pct(bigDir.size, v.total_size)} · ${num(bigDir.file_count)} ${t("files")}`, + row.append(kpi(t("kBiggestFolder"), baseName(bigDir.path), `${sizeText(bigDir, t("notMeasured"))} · ${pct(bigDir.size, v.total_size)} · ${countText(bigDir, num)} ${t("files")}`, { role: "button", tabindex: "0", style: { cursor: "pointer" }, onclick: () => navigate(j, bigDir.path) }, "name")); } const bigFile = v.top_files[0]; @@ -1071,6 +1080,7 @@ async function autoCompare(j) { // folder on screen against the newest earlier snapshot. async function openCompare(j) { j.tab = "changes"; + if (j.partial) return renderMain(); if (j.path === j.root && j.auto?.state === "done") { j.diff = j.auto.diff; j.sinceAt = j.diff.old_taken_at; @@ -1155,10 +1165,10 @@ function drawTreemap(container, j) { node.style.height = `${Math.max(0, h)}px`; container.append(node); }; - const label = (name, size, share, w, h, stacked) => { + const label = (name, item, share, w, h, stacked) => { if (w < 46 || h < 18) return []; const out = [el("span", { class: "tn", text: name })]; - if (stacked ? h > 34 : w > 150) out.push(el("span", { class: "ts", text: `${human(size)}${share ? ` · ${share}` : ""}` })); + if (stacked ? h > 34 : w > 150) out.push(el("span", { class: "ts", text: `${sizeText(item, t("notMeasured"))}${share ? ` · ${share}` : ""}` })); return out; }; const decorate = (node, path) => { @@ -1170,10 +1180,10 @@ function drawTreemap(container, j) { }; const folderTip = (item, parentSize) => () => { const out = [el("b", { text: item.path ? baseName(item.path) : item.label }), item.path ? el("div", { class: "tp", text: item.path }) : null, - tipRow(t("colSize"), human(item.size)), + tipRow(t("colSize"), sizeText(item, t("notMeasured"))), tipRow(`${t("colPct")} ${t("shareOfView")}`, pct(item.size, v.total_size))]; if (item.path && parentSize && parentSize !== v.total_size) out.push(tipRow(`${t("colPct")} ${baseName(parentPath(item.path) || "")}`, pct(item.size, parentSize))); - if (item.file_count != null) out.push(tipRow(t("colFiles"), num(item.file_count))); + if (item.file_count != null) out.push(tipRow(t("colFiles"), countText(item, num))); const c = diffMap?.map.get(String(item.path)); if (c) out.push(tipRow(t("colDelta"), c.added ? t("newBadge") : humanDelta(c.delta))); return out; @@ -1185,7 +1195,7 @@ function drawTreemap(container, j) { if (!it.path) { it.label = it.rest === "files" ? `${num(v.root_files_count)} ${t("filesHere")}` : t("otherFolders"); const node = el("div", { class: "tile l1 leaf rest" }); - node.append(el("div", { class: "thead" }, label(it.label, it.size, pct(it.size, v.total_size), w, h, true))); + node.append(el("div", { class: "thead" }, label(it.label, it, pct(it.size, v.total_size), w, h, true))); tip(node, folderTip(it)); place(node, x, y, w, h); continue; @@ -1193,7 +1203,7 @@ function drawTreemap(container, j) { const framed = w > 150 && h > 90 && it.nested.length > 0; const node = el("button", { type: "button", class: `tile l1${framed ? "" : " leaf"}`, onclick: () => navigate(j, it.path) }); node.style.setProperty("--s", String(Math.sqrt(it.size / maxL1))); - node.append(el("div", { class: "thead" }, label(baseName(it.path), it.size, pct(it.size, v.total_size), w, h, !framed))); + node.append(el("div", { class: "thead" }, label(baseName(it.path), it, pct(it.size, v.total_size), w, h, !framed))); decorate(node, it.path); tip(node, folderTip(it)); place(node, x, y, w, h); @@ -1206,13 +1216,13 @@ function drawTreemap(container, j) { const x2 = r2.x + 1, y2 = r2.y + 1, w2 = r2.w - 2, h2 = r2.h - 2; if (!n2.path) { n2.label = t("filesSmaller"); - const rest = el("div", { class: "tile l2 rest" }, label(n2.label, n2.size, null, w2, h2, true)); + const rest = el("div", { class: "tile l2 rest" }, label(n2.label, n2, null, w2, h2, true)); tip(rest, folderTip(n2, it.size)); place(rest, x2, y2, w2, h2); continue; } const t2 = el("button", { type: "button", class: "tile l2", onclick: (e) => { e.stopPropagation(); navigate(j, n2.path); } }, - label(baseName(n2.path), n2.size, null, w2, h2, true)); + label(baseName(n2.path), n2, null, w2, h2, true)); t2.style.setProperty("--s", String(Math.sqrt(n2.size / maxL2))); tip(t2, folderTip(n2, it.size)); place(t2, x2, y2, w2, h2); @@ -1267,9 +1277,9 @@ function foldersTab(j, v, body, diffMap) { tb.append(el("tr", { class: cls, onclick: () => navigate(j, d.path) }, el("td", { class: "namecell" }, el("div", { class: "nm", text: `📁 ${baseName(d.path)}` }), el("div", { class: "pth", text: d.path })), el("td", { class: "barcell" }, shareBar(d.size, v.total_size)), - el("td", { class: "num", text: human(d.size) }), - el("td", { class: "num", text: pct(d.size, v.total_size) }), - el("td", { class: "num", text: num(d.file_count) }), + el("td", { class: "num", text: sizeText(d, t("notMeasured")) }), + el("td", { class: "num", text: unmeasured(d) ? "—" : pct(d.size, v.total_size) }), + el("td", { class: "num", text: countText(d, num) }), diffMap ? el("td", { class: "num" }, el("span", { class: "delta", text: delta })) : null, el("td", { class: "acts" }, delButton(d.path)))); } @@ -1397,7 +1407,7 @@ function sizesTab(v, body) { // Every complete scan is stored as a snapshot of its root (every folder and // file, compressed). Compare the folder on screen against an earlier one. async function changesTab(j, v, body, tools) { - if (v.cancelled) return body.append(el("div", { class: "muted", text: t("cancelledNotice") })); + if (j.partial) return body.append(el("div", { class: "muted", text: t(v.cancelled ? "cancelledNotice" : "depthNotice") })); if (!j.history) { try { j.history = (await invoke("list_snapshots", { job: j.id })).snapshots || []; diff --git a/gui/ui/lib.js b/gui/ui/lib.js index ad103c3..431e1f7 100644 --- a/gui/ui/lib.js +++ b/gui/ui/lib.js @@ -15,6 +15,27 @@ return i === 0 ? `${bytes} B` : `${s.toFixed(1)} ${u[i]}`; } + // Whether a folder entry was not read at all (beyond the depth limit). An + // entry without `complete` was read in full. + function unmeasured(item) { + return item.complete === false && !item.size; + } + + // A folder's size as far as it is known: "≥ 1.5 KB" when part of it was not + // read, `notMeasured` when none of it was. + function sizeText(item, notMeasured) { + if (item.complete !== false) return human(item.size); + return unmeasured(item) ? notMeasured : `≥ ${human(item.size)}`; + } + + // A folder's file count, formatted by `fmt`, the same way ("—" when not + // read at all). + function countText(item, fmt) { + const n = fmt(item.file_count); + if (item.complete !== false) return n; + return unmeasured(item) ? "—" : `≥ ${n}`; + } + // A signed byte difference: +1.5 KB / -1.5 KB / 0 B. function humanDelta(delta) { if (!delta) return "0 B"; @@ -396,7 +417,7 @@ } return { - human, humanDelta, fmtMs, baseName, fmtDate, crumbs, squarify, deltaTint, changesByPath, + human, humanDelta, unmeasured, sizeText, countText, fmtMs, baseName, fmtDate, crumbs, squarify, deltaTint, changesByPath, pct, EXCLUDE_PRESETS, DEFAULT_SETTINGS, mergeSettings, activeExcludes, categoryOf, categorize, bucketLabel, jobsToStart, driveKey, mapItems, parentPath, isUnder, SETTINGS_VERSION, isProtectedPath, folderSets, protectedExcludes, dropReclaimed, diff --git a/gui/ui/lib.test.js b/gui/ui/lib.test.js index 9c8b0b6..6bd46c5 100644 --- a/gui/ui/lib.test.js +++ b/gui/ui/lib.test.js @@ -15,6 +15,25 @@ test("human formats bytes like the Rust side", () => { assert.equal(lib.human(1024 ** 6), "1024.0 PB"); // capped at PB }); +test("sizes of folders the depth limit cut off are never shown as empty", () => { + const fmt = (n) => String(n); + const full = { size: 1536, file_count: 3 }; + assert.equal(lib.sizeText(full, "n/m"), "1.5 KB"); + assert.equal(lib.countText(full, fmt), "3"); + assert.equal(lib.unmeasured(full), false); + const empty = { size: 0, file_count: 0, complete: true }; + assert.equal(lib.sizeText(empty, "n/m"), "0 B"); + assert.equal(lib.countText(empty, fmt), "0"); + const partly = { size: 1536, file_count: 3, complete: false }; + assert.equal(lib.sizeText(partly, "n/m"), "≥ 1.5 KB"); + assert.equal(lib.countText(partly, fmt), "≥ 3"); + assert.equal(lib.unmeasured(partly), false); + const unread = { size: 0, file_count: 0, complete: false }; + assert.equal(lib.sizeText(unread, "n/m"), "n/m"); + assert.equal(lib.countText(unread, fmt), "—"); + assert.equal(lib.unmeasured(unread), true); +}); + test("humanDelta carries the sign", () => { assert.equal(lib.humanDelta(0), "0 B"); assert.equal(lib.humanDelta(1536), "+1.5 KB");