diff --git a/ansible/roles/developer-rust/README.md b/ansible/roles/developer-rust/README.md index d322e6c..90e0c87 100644 --- a/ansible/roles/developer-rust/README.md +++ b/ansible/roles/developer-rust/README.md @@ -85,9 +85,9 @@ binaries, so `cargo run`, IDEs and anything globbing for a built artefact are unaffected. `hyperi-rust-cache-prune` then bounds that pool on a schedule -- a systemd timer -on Linux, a launchd agent on macOS, daily and at idle IO priority. It drops -workspaces not built for `rust_cache_max_age_days`, then evicts the oldest by -build time until the pool is under `rust_cache_build_dir_max`. +on Linux, a launchd agent on macOS, daily and at idle IO priority, as the user. +It drops workspaces not built for `rust_cache_max_age_days`, then evicts the +oldest by build time until the pool is under `rust_cache_build_dir_max`. It touches no project `target/`, and reports the self-capping caches without pruning them. @@ -97,16 +97,28 @@ derived from it chases itself downward. That puts a 692G build box at 115G and a 256G laptop at the floor, so one default suits both. Set `rust_cache_build_dir_max` to an explicit size to override it. -**The ceiling binds while the tool runs, not between runs.** A pool that grows -faster than the schedule spends the gap above it, so the prune runs daily and an -hourly guard backs it up -- one statvfs while the disk has room, a prune to the -same ceiling once free space falls below `rust_cache_prune_free_floor` (20%). +On a filesystem of its own (`rust_cache_root` on a dedicated volume), `auto` +means no ceiling. A sixth of a 512G cache volume sat under the working set of +several concurrent build sessions, so every nightly prune evicted live +workspaces and the rebuild refilled the volume. The guard bounds it there. -A guard run that finds the pool already under its ceiling stops and says so. The -space went somewhere the prune does not own, and naming that is more use than -evicting artefacts that were not the cause. Set -`rust_cache_prune_guard_enabled: false` to drop the guard, or -`rust_cache_prune_schedule_weekday` to go back to weekly. +## The free-space guard + +**The ceiling binds while the tool runs, not between runs.** So a guard runs +every five minutes -- one statvfs while the disk has room. Below +`rust_cache_prune_free_floor` (15%) it evicts the oldest workspaces, past the +ceiling if need be, until `rust_cache_prune_free_target` (25%) is free. + +**No run evicts a workspace a build is using.** Cargo holds an exclusive flock +on a lock file in each profile directory of the build-dir for the whole build +(`.cargo-build-lock`, or `.cargo-lock` in 1.91). The prune takes it without +waiting, skips the workspace if it is held, and holds it through the delete. + +If the pool runs out first and Docker's data root is on the same filesystem, +the guard has Docker prune its build cache unused for 3 days, then unused images +created over a week ago. Docker's own commands, as the user, never sudo, and +never `daemon.json`. `rust_cache_prune_docker: false` switches that off, and +`rust_cache_prune_guard_enabled: false` drops the guard. ## Which sccache builds actually use diff --git a/ansible/roles/developer-rust/defaults/main.yml b/ansible/roles/developer-rust/defaults/main.yml index 1937040..2be1a24 100644 --- a/ansible/roles/developer-rust/defaults/main.yml +++ b/ansible/roles/developer-rust/defaults/main.yml @@ -66,8 +66,10 @@ rust_cache_ccache_max: "10G" # which covers a container or CI runner that mounts the tree elsewhere. # It does NOT unify sibling checkouts: those differ by directory name, not root. rust_cache_sccache_basedirs: "{{ user_home }}" -# `auto` is a sixth of the filesystem holding the pool, with a 40G floor. An -# explicit size (40G, 120G) overrides it. +# `auto` is a sixth of the filesystem holding the pool, with a 40G floor, while +# the pool shares a filesystem with the user's home. On a filesystem of its own +# it means no ceiling, and the guard's free-space floor and target bound it. An +# explicit size (40G, 120G) overrides both. rust_cache_build_dir_max: "auto" rust_cache_max_age_days: 14 @@ -78,17 +80,24 @@ rust_cache_max_age_days: 14 rust_cache_prune_schedule_weekday: "" rust_cache_prune_schedule_hour: 3 -# Backstop for growth that outruns the daily prune. One statvfs an hour, and a -# prune only below the floor. +# Backstop for growth that outruns the daily prune. One statvfs every five +# minutes: a busy build box has written tens of gigabytes within an hour. # -# A percentage because one size cannot suit a laptop and a build box. +# Below the floor it drops the least-recently-built workspaces, past the ceiling +# if need be, until the target is free, skipping any a build is using. The gap +# between floor and target stops the next builds pushing straight back under. # -# The guard prunes to the ordinary ceiling and no further. A pool already inside -# the ceiling means the space went elsewhere, and the run reports that. +# Percentages because one size cannot suit a laptop and a build box. rust_cache_prune_guard_enabled: true -rust_cache_prune_free_floor: "20%" -rust_cache_prune_guard_schedule: "hourly" -rust_cache_prune_guard_interval_seconds: 3600 +rust_cache_prune_free_floor: "15%" +rust_cache_prune_free_target: "25%" +rust_cache_prune_guard_interval_seconds: 300 + +# When the pool alone cannot reach the target, the guard asks Docker to prune +# its build cache unused for 3 days, then unused images created over a week +# ago. Only when Docker's data root is on the pool's filesystem, and only as the +# user through Docker's own commands -- no sudo, and daemon.json is not touched. +rust_cache_prune_docker: true # macOS launchd only; systemd takes its unit name from the file. rust_cache_prune_label: "io.hyperi.rust-cache-prune" diff --git a/ansible/roles/developer-rust/files/hyperi-rust-cache-prune b/ansible/roles/developer-rust/files/hyperi-rust-cache-prune index 1469fb8..2b2f016 100644 --- a/ansible/roles/developer-rust/files/hyperi-rust-cache-prune +++ b/ansible/roles/developer-rust/files/hyperi-rust-cache-prune @@ -1,12 +1,14 @@ #!/usr/bin/env python3 -"""Keep the Rust build cache under a fixed ceiling. +"""Keep the pooled Rust build cache, and the disk under it, inside their bounds. Run it yourself: hyperi-rust-cache-prune # report, then ask before deleting hyperi-rust-cache-prune --check # report only, delete nothing - hyperi-rust-cache-prune --yes # no prompt (how the timer invokes it) - hyperi-rust-cache-prune --if-free-below 20% # only when the disk is tight + hyperi-rust-cache-prune --yes # no prompt (how the timers invoke it) + hyperi-rust-cache-prune --if-free-below 15% --free-target 25% + # only when the disk is tight, and + # then until a quarter of it is free Why this exists --------------- @@ -22,18 +24,36 @@ and this is what bounds it. How it decides what to go ------------------------- -Two passes over the pool, in order: +Oldest first, in one ordering: 1. Age. A workspace not built for `--max-age-days` is dropped outright. The cost of being wrong is one slow rebuild. -2. Size. If the pool is still over the ceiling, drop least-recently-built - workspaces until it is under. +2. Ceiling. While the pool is over `--max-size`, drop the least-recently-built + workspace. +3. Free space. While the filesystem has less free than `--free-target`, keep + dropping the least-recently-built workspace, whatever the ceiling says. Least-recently-built, not least-recently-used: a read does not update mtime, so a workspace you compile against daily and one you only link against look the same. Age is measured from the newest thing cargo wrote, which is the closest honest proxy. +A workspace a build is using right now is never dropped. For the whole of a +build, cargo holds an exclusive flock on a lock file in each profile directory +of the build-dir: `.cargo-build-lock`, or `.cargo-lock` in cargo 1.91 +(src/cargo/core/compiler/layout.rs, `Layout::new`). This tool takes the same +locks without waiting, leaves the workspace alone when any is held, and keeps +them through the delete, so a build that starts meanwhile waits for it. Cargo +takes no lock on NFS, so a pool there has nothing to detect. + +The ceiling +----------- +`auto` is a sixth of the filesystem, floor 40G, while the pool shares a +filesystem with the user's home. On a filesystem of its own, such as a volume +given over to caches, `auto` means no ceiling at all: a fixed share of such a +volume evicts the live working set, and the free-space floor and target bound +the pool there instead. + When it runs at all ------------------- The ceiling is enforced when the tool runs and at no other moment, so a pool @@ -45,13 +65,20 @@ be scheduled often enough to matter and still do nothing almost every time: if the filesystem holding the pool has more free space than the floor, it exits before walking anything. -The floor takes a percentage as well as a byte count, because the same number -cannot be right on a 256G laptop and a 692G build box. +Below the floor, a guarded run prunes until `--free-target` is free, or back to +the floor itself when no target is given. The gap between floor and target is +what stops the next few builds pushing the disk straight back under the floor. + +The floor and target take a percentage as well as a byte count, because the +same number cannot be right on a 256G laptop and a 692G build box. -A guarded run that finds the pool already inside its ceiling does not go -looking for more to delete. It says the pool is not where the space went and -stops. Reclaiming artefacts that were not the problem would be the wrong -answer to a disk filled by something else. +Docker +------ +When the pool runs out before the target is met, and Docker keeps its data on +the same filesystem, the run asks Docker to prune its own build cache unused for +three days and then, if still short, unused images created over a week ago. It +does that as the invoking user through Docker's own commands: never sudo, and +never the daemon's config. `--no-docker` switches it off. What it does NOT touch ---------------------- @@ -65,9 +92,8 @@ It does not touch the cargo registry or the sccache/ccache stores either. Those have their own eviction and are reported on only. """ -from __future__ import annotations - import argparse +import fcntl import os import re import shutil @@ -92,6 +118,21 @@ AUTO_FLOOR = "40G" # artefacts and is the unit that gets evicted. POOL_DEPTH = 2 +# The build-dir lock is `.cargo-build-lock` in current cargo and was `.cargo-lock` in 1.91. +BUILD_LOCK_NAMES = (".cargo-build-lock", ".cargo-lock") +# Reaches , / and a tool's own nested build dir such as +# llvm-cov-target//. +BUILD_LOCK_DEPTH = 4 + +# Oldest data first: build cache nothing has used for three days, then unused images created +# over a week ago. +DOCKER_PRUNES = ( + ("build cache", ("builder", "prune", "-f", "--filter", "until=72h")), + ("unused images", ("image", "prune", "-af", "--filter", "until=168h")), +) +# A prune over a large image store takes minutes, and the guard's unit allows the whole run 30. +DOCKER_TIMEOUT = 900 + _SIZE = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*([KMGT]?)i?B?\s*$", re.IGNORECASE) _PERCENT = re.compile(r"^\s*(\d+(?:\.\d+)?)\s*%\s*$") _MULTIPLIER = {"": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4} @@ -103,6 +144,7 @@ class Reporter: def __init__(self) -> None: self.freed = 0 self.warnings: list[str] = [] + self.busy: list[str] = [] def info(self, msg: str) -> None: print(f" {msg}") @@ -132,7 +174,7 @@ def parse_size(text: str) -> int: return int(float(match.group(1)) * _MULTIPLIER[match.group(2).upper()]) -def parse_size_or_auto(text: str): +def parse_size_or_auto(text: str) -> int | str: """Accept a size, or the literal `auto` to derive one from the disk. Returned as the string "auto" rather than a number because the filesystem @@ -143,7 +185,7 @@ def parse_size_or_auto(text: str): return parse_size(text) -def parse_size_or_percent(text: str): +def parse_size_or_percent(text: str) -> tuple[str, float]: """A byte count, or a share of the filesystem written as a percentage. Returned as a (kind, value) pair rather than a number because a percentage @@ -160,7 +202,13 @@ def parse_size_or_percent(text: str): return ("bytes", parse_size(text)) -def above_free_floor(pool: Path, floor, rep: Reporter) -> bool: +def resolve_threshold(spec: tuple[str, float], total: int) -> int: + """Turn a (kind, value) pair from parse_size_or_percent into bytes of one filesystem.""" + kind, value = spec + return int(total * value / 100) if kind == "percent" else int(value) + + +def above_free_floor(pool: Path, floor: tuple[str, float], rep: Reporter) -> bool: """Has the filesystem holding the pool got more free space than the floor? One statvfs, called before anything walks the pool, so a guarded run that @@ -168,7 +216,6 @@ def above_free_floor(pool: Path, floor, rep: Reporter) -> bool: blocks are not ours to spend and counting them would let the guard sit quiet while writes are already failing. """ - kind, value = floor try: usage = shutil.disk_usage(pool) except OSError as exc: @@ -177,7 +224,7 @@ def above_free_floor(pool: Path, floor, rep: Reporter) -> bool: rep.warn(f"could not measure free space at {pool} ({exc}) -- pruning anyway") return False - threshold = int(usage.total * value / 100) if kind == "percent" else value + threshold = resolve_threshold(floor, usage.total) where = f"{human(usage.free)} free of {human(usage.total)}" if usage.free > threshold: rep.info(f"{where}, above the {human(threshold)} floor -- nothing to do") @@ -186,10 +233,38 @@ def above_free_floor(pool: Path, floor, rep: Reporter) -> bool: return False -def resolve_max_size(value, pool: Path, rep: Reporter) -> int: - """Turn `auto` into bytes for the filesystem that actually holds the pool.""" - if value != "auto": +def measure_free(path: Path) -> int | None: + """Free bytes on the filesystem holding path, or None when statvfs fails.""" + try: + return shutil.disk_usage(path).free + except OSError: + return None + + +def same_filesystem(first: Path, second: Path) -> bool | None: + """Whether two paths sit on one filesystem, or None when either cannot be stat'd. + + stat needs no read access to the directory itself, so this also answers for + a data root owned by root. + """ + try: + return os.stat(first).st_dev == os.stat(second).st_dev + except OSError: + return None + + +def resolve_max_size(value: int | str, pool: Path, rep: Reporter) -> int | None: + """Turn `auto` into bytes for the filesystem that holds the pool, or None for no ceiling.""" + if isinstance(value, int): return value + # A pool off the home filesystem is on a volume given over to caches, where only free + # space is a meaningful bound. + if same_filesystem(pool, Path.home()) is False: + rep.info( + "ceiling: none (auto -- the pool has a filesystem of its own, " + "so the free-space floor and target bound it)" + ) + return None floor = parse_size(AUTO_FLOOR) try: total = shutil.disk_usage(pool).total @@ -204,6 +279,38 @@ def resolve_max_size(value, pool: Path, rep: Reporter) -> int: return derived +def resolve_free_target( + target: tuple[str, float] | None, + floor: tuple[str, float] | None, + pool: Path, + rep: Reporter, +) -> int | None: + """The free space a run prunes towards, in bytes, or None when it has none. + + Given only a floor, the target is the floor. A target under the floor is + raised to it, since stopping there leaves the disk below the floor. + """ + spec = target if target is not None else floor + if spec is None: + return None + try: + total = shutil.disk_usage(pool).total + except OSError as exc: + rep.warn(f"could not measure the filesystem at {pool} ({exc}) -- no free-space target") + return None + wanted = resolve_threshold(spec, total) + if floor is not None: + minimum = resolve_threshold(floor, total) + if wanted < minimum: + rep.warn( + f"free target {human(wanted)} is below the {human(minimum)} floor " + "-- using the floor" + ) + wanted = minimum + rep.info(f"free-space target: {human(wanted)} of {human(total)}") + return wanted + + def human(size: int) -> str: value = float(size) for unit in ("B", "K", "M", "G", "T"): @@ -352,28 +459,140 @@ def find_workspaces(pool: Path) -> list[Workspace]: return [Workspace(leaf) for leaf in leaves] +def build_locks(workspace: Path) -> list[Path]: + """Every cargo build-dir lock file under one workspace's artefacts. + + A directory holding a lock is a profile directory, and what sits below it + is cargo's output, so the search stops there. + """ + found: list[Path] = [] + level = [workspace] + for _ in range(BUILD_LOCK_DEPTH): + below: list[Path] = [] + for directory in level: + locks = [directory / name for name in BUILD_LOCK_NAMES] + present = [lock for lock in locks if lock.is_file()] + if present: + found.extend(present) + continue + try: + with os.scandir(directory) as entries: + below.extend( + Path(entry.path) for entry in entries if entry.is_dir(follow_symlinks=False) + ) + except OSError: + continue + level = below + return found + + +def take_build_locks(workspace: Path) -> list[int] | None: + """Take every build lock in a workspace without waiting. + + Returns the held descriptors, or None when a build holds any of them. A lock + file that cannot be opened or flocked protects nothing, so it is passed over, + as cargo passes over a filesystem that does not support locking. + """ + held: list[int] = [] + for lock in build_locks(workspace): + try: + fd = os.open(lock, os.O_RDONLY) + except OSError: + continue + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + os.close(fd) + release_build_locks(held) + return None + except OSError: + os.close(fd) + continue + held.append(fd) + return held + + +def release_build_locks(held: list[int]) -> None: + """Drop locks taken by take_build_locks, since closing a descriptor releases its flock.""" + for fd in held: + os.close(fd) + + +class FreeSpaceTarget: + """Whether the pool's filesystem has reached its free-space target yet. + + Credited with what this run has freed as well as re-measured, and the larger + answer wins. A filesystem can report deleted space as free only after its + next commit, and a re-measure alone would then evict far past the target. A + dry run deletes nothing, so the credit is all it has. + """ + + def __init__(self, pool: Path, target: int, dry_run: bool, rep: Reporter) -> None: + self.pool = pool + self.target = target + self.dry_run = dry_run + self.rep = rep + self.baseline = rep.freed + self.start = measure_free(pool) + if self.start is None: + rep.warn( + f"could not measure free space at {pool} -- the free-space target is not applied" + ) + + def free(self) -> int | None: + if self.start is None: + return None + credited = self.start + self.rep.freed - self.baseline + if self.dry_run: + return credited + measured = measure_free(self.pool) + return credited if measured is None else max(measured, credited) + + def short(self) -> bool: + free = self.free() + return free is not None and free < self.target + + def remove(workspace: Workspace, reason: str, dry_run: bool, rep: Reporter) -> bool: - """Drop one workspace. False means the bytes are still on disk. + """Drop one workspace unless a build is using it. False means the bytes are still on disk. The caller has to know, because counting a failed removal against the pool total reports a cache back under its ceiling while it is still over. """ - if dry_run: - rep.info(f"[check] would drop {workspace.path.name} ({human(workspace.size)}, {reason})") + held = take_build_locks(workspace.path) + if held is None: + rep.busy.append(workspace.path.name) + rep.info(f"kept {workspace.path.name} ({human(workspace.size)}) -- a build is using it") + return False + # Held through the delete, so a build that starts meanwhile waits instead of writing into a + # tree being removed. + try: + if dry_run: + rep.info( + f"[check] would drop {workspace.path.name} ({human(workspace.size)}, {reason})" + ) + rep.freed += workspace.size + return True + try: + shutil.rmtree(workspace.path) + except OSError as exc: + rep.warn(f"could not remove {workspace.path}: {exc}") + return False + rep.change(f"dropped {workspace.path.name} ({human(workspace.size)}, {reason})") rep.freed += workspace.size return True - try: - shutil.rmtree(workspace.path) - except OSError as exc: - rep.warn(f"could not remove {workspace.path}: {exc}") - return False - rep.change(f"dropped {workspace.path.name} ({human(workspace.size)}, {reason})") - rep.freed += workspace.size - return True + finally: + release_build_locks(held) def prune_pool( - pool: Path, dry_run: bool, rep: Reporter, *, max_size: int, max_age_days: int + pool: Path, + dry_run: bool, + rep: Reporter, + *, + max_size: int | None, + max_age_days: int, + target: FreeSpaceTarget | None = None, ) -> int: """Bound the pool, and return what it still occupies. @@ -386,9 +605,8 @@ def prune_pool( return 0 total = sum(item.size for item in workspaces) - rep.info( - f"{pool}: {len(workspaces)} workspace(s), {human(total)} of {human(max_size)} ceiling" - ) + ceiling = f"of {human(max_size)} ceiling" if max_size is not None else "no ceiling" + rep.info(f"{pool}: {len(workspaces)} workspace(s), {human(total)} {ceiling}") stale = [item for item in workspaces if item.age_days > max_age_days] for item in stale: @@ -397,21 +615,24 @@ def prune_pool( remaining = [item for item in workspaces if item not in stale] - if total <= max_size: - rep.info(f"under the ceiling at {human(total)}") - return total - # Oldest first: the least-recently-built workspace is the cheapest to lose. for item in sorted(remaining, key=lambda entry: entry.mtime): - if total <= max_size: + over_ceiling = max_size is not None and total > max_size + short_of_space = target is not None and target.short() + if not over_ceiling and not short_of_space: break - if remove(item, f"over ceiling, {human(total)} used", dry_run, rep): + reason = ( + f"over ceiling, {human(total)} used" if over_ceiling else "below the free-space target" + ) + if remove(item, reason, dry_run, rep): total -= item.size - if total > max_size: + if max_size is not None and total > max_size: rep.warn(f"still {human(total)} after pruning everything eligible") - else: + elif max_size is not None: rep.info(f"under the ceiling at {human(total)}") + if rep.busy: + rep.info(f"{len(rep.busy)} workspace(s) kept because a build is using them") if not dry_run: drop_empty_shards(pool) @@ -432,6 +653,91 @@ def drop_empty_shards(pool: Path) -> None: continue +def docker_root_dir(docker: str, rep: Reporter) -> Path | None: + """Where the daemon keeps its data, or None when this user cannot reach it without sudo.""" + try: + proc = subprocess.run( + [docker, "info", "--format", "{{.DockerRootDir}}"], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=60, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + rep.warn(f"docker: could not ask the daemon where it keeps its data ({exc})") + return None + root = proc.stdout.strip() + if proc.returncode != 0 or not root: + detail = (proc.stderr or proc.stdout).strip().splitlines() + reason = detail[-1] if detail else f"exit {proc.returncode}" + rep.info(f"docker: daemon not reachable as this user, leaving it alone ({reason})") + return None + return Path(root) + + +def run_docker_prune(label: str, argv: list[str], rep: Reporter) -> None: + """Run one of Docker's own prunes, reporting what it reclaimed and warning on any failure.""" + command = " ".join(["docker", *argv[1:]]) + try: + proc = subprocess.run( + argv, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=DOCKER_TIMEOUT, + check=False, + ) + except (OSError, subprocess.SubprocessError) as exc: + rep.warn(f"docker: `{command}` did not complete ({exc})") + return + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip().splitlines() + rep.warn( + f"docker: `{command}` exited {proc.returncode}" + (f": {detail[-1]}" if detail else "") + ) + return + totals = [line.strip() for line in proc.stdout.splitlines() if line.strip().startswith("Total")] + rep.change(f"docker {label}: {totals[-1] if totals else 'pruned'}") + + +def docker_fallback(pool: Path, target: FreeSpaceTarget, dry_run: bool, rep: Reporter) -> None: + """Have Docker prune its own caches when they share the pool's filesystem. + + Only as the invoking user and only through Docker's own commands: a user who + cannot reach the daemon without sudo gets nothing done here, by design. + """ + docker = shutil.which("docker") + if docker is None: + rep.info("docker: not on PATH") + return + root = docker_root_dir(docker, rep) + if root is None: + return + shared = same_filesystem(root, pool) + if shared is None: + rep.info(f"docker: data root {root} is not visible from this host, leaving it alone") + return + if not shared: + rep.info( + f"docker: data root {root} is on another filesystem, so pruning it frees nothing here" + ) + return + + commands = [(label, [docker, *args]) for label, args in DOCKER_PRUNES] + if dry_run: + for index, (_, argv) in enumerate(commands): + when = "would run" if index == 0 else "then, if still short, would run" + rep.info(f"[check] {when}: docker {' '.join(argv[1:])}") + return + for label, argv in commands: + if not target.short(): + return + run_docker_prune(label, argv, rep) + + def report_self_capping_caches(rep: Reporter) -> None: """Show the caches that bound themselves, so a disk problem can be placed. @@ -500,7 +806,7 @@ def report_self_capping_caches(rep: Reporter) -> None: def main() -> int: parser = argparse.ArgumentParser( prog="hyperi-rust-cache-prune", - description="Keep the pooled Rust build cache under a fixed ceiling.", + description="Keep the pooled Rust build cache, and the disk under it, inside their bounds.", ) parser.add_argument("--check", action="store_true", help="report only, delete nothing") parser.add_argument("--yes", action="store_true", help="do not prompt before deleting") @@ -510,8 +816,9 @@ def main() -> int: default=DEFAULT_MAX_SIZE, metavar="SIZE", help=( - f"ceiling for the pool, or `auto` to take a {AUTO_FRACTION}th of the " - f"filesystem with a {AUTO_FLOOR} floor (default: {DEFAULT_MAX_SIZE})" + f"ceiling for the pool, or `auto`: a {AUTO_FRACTION}th of the filesystem " + f"with a {AUTO_FLOOR} floor, or no ceiling when the pool has a filesystem " + f"of its own (default: {DEFAULT_MAX_SIZE})" ), ) parser.add_argument( @@ -528,7 +835,27 @@ def main() -> int: metavar="SIZE|PERCENT", help=( "do nothing unless the filesystem holding the pool has less than this " - "free (e.g. 20%% or 80G) -- how the hourly guard invokes it" + "free (e.g. 15%% or 80G) -- how the guard invokes it" + ), + ) + parser.add_argument( + "--free-target", + type=parse_size_or_percent, + default=None, + metavar="SIZE|PERCENT", + help=( + "keep dropping the least-recently-built workspace, past the ceiling if " + "need be, until the filesystem has this much free (default with " + "--if-free-below: the floor)" + ), + ) + parser.add_argument( + "--docker", + action=argparse.BooleanOptionalAction, + default=True, + help=( + "when the pool alone cannot reach the free-space target, prune Docker's " + "build cache and old unused images if they share its filesystem" ), ) parser.add_argument( @@ -585,10 +912,18 @@ def main() -> int: return 0 max_size = resolve_max_size(args.max_size, pool, rep) + free_target = resolve_free_target(args.free_target, args.if_free_below, pool, rep) if not args.check and not args.yes: - print(f"\nThis deletes pooled build artefacts over {human(max_size)} " - f"or idle more than {args.max_age_days} days.") + bounds = f"idle more than {args.max_age_days} days" + if max_size is not None: + bounds += f", over the {human(max_size)} ceiling" + if free_target is not None: + bounds += f", and oldest first until {human(free_target)} is free" + print(f"\nThis deletes pooled build artefacts {bounds}.") + if free_target is not None and args.docker: + print("If the pool alone is not enough, it also has Docker prune old build") + print("cache and unused images on the same disk.") print("Projects rebuild them on demand. Re-run with --check to see the detail first.") try: if input("Proceed? [y/N] ").strip().lower() not in {"y", "yes"}: @@ -598,25 +933,36 @@ def main() -> int: print("\nNothing done.") return 0 + target = ( + FreeSpaceTarget(pool, free_target, args.check, rep) if free_target is not None else None + ) + rep.step("Pooled build artefacts") - remaining = prune_pool( + prune_pool( pool, args.check, rep, max_size=max_size, max_age_days=args.max_age_days, + target=target, ) - # A guarded run ending inside the ceiling means the pool is not what filled - # the disk. Tested on what the pool still holds, not on what was freed -- a - # failed eviction also frees nothing. - if guarded and remaining <= max_size: - rep.warn( - "below the free floor with the pool already inside its ceiling -- " - "nothing here to reclaim. Whatever filled the disk is outside what " - "this tool bounds, and the caches reported below are the only other " - "ones it can see." - ) + if target is not None and target.short(): + rep.step("Docker (the pool alone did not reach the free-space target)") + if args.docker: + docker_fallback(pool, target, args.check, rep) + else: + rep.info("switched off (--no-docker)") + free = target.free() + # A dry run cannot credit what Docker would free, so only a real run can say the + # target was missed. + if not args.check and free is not None and free < target.target: + rep.warn( + f"{human(free)} free, still below the {human(target.target)} target " + "after everything this tool may reclaim. Whatever filled the disk is " + "outside what it bounds, and the caches reported below are the only " + "other ones it can see." + ) rep.step("Self-capping caches (reported, not pruned)") try: @@ -625,7 +971,7 @@ def main() -> int: rep.warn(f"could not report the other caches: {exc}") verb = "would free" if args.check else "freed" - print(f"\n{verb} {human(rep.freed)}") + print(f"\n{verb} {human(rep.freed)} of pooled build artefacts") if rep.warnings: print(f"Finished with {len(rep.warnings)} warning(s):") for warning in rep.warnings: diff --git a/ansible/roles/developer-rust/tasks/cache.yml b/ansible/roles/developer-rust/tasks/cache.yml index 863aa56..b0b4484 100644 --- a/ansible/roles/developer-rust/tasks/cache.yml +++ b/ansible/roles/developer-rust/tasks/cache.yml @@ -7,8 +7,9 @@ # not track build artefacts and nothing upstream ever reclaims them. # # Two schedules, because a ceiling only binds while the tool runs. The daily -# prune is the routine path. The guard runs hourly, costs one statvfs when the -# disk has room, and prunes when free space drops below the floor. +# prune is the routine path. The guard runs every five minutes as the user, +# costs one statvfs when the disk has room, and below the floor prunes until the +# free-space target is met. # # The whole file is optional: a machine with no cap still builds, so nothing # here may fail the run. @@ -65,6 +66,7 @@ group: root mode: '0644' become: true + register: developer_rust_guard_timer when: rust_cache_prune_guard_enabled | bool # Check mode does not write the unit files above, so systemd has nothing to @@ -78,11 +80,13 @@ become: true when: not ansible_check_mode + # Restarted when its schedule changes: a restart re-arms OnBootSec, which is + # already past, so the guard runs once straight away on the new schedule. - name: Enable and start the rust cache prune guard timer ansible.builtin.systemd: name: hyperi-rust-cache-prune-guard.timer enabled: true - state: started + state: "{{ 'restarted' if developer_rust_guard_timer is changed else 'started' }}" daemon_reload: true become: true when: diff --git a/ansible/roles/developer-rust/templates/hyperi-rust-cache-prune-guard.service.j2 b/ansible/roles/developer-rust/templates/hyperi-rust-cache-prune-guard.service.j2 index 7babee9..69d6292 100644 --- a/ansible/roles/developer-rust/templates/hyperi-rust-cache-prune-guard.service.j2 +++ b/ansible/roles/developer-rust/templates/hyperi-rust-cache-prune-guard.service.j2 @@ -7,9 +7,9 @@ User={{ actual_user }} # A systemd unit inherits none of the user's shell environment, and the pruner # reads $CARGO_HOME to find the config that names the pool. Environment="CARGO_HOME={{ developer_rust_cargo_home }}" -# Same prune with the same ceiling, gated on free space. Above the floor it -# exits after one statvfs without walking the pool. -ExecStart=/usr/local/bin/hyperi-rust-cache-prune --yes --max-size {{ rust_cache_build_dir_max }} --max-age-days {{ rust_cache_max_age_days }}{{ ' --cache-root ' ~ rust_cache_root if rust_cache_root else '' }} --if-free-below {{ rust_cache_prune_free_floor }} +# Above the floor it exits after one statvfs without walking the pool. Below it, +# it prunes until the target is free, then falls back to Docker's own prunes. +ExecStart=/usr/local/bin/hyperi-rust-cache-prune --yes --max-size {{ rust_cache_build_dir_max }} --max-age-days {{ rust_cache_max_age_days }}{{ ' --cache-root ' ~ rust_cache_root if rust_cache_root else '' }} --if-free-below {{ rust_cache_prune_free_floor }} --free-target {{ rust_cache_prune_free_target }}{{ '' if rust_cache_prune_docker | bool else ' --no-docker' }} # Walking a large pool is IO-bound; it must never compete with an active build. Nice=19 IOSchedulingClass=idle diff --git a/ansible/roles/developer-rust/templates/hyperi-rust-cache-prune-guard.timer.j2 b/ansible/roles/developer-rust/templates/hyperi-rust-cache-prune-guard.timer.j2 index 9baa0af..b450e26 100644 --- a/ansible/roles/developer-rust/templates/hyperi-rust-cache-prune-guard.timer.j2 +++ b/ansible/roles/developer-rust/templates/hyperi-rust-cache-prune-guard.timer.j2 @@ -2,11 +2,11 @@ Description=Watch free space for the pooled Rust build cache [Timer] -OnCalendar={{ rust_cache_prune_guard_schedule }} -# Persistent so a box that was off catches up once, not once per missed hour. -Persistent=true -# Short jitter only. A guard that fires late is a guard that fired too late. -RandomizedDelaySec=5m +# Monotonic rather than calendar: OnBootSec already in the past fires at once +# when the timer starts, then OnUnitActiveSec repeats from each run. Persistent= +# applies to OnCalendar= only, so there is no missed-run catch-up to configure. +OnBootSec={{ rust_cache_prune_guard_interval_seconds | int }}s +OnUnitActiveSec={{ rust_cache_prune_guard_interval_seconds | int }}s [Install] WantedBy=timers.target diff --git a/ansible/roles/developer-rust/templates/io.hyperi.rust-cache-prune-guard.plist.j2 b/ansible/roles/developer-rust/templates/io.hyperi.rust-cache-prune-guard.plist.j2 index 8dd9064..cb2ec62 100644 --- a/ansible/roles/developer-rust/templates/io.hyperi.rust-cache-prune-guard.plist.j2 +++ b/ansible/roles/developer-rust/templates/io.hyperi.rust-cache-prune-guard.plist.j2 @@ -19,6 +19,11 @@ {% endif %} --if-free-below {{ rust_cache_prune_free_floor }} + --free-target + {{ rust_cache_prune_free_target }} +{% if not rust_cache_prune_docker | bool %} + --no-docker +{% endif %}