diff --git a/ansible/roles/developer-rust/README.md b/ansible/roles/developer-rust/README.md index 75efe19..d322e6c 100644 --- a/ansible/roles/developer-rust/README.md +++ b/ansible/roles/developer-rust/README.md @@ -135,14 +135,15 @@ developer installed is their call. `build.build-dir` is stable from Rust 1.91. On an older toolchain the setup tool says so and leaves the per-project layout alone, so the default stays safe. -## Bounding concurrent builds +## The build governor `hyperi-rust-govern` is installed as `~/.local/bin/cargo`, ahead of the real cargo on PATH, so a developer or an agent who knows none of this runs -`cargo build` and is governed: it holds one of N slots sized from the memory -budget, and on Linux with a live user manager also lands in `rust-build.slice`. -How N is chosen, what happens at saturation, and why it needs the `zram_swap` -role are in [docs/rust-build-governor.md](../../../docs/rust-build-governor.md). +`cargo build` and is governed: nice 19, and on Linux with a live user manager +`rustbuild.slice`, which bounds memory and sits below the desktop for CPU. +Which instrument yields on which host, and why it needs the `zram_swap` role, +are in +[docs/rust-build-governor.md](../../../docs/rust-build-governor.md). The host-wide picture is [docs/concurrent-dev-cache.md](../../../docs/concurrent-dev-cache.md). diff --git a/ansible/roles/developer-rust/defaults/main.yml b/ansible/roles/developer-rust/defaults/main.yml index 88f3c93..1937040 100644 --- a/ansible/roles/developer-rust/defaults/main.yml +++ b/ansible/roles/developer-rust/defaults/main.yml @@ -107,32 +107,18 @@ rust_sccache_service_manage: true # Concurrent cargo/rustc across several projects can OOM a host outright -- # observed taking down a 30G build box. Opt-OUT: disabling it removes the # shim, the slice and the config. -rust_governor_enabled: true - -# How many rust builds may be resident at once, per user. A memory semaphore, -# not a queue length: memory is spent per CRATE rather than per job, so the -# thing worth bounding is concurrent BUILDS and the bound is the slice's memory -# budget divided by what one build costs. # -# auto derive from this host's RAM (see rust_governor_build_allowance_gb) -# pin the count; 1 is a global mutex, which is what this used to be -# 0 no semaphore at all +# MEMORY is BOUNDED, by rustbuild.slice: MemoryHigh throttles by reclaim and +# MemoryMax kills the build rather than the host. Both are percentages of the +# host's own RAM, so one number fits a laptop and a build box. # -# `auto` on a 32 GB host computes 1, which is the check that the model -# reproduces the behaviour that is known to work. -rust_governor_slots: auto - -# What one build is assumed to cost, in GB: the largest single rustc plus -# headroom. Derived from an 11.6 GB peak on one workspace, so it is a starting -# point rather than a constant -- a codebase whose memory scales with the job -# count instead of with one huge crate will want a different number. -rust_governor_build_allowance_gb: 14 - -# Seconds to wait for a slot. On expiry the build proceeds at the FLOOR job -# count, still inside the slice -- it degrades, it is not released. Waiting the -# full time and then building unbounded would drop the limit at exactly the -# moment contention is highest. -rust_governor_lock_wait_seconds: 1800 +# CPU is not bounded at all -- it is DEFERRED. Builds run at nice 19 under a +# below-desktop CPUWeight, so they take every core while nobody else wants one +# and drop to about a quarter the moment the desktop does. Nothing withholds +# cores and nothing sets a job count: cargo's own default is already every +# core, and rationing CPU costs throughput on an idle machine to buy +# responsiveness that only matters on a busy one. +rust_governor_enabled: true # Whether governed builds run with CARGO_INCREMENTAL=0, so sccache can cache # them. sccache refuses to cache any rustc call carrying -C incremental. @@ -160,24 +146,17 @@ rust_governor_memory_high_pct: 50 rust_governor_memory_max_pct: 70 rust_governor_swap_max_pct: 25 -# Weight is contention-only: the desktop always wins under load, and an idle -# machine still gives builds full speed. 100 is the systemd default weight. +# Contention-only: the desktop wins under load, and an idle machine still gives +# builds every core. 100 is the systemd default, so 30 is a bit under a third of +# the desktop's share when both want the CPU at once. This, not the shim's nice +# 19, is what makes a build yield on Linux -- rustbuild.slice.j2 says why. rust_governor_cpu_weight: 30 -# Cores held back from builds on any host with 4 or more; smaller hosts keep -# every core. Taken off the top BEFORE the remainder is divided between -# concurrent builds, so the desktop keeps its share at every slot count. -rust_governor_cpu_reserve_cores: 2 - -# macOS has no cgroups; builds run under this taskpolicy QoS clamp instead. -# `utility` sits below the desktop without background's disk-IO throttle and -# E-core pinning. -rust_governor_macos_qos: utility - # The user manager's runtime directory, for every `systemctl --user` call the # role makes. Defined once because the fallback is wrong on a fleet box: # actual_user_uid is set by the developer role's init, which does not run under # a --tags run, so every call then falls back to the CONNECTION user's uid. # When that is fixed it is fixed here, in one place. +developer_rust_uid: "{{ actual_user_uid | default(ansible_facts['user_uid']) }}" developer_rust_user_env: - XDG_RUNTIME_DIR: "/run/user/{{ actual_user_uid | default(ansible_facts['user_uid']) }}" + XDG_RUNTIME_DIR: "/run/user/{{ developer_rust_uid }}" diff --git a/ansible/roles/developer-rust/files/hyperi-rust-govern b/ansible/roles/developer-rust/files/hyperi-rust-govern index de8637b..67a9624 100644 --- a/ansible/roles/developer-rust/files/hyperi-rust-govern +++ b/ansible/roles/developer-rust/files/hyperi-rust-govern @@ -1,53 +1,31 @@ #!/usr/bin/env bash -# Governs Rust builds so they cannot take the machine down: at most N builds at -# a time per user, the whole process tree inside the rust-build.slice budget, -# and priority below the desktop. +# Governs Rust builds so they cannot take the machine down: every build runs at +# nice 19, and on Linux with a live user manager inside rustbuild.slice, whose +# memory budget and CPU weight the developer-rust role deploys. No core is +# withheld and no job count is set -- docs/rust-build-governor.md has the model. # -# N is a MEMORY semaphore, not a queue length. Memory is spent per CRATE, not -# per job -- one enormous crate compiles as a single rustc however high -j goes -# -- so the bound is the slice's memory budget divided by what one build costs. -# A small host computes N=1, which is a global mutex. -# -# Installed as ~/.local/bin/cargo, which sits ahead of the cargo bin directory -# on the SOE PATH, so cargo and everything it spawns is captured. -# sccache-hosted compiles are captured separately: hyperi-sccache.service -# carries Slice=rust-build.slice. +# Installed as ~/.local/bin/cargo, ahead of the cargo bin directory on the SOE +# PATH, so cargo and everything it spawns is captured. sccache-hosted compiles +# are captured by hyperi-sccache.service's own Slice= and Nice=. # # HYPERI_RUST_GOVERNOR=off cargo build # bypass one invocation # hyperi-rust-govern [args...] # govern any other command # -# cargo run holds its slot for the program's whole lifetime, because cargo -# stays resident wrapping the binary. Under a semaphore that costs 1/N of -# capacity rather than everything, but the bypass is still right for a -# long-running `cargo run` service. -# # Written for bash 3.2: stock macOS runs this. set -u # HOME is unset in a system unit, in a container whose uid has no passwd entry, # and under env -i. Unset, set -u aborts at the first $HOME below before any -# tool runs; empty degrades to an unusable slot directory and an ungoverned -# build, which is the right failure. +# tool runs; empty degrades to an unreadable config path, which is the right +# failure. HOME="${HOME-}" -CONF="${XDG_CONFIG_HOME:-$HOME/.config}/hyperi/rust-governor.conf" +# The same path the role writes, so a host that sets XDG_CONFIG_HOME still +# reads the deployed file. +CONF="$HOME/.config/hyperi/rust-governor.conf" # shellcheck source=/dev/null [ -r "$CONF" ] && . "$CONF" -# auto = derive from this host's RAM. A positive integer pins the count. 0/off -# disables the semaphore entirely. 1 is the old global mutex. -SLOTS="${HYPERI_RUST_GOVERN_SLOTS:-auto}" -RESERVE="${HYPERI_RUST_GOVERN_CPU_RESERVE:-2}" -QOS="${HYPERI_RUST_GOVERN_MACOS_QOS:-utility}" -LOCK_WAIT="${HYPERI_RUST_GOVERN_LOCK_WAIT:-1800}" -# Must match rust-build.slice's MemoryHigh. Both resolve a percentage of the -# same memory total -- systemd against MemTotal, this script against the cgroup -# limit where one is set and MemTotal otherwise -- so on a bare host they agree. -MEM_PCT="${HYPERI_RUST_GOVERN_MEMORY_HIGH_PCT:-50}" -# What one build costs: the largest single rustc plus headroom. Derived from a -# 11.6 GB peak on one workspace, so it is a starting point, not a constant. -ALLOWANCE_GB="${HYPERI_RUST_GOVERN_BUILD_ALLOWANCE_GB:-14}" - TOOL="$(basename "$0")" if [ "$TOOL" = "hyperi-rust-govern" ]; then if [ "$#" -eq 0 ]; then @@ -93,276 +71,8 @@ if [ "${HYPERI_RUST_GOVERNOR:-on}" = "off" ] || [ -n "${HYPERI_RUST_GOVERNED:-}" exec "$REAL" "$@" fi -if command -v nproc >/dev/null 2>&1; then - cores="$(nproc)" -else - cores="$(sysctl -n hw.ncpu 2>/dev/null || echo 2)" -fi -case "$cores" in - ''|*[!0-9]*) cores=2 ;; -esac - -# Leading zeros are stripped before every numeric guard: `08` passes a digit -# test and then aborts bash arithmetic as an invalid octal constant, which -# under set -u leaves N empty and silently removes the protection. -RESERVE="${RESERVE#"${RESERVE%%[!0]*}"}" -[ -z "$RESERVE" ] && RESERVE=0 -case "$RESERVE" in - *[!0-9]*) RESERVE=2 ;; -esac -case "$LOCK_WAIT" in - ''|*[!0-9]*) LOCK_WAIT=1800 ;; -esac -MEM_PCT="${MEM_PCT#"${MEM_PCT%%[!0]*}"}" -[ -z "$MEM_PCT" ] && MEM_PCT=0 -case "$MEM_PCT" in - *[!0-9]*) MEM_PCT=50 ;; -esac -ALLOWANCE_GB="${ALLOWANCE_GB#"${ALLOWANCE_GB%%[!0]*}"}" -case "$ALLOWANCE_GB" in - ''|*[!0-9]*) ALLOWANCE_GB=14 ;; -esac -# taskpolicy's clamp names. That path has no fallback, so anything else would -# make every cargo on a Mac fail with a usage error. -case "$QOS" in - background|utility|maintenance|default) ;; - *) QOS=utility ;; -esac - -# Cores builds may use at all. The reserve is a property of the HOST -- keep -# the desktop responsive -- so it comes off the top, before anything is divided -# between concurrent builds. A box under 4 cores keeps them all. -pool="$cores" -if [ "$cores" -ge 4 ]; then - pool=$((cores - RESERVE)) -fi -[ "$pool" -lt 1 ] && pool=1 - -# Read the memory total rather than asking systemd. `systemctl show` is a D-Bus -# round trip on every cargo invocation, reports `infinity` when the slice is not -# loaded, and does not exist on macOS or in a container -- all three of which -# this script has to keep working in. Reproducing systemd's own arithmetic -# against the same total gives the same answer with none of that. -# -# The cgroup limit is preferred where one is set, because /proc/meminfo reports -# the HOST's memory inside a container: a 4 GB container on a 256 GB box would -# otherwise admit eight builds against an allowance it does not have. -mem_total_kb() { - limit="$(cat /sys/fs/cgroup/memory.max 2>/dev/null || true)" - case "$limit" in - ''|max|*[!0-9]*) ;; - *) echo $((limit / 1024)); return 0 ;; - esac - if [ -r /proc/meminfo ]; then - awk '/^MemTotal:/ { print $2; exit }' /proc/meminfo 2>/dev/null - return 0 - fi - if command -v sysctl >/dev/null 2>&1; then - bytes="$(sysctl -n hw.memsize 2>/dev/null)" - case "$bytes" in - ''|*[!0-9]*) return 0 ;; - esac - echo $((bytes / 1024)) - fi -} - -# N = min(memory budget / per-build allowance, pool / 2). -# -# The pool/2 term is not a safety margin: admitting more concurrent builds than -# there are cores to give each one a couple of jobs makes every build slower -# for no gain, and it is what makes the jobs arithmetic below unable to reach 0. -compute_slots() { - total_kb="$(mem_total_kb)" - case "$total_kb" in - ''|*[!0-9]*) echo 1; return 0 ;; - esac - - # Divide before multiplying: the loss is under a percent and it cannot - # overflow on a 32-bit shell. - high_kb=$(( (total_kb / 100) * MEM_PCT )) - allowance_kb=$(( ALLOWANCE_GB * 1024 * 1024 )) - n=$(( high_kb / allowance_kb )) - - n_cpu=$(( pool / 2 )) - [ "$n" -gt "$n_cpu" ] && n="$n_cpu" - [ "$n" -lt 1 ] && n=1 - echo "$n" -} - -# The same strip as the guards above, on a copy: `auto` has to survive it. -SLOTS_N="${SLOTS#"${SLOTS%%[!0]*}"}" -[ -z "$SLOTS_N" ] && SLOTS_N=0 - -# Anything unrecognised falls to 1 rather than to 0: a typo in the config must -# not silently remove the protection. That also catches Jinja rendering a -# leftover boolean `rust_governor_serialize: true` as `True`, which means -# serialise, which is one slot. -case "$SLOTS" in - auto|AUTO) N="$(compute_slots)" ;; - 0|off|OFF|false|False|FALSE|no|No|NO) N=0 ;; - *[!0-9]*) N=1 ;; - *) N="$SLOTS_N" ;; -esac - -# Jobs scale with the number of neighbours the semaphore admits, rather than -# assuming sole occupancy. Floored at 2: a measured single-job build took 2.3x -# as long, which is never the right computed default. -if [ "$N" -ge 1 ]; then - jobs=$((pool / N)) -else - jobs="$pool" -fi -[ "$jobs" -lt 2 ] && jobs=2 -[ "$pool" -lt 2 ] && jobs=1 - export HYPERI_RUST_GOVERNED=1 -# Never /tmp. A world-writable slot directory lets any local account take an -# exclusive flock on every slot file -- read access is enough for flock(2) -- -# and pin every build on the host at the degraded job count. It also lets one -# be pre-created unwritable, which is worse. ~/.cache is the fallback because -# XDG_RUNTIME_DIR is absent for cron, containers, CI and plain `ssh host cargo`. -# -# Tested for emptiness, not just for being unset: an XDG_RUNTIME_DIR set to "" -# passes :- and yields a path at the filesystem root. -if [ -n "${XDG_RUNTIME_DIR:-}" ]; then - SLOT_BASE="$XDG_RUNTIME_DIR" -else - SLOT_BASE="${XDG_CACHE_HOME:-$HOME/.cache}/hyperi" -fi -SLOT_DIR="$SLOT_BASE/hyperi-rust-govern-$(id -u).slots" -uname_s="$(uname -s)" - -slot=0 - -notify_wait() { - echo "hyperi-rust-govern: all $N build slots are taken -- waiting (HYPERI_RUST_GOVERNOR=off to bypass)" >&2 -} - -# No holder PIDs are reported. fuser on the slot files lists every process with -# the fd open, which is waiters and the reporting process too, so the number was -# consistently larger than the number of builds and told nobody anything. -notify_degraded() { - echo "hyperi-rust-govern: no slot after ${LOCK_WAIT}s -- building at -j$jobs, capped at ${ALLOWANCE_GB}G" >&2 -} - -# One fd over N files: reopening fd 9 closes the previous description and -# releases any flock on it, so slot choice is which FILE you open. That keeps -# this to bash 3.2 with no eval and no {var}> named descriptors. -# -# `<>` rather than `>`: `>` truncates, so probing a busy slot would wipe the -# holder's file on the way past. -take_slot() { - i=0 - while [ "$i" -lt "$N" ]; do - n=$(( (start_at + i - 1) % N + 1 )) - i=$((i + 1)) - # No 2>/dev/null on an exec redirect: on success it would persist and - # silence the build's own stderr. - exec 9<>"$SLOT_DIR/slot.$n" || continue - if flock "$@" 9; then - slot="$n" - return 0 - fi - done - return 1 -} - -# A slot directory that cannot be used is NOT contention, and must not be -# mistaken for it: every lock would fail instantly, the wait loop would spin its -# whole budget in zero seconds printing a line per attempt, and the build would -# end at the degraded job count on a completely idle host. Checked once, and -# treated as ungoverned so the computed job count survives. -# -# The file open is probed as well as the mkdir: mkdir -p succeeds on a directory -# that already exists whether or not it is writable, and it is the open inside -# take_slot that has to work. ENOSPC and a read-only remount both land here. -# -# Created 0700, because a traversable home is not a private one: read access is -# all flock(2) needs, so a 0755 $HOME with a 0775 cache directory lets any local -# account hold every slot and pin every build at the degraded job count. -if [ "$N" -ge 1 ] && { ! (umask 077; mkdir -p "$SLOT_DIR") || ! : >>"$SLOT_DIR/slot.1"; } 2>/dev/null; then - echo "hyperi-rust-govern: cannot use $SLOT_DIR -- building ungoverned at -j$jobs" >&2 - N=0 -fi - -if [ "$N" -ge 1 ] && command -v flock >/dev/null 2>&1; then - # Spread waiters across slots rather than having every arrival stampede - # slot 1 and hand it to whoever wakes first. - start_at=$(( ($$ % N) + 1 )) - - if ! take_slot -n; then - notify_wait - waited=0 - # Short blocking waits cycled across every slot, so a waiter takes - # whichever frees first. One pass costs up to 2N seconds. - while [ "$waited" -lt "$LOCK_WAIT" ]; do - take_slot -w 2 && break - waited=$(( waited + 2 * N )) - done - fi -elif [ "$N" -ge 1 ]; then - # macOS ships no flock(1). A pid-stamped mkdir lock is atomic everywhere; - # a holder that died without cleaning up is detected and cleared. The trap - # is not inherited by children, so this path never had the daemon problem - # the flock path needed 9>&- to fix -- but a trap does not fire on SIGKILL, - # which is why the liveness check below is still required. - start_at=$(( ($$ % N) + 1 )) - notified=0 - waited=0 - while [ "$waited" -lt "$LOCK_WAIT" ]; do - i=0 - while [ "$i" -lt "$N" ]; do - n=$(( (start_at + i - 1) % N + 1 )) - i=$((i + 1)) - if mkdir "$SLOT_DIR/slot.$n.d" 2>/dev/null; then - # Stamped BEFORE the loop exits, so a reclaimer can never see a - # live holder's directory with no pid in it and clear it. - printf '%s\n' "$$" >"$SLOT_DIR/slot.$n.d/pid" - slot="$n" - break - fi - holder="$(cat "$SLOT_DIR/slot.$n.d/pid" 2>/dev/null || true)" - [ -n "$holder" ] || continue - kill -0 "$holder" 2>/dev/null && continue - # Rename rather than remove: exactly one reclaimer can win the mv, - # so two waiters cannot both clear and hand the same slot to two - # different builds. - if mv "$SLOT_DIR/slot.$n.d" "$SLOT_DIR/slot.$n.stale.$$" 2>/dev/null; then - rm -rf "$SLOT_DIR/slot.$n.stale.$$" - fi - done - [ "$slot" -ne 0 ] && break - if [ "$notified" -eq 0 ]; then - notify_wait - notified=1 - fi - sleep 2 - waited=$((waited + 2)) - done - if [ "$slot" -ne 0 ]; then - # Single quotes: expanded when the trap fires, not here. - trap 'rm -rf "$SLOT_DIR/slot.$slot.d"' EXIT INT TERM - fi -fi - -# No slot after the wait: degrade, do not release. The old behaviour proceeded -# unserialised after the timeout, dropping the bound at the exact moment -# contention was highest. The floor job count alone would not bound memory -- -# it is spent per crate, and one big crate peaks the same at -j2 as at -j30 -- -# so the systemd path below also caps the build at one allowance. -SLOTLESS_CAP="" -if [ "$slot" -eq 0 ] && [ "$N" -ge 1 ]; then - jobs=2 - [ "$pool" -lt 2 ] && jobs=1 - SLOTLESS_CAP="--property=MemoryMax=${ALLOWANCE_GB}G" - notify_degraded -fi - -# After acquisition, never before: a build that had to degrade needs the lower -# count, and a caller's own CARGO_BUILD_JOBS still wins over both. -export CARGO_BUILD_JOBS="${CARGO_BUILD_JOBS:-$jobs}" - # sccache cannot cache a rustc call carrying -C incremental. Off by default: # cargo never builds dependencies incrementally, so this only makes WORKSPACE # crates cacheable, and it costs incremental on those same crates. Worth it on a @@ -374,38 +84,18 @@ if [ "${HYPERI_RUST_GOVERN_NO_INCREMENTAL:-0}" = "1" ]; then export CARGO_INCREMENTAL="${CARGO_INCREMENTAL:-0}" fi -# Linux with a live user manager: the scope joins rust-build.slice, whose -# memory/CPU limits are deployed by the developer-rust role. -# -# Run and wait rather than exec, and close fd 9 for the child. The fd is what -# holds the slot, and exec handed it to every descendant -- so any daemon the -# build started kept the slot for its own lifetime. Waiting costs one bash -# process, outside the scope; the kernel still releases the slot on a crash, -# because flock is held against the open file description. -# -# XDG_RUNTIME_DIR must be non-empty, not merely defaulted: systemd-run --user -# cannot reach the bus without it even when the socket is found by path. -if [ "$uname_s" = "Linux" ] && command -v systemd-run >/dev/null 2>&1 \ - && [ -n "${XDG_RUNTIME_DIR:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ]; then - systemd-run --user --scope --quiet --collect \ - --slice=rust-build.slice ${SLOTLESS_CAP:+"$SLOTLESS_CAP"} \ - -- "$REAL" "$@" 9>&- - exit "$?" -fi - -# macOS: QoS clamp puts every thread below the desktop's priority. No memory -# cap here -- taskpolicy's jetsam limit is per-process and cannot bound the -# tree -- so on that platform the semaphore and the job cap ARE the memory -# model, with nothing behind them. -# -# 9>&- on these two as well: a Mac with Homebrew util-linux has flock(1), so it -# takes the flock path above and reaches here holding fd 9. Closing an fd that -# was never opened is a no-op, so this is safe on the mkdir path too. -if [ "$uname_s" = "Darwin" ] && command -v taskpolicy >/dev/null 2>&1; then - taskpolicy -c "$QOS" "$REAL" "$@" 9>&- - exit "$?" +# Linux with a live user manager: the scope joins rustbuild.slice, with nice +# inside it so the whole tree carries it. A bus socket left by a dead session +# or a container image is not a manager answering, so one round trip settles +# that before the exec. +if [ "$(uname -s)" = "Linux" ] && command -v systemd-run >/dev/null 2>&1 \ + && [ -n "${XDG_RUNTIME_DIR:-}" ] && [ -S "$XDG_RUNTIME_DIR/bus" ] \ + && systemctl --user show --property=Version >/dev/null 2>&1; then + exec systemd-run --user --scope --quiet --collect \ + --slice=rustbuild.slice \ + -- nice -n 19 "$REAL" "$@" fi -# No systemd user manager (container, degraded session): priority only. -nice -n 19 "$REAL" "$@" 9>&- -exit "$?" +# Everywhere else -- macOS, a container, a degraded session: priority only, no +# memory bound. +exec nice -n 19 "$REAL" "$@" diff --git a/ansible/roles/developer-rust/tasks/governor.yml b/ansible/roles/developer-rust/tasks/governor.yml index 792c5ac..5982369 100644 --- a/ansible/roles/developer-rust/tasks/governor.yml +++ b/ansible/roles/developer-rust/tasks/governor.yml @@ -1,29 +1,41 @@ --- -# The rust build governor: at most N builds at a time per user, N derived from -# the slice's memory budget, the whole tree in the rust-build.slice budget, -# priority below the desktop. Opt-OUT -- `rust_governor_enabled: false` -# removes everything it deployed. +# The rust build governor: builds run at nice 19 inside rustbuild.slice, which +# bounds their memory and puts them below the desktop for CPU. Opt-OUT -- +# `rust_governor_enabled: false` removes everything it deployed. # # The shim shadows ~/.cargo/bin/cargo from ~/.local/bin, which the SOE PATH # puts first. sccache is captured separately: compiles execute inside the -# sccache SERVER, so hyperi-sccache.service carries Slice=rust-build.slice -# (see hyperi-sccache.service.j2), or the budget would miss most of the work. - -# rust_governor_serialize was a boolean when the governor was a global mutex. -# It is now a count, under a name that says so. Failing here rather than -# ignoring it is deliberate: a host that still sets `false` believes it has -# disabled serialisation, and silently applying `auto` instead would hand it -# eight concurrent builds. A setting that is quietly ignored is the same defect -# class as a config written to a directory the tool never reads. -- name: Fail on the retired rust_governor_serialize variable +# sccache SERVER, so hyperi-sccache.service carries Slice=rustbuild.slice and +# its own Nice (see hyperi-sccache.service.j2), or the budget and the priority +# would miss most of the work. + +# Retired settings fail the converge rather than being ignored. A host that sets +# rust_governor_slots: 1 believes it has one build at a time, and silently +# dropping the setting would hand it as many as it starts. +- name: Fail on retired build-governor variables ansible.builtin.fail: msg: >- - rust_governor_serialize has been replaced by rust_governor_slots, which - takes a COUNT rather than a boolean. `true` becomes - rust_governor_slots: 1 (one build at a time, the old behaviour), - `false` becomes rust_governor_slots: 0, and rust_governor_slots: auto - derives the count from this host's RAM. - when: rust_governor_serialize is defined + {{ developer_rust_retired_governor_vars | join(', ') }}: retired. Builds + are no longer rationed -- there is no semaphore, no CPU reserve and no + job count, so nothing reads these. Memory is bounded by rustbuild.slice + (rust_governor_memory_high_pct / _memory_max_pct) and CPU is deferred by + nice 19 plus rust_governor_cpu_weight. Delete the setting. + vars: + developer_rust_retired_governor_vars: "{{ lookup('ansible.builtin.varnames', + '^rust_governor_(serialize|slots|cpu_reserve_cores|build_allowance_gb|lock_wait_seconds|macos_qos)$', + wantlist=True) }}" + when: developer_rust_retired_governor_vars | length > 0 + +# The retired semaphore kept its slot files here; nothing reads them now. +- name: Remove the retired build-slot directory + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - "{{ user_home }}/.cache/hyperi/hyperi-rust-govern-{{ developer_rust_uid }}.slots" + - "/run/user/{{ developer_rust_uid }}/hyperi-rust-govern-{{ developer_rust_uid }}.slots" + become: "{{ ansible_facts['distribution'] != 'MacOSX' }}" + become_user: "{{ actual_user }}" - name: Deploy the rust build governor when: rust_governor_enabled | bool @@ -103,10 +115,10 @@ become_user: "{{ actual_user }}" when: ansible_facts['system'] == 'Linux' - - name: Install the rust-build slice + - name: Install the rustbuild slice ansible.builtin.template: - src: rust-build.slice.j2 - dest: "{{ user_home }}/.config/systemd/user/rust-build.slice" + src: rustbuild.slice.j2 + dest: "{{ user_home }}/.config/systemd/user/rustbuild.slice" owner: "{{ actual_user }}" mode: '0644' become: true @@ -114,6 +126,17 @@ register: developer_rust_governor_slice when: ansible_facts['system'] == 'Linux' + # The dash put the old unit under an auto-created rust.slice, where its + # CPUWeight faced no sibling -- see rustbuild.slice.j2. + - name: Remove the dash-named slice this replaced + ansible.builtin.file: + path: "{{ user_home }}/.config/systemd/user/rust-build.slice" + state: absent + become: true + become_user: "{{ actual_user }}" + register: developer_rust_governor_old_slice + when: ansible_facts['system'] == 'Linux' + - name: Reload the user manager so the slice limits are live ansible.builtin.systemd: daemon_reload: true @@ -124,7 +147,7 @@ when: - ansible_facts['system'] == 'Linux' - not ansible_check_mode - - developer_rust_governor_slice is changed + - developer_rust_governor_slice is changed or developer_rust_governor_old_slice is changed failed_when: false # MemoryHigh throttles by reclaim, and with no swap the only reclaimable @@ -206,10 +229,13 @@ failed_when: false when: ansible_facts['distribution'] == 'MacOSX' - - name: Remove the rust-build slice + - name: Remove the rustbuild slice and the dash-named one it replaced ansible.builtin.file: - path: "{{ user_home }}/.config/systemd/user/rust-build.slice" + path: "{{ user_home }}/.config/systemd/user/{{ item }}" state: absent + loop: + - rustbuild.slice + - rust-build.slice become: true become_user: "{{ actual_user }}" register: developer_rust_governor_slice_removed diff --git a/ansible/roles/developer-rust/templates/hyperi-sccache.service.j2 b/ansible/roles/developer-rust/templates/hyperi-sccache.service.j2 index c30c939..cf4b83b 100644 --- a/ansible/roles/developer-rust/templates/hyperi-sccache.service.j2 +++ b/ansible/roles/developer-rust/templates/hyperi-sccache.service.j2 @@ -11,7 +11,10 @@ Type=simple {% if rust_governor_enabled | default(true) %} # Compiles execute inside this server, not inside cargo, so the server must # live in the governed slice or the rust-build budget misses most of the work. -Slice=rust-build.slice +# The same goes for priority: a compile here inherits nothing from the nice 19 +# the shim put on cargo, so the server carries its own. +Slice=rustbuild.slice +Nice=19 {% endif %} Environment=SCCACHE_START_SERVER=1 Environment=SCCACHE_NO_DAEMON=1 diff --git a/ansible/roles/developer-rust/templates/rust-governor.conf.j2 b/ansible/roles/developer-rust/templates/rust-governor.conf.j2 index 2f04d06..060dff4 100644 --- a/ansible/roles/developer-rust/templates/rust-governor.conf.j2 +++ b/ansible/roles/developer-rust/templates/rust-governor.conf.j2 @@ -1,22 +1,12 @@ # Deployed by hyperi-developer (developer-rust role). Edits are overwritten. # Read by hyperi-rust-govern at the start of every governed invocation. # -# Policy only. Every size is resolved by the shim against the live host, so a -# resized VM needs no re-converge -- the same bargain the CPU reserve makes. +# Written as a DEFAULT rather than an assignment, and that is a property to +# keep: the shim sources this file BEFORE reading its variables, so a plain +# assignment here would silently beat anything the caller exported, leaving no +# way to tune one invocation from the shell and no way to test the shim against +# a real deployed config. # -# Each value is written as a DEFAULT rather than an assignment. The shim sources -# this file before reading its variables, so a plain assignment here would -# silently beat anything the caller set, leaving no way to tune one invocation -# and no way to test the shim against a real deployed config. -HYPERI_RUST_GOVERN_SLOTS="${HYPERI_RUST_GOVERN_SLOTS:-{{ rust_governor_slots }}}" -HYPERI_RUST_GOVERN_CPU_RESERVE="${HYPERI_RUST_GOVERN_CPU_RESERVE:-{{ rust_governor_cpu_reserve_cores }}}" -HYPERI_RUST_GOVERN_MACOS_QOS="${HYPERI_RUST_GOVERN_MACOS_QOS:-{{ rust_governor_macos_qos }}}" -# Seconds to wait for a build slot before building at the floor job count. -HYPERI_RUST_GOVERN_LOCK_WAIT="${HYPERI_RUST_GOVERN_LOCK_WAIT:-{{ rust_governor_lock_wait_seconds }}}" -# Matches rust-build.slice's MemoryHigh so the shim's slot count and systemd's -# memory ceiling are derived from the same number. -HYPERI_RUST_GOVERN_MEMORY_HIGH_PCT="${HYPERI_RUST_GOVERN_MEMORY_HIGH_PCT:-{{ rust_governor_memory_high_pct }}}" -HYPERI_RUST_GOVERN_BUILD_ALLOWANCE_GB="${HYPERI_RUST_GOVERN_BUILD_ALLOWANCE_GB:-{{ rust_governor_build_allowance_gb }}}" # sccache cannot cache an incremental rustc call, and cargo's dev profile turns # incremental on by default. -HYPERI_RUST_GOVERN_NO_INCREMENTAL="${HYPERI_RUST_GOVERN_NO_INCREMENTAL:-{{ '1' if rust_governor_no_incremental else '0' }}}" +HYPERI_RUST_GOVERN_NO_INCREMENTAL="${HYPERI_RUST_GOVERN_NO_INCREMENTAL:-{{ '1' if rust_governor_no_incremental | bool else '0' }}}" diff --git a/ansible/roles/developer-rust/templates/rust-build.slice.j2 b/ansible/roles/developer-rust/templates/rustbuild.slice.j2 similarity index 50% rename from ansible/roles/developer-rust/templates/rust-build.slice.j2 rename to ansible/roles/developer-rust/templates/rustbuild.slice.j2 index 347cc9b..6c92c07 100644 --- a/ansible/roles/developer-rust/templates/rust-build.slice.j2 +++ b/ansible/roles/developer-rust/templates/rustbuild.slice.j2 @@ -2,18 +2,18 @@ [Unit] Description=Governed Rust builds (hyperi-developer) -# systemd reads `-` as a hierarchy separator, so this unit lands under an -# auto-created `rust.slice` that nothing manages and nothing limits. Harmless, -# because the limits below bind on this slice -- but limits set on "rust.slice" -# expecting them to reach builds would land on an empty wrapper. +# No dash in the name, on purpose. systemd reads `-` as a hierarchy separator: +# a `rust-build.slice` sits alone under an auto-created `rust.slice`, and since +# CPUWeight is relative to SIBLINGS it is that wrapper's default weight of 100, +# not the 30 below, that faces app.slice. Dash-free, this slice is a direct +# child of the user manager and a true sibling of the desktop's slices. [Slice] # Percentages resolve against this host's own RAM and swap, so one unit fits # every machine. Both capture prongs land here: cargo trees via the # hyperi-rust-govern scope, sccache-hosted compiles via Slice= on -# hyperi-sccache.service. -# -# MemoryHigh is also what the shim divides to size its build semaphore, so the -# two are derived from one number and cannot drift. +# hyperi-sccache.service. This is the ONLY memory bound on a build: nothing +# queues builds or caps their job count, so a build over budget is throttled +# here and, at MemoryMax, killed here -- the build, never the host. # # MemorySwapMax only means anything where swap exists. On a swapless host # MemoryHigh has nothing but page cache to reclaim, so it stalls a build @@ -22,8 +22,11 @@ MemoryHigh={{ rust_governor_memory_high_pct }}% MemoryMax={{ rust_governor_memory_max_pct }}% MemorySwapMax={{ rust_governor_swap_max_pct }}% # Yields to the desktop under contention; no effect while the machine is idle. +# This, not nice, is what makes a build defer on Linux: cgroup v2 splits CPU +# between sibling cgroups by weight, and nice only orders threads inside their +# own cgroup. The shim's nice 19 covers hosts where the cpu controller is not +# delegated and this line is silently ignored. # No CPU quota anywhere: a quota wastes cores whenever a session sits idle, # whereas weight gives proportional share under contention and the whole box -# when nothing is competing. Parallelism is bounded by CARGO_BUILD_JOBS, which -# the shim scales to the number of neighbours the semaphore admits. +# when nothing is competing. Parallelism is cargo's own default, every core. CPUWeight={{ rust_governor_cpu_weight }} diff --git a/ansible/roles/zram_swap/README.md b/ansible/roles/zram_swap/README.md index f7a1156..f793be5 100644 --- a/ansible/roles/zram_swap/README.md +++ b/ansible/roles/zram_swap/README.md @@ -23,7 +23,7 @@ It is somewhere for the throttle to push, which is why a few GB is the right size and a disk-backed swap file is not a substitute. This matters most on a host running `developer-rust`'s build governor, whose -`rust-build.slice` sets `MemoryHigh` and `MemorySwapMax` as percentages of RAM. +`rustbuild.slice` sets `MemoryHigh` and `MemorySwapMax` as percentages of RAM. Without swap, those limits do not degrade a build -- they wedge it. ## What it touches diff --git a/docs/concurrent-dev-cache.md b/docs/concurrent-dev-cache.md index 2b4c454..da601f3 100644 --- a/docs/concurrent-dev-cache.md +++ b/docs/concurrent-dev-cache.md @@ -1,7 +1,8 @@ # Many build sessions on one host -Rust builds are bounded by a memory semaphore, the pooled build artefacts by a -ceiling derived from the disk, and the compiler caches by fixed byte ceilings. +Rust builds are bounded by a memory budget on a systemd slice, the pooled build +artefacts by a ceiling derived from the disk, and the compiler caches by fixed +byte ceilings. Nothing beyond those three is bounded here at all. This is what each means, and where the edges are. @@ -25,41 +26,41 @@ answer. single `rustc` however high `-j` goes, so the job count sets how many *crates* build at once while the largest crate sets a floor no job setting goes under. On a 32-core workstation with 246 GB RAM the peak resident size of one `rustc` was -11.6 GB. A host that admits only one build at a time arrives at that number by -fitting one such process, not by a policy about job counts. +11.6 GB. That is why the bound is a memory budget on the whole build tree and +not a policy about job counts: a job count cannot bound what one process costs. Someone arriving cold with a new project gets the whole mechanism with no opt-in, which is why it is a shim on PATH rather than a setting to remember. -## The governor admits N builds at once, sized from the host's own memory +## The governor bounds memory and defers CPU, and rations neither `hyperi-rust-govern` is installed as `~/.local/bin/cargo`, ahead of the real -cargo, so a build holds one of N slots for its lifetime. On Linux with a live -user manager it also runs in `rust-build.slice`, whose memory and CPU limits are -the other half of the model. Where there is no user bus the shim falls back to a -QoS clamp on macOS or to `nice`, leaving the semaphore and the job cap as the -bounds. Setting `rust_governor_slots: 0` disables the semaphore entirely. +cargo. Every build runs at nice 19, and on Linux with a live user manager it also +runs in `rustbuild.slice`, whose memory budget is a percentage of the host's own +RAM and whose CPU weight sits below the desktop's. Where there is no live user +manager the shim falls back to `nice` alone, with no memory bound behind it. ```mermaid flowchart TB - RAM[cgroup limit or MemTotal] -->|scaled by the MemoryHigh percentage| Budget[Memory budget] - Cores[Cores on the host] -->|drops the CPU reserve| Pool[Usable cores] - Budget -->|divides by the per-build allowance| N[Slot count N] - Pool -->|caps N at half the pool| N - N -->|divides the pool into| Jobs[CARGO_BUILD_JOBS] - N -->|creates| Slots[N slot files] - Jobs -->|sets -j for| Scope[Governed build] - Slots -->|admits one build to| Scope + Cargo[cargo build] --> Shim[hyperi-rust-govern] + Shim -->|Linux with a user bus| Scope[systemd scope in rustbuild.slice
nice 19] + Shim -->|macOS, container, no bus| Nice[nice 19 only] + Scope --> Mem[MemoryHigh throttles
MemoryMax kills the build] + Scope --> Weight[CPUWeight below the desktop] + Sccache[hyperi-sccache.service
Slice + Nice=19] --> Mem + Sccache --> Weight ``` -Both numbers are computed by the shim at run time, reproducing the arithmetic -systemd does for `MemoryHigh=%` against the same total, so the slot count -and the memory ceiling cannot drift and a resized host needs no re-converge. The -check that the model reproduces behaviour known to work is that `auto` computes -N=1 on a 32 GB host - the global mutex this was before it was a semaphore. +Nothing withholds cores and nothing sets a job count - cargo's own default is +already every core. A reserve or a computed `-j` is paid on an idle machine as +well as a busy one, and buys nothing that yielding does not buy when it is +needed. The semaphore this replaced did both, built at `-j3` on a 32-core box, +and was shared state two sessions could disagree about. With nothing computed +there is nothing to keep in sync. -Slot mechanics, what happens at saturation, and why no CPU quota is set anywhere -are in [rust-build-governor.md](rust-build-governor.md). +Which instrument makes a build yield on which host, and why `CPUWeight` rather +than nice is the Linux one, are in +[rust-build-governor.md](rust-build-governor.md). ## The toolchain location is read from the host, never assumed @@ -179,12 +180,11 @@ to compiling. Which hosts want it on is in of PID 1 and container processes are created under its tree: its levers are `cgroup-parent` in `daemon.json`, the per-container flag of the same name, and compose's `cgroup_parent`. -- **A build that starts alone keeps the crowded job count.** `CARGO_BUILD_JOBS` - is fixed when the process starts, so a lone build on an idle host still runs at - the shared count. Pass your own value for a known-solo run. -- **`RUST_TEST_THREADS` is unbounded.** The shim caps `CARGO_BUILD_JOBS`, but - libtest defaults its harness parallelism to the visible CPU count, so a - governed `cargo test` still spawns that many test threads. -- **The per-build allowance comes from one workspace.** 14 GB is an 11.6 GB peak - plus headroom, measured once. A codebase whose memory scales with the job count - rather than with one huge crate wants a different number. +- **Builds are not queued.** Several started at once all run at every core and + share the slice's memory budget, so on a small-RAM host one may be killed at + `MemoryMax` rather than wait its turn: a failed build, retried, not a failed + host. +- **Nothing bounds memory where there is no user bus.** A container or a + degraded session gets nice 19 and nothing else. +- **Disk priority is untouched.** `ionice` binds only under BFQ, and NVMe hosts + run `none` or `mq-deadline`, where it does nothing. diff --git a/docs/install-matrix.md b/docs/install-matrix.md index 05c8493..6d13eab 100644 --- a/docs/install-matrix.md +++ b/docs/install-matrix.md @@ -215,7 +215,7 @@ is the meta-role pulling them all. | sccache | all | github-binary, latest each run (Tier 3) / brew | | cargo-sweep | all | cargo-binstall / cargo | | hyperi-rust-setup, hyperi-rust-cache-prune | all | role file -> `/usr/local/bin` | -| build governor (`hyperi-rust-govern` + `rust-build.slice`) | all | role file + user unit | +| build governor (`hyperi-rust-govern` + `rustbuild.slice`) | all | role file + user unit | Every cargo tool is installed with `--locked`, and `hyperi-update` refreshes them with `cargo install-update -a --locked`. `cargo install` ignores the crate's @@ -242,15 +242,14 @@ full toolchain converge: The build governor is the same shape (`--tags rust-governor`), and opt-OUT (`-e rust_governor_enabled=false` removes it). Concurrent cargo/rustc across -several projects can OOM a host outright, so: one build at a time per user -(later ones wait on a lock), the whole tree -- sccache-hosted compiles -included -- inside `rust-build.slice` with memory capped as a percentage of -the host's RAM, CPU capped at cores minus two, and priority below the -desktop. macOS has no cgroups and gets the lock, the job cap and a -`taskpolicy` QoS clamp instead -- no hard memory ceiling. -`HYPERI_RUST_GOVERNOR=off` bypasses one invocation. - -That tag installs both tools, writes the caps, and schedules the prune +several projects can OOM a host outright, so: the whole tree -- sccache-hosted +compiles included -- runs inside `rustbuild.slice` with memory capped as a +percentage of the host's RAM, at nice 19 and a CPU weight below the desktop's. +No core is withheld and no job count is set: builds take every core until +something else wants one. macOS has no cgroups and gets nice 19 alone -- no +hard memory ceiling. `HYPERI_RUST_GOVERNOR=off` bypasses one invocation. + +The `rust-cache` tag installs both tools, writes the caps, and schedules the prune (systemd timer on Linux, launchd agent on macOS). Sizes come from `rust_cache_*` in the role defaults; a build box overrides them per host. diff --git a/docs/rust-build-governor.md b/docs/rust-build-governor.md index fecd117..a9674ea 100644 --- a/docs/rust-build-governor.md +++ b/docs/rust-build-governor.md @@ -1,69 +1,128 @@ -# Bounding concurrent Rust builds +# The Rust build governor `hyperi-rust-govern` is installed by the `developer-rust` role as `~/.local/bin/cargo`, ahead of the real cargo on PATH, so a developer or an -agent who knows none of this runs `cargo build` and is governed. It holds one of -N build slots for the build's lifetime, and on Linux with a live user manager it -also places the build in `rust-build.slice`. - -## How N is chosen - -**N is a memory semaphore, not a queue length.** Memory is spent per *crate*, -not per job: one enormous crate compiles as a single `rustc` however high `-j` -goes, so what is worth bounding is how many builds are resident at once. - - N = min(MemoryHigh / per-build allowance, cores-minus-reserve / 2) - jobs = cores-minus-reserve / N (floored at 2) - -Both are computed by the shim at run time, from the cgroup memory limit where -one is set and `/proc/meminfo` otherwise, reproducing the arithmetic systemd -does for `MemoryHigh=%` against the same total. The slot count and the -memory ceiling therefore cannot drift, and a resized VM needs no re-converge. - -| host RAM | MemoryHigh (50%) | N | jobs each (32 cores) | -|---|---|---|---| -| 256 GB | 128G | 8 | 3 | -| 128 GB | 64G | 4 | 7 | -| 64 GB | 32G | 2 | 15 | -| 32 GB | 16G | 1 | 30 | - -The 32 GB row is a global mutex -- what this was before it was a semaphore -- -and is the check that the model reproduces behaviour known to work. -`rust_governor_slots: 1` pins that everywhere. - -The per-build allowance (`rust_governor_build_allowance_gb`, 14 GB) is the -largest single `rustc` observed plus headroom, taken from one workspace. A -codebase whose memory scales with the job count rather than with one huge crate -wants a different number. - -The CPU reserve comes off the top *before* the remainder is divided, so the -desktop keeps its share at every slot count. No CPU quota is set anywhere: a -quota wastes cores whenever a session idles, while `CPUWeight` gives -proportional share under contention and the whole box when nothing competes. - -## At saturation - -**A build degrades, it is not released.** A build that cannot get a slot within -`rust_governor_lock_wait_seconds` proceeds at the floor job count -- and on -Linux with a live user manager, inside the slice under its own `MemoryMax` of -one allowance. Waiting the full timeout -and then building unbounded would drop the limit at exactly the moment -contention is highest. - -**On Linux a slot is released by the kernel, not by cleanup.** It is an -`flock` on an open file description, so a killed or OOM-killed build frees its -slot with no reaper and no stale-lock detection. The fd is closed for the build -itself (`9>&-`), so a daemon the build starts cannot inherit the slot. macOS -ships no `flock(1)`, so there a slot is a directory held by an exit trap with a -liveness check on the recorded pid: a SIGKILLed holder is reclaimed by the next -arrival rather than by the kernel. - -A build that starts alone keeps the crowded job count: `CARGO_BUILD_JOBS` is -fixed when the process starts. For a known-solo run, pass `CARGO_BUILD_JOBS` -yourself -- the shim respects a caller's value over its own. +agent who knows none of this runs `cargo build` and is governed. Every build +runs at nice 19, and on Linux with a live user manager it also runs inside +`rustbuild.slice`. Only `cargo` is shimmed: a bare `rustc`, or anything run +through `rustup run`, is ungoverned unless wrapped in `hyperi-rust-govern`. + +## Memory is bounded, CPU is deferred + +Two instruments, answering two different questions. + +**Memory is bounded by the slice.** `MemoryHigh` (50% of the host's RAM) +throttles a build by reclaim, `MemoryMax` (70%) kills it, and `MemorySwapMax` +(25%) caps what it may push to swap. Percentages, so one unit fits a laptop and +a build box, and the build dies before the host does. sccache-hosted compiles +sit inside the same budget because `hyperi-sccache.service` carries +`Slice=rustbuild.slice`. + +**CPU is not bounded at all -- it is deferred.** Nothing withholds cores and +nothing sets a job count: cargo's own default is already every core. A build +takes the whole machine while nobody else wants it, and drops to about a +quarter of it the moment the desktop does. Rationing CPU -- a reserve, a +computed `-j` -- is paid on an idle machine as well as a busy one, and buys +nothing that yielding does not buy at the moment it is needed. The semaphore +this replaced did exactly that: on a 32-core box it withheld two cores and +divided the rest between the eight builds it would admit, so every build ran +at `-j3`. + +```mermaid +flowchart LR + Cargo[cargo build] --> Shim[hyperi-rust-govern] + Shim -->|Linux with a live user manager| Scope[systemd-run --scope
nice 19] + Shim -->|macOS, container, no manager| Nice[nice 19] + Scope --> Slice[rustbuild.slice
MemoryHigh / MemoryMax
CPUWeight] + Sccache[hyperi-sccache.service
Nice=19] --> Slice +``` + +## Which instrument makes a build yield + +| host | what defers the build | what bounds memory | +|---|---|---| +| Linux, cpu controller delegated to the user manager | `CPUWeight=30` on the slice, against the desktop's `app.slice` | the slice | +| Linux, cpu controller not delegated | nice 19, against everything in the same cpu cgroup -- the desktop included | the slice | +| Linux with no live user manager (container, stale bus socket, degraded session) | nice 19, within the build's own cpu cgroup | nothing | +| macOS | nice 19 | nothing | + +`CPUWeight`, not nice, is the Linux instrument. In cgroup v2 the CPU split +between sibling cgroups is decided by `cpu.weight`, and a task's nice value +only orders threads inside its own cgroup. The slice is a direct child of the +user manager, so its siblings are `app.slice` and `session.slice` -- the +desktop -- and 30 against their 100 is the split under contention. The name +has no dash on purpose: systemd reads `-` as a hierarchy separator, so a +`rust-build.slice` would sit alone under an auto-created `rust.slice` whose +default weight of 100 is what would actually face the desktop. + +What the weight does not reach is another login session. An ssh login is a +`session-N.scope` beside the whole user manager, not inside it, so a build and +a second ssh session split the CPU evenly whatever the slice says. A weight on +`user@UID.service` itself would change that, and that is a system unit, not +this role's. + +nice is what remains where the weight cannot apply -- no cpu controller +delegated, no live user manager, macOS. It orders the build against everything +sharing the build's own cpu cgroup and reaches nothing outside it. With the cpu +controller not delegated into the user manager, the build and the desktop's +processes are in that same cgroup, and nice 19 is what defers one to the other. +A desktop app in another login session is still untouched by it -- the same +boundary that stops `CPUWeight`. + +## What it does not do + +**It does not queue builds.** Several builds started at once all run, at every +core, and share the slice's memory budget. On a small-RAM host that can mean one +is killed at `MemoryMax` instead of waiting its turn -- a failed build, retried, +rather than a failed host. Nothing queues them, and nothing ever did so +reliably: the slot count was shared state two sessions could compute +differently. + +**It does not touch disk priority.** `ionice` was considered and dropped: the +idle class only binds under BFQ, and NVMe hosts run `none` or `mq-deadline`, +where it does nothing. + +**It does not cap test threads.** libtest defaults `RUST_TEST_THREADS` to the +visible CPU count, which is now also what the build itself uses. + +## Bypass and tuning + + HYPERI_RUST_GOVERNOR=off cargo build # bypass one invocation + hyperi-rust-govern [args...] # govern any other command + +The bypass covers cargo's own process tree. Compiles that sccache hosts run +inside `hyperi-sccache.service`, which keeps its slice and its nice whatever +the caller set. A service started with `cargo run` keeps the nice and the +memory bound for its whole life, so bypass that one. + +`-e rust_governor_enabled=false` removes the lot -- shim, slice and config -- +rather than merely stopping it. To see what a host actually got: + + systemctl --user show rustbuild.slice -p CPUWeight -p MemoryHigh -p MemoryMax + +The role writes `~/.config/hyperi/rust-governor.conf`, which the shim sources +first. The value in it is written as `${VAR:-default}` on purpose, so an +export in the shell beats the deployed file for one invocation and the shim can +be tested against a real config. Nothing per-machine is needed: the memory +percentages resolve against the host's own RAM, and there is no core count to +tune. + +Five role variables went with the rationing: `rust_governor_slots`, +`rust_governor_cpu_reserve_cores`, `rust_governor_build_allowance_gb`, +`rust_governor_lock_wait_seconds` and `rust_governor_macos_qos` +(`rust_governor_serialize` before them). A host that still sets one fails the +converge rather than having it ignored: a host asking for one build at a time +would otherwise silently get as many as it starts. Delete the setting. ## What it needs +**Enough RAM for the biggest crate.** `MemoryMax` is 70% of the host's RAM, so +a 16 GB laptop gives the whole slice 11.2 GB -- and one `rustc` on a large +crate has been measured at 11.6 GB. That build is killed, and cargo reports the +rustc process dying on signal 9 rather than an out-of-memory error. The levers +are bypassing the governor for that one build, or building it on a box with +more RAM. + **Swap.** `MemoryHigh` throttles by reclaim, and on a swapless host the only reclaimable memory is page cache -- so a build past the line stalls rather than slows. The `zram_swap` role is the other half, and a converge onto a swapless diff --git a/tools/tests/test_rust_govern.py b/tools/tests/test_rust_govern.py new file mode 100644 index 0000000..6610f03 --- /dev/null +++ b/tools/tests/test_rust_govern.py @@ -0,0 +1,205 @@ +"""Tests for hyperi-rust-govern, the cargo shim. + +The shim is bash, so it is exercised as a subprocess against a fake cargo that +reports what it was handed. XDG_RUNTIME_DIR is removed from every run, which +forces the no-user-bus path on any host: the systemd scope path needs a live +user manager and is covered by the role's converge on a real machine, not here. +""" + +import os +import shutil +import socket +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPT = ( + Path(__file__).resolve().parents[2] + / "ansible/roles/developer-rust/files/hyperi-rust-govern" +) + +FAKE_CARGO = """#!/bin/sh +echo "nice=$(nice)" +echo "args=$*" +echo "governed=${HYPERI_RUST_GOVERNED:-unset}" +echo "jobs=${CARGO_BUILD_JOBS:-unset}" +echo "incremental=${CARGO_INCREMENTAL:-unset}" +exit 42 +""" + +pytestmark = pytest.mark.skipif( + sys.platform == "win32" + or shutil.which("bash") is None + or shutil.which("nice") is None, + reason="the shim is a POSIX bash script and needs nice(1)", +) + + +def parse(stdout): + return dict(line.split("=", 1) for line in stdout.splitlines() if "=" in line) + + +@pytest.fixture +def host(tmp_path): + """A fake cargo home, an empty config dir, and a minimal environment. + + Built from scratch rather than copied from the process: no XDG_RUNTIME_DIR + means the no-user-bus path on every host, and nothing from the runner's + environment (a CI secret, say) can reach a failure report. + """ + cargo_home = tmp_path / "cargo-home" + (cargo_home / "bin").mkdir(parents=True) + fake = cargo_home / "bin" / "cargo" + fake.write_text(FAKE_CARGO, encoding="utf-8", newline="\n") + fake.chmod(0o755) + env = { + "PATH": os.environ["PATH"], + "HOME": str(tmp_path), + "CARGO_HOME": str(cargo_home), + } + return {"tmp": tmp_path, "cargo_home": cargo_home, "env": env} + + +def run(host, *args, env_extra=None, script=SCRIPT): + """Run the shim under bash: role files are tracked 0644 and made executable at deploy.""" + env = dict(host["env"]) + if env_extra: + for k, v in env_extra.items(): + if v is None: + env.pop(k, None) + else: + env[k] = v + return subprocess.run( + ["bash", str(script), *args], + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + + +def baseline_nice(host): + out = subprocess.run( + ["nice"], env=host["env"], capture_output=True, text=True, check=True + ) + return int(out.stdout.strip()) + + +def write_conf(host, body): + conf_dir = host["tmp"] / ".config" / "hyperi" + conf_dir.mkdir(parents=True, exist_ok=True) + (conf_dir / "rust-governor.conf").write_text(body, encoding="utf-8", newline="\n") + + +def test_governed_build_runs_at_nice_19_with_no_job_count(host): + result = run(host, "cargo", "build", "--release") + got = parse(result.stdout) + assert result.returncode == 42, result.stderr + assert got["nice"] == "19" + assert got["args"] == "build --release" + assert got["governed"] == "1" + assert got["jobs"] == "unset" + + +def test_bypass_runs_the_real_tool_untouched(host): + result = run(host, "cargo", "run", env_extra={"HYPERI_RUST_GOVERNOR": "off"}) + got = parse(result.stdout) + assert result.returncode == 42 + assert got["nice"] == str(baseline_nice(host)) + assert got["governed"] == "unset" + + +def test_already_governed_tree_is_not_governed_twice(host): + """cargo re-invokes cargo for build scripts and proxies; the marker stops recursion.""" + result = run(host, "cargo", "check", env_extra={"HYPERI_RUST_GOVERNED": "1"}) + got = parse(result.stdout) + assert result.returncode == 42 + assert got["nice"] == str(baseline_nice(host)) + + +def test_conf_turns_incremental_off_and_the_caller_still_wins(host): + write_conf( + host, + 'HYPERI_RUST_GOVERN_NO_INCREMENTAL="${HYPERI_RUST_GOVERN_NO_INCREMENTAL:-1}"\n', + ) + assert parse(run(host, "cargo", "build").stdout)["incremental"] == "0" + assert ( + parse(run(host, "cargo", "build", env_extra={"CARGO_INCREMENTAL": "1"}).stdout)[ + "incremental" + ] + == "1" + ) + assert ( + parse( + run( + host, + "cargo", + "build", + env_extra={"HYPERI_RUST_GOVERN_NO_INCREMENTAL": "0"}, + ).stdout + )["incremental"] + == "unset" + ) + + +def test_incremental_is_left_alone_without_a_conf(host): + assert parse(run(host, "cargo", "build").stdout)["incremental"] == "unset" + + +def test_symlink_named_cargo_on_path_resolves_the_real_cargo(host): + """The role installs the shim as ~/.local/bin/cargo; it must not exec itself.""" + shimbin = host["tmp"] / "shimbin" + shimbin.mkdir() + link = shimbin / "cargo" + link.symlink_to(SCRIPT) + result = run( + host, + "build", + env_extra={"PATH": f"{shimbin}{os.pathsep}{host['env']['PATH']}"}, + script=link, + ) + got = parse(result.stdout) + assert result.returncode == 42, result.stderr + assert got["nice"] == "19" + assert got["args"] == "build" + + +def test_missing_real_tool_exits_127(host): + result = run(host, "no-such-tool", "anything") + assert result.returncode == 127 + assert "no real 'no-such-tool'" in result.stderr + + +def test_no_command_is_a_usage_error(host): + result = run(host) + assert result.returncode == 64 + assert "usage:" in result.stderr + + +def test_dead_user_bus_falls_back_to_nice(host): + """A bus socket left behind by a dead session must not fail the build. + + The file passes the -S test; only the reachability probe tells the shim the + manager is gone, and the build has to land on the plain nice path. + """ + deadrun = host["tmp"] / "deadrun" + deadrun.mkdir() + sock = socket.socket(socket.AF_UNIX) + sock.bind(str(deadrun / "bus")) + sock.close() + result = run(host, "cargo", "build", env_extra={"XDG_RUNTIME_DIR": str(deadrun)}) + got = parse(result.stdout) + assert result.returncode == 42, result.stderr + assert got["nice"] == "19" + assert got["governed"] == "1" + + +def test_runs_with_home_unset(host): + """A system unit or env -i has no HOME; set -u must not abort the shim.""" + result = run(host, "cargo", "build", env_extra={"HOME": None}) + assert result.returncode == 42, result.stderr + assert parse(result.stdout)["nice"] == "19"