diff --git a/scripts/bench/memory.py b/scripts/bench/memory.py new file mode 100755 index 000000000..591d1b0a7 --- /dev/null +++ b/scripts/bench/memory.py @@ -0,0 +1,530 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +""" +memory.py - peak-memory harness for one (variant, sample) cell at a time. + +Every memory branch in notes/memory-reduction-plan.md is accepted on a number, +not on "it got better", and those numbers are only comparable if they come from +the same harness, run one process at a time (a 16 GB host OOMs and corrupts +peaks otherwise). This is that harness: it runs transcribe-cli serially, records +peak RSS, the platform's own footprint number, and the scheduler scratch the +run committed, and appends one CSV row per cell. + +Usage: + # one variant, one sample + uv run scripts/bench/memory.py --models granite-4.0-1b-speech --samples jfk + + # a family sweep at the plan's reference length, warm (2 runs, keep the 2nd) + uv run scripts/bench/memory.py \\ + --models granite-4.0-1b-speech,granite-speech-4.1-2b,canary-qwen-2.5b \\ + --samples dots-full --repeat 2 + + # duration scaling (generates a cut with ffmpeg if missing) + uv run scripts/bench/memory.py --models sensevoice --samples dots-full \\ + --duration 1200 + + # A/B a branch-specific switch; --env for library env vars, `--` for CLI flags + uv run scripts/bench/memory.py --models granite-4.0-1b-speech --samples jfk \\ + --label pack-gate-up-copy --env TRANSCRIBE_NO_GATE_UP_ALIAS=1 + uv run scripts/bench/memory.py --models qwen3-asr-0.6b --samples jfk \\ + --backend cpu -- --n-ctx 4096 + +Everything after `--` is passed straight to transcribe-cli, so branch-specific +CLI switches work without teaching this script about them. + +Output: reports/memory//