diff --git a/docs/phase5-issue-watcher.md b/docs/phase5-issue-watcher.md index 6cb37a7..68266ed 100644 --- a/docs/phase5-issue-watcher.md +++ b/docs/phase5-issue-watcher.md @@ -307,6 +307,9 @@ Continuous mode logs every sweep and shuts down cleanly on SIGINT/SIGTERM Run one continuous watcher on each machine, using the same GitHub owner, repository, and label, but a different stable `--runner-id` for each machine. +Full install/runbook (node prerequisites, credentials, systemd unit, capacity +sizing, claim discipline, stale-claim recovery): see +[`docs/runner-pool-node.md`](./runner-pool-node.md). The `--max-concurrent-sessions` setting is the capacity limit for that machine. `--runner-assignee` is the GitHub login displayed on issues while a claim is active; it must be configured with `--runner-id`. diff --git a/docs/runner-pool-node.md b/docs/runner-pool-node.md new file mode 100644 index 0000000..e03548a --- /dev/null +++ b/docs/runner-pool-node.md @@ -0,0 +1,134 @@ +# Runner pool — home node runbook (issue #32) + +Claim-based two-machine pool: the home node (`joe@192.168.1.252`) and the +workstation each run one continuous `headlesscode watch` with a distinct +`--runner-id`. Jobs (issues carrying the watcher label) go to whichever +machine has capacity. There is no central balancer and no new infra — the +GitHub claim refs (`headlesscode-claims/issue-N`, see +[`src/watcher/claims.ts`](../src/watcher/claims.ts)) are the coordination +primitive. The alternative (central dispatcher SSH-dispatching to the node) +was decided against: claim-based balancing is the implemented design. + +## 1. Install (home node) + +On the node, as the service user: + +```bash +# 1. Prerequisites: Node.js 22.19+, git, gh +node --version # >= 22.19 +git --version +gh --version # needed by worker workflows that shell out to gh + +# 2. Clone + install headlesscode deps +git clone /opt/headlesscode +cd /opt/headlesscode +npm ci + +# 3. Clone the target repo the watcher will spawn worktrees in +git clone /opt/my-repo + +# 4. Optional CLI shim +bash /opt/headlesscode/scripts/install-cli.sh +``` + +`gh` is not used by the watcher itself (it uses the GitHub REST API), but +worker jobs invoke the GitHub CLI — install and authenticate it on the node +the same way as on the workstation. + +## 2. Credentials (per machine, never copied) + +Create one fine-grained token per machine with repository **Contents +read/write** (claim refs) and **Issues read/write** (lifecycle labels, +assignees, comments). Store it in a protected environment file, never in the +unit file or command line: + +```bash +sudo install -o headlesscode -g headlesscode -m 0600 /dev/null /etc/headlesscode/runner.env +# then edit /etc/headlesscode/runner.env: +# GH_TOKEN=ghp_... +# HEADLESSCODE_RUNNER_ID=home-node +# HEADLESSCODE_RUNNER_ASSIGNEE= +``` + +Do not copy model-provider credentials between machines; provision each +runner's credentials through the deployment's approved secret mechanism. + +## 3. Run (systemd, one watcher per machine) + +```ini +# /etc/systemd/system/headlesscode-watch.service (home node) +[Unit] +Description=headlesscode GitHub issue watcher (runner-pool node) +After=network-online.target + +[Service] +Type=simple +User=headlesscode +EnvironmentFile=/etc/headlesscode/runner.env +WorkingDirectory=/opt/headlesscode +ExecStart=/usr/bin/npx --prefix /opt/headlesscode tsx /opt/headlesscode/src/cli.ts watch --owner my-org --repo /opt/my-repo --label needs-agent --runner-id home-node --runner-assignee --max-concurrent-sessions 1 --poll-interval-ms 60000 +Restart=on-failure + +[Install] +WantedBy=multi-user.target +``` + +The workstation uses the same unit with a different `--runner-id` +(e.g. `workstation`) and its own env file. `HEADLESSCODE_RUNNER_ID` / +`HEADLESSCODE_RUNNER_ASSIGNEE` env vars may replace the two flags. + +## 4. Per-machine capacity cap + +`--max-concurrent-sessions` (default `$HEADLESSCODE_MAX_CONCURRENT_SESSIONS` +or 3) is the **per-machine** cap in pool mode: orchestrator groups in +`spawned`/`running` + watcher in-flight entries + active claim worktrees held +by this runner. Issues beyond it stay `pending` and are picked up on a later +sweep once slots free up. Size each machine independently (e.g. node `1`, +workstation `2`–`3`). Keep exactly one pool watcher per machine: the watcher +takes an exclusive lock under `$XDG_STATE_HOME/headlesscode` +(or `~/.local/state/headlesscode`) and two local watchers with different +runner ids would each consume the full cap independently. + +Lease tuning: `--claim-lease-ms` (default 120000, valid 30000..900000; +env `HEADLESSCODE_CLAIM_LEASE_MS`). The poll interval must be at most half +the lease so the watcher renews in time — the watcher rejects a longer +interval at startup. + +## 5. Claim discipline + +- **Claimed = assignee set + active claim ref.** The assignee + (`--runner-assignee`) is a visible indicator only; the authoritative record + is the `headlesscode-claims/issue-N` ref. Runners acquire via atomic + ref-creation / non-forced fast-forward CAS, renew every sweep, and record + their nonce + epoch in local watcher state. +- **Do not delete claim refs.** A released ref stays in place so the next + acquisition increments the epoch and cannot reuse an old claim (no ABA). +- **Pool mode needs continuous operation.** `--run-once` is rejected with + `--runner-id` because a single sweep cannot renew leases. +- **Renewal failure is fail-closed.** When GitHub cannot confirm a renewal, + the watcher stops local worker process groups where possible and keeps the + claim active — it never releases a claim it cannot prove is torn down. + +## 6. Stale-claim recovery + +Expired active claims are **quarantined, not reassigned**: a runner may +release only a reservation that never dispatched (`stage: reserved`, +`release(..., { kind: "never-spawned" })`). Once a worker could have started, +worktree markers (PID/PGID files, `.harness.done`, sandbox markers) are not +trustworthy teardown proof — they live in worker-writable trees. + +Recovery procedure: + +1. Verify out of band that the old host's worker process groups are absent + **and** any guest resources are gone. If the machine cannot be inspected, + leave the claim quarantined — do not infer teardown from lease expiry. +2. An authorized operator calls `GitHubIssueClaimStore.recoverExpired()` + (see [`src/watcher/claims.ts`](../src/watcher/claims.ts)) with the current + ref tip, runner id, nonce, epoch, operator login, bounded evidence text, + and explicit terminal-state confirmation. The method rejects an unexpired + lease or a changed tip, and records the operator + evidence in the released + claim commit. That API records the operator's declaration — it cannot + independently prove remote teardown. + +See [Phase 5 watcher](./phase5-issue-watcher.md) § "Claim-based runner pool" +for the full semantics. diff --git a/src/watcher/cli.ts b/src/watcher/cli.ts index 86c947b..c6572b4 100644 --- a/src/watcher/cli.ts +++ b/src/watcher/cli.ts @@ -69,6 +69,10 @@ Options: mode and GitHub Contents read/write + Issues read/write. --runner-assignee GitHub account shown as assignee while a claim is active (required with --runner-id) + --claim-lease-ms Claim lease duration in runner-pool mode (default: + 120000; valid range 30000..900000). The poll + interval must be at most half the lease so claims + renew before expiry --state-file Durable idempotency state (default: /.worktrees/.watcher-state.json) --mode Harness mode for workers (default: code; forwarded @@ -98,6 +102,8 @@ Environment: HEADLESSCODE_MAX_CONCURRENT_SESSIONS Session cap (default 3; per-machine in pool mode) HEADLESSCODE_RUNNER_ID / HEADLESSCODE_RUNNER_ASSIGNEE Enable claim-based multi-machine runner mode + HEADLESSCODE_CLAIM_LEASE_MS Claim lease duration in ms (default 120000; + valid range 30000..900000) ` export interface WatchCliOptions { @@ -112,6 +118,8 @@ export interface WatchCliOptions { maxConcurrentSessions: number runnerId?: string runnerAssignee?: string + /** Claim lease duration in runner-pool mode (default 120000; valid 30000..900000). */ + claimLeaseMs?: number stateFile?: string mode: string executionProvider: "local" | "openshell" @@ -137,6 +145,7 @@ export function parseWatchArgs(argv: string[]): { options: WatchCliOptions; erro maxConcurrentSessions: 3, ...(process.env.HEADLESSCODE_RUNNER_ID ? { runnerId: process.env.HEADLESSCODE_RUNNER_ID } : {}), ...(process.env.HEADLESSCODE_RUNNER_ASSIGNEE ? { runnerAssignee: process.env.HEADLESSCODE_RUNNER_ASSIGNEE } : {}), + ...(process.env.HEADLESSCODE_CLAIM_LEASE_MS ? { claimLeaseMs: Number(process.env.HEADLESSCODE_CLAIM_LEASE_MS) } : {}), mode: "code", executionProvider: process.env.HEADLESSCODE_EXECUTION_PROVIDER === "openshell" ? "openshell" : "local", qa: false, @@ -176,6 +185,15 @@ export function parseWatchArgs(argv: string[]): { options: WatchCliOptions; erro options.runnerId = v break } + case "--claim-lease-ms": { + const v = next() + const n = v === undefined ? Number.NaN : Number(v) + if (!Number.isInteger(n) || n < 30_000 || n > 900_000) { + return { options, error: "--claim-lease-ms requires an integer in 30000..900000" } + } + options.claimLeaseMs = n + break + } case "--security-profile": { const v = next() try { options.securityProfile = resolveSecurityProfile(v) } @@ -312,6 +330,7 @@ export function parseWatchArgs(argv: string[]): { options: WatchCliOptions; erro catch (error) { return { options, error: error instanceof Error ? error.message : String(error) } } } if (options.runnerId && !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(options.runnerId)) return { options, error: "HEADLESSCODE_RUNNER_ID must be a safe stable machine identifier" } + if (options.claimLeaseMs !== undefined && (!Number.isInteger(options.claimLeaseMs) || options.claimLeaseMs < 30_000 || options.claimLeaseMs > 900_000)) return { options, error: "HEADLESSCODE_CLAIM_LEASE_MS must be an integer in 30000..900000" } if (!!options.runnerId !== !!options.runnerAssignee) return { options, error: "--runner-id and --runner-assignee must be set together" } if (options.runnerId && options.runOnce && !options.dryRun) return { options, error: "claim-based runner pool requires continuous mode; --run-once cannot maintain leases" } if (options.securityProfile === "contained") { @@ -448,6 +467,7 @@ export async function watchMain(argv: string[]): Promise { maxConcurrentSessions: options.maxConcurrentSessions, runnerId: options.runnerId, runnerAssignee: options.runnerAssignee, + ...(options.claimLeaseMs !== undefined ? { claimLeaseMs: options.claimLeaseMs } : {}), retryFailed: options.retryFailed, dryRun: options.dryRun, mode: options.mode,