From 91942b9ff0c298cd1d3a22b903e3bb2ab729556f Mon Sep 17 00:00:00 2001 From: Vikas Singhal Date: Fri, 18 Sep 2026 10:40:07 +0530 Subject: [PATCH] feat(sessions): a rate-limited pool queues a run instead of crashing it (v0.448.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When every runtime account is limited, rotation falls through to the box's own login — which on most boxes has never been used and holds no tokens — so the launch pre-flight refused the run and reported a crashed session. On live instawp 21 of the last 22 `crashed` sessions were this: each card told an admin to re-login a credential that was not the problem, while the accounts that were reset themselves within the hour, and two humans re-clicked take-over into the same card. A temporary exhaustion (every account limited, with a reset time) now parks the launch as `queued` and the 60s sweep launches it when an account frees up. A pool with no moment to retry at — empty, or all disabled — still refuses as before; a wait past 6h gives up with the original credential reason so a weekly cap cannot hide a run forever; a `queued` row left by a restart is reported, not left waiting. Stop calls off a wait; attach says the run has not started. Also: `expiresAt: 0` is a record with no tokens, not a date — it no longer renders as "expired on 1970-01-01" on cards and admin alerts. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 21 ++++ package-lock.json | 4 +- package.json | 4 +- scripts/capacity-queue-test.cjs | 183 ++++++++++++++++++++++++++++++++ src/server.ts | 6 ++ src/terminal.ts | 168 +++++++++++++++++++++++++++-- web/src/App.tsx | 19 ++-- web/src/lib/api.ts | 8 +- 8 files changed, 391 insertions(+), 22 deletions(-) create mode 100644 scripts/capacity-queue-test.cjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 4df15eaf..e18eb302 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,27 @@ new version heading in the same commit. ## [Unreleased] +## [0.448.0] - 2026-09-18 +### Added +- **A run that only lacks a free account now waits for one instead of crashing.** When every runtime + account is rate-limited, rotation falls through to the box's own login — which on most boxes has never + been used and holds no tokens — so the launch pre-flight refused the run and reported it as a crashed + session. Live instawp: 21 of its last 22 `crashed` sessions were this, each card telling an admin to + re-login a credential that was not the problem, while the accounts that were reset themselves within the + hour. A temporary exhaustion now parks the run as **queued** (a new session status, with its own dot and + filter in Sessions), and the 60s sweep launches it by itself the moment an account frees up. A pool with + no reset time to wait for — empty, or all disabled — still refuses as before, and a wait past 6 hours + gives up with the original credential reason, so a weekly cap can't hide a run forever. Pinned by + `scripts/capacity-queue-test.cjs`. + **For admins:** A session that can't start because every Claude account is at its rate limit now says + "queued" and starts on its own when one resets, instead of showing up as a crash. [Open Sessions](#/sessions) + +### Fixed +- **A credential record with no tokens no longer reports "expired on 1970-01-01".** `expiresAt: 0` is what + a record with neither an access nor a refresh token stores, not a date; every card and admin alert about + the box default on instawp rendered it as an expiry that never happened. It now reads "holds no usable + login — its credential record carries neither an access token nor a refresh token". + ## [0.447.1] - 2026-09-17 ### Fixed - **The unattended-run brief no longer tells agents three things that stopped being true.** It said diff --git a/package-lock.json b/package-lock.json index 79953037..60b78cde 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.447.1", + "version": "0.448.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.447.1", + "version": "0.448.0", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index b3aa4fe9..a1fdf941 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.447.1", + "version": "0.448.0", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", @@ -27,7 +27,7 @@ "check-deps": "bash scripts/install-deps.sh --check", "dev": "ts-node src/cli.ts serve", "demo:dev": "ts-node src/demo.ts", - "test:governance": "node scripts/version-sync-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs", + "test:governance": "node scripts/version-sync-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs", "test:alert-staleness": "node scripts/alert-staleness-test.cjs", "test:deps": "node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-login-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs", "test:dm-continuity": "node scripts/dm-continuity-test.cjs", diff --git a/scripts/capacity-queue-test.cjs b/scripts/capacity-queue-test.cjs new file mode 100644 index 00000000..de6ee44f --- /dev/null +++ b/scripts/capacity-queue-test.cjs @@ -0,0 +1,183 @@ +#!/usr/bin/env node +/* The capacity queue — a launch parked because every runtime account is rate-limited, instead of crashed. + * + * Why this exists — instawp, 2026-09-17: 21 of the tenant's last 22 `crashed` sessions were not crashes. + * Both pool accounts hit their 5-hour session limit within an hour of each other, a third was at its + * weekly cap and a fourth was disabled, so rotation had nothing to hand out and every launch fell through + * to the box default `~/.claude` — a login that had never been used and held no tokens at all. The + * pre-flight refused each run (correctly: it could not authenticate) and reported it as a crashed session + * whose card told an admin to re-login a credential that was not the problem, while the accounts that + * WERE the problem reset themselves within the hour. Two humans re-clicked take-over and got the same + * card twice. + * + * Pinned here: + * 1. exhausted-but-recovering pool + unusable fallback → `queued`, audited, with a card that names the + * reset time — and NOT a crash; + * 2. the same block with no reset to wait for (empty pool / all disabled) still crashes, because there + * is no moment to retry at; + * 3. the retry sweep launches the parked run once an account frees up, and resolves its card; + * 4. a wait past the ceiling gives up and crashes with the ORIGINAL credential reason — late, never + * silent; + * 5. a `queued` row this process holds no spec for (a restart mid-wait) is reported, not left waiting; + * 6. stop works on a queued run (there is no pane to kill, but there is a wait to call off); + * 7. `expiresAt: 0` never renders as "expired on 1970-01-01" — that is a record with no token in it. + * + * Isolated home; the backend and the actual launch are stubbed, so no tmux and no claude are needed. */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); +const HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'aos-capacity-test-')); +process.env.AGENT_OS_HOME = HOME; +process.env.AGENT_OS_TENANT = 'testco'; +process.env.AOS_NO_TTYD = '1'; +delete process.env.AGENT_OS_SECRET_KEY; + +let pass = 0, fail = 0; +const assert = (c, name, d) => c ? (pass++, console.log(` \x1b[32m✓\x1b[0m ${name}`)) : (fail++, console.log(` \x1b[31m✗ ${name}\x1b[0m${d ? ' — ' + d : ''}`)); + +const { loadAgentOS } = require(path.join(ROOT, 'dist/kernel.js')); +const { TerminalManager } = require(path.join(ROOT, 'dist/terminal.js')); + +const aos = loadAgentOS(); +const tm = new TerminalManager(aos, 'http://127.0.0.1:0', path.join(HOME, 'tmux.sock')); +tm.backend.kill = () => {}; +tm.backend.hasClient = () => false; +tm.backend.aliveNames = () => new Set(); + +// The launch itself is the one thing we never want to actually do — record the spec instead. +const launched = []; +tm.launchAgentRuntime = (spec) => { launched.push(spec.id); }; + +const MIN = 60_000; +// A credential dir in the shape the instawp box's `~/.claude` was in: a record with neither token, which +// stores `expiresAt: 0` — present enough to be read, dead enough to authenticate nothing. +const deadDir = path.join(HOME, 'dead-claude'); +fs.mkdirSync(deadDir, { recursive: true }); +fs.writeFileSync(path.join(deadDir, '.credentials.json'), JSON.stringify({ claudeAiOauth: { expiresAt: 0, scopes: ['user:inference'] } })); + +let n = 0; +const mkSession = () => { + const id = 'ts_' + (++n); + aos.db.prepare("INSERT INTO term_sessions (id,agent,title,task,tmux,status,headless,resident,spawned_by,created_at,updated_at) VALUES (?,?,?,?,?,'running',0,0,'m_alice',?,?)") + .run(id, 'support-ops', 't', 'x', 'aos-' + id, Date.now(), Date.now()); + return id; +}; +const spec = (id) => ({ + id, agent: 'support-ops', task: 'x', secret: 's', hasSlack: false, hasDiscord: false, hasClickup: false, + hasTelegram: false, headless: false, resident: false, resume: false, claudeSessionId: null, +}); +const statusOf = (id) => aos.db.prepare('SELECT status s FROM term_sessions WHERE id=?').get(id).s; +const cardsFor = (id) => aos.db.prepare('SELECT type,title,body,status,outcome FROM messages WHERE session_id=? ORDER BY created_at').all(id); +const auditTypes = (id) => aos.db.prepare('SELECT type FROM audit_events WHERE run_id=?').all(id).map((r) => r.type); +const preflight = (id) => tm.assertCredentialsUsable({ CLAUDE_CONFIG_DIR: deadDir }, spec(id), 'claude-code'); + +const addLimited = (name, untilMs) => { + const dir = path.join(HOME, 'acct-' + name); + fs.mkdirSync(dir, { recursive: true }); + aos.runtimeAccounts.add({ runtime: 'claude-code', name, kind: 'oauth', configDir: dir }); + aos.runtimeAccounts.markLimited('claude-code', name, untilMs); +}; + +console.log('\n\x1b[1m1) An exhausted pool parks the run — it does not crash it\x1b[0m'); +addLimited('tools', Date.now() + 40 * MIN); +addLimited('tools2', Date.now() + 46 * MIN); +const parked = mkSession(); +{ + const proceed = preflight(parked); + assert(proceed === false, 'the launch does not proceed (there is still nothing to authenticate with)'); + assert(statusOf(parked) === 'queued', 'the session is queued, not crashed', statusOf(parked)); + const types = auditTypes(parked); + assert(types.includes('session.launch.queued'), 'audited as session.launch.queued', types.join(',')); + assert(!types.includes('session.launch.refused'), 'NOT audited as a refusal — nothing is wrong with this box'); + const card = cardsFor(parked).find((c) => c.title.startsWith('Waiting for capacity')); + assert(Boolean(card), 'its owner gets a "Waiting for capacity" card'); + assert(card && card.outcome !== 'crashed', 'the card does not claim the run crashed'); + assert(card && /rate limit/.test(card.body) && /starts on its own/.test(card.body), 'the card says why and that it self-starts', card && card.body); + assert(card && /in about 40 min/.test(card.body), 'the card names how long the wait is', card && card.body); +} + +console.log('\n\x1b[1m2) With no reset to wait for, a dead credential still crashes\x1b[0m'); +{ + // Same dead fallback, but the pool can offer no moment to retry at: every account disabled. + aos.runtimeAccounts.setEnabled('claude-code', 'tools', false); + aos.runtimeAccounts.setEnabled('claude-code', 'tools2', false); + const doomed = mkSession(); + const proceed = preflight(doomed); + assert(proceed === false, 'the launch does not proceed'); + assert(statusOf(doomed) === 'crashed', 'an all-disabled pool is a refusal, not a queue', statusOf(doomed)); + assert(auditTypes(doomed).includes('session.launch.refused'), 'audited as a refusal'); + aos.runtimeAccounts.setEnabled('claude-code', 'tools', true); + aos.runtimeAccounts.setEnabled('claude-code', 'tools2', true); +} + +console.log('\n\x1b[1m3) The retry sweep launches the parked run once an account frees up\x1b[0m'); +{ + tm.retryCapacityQueue(); + assert(statusOf(parked) === 'queued' && launched.length === 0, 'still limited → still waiting, nothing launched'); + aos.runtimeAccounts.clearLimit('claude-code', 'tools'); + tm.retryCapacityQueue(); + assert(launched.includes(parked), 'an available account launches it'); + assert(statusOf(parked) === 'running', 'and the row goes back to running', statusOf(parked)); + assert(auditTypes(parked).includes('session.launch.dequeued'), 'audited as session.launch.dequeued'); + const card = cardsFor(parked).find((c) => c.title.startsWith('Waiting for capacity')); + assert(card && card.status === 'resolved', 'the waiting card is resolved — the wait is over', card && card.status); + tm.retryCapacityQueue(); + assert(launched.filter((x) => x === parked).length === 1, 'and it is launched exactly once'); +} + +console.log('\n\x1b[1m4) A wait past the ceiling gives up — late, but never silent\x1b[0m'); +{ + aos.runtimeAccounts.markLimited('claude-code', 'tools', Date.now() + 7 * 24 * 3600_000); // a WEEKLY cap + aos.runtimeAccounts.markLimited('claude-code', 'tools2', Date.now() + 7 * 24 * 3600_000); + const stale = mkSession(); + preflight(stale); + assert(statusOf(stale) === 'queued', 'parked first'); + const entry = tm.capacityQueue.get(stale); + entry.queuedAt = Date.now() - 7 * 3600_000; // waited 7h + tm.retryCapacityQueue(); + assert(statusOf(stale) === 'crashed', 'past the 6h ceiling the run crashes', statusOf(stale)); + const types = auditTypes(stale); + assert(types.includes('session.launch.queue.expired'), 'audited as session.launch.queue.expired', types.join(',')); + assert(types.includes('session.launch.refused'), 'and it falls back to the original credential refusal'); + assert(!tm.capacityQueue.has(stale), 'and it leaves the queue'); +} + +console.log('\n\x1b[1m5) A queued row with no spec in memory (a restart mid-wait) is reported\x1b[0m'); +{ + const orphan = mkSession(); + aos.db.prepare("UPDATE term_sessions SET status='queued' WHERE id=?").run(orphan); + tm.retryCapacityQueue(); + assert(statusOf(orphan) === 'crashed', 'it is not left waiting forever', statusOf(orphan)); + assert(auditTypes(orphan).includes('session.launch.queue.orphaned'), 'audited as orphaned'); + const card = cardsFor(orphan).find((c) => c.title.startsWith('Did not start')); + assert(Boolean(card) && /never started/.test(card.body), 'its owner is told it never started', card && card.body); + assert(card && /nothing was billed/.test(card.body), 'and that nothing was billed'); +} + +console.log('\n\x1b[1m6) Stop works on a queued run\x1b[0m'); +{ + const waiting = mkSession(); + preflight(waiting); + assert(statusOf(waiting) === 'queued', 'parked first'); + assert(tm.stopSession(waiting, 'alice@example.com') === true, 'stop returns true'); + assert(statusOf(waiting) === 'stopped', 'the row is stopped, not left queued', statusOf(waiting)); + assert(!tm.capacityQueue.has(waiting), 'and dropped from the queue'); + const before = launched.length; + tm.retryCapacityQueue(); + assert(launched.length === before, 'a stopped run is never launched by a later sweep'); + assert(statusOf(waiting) === 'stopped', 'and is not re-reported as a crash by the orphan pass', statusOf(waiting)); +} + +console.log('\n\x1b[1m7) A credential with no tokens is not a 1970 expiry\x1b[0m'); +{ + const crashed = aos.db.prepare("SELECT session_id id, body FROM messages WHERE body LIKE '%no usable login%' OR body LIKE '%1970%'").all(); + assert(crashed.length > 0, 'the refusal cards are on the record to check'); + assert(!crashed.some((c) => /1970/.test(c.body)), 'no card claims an expiry on 1970-01-01', JSON.stringify(crashed.map((c) => c.body.slice(0, 90)))); + assert(crashed.every((c) => /holds no usable login/.test(c.body)), 'they say the credential holds no usable login instead'); +} + +try { fs.rmSync(HOME, { recursive: true, force: true }); } catch { /* best effort */ } +console.log(`\n${fail ? '\x1b[31m' : '\x1b[32m'}${pass} passed, ${fail} failed\x1b[0m\n`); +process.exit(fail ? 1 : 0); diff --git a/src/server.ts b/src/server.ts index fbfa799a..5f3ad16f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -473,6 +473,9 @@ export function startServer(port = Number(process.env.PORT) || 3010): http.Serve const reaper = setInterval(() => registry.forEach((rt) => { try { requestMetrics.phase('reaper:idleSpaces', () => rt.tm.reapIdleSpaces()); } catch { /* never let the sweep crash */ } try { requestMetrics.phase('reaper:idleSessions', () => rt.tm.reapIdleSessions()); } catch { /* idle reaper (warm chat + unattended backstop) — never crash the sweep */ } + // The inverse of the reaper: runs parked because every runtime account was rate-limited, launched as + // soon as one resets. Same 60s cadence, because a limit reset is only ever known to the minute. + try { requestMetrics.phase('reaper:capacityQueue', () => rt.tm.retryCapacityQueue()); } catch { /* a parked launch must never crash the sweep */ } }), 60_000); reaper.unref?.(); // Process janitor: reap ttyd/tmux left behind pointing at tmux sockets that no longer exist, plus agent @@ -3644,6 +3647,9 @@ async function handle(os: AgentOS, tm: TerminalManager, autos: Automations, req: // be attachable — the console renders its transcript read-only instead. The stay-paused sentinel // already makes attach.sh refuse; this is the honest error rather than a terminal that opens blank. if (tm.isPaused(id)) return sendJson(res, 409, { error: 'this session is paused — resume it to use its terminal' }); + // Same shape, different reason: a queued run has no pane yet because it hasn't launched. Attaching + // would run attach.sh against a session that was deliberately not started, so say so instead. + if (tm.isQueuedForCapacity(id)) return sendJson(res, 409, { error: 'this session has not started yet — every runtime account is rate-limited; it starts on its own when one resets' }); try { const attachUrl = await tm.attachUrl(id); return sendJson(res, attachUrl ? 200 : 404, attachUrl ? { url: attachUrl } : { error: 'unknown session' }); diff --git a/src/terminal.ts b/src/terminal.ts index 6cf1b20f..5106f01e 100644 --- a/src/terminal.ts +++ b/src/terminal.ts @@ -2354,6 +2354,12 @@ export class TerminalManager { return this.db.prepare("SELECT 1 FROM term_sessions WHERE id = ? AND status = 'paused'").get(sessionId) != null; } + /** Parked at launch because every runtime account was rate-limited — no pane, no transcript, nothing to + * attach to. The server launches it itself when an account resets (see {@link retryCapacityQueue}). */ + isQueuedForCapacity(sessionId: string): boolean { + return this.db.prepare("SELECT 1 FROM term_sessions WHERE id = ? AND status = 'queued'").get(sessionId) != null; + } + reachable(sessionId: string): boolean { if (this.launching.has(sessionId)) return true; // scheduled; its pane is imminent const r = this.db.prepare('SELECT tmux, status FROM term_sessions WHERE id = ?').get<{ tmux: string; status: string }>(sessionId); @@ -4158,22 +4164,47 @@ export class TerminalManager { private lastCredentialAlertAt = 0; private static readonly CREDENTIAL_ALERT_COOLDOWN_MS = 30 * 60_000; - /** Launch pre-flight for the run's credentials. True = proceed. False = the launch was REFUSED and the - * session has already been marked crashed and explained to its owner; the caller must return. + /** Launch pre-flight for the run's credentials. True = proceed. False = the launch did NOT happen and + * the session has already been queued (temporary) or marked crashed and explained to its owner + * (permanent); the caller must return. * * Fails CLOSED, unlike every other credential path here, because the alternative isn't a degraded run — * it's a run that cannot authenticate at all. Falling through to the box default (the fail-open move * everywhere else) does not help either: on macOS the box default reads through the SAME locked - * keychain. */ - private assertCredentialsUsable(env: Record, o: { id: string; agent: string }, runtime: CodingRuntimeId): boolean { + * keychain. + * + * But "closed" is not the same as "failed". A pool whose accounts are all at their rate limit is a + * capacity condition with a KNOWN reset time, and the box default we then fall through to is, on most + * boxes, a login nobody has used since install. Crashing there reports an outage that is really a + * queue: live instawp, 2026-09-17 — 21 of its last 22 `crashed` sessions were this, each card telling + * an admin to re-login a credential that was not the problem, while the accounts that WERE the problem + * reset themselves within the hour. So a temporary exhaustion parks the run instead (see + * {@link queueForCapacity}); only a credential that no clock will fix still crashes. */ + private assertCredentialsUsable(env: Record, o: LaunchSpec, runtime: CodingRuntimeId): boolean { let blocked: CredentialBlock | null = null; try { blocked = preflightCredential(runtime, env); } catch { return true; } // a probe that can't run must never block a launch if (!blocked) return true; + // Only when rotation had somewhere to go and every stop is parked WITH a reset time. A pool that is + // empty, all-disabled, or parked with no recorded reset gives no moment to retry at, so it is not a + // queue — it is the refusal it always was. + const pool = this.poolCapacity(runtime); + if (pool && pool.until > Date.now()) return this.queueForCapacity(o, runtime, pool, blocked); this.refuseForCredential(o.id, o.agent, runtime, blocked); return false; } + /** Is this runtime's pool exhausted-but-recovering — every enabled account rate-limited, with a reset + * time to wait for? Null when rotation is inert (no pool), something is available, or nothing carries a + * reset. Shared by the launch pre-flight and the queue's retry so the two agree on what "exhausted" is. */ + private poolCapacity(runtime: CodingRuntimeId): { until: number; accounts: number } | null { + try { + const all = this.os.runtimeAccounts.allLimited(runtime); + if (!all.limited || !all.until) return null; + return { until: all.until, accounts: this.os.runtimeAccounts.enabledCount(runtime) }; + } catch { return null; } + } + /** * The same question, asked by the RESUME path instead of the launch path. * @@ -4202,9 +4233,19 @@ export class TerminalManager { * reply and the audit, so an operator never has to reconcile two accounts of the same refusal. */ private static credentialBlockWhy(runtime: CodingRuntimeId, b: CredentialBlock): string { const label = CODING_RUNTIMES[runtime].label; - return b.reason === 'keychain_locked' - ? `the macOS login keychain is locked, so ${label} cannot read the credential for ${b.dir} — this run would start, authenticate as nobody and end with no work done` - : `the login in ${b.dir} expired on ${new Date(b.expiredAt).toISOString().slice(0, 16).replace('T', ' ')} UTC and has no refresh token left, so ${label} would start, get "Login expired · Please run /login" on its first call and end with no work done`; + if (b.reason === 'keychain_locked') { + return `the macOS login keychain is locked, so ${label} cannot read the credential for ${b.dir} — this run would start, authenticate as nobody and end with no work done`; + } + return `${TerminalManager.expiryPhrase(b.expiredAt, b.dir)}, so ${label} would start, get "Login expired · Please run /login" on its first call and end with no work done`; + } + + /** How a dead credential reads in one clause. `expiresAt: 0` is not a date — it is what a record with no + * token in it stores, and rendering it as one produced "expired on 1970-01-01 00:00 UTC" on every card + * the instawp box raised, which sends an admin looking for an expiry event that never happened. */ + private static expiryPhrase(expiredAt: number, dir: string): string { + return expiredAt > 0 + ? `the login in ${dir} expired on ${new Date(expiredAt).toISOString().slice(0, 16).replace('T', ' ')} UTC and has no refresh token left` + : `${dir} holds no usable login — its credential record carries neither an access token nor a refresh token`; } /** Record a refused run: audit, crash the row, tell its owner, badge the pool account, alert admins. @@ -4248,12 +4289,118 @@ export class TerminalManager { topic: 'credentials-expired', type: 'notification', title: `Agent runs are blocked — the ${label} login has expired`, - body: `The login in ${b.dir} expired on ${new Date(b.expiredAt).toISOString().slice(0, 16).replace('T', ' ')} UTC and carries no refresh token, so every session started with it would get "Login expired · Please run /login" on its first call. Runs are being refused rather than started and left to fail silently.\n\nSign that credential in again on the box:\n\n CLAUDE_CONFIG_DIR=${b.dir} claude /login\n\nOr add a working account under Settings → Runtime → Runtime accounts, which is what sessions rotate onto when the box default is unusable.`, + body: `${TerminalManager.expiryPhrase(b.expiredAt, b.dir).replace(/^the login/, 'The login')}, so every session started with it would get "Login expired · Please run /login" on its first call. Runs are being refused rather than started and left to fail silently.\n\nSign that credential in again on the box:\n\n CLAUDE_CONFIG_DIR=${b.dir} claude /login\n\nOr add a working account under Settings → Runtime → Runtime accounts, which is what sessions rotate onto when the box default is unusable.`, audience: { kind: 'admins' }, }); } catch { /* the audit line above is the durable record */ } } + /** + * ── THE CAPACITY QUEUE ───────────────────────────────────────────────────────────────────────── + * + * A run that only lacks a free account is not a failed run; it is an early one. Park it here, retry it + * from the 60s sweep, and launch it the moment rotation can serve it. + * + * Deliberately IN MEMORY. A queued entry is a launch that has not started — no pane, no transcript, no + * cost — so the cheapest correct behaviour across a restart is to forget it and tell its owner, which + * is exactly what {@link retryCapacityQueue} does for any `queued` row it no longer holds a spec for. + * Persisting the spec would buy a launch that survives a deploy at the price of a second source of + * truth for what a session's environment was, and the environment (secrets, mints, member identity) is + * rebuilt at launch anyway. + * + * The ceiling matters more than the retry. A five-hour session window resets within the hour; a WEEKLY + * quota does not, and a run silently waiting out six days is the same silence as a crash with better + * manners. Past {@link CAPACITY_QUEUE_MAX_MS} the wait is given up on and the run crashes with the + * credential reason it originally had, so the failure is late but never invisible. + */ + private readonly capacityQueue = new Map(); + + /** How long a run may wait for an account before the wait itself is treated as the failure. */ + private static readonly CAPACITY_QUEUE_MAX_MS = 6 * 60 * 60_000; + + /** Park a launch until an account frees up. Always returns false — the caller's launch does not proceed. */ + private queueForCapacity(o: LaunchSpec, runtime: CodingRuntimeId, pool: { until: number; accounts: number }, blocked: CredentialBlock): boolean { + const now = Date.now(); + this.capacityQueue.set(o.id, { spec: o, runtime, queuedAt: now, until: pool.until, blocked }); + this.db.prepare("UPDATE term_sessions SET status = 'queued', busy_since = NULL, updated_at = ? WHERE id = ?").run(now, o.id); + this.audit(o.id, o.agent, 'session.launch.queued', { runtime, reason: 'every account is rate-limited', accounts: pool.accounts, until: pool.until, dir: blocked.dir }); + const label = CODING_RUNTIMES[runtime].label; + this.addMessage({ + type: 'notification', sessionId: o.id, agent: o.agent, status: 'open', + title: `Waiting for capacity — ${o.agent}`, + body: `All ${pool.accounts} ${label} ${pool.accounts === 1 ? 'account is' : 'accounts are'} at their rate limit, so this run has not started yet. It starts on its own once the first one resets${TerminalManager.resetPhrase(pool.until, now)}. Nothing has been lost — no turn ran and nothing was billed.`, + audienceKind: 'sessionOwner', audienceId: o.id, + }); + return false; + } + + /** " at 14:14 UTC (in about 40 min)" — the two halves an operator actually wants: when, and how long. */ + private static resetPhrase(until: number, now: number): string { + const mins = Math.max(0, Math.round((until - now) / 60_000)); + const when = new Date(until).toISOString().slice(11, 16); + const howLong = mins < 60 ? `in about ${mins} min` : `in about ${Math.round(mins / 60)}h`; + return ` at ${when} UTC (${howLong})`; + } + + /** + * Retry every parked launch. Run from the process-wide 60s sweep (server.ts), never throws. + * + * Three outcomes per entry: the session is no longer waiting (stopped by a human, or the row is gone) → + * drop it; capacity is back → launch it; the ceiling passed → crash it with its original reason. Plus a + * fourth for the rows this process does not own: a `queued` row with no spec in memory is a launch its + * server restarted out from under, which can only be reported, not resumed. + */ + retryCapacityQueue(): void { + const now = Date.now(); + try { + for (const [id, q] of [...this.capacityQueue]) { + const row = this.db.prepare('SELECT status, agent FROM term_sessions WHERE id = ?').get<{ status: string; agent: string }>(id); + if (!row || row.status !== 'queued') { this.capacityQueue.delete(id); continue; } + if (now - q.queuedAt >= TerminalManager.CAPACITY_QUEUE_MAX_MS) { + this.capacityQueue.delete(id); + this.audit(id, q.spec.agent, 'session.launch.queue.expired', { runtime: q.runtime, waitedMs: now - q.queuedAt }); + this.refuseForCredential(id, q.spec.agent, q.runtime, q.blocked); + continue; + } + // Still exhausted → keep waiting. `poolCapacity` re-reads the pool (recover() un-parks accounts + // whose reset has passed), so this is the same question the pre-flight asked, asked again. + if (this.poolCapacity(q.runtime)) continue; + this.capacityQueue.delete(id); + this.audit(id, q.spec.agent, 'session.launch.dequeued', { runtime: q.runtime, waitedMs: now - q.queuedAt }); + this.db.prepare("UPDATE term_sessions SET status = 'running', updated_at = ? WHERE id = ? AND status = 'queued'").run(now, id); + this.resolveQueuedCard(id); + this.launchAgentRuntime(q.spec); + } + this.expireOrphanedQueued(now); + } catch { /* a queue that can't be swept must never break the sweep */ } + } + + /** `queued` rows this process has no spec for — a restart during the wait. Reported, not resumed: the + * owner is told the run never started, rather than left with a row that will wait forever. */ + private expireOrphanedQueued(now: number): void { + const rows = this.db.prepare("SELECT id, agent FROM term_sessions WHERE status = 'queued'").all<{ id: string; agent: string }>(); + for (const r of rows) { + if (this.capacityQueue.has(r.id)) continue; + this.db.prepare("UPDATE term_sessions SET status = 'crashed', busy_since = NULL, updated_at = ? WHERE id = ?").run(now, r.id); + this.audit(r.id, r.agent, 'session.launch.queue.orphaned', { reason: 'the server restarted while this run was waiting for a runtime account' }); + this.resolveQueuedCard(r.id); + this.addMessage({ + type: 'completed', sessionId: r.id, agent: r.agent, status: 'open', outcome: 'crashed', + title: `Did not start — ${r.agent}`, + body: 'This run was waiting for a rate-limited runtime account to reset when the server restarted, so it never started. Nothing ran and nothing was billed — start it again when you want it.', + audienceKind: 'sessionOwner', audienceId: r.id, + }); + } + } + + /** Close the "waiting for capacity" card once the wait is over, either way — it is a status, and a + * status that outlives the thing it described is just noise in an inbox. */ + private resolveQueuedCard(sessionId: string): void { + try { + this.db.prepare("UPDATE messages SET status = 'resolved' WHERE session_id = ? AND type = 'notification' AND status = 'open' AND title LIKE 'Waiting for capacity%'").run(sessionId); + } catch { /* advisory */ } + } + /** Select a rotation-pool account for this runtime and point the session's credentials at it, via the * runtime's own env vars (`CODING_RUNTIMES[runtime].credentialEnv`). Records which account the run used * (`term_sessions.runtime_account`) so limit detection at teardown can park the right one. No-op — leaving @@ -8847,8 +8994,11 @@ export class TerminalManager { // `paused` rides along with `running`: pausing is not a decision to end the run, so "stop" is still // available on a paused session and is how you say "I'm not coming back" — it writes the episode and // clears the paused stamp, which `running` alone would have left on the row forever. - if (r.status === 'running' || r.status === 'paused') { + // `queued` rides along too: a run parked for capacity has no pane to kill, but "stop" is how a human + // says they are no longer waiting. The status flip is what the retry sweep reads to drop the entry. + if (r.status === 'running' || r.status === 'paused' || r.status === 'queued') { this.db.prepare("UPDATE term_sessions SET status = 'stopped', busy_since = NULL, paused_at = NULL, paused_by = NULL, updated_at = ? WHERE id = ?").run(Date.now(), sessionId); + if (r.status === 'queued') { this.capacityQueue.delete(sessionId); this.resolveQueuedCard(sessionId); } } this.clearNotifications(sessionId); // The agent that asked is now dead — no one can answer its open questions or act on its approvals. diff --git a/web/src/App.tsx b/web/src/App.tsx index f84370d8..fccd19f4 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -114,7 +114,7 @@ function mergeSessionRows(prev: Session[], fresh: Session[]): Session[] { * * `headless` is deliberately NOT in the dot any more: the hollow ring now means "not busy", and the * unattended/interactive axis has its own marker (ModeBadge / the sidebar's Cpu glyph). */ -type SessionState = 'waiting' | 'working' | 'idle' | 'paused' | 'stopped' | 'crashed' | 'done' +type SessionState = 'waiting' | 'working' | 'idle' | 'queued' | 'paused' | 'stopped' | 'crashed' | 'done' /** Resolve a session's state. `waiting` may be forced by the caller — the console unions the * server-authoritative `s.blocked` with open `notification` cards (a runtime permission prompt raises @@ -122,8 +122,10 @@ type SessionState = 'waiting' | 'working' | 'idle' | 'paused' | 'stopped' | 'cra const sessionState = (s: Session, waiting = false): SessionState => waiting || s.blocked ? 'waiting' : isLive(s) ? (s.working ? 'working' : 'idle') - // `paused` is checked before the terminal states and is deliberately NOT one of them: the run hasn't - // finished, it is suspended mid-conversation and one click from being live again. + // `queued` and `paused` are both checked before the terminal states and are deliberately NOT ones: + // a queued run has not started yet (every runtime account is rate-limited — it launches itself when + // one resets), and a paused one is suspended mid-conversation, one click from being live again. + : s.status === 'queued' ? 'queued' : s.status === 'paused' ? 'paused' : s.status === 'stopped' ? 'stopped' : s.status === 'crashed' ? 'crashed' @@ -193,6 +195,7 @@ const STATE_META: Record (outcome ? VERDICT_ const resultLabel = (s: Session, waiting = false): string => { if (waiting || s.blocked || isLive(s)) return statusLabel(s, waiting) // A paused run has no result — it hasn't finished. Say the status, don't reach for an outcome. - if (s.status === 'crashed' || s.status === 'stopped' || s.status === 'paused') return s.status + if (s.status === 'crashed' || s.status === 'stopped' || s.status === 'paused' || s.status === 'queued') return s.status if (!s.outcome) return s.status // not stamped yet — fall back to the process view const v = verdictOf(s.outcome) return v ? VERDICT_META[v].label : s.outcome // an unmapped value prints as the agent wrote it @@ -292,6 +295,7 @@ const resultLabel = (s: Session, waiting = false): string => { const resultTone = (s: Session, waiting = false): string => { if (waiting || s.blocked || isLive(s)) return STATE_META[sessionState(s, waiting)].tone if (s.status === 'crashed') return 'text-red-600' + if (s.status === 'queued') return STATE_META.queued.tone if (s.status === 'paused') return STATE_META.paused.tone const v = verdictOf(s.outcome) if (v && v !== 'none') return VERDICT_META[v].tone @@ -426,7 +430,7 @@ const sessionSource = (s: Session): SessionSource => { // `chains` is not a lifecycle state — it narrows to sessions that took part in a HAND-OFF (a caller // that delegated, or a delegate). It rides in this filter because that's where people already look to // cut the list down, and it's resolved in `filtered` (it needs the whole list to know who called whom). -type SessionStatusFilter = 'all' | 'live' | 'working' | 'blocked' | 'chains' | 'done' | 'paused' | 'stopped' | 'crashed' +type SessionStatusFilter = 'all' | 'live' | 'working' | 'blocked' | 'chains' | 'done' | 'queued' | 'paused' | 'stopped' | 'crashed' const matchesStatus = (s: Session, f: SessionStatusFilter): boolean => f === 'all' || f === 'chains' ? true // `chains` is applied separately — it needs the whole list : f === 'live' ? isLive(s) @@ -437,7 +441,7 @@ const matchesStatus = (s: Session, f: SessionStatusFilter): boolean => // Filter labels — shared by the dropdown options AND the collapsed trigger (base-ui's SelectValue // renders the raw value unless given a formatter, so the two must read from one source). const SESSION_STATUS_LABELS: Record = - { all: 'All statuses', live: 'Live', working: 'Working', blocked: 'Needs you', chains: 'Hand-offs', done: 'Done', paused: 'Paused', stopped: 'Stopped', crashed: 'Crashed' } + { all: 'All statuses', live: 'Live', working: 'Working', blocked: 'Needs you', chains: 'Hand-offs', done: 'Done', queued: 'Queued', paused: 'Paused', stopped: 'Stopped', crashed: 'Crashed' } const SESSION_SOURCE_LABELS: Record<'all' | SessionSource, string> = { all: 'All sources', member: 'Member', automation: 'Automation', task: 'Task', chat: 'Chat' } @@ -4176,6 +4180,8 @@ const nodeState = (n: ChainNode): { label: string; tone: string; icon: LucideIco if (nodeLive(n)) return shared(n.working ? 'working' : 'idle', true) if (n.duplicateOf) return { label: 'duplicate', tone: 'text-amber-600', icon: CopyIcon, anim: '' } if (n.status === 'crashed') return shared('crashed') + // A queued run has no outcome to look up — it never started. + if (n.status === 'queued') return shared('queued') // Before the outcome lookup: a paused run may carry an outcome from an earlier `report` in the same // conversation, and printing that would claim the run is over when it is suspended mid-flight. if (n.status === 'paused') return shared('paused') @@ -12713,6 +12719,7 @@ function fmtBytes(n: number): string { function runVerdict(r: TaskRun): { label: string; cls: string; role: StatusRole } { if (r.alive) return { label: STATE_META.working.label, cls: STATE_META.working.tone, role: 'busy' } if (r.status === 'paused') return { label: STATE_META.paused.label, cls: STATE_META.paused.tone, role: 'paused' } + if (r.status === 'queued') return { label: STATE_META.queued.label, cls: STATE_META.queued.tone, role: 'queued' } const v = verdictOf((r.outcome || '').toLowerCase()) if (v && v !== 'none') return { label: VERDICT_META[v].label, cls: VERDICT_META[v].tone, role: v === 'success' ? 'ok' : v === 'partial' ? 'partial' : 'failed' } if (r.status === 'crashed') return { label: STATE_META.crashed.label, cls: STATE_META.crashed.tone, role: 'crashed' } diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 693cbe05..165e0dab 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -398,9 +398,11 @@ export interface Session { title: string task: string tmux: string - /** `paused` is the one value that is neither live nor finished: a human suspended the run, its agent - * was killed (freeing its memory) and its conversation is on disk waiting for `unpauseSession`. */ - status: 'running' | 'done' | 'stopped' | 'crashed' | 'paused' + /** Two values are neither live nor finished. `paused`: a human suspended the run, its agent was killed + * (freeing its memory) and its conversation is on disk waiting for `unpauseSession`. `queued`: the run + * has not started at all — every runtime account was rate-limited at launch, so the server parked it and + * launches it itself once one resets. */ + status: 'running' | 'done' | 'stopped' | 'crashed' | 'paused' | 'queued' /** True when the tmux pane is alive now, regardless of the stored lifecycle `status` (an interactive * session that reported `done` keeps a live pane). Undefined when the server couldn't poll tmux. */ alive?: boolean