From d15aece9841cdf5ad5619d181451a8bdd7367a42 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Thu, 29 Oct 2026 23:14:58 -0400 Subject: [PATCH 1/9] test(run-state): add model checker for the run-state kernel --- docs/architecture/task-lifecycle-model.md | 35 +- .../task-lifecycle-target-architecture.md | 2 +- package.json | 2 +- scripts/check-run-state-kernel.ts | 835 ++++++++++++++++++ 4 files changed, 869 insertions(+), 5 deletions(-) create mode 100644 scripts/check-run-state-kernel.ts diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index a6fc40494d..377d41b8c8 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -6,7 +6,7 @@ Zoo Code checks task lifecycle protocols through one umbrella command for indepe pnpm lifecycle:model-check ``` -The baseline command runs eight independent bounded checks in sequence: +The baseline command runs nine independent bounded checks in sequence: 1. the persisted task delegation lifecycle; 2. shared-store concurrency across task-history hosts; @@ -14,8 +14,9 @@ The baseline command runs eight independent bounded checks in sequence: 4. the task cleanup protocol; 5. request-stream parser scoping; 6. completion persistence; -7. delegated-mode reader refinement; and -8. tool-turn continuation. +7. delegated-mode reader refinement; +8. tool-turn continuation; and +9. the in-memory run-state kernel. The planned two-sibling fan-out protocol is intentionally outside the baseline and CI umbrella. Run it explicitly with `pnpm fanout-protocol:model-check`; it describes optional future functionality, not current production coverage. @@ -205,6 +206,7 @@ These are safety claims within the documented bounds. The checks do not claim li | Cleanup | Abstract bounded universal plus adapter tests | Abort, disposal, settlement, rejection, and provider shutdown are modeled as protocol/environment actions. | No direct execution of all production cleanup methods, filesystem/editor promises, timing liveness, fairness, or arbitrary task counts. | | Parser request scope | Production-backed bounded schedule replay | The checker executes production parser APIs across 924 order-preserving schedules for two scopes. | Assumes callers stop invoking a finalized scope; transport behavior, arbitrary request counts, indices, and malformed histories are outside the claim. | | Completion persistence | Abstract bounded universal plus production tests and one fresh-host E2E path | The model abstracts persistence as a durable phase with at most two write starts; production guards and retry paths are tested separately. | Production permits more retries; no power-loss/filesystem proof, fairness, arbitrary retry count, complete delegated fallback, provider status metadata, or downstream event-consumer model. | +| Run-state kernel | Production-backed bounded universal | The explorer calls the production `nextRunState` and compares each result with a transcription of the design-doc guard and effect table. | Models one task's run state only. Excludes persistence, the actor mailbox, and the wiring of `Task.ts` to the kernel. Witness and source-pair checks have no self-test. | The production mapping above names primary lifecycle transitions, not every mutation or consumer. Generic store upserts, reconciliation, repair replay, migrations, tool entry points, webview/public API abandonment, provider status updates, and public `TaskCompleted` re-emission remain outside the persisted reducer graph unless explicitly named by a submodel or focused test. For task-local mode, the consumers named under #921/#1623 are a confirmed set, not an exhaustive repository-wide inventory; other mode-sensitive tools must be audited before claiming universal reader isolation. @@ -271,8 +273,35 @@ Completion-readiness changes belong in `scripts/check-completion-persistence.ts` Parser request scoping is one such independent bounded submodel within the umbrella suite. Extend `scripts/check-native-tool-call-parser-scoping.ts` and its focused architecture document instead of adding parser state or transitions to `taskLifecycle.ts` or the persisted lifecycle state graph. +Run-state kernel changes belong in `scripts/check-run-state-kernel.ts`. Update the guard and effect table in the checker together with the Transitions table in the target-architecture document, and add a known-unsafe kernel for each new check. + ## Test layering Keep reducer permutations in this model and focused Vitest suites. The real VS Code extension-host suite using a mocked provider in `apps/vscode-e2e/src/suite/subtasks.test.ts` already covers the boundaries the pure explorer cannot: task creation and rehydration, persisted parent-child state, cancellation during a delayed provider stream, interrupted-child resume, abandonment followed by a real resume/save/completion cycle, pending approvals across leave/return, and scheduler-driven resume. `restart-persistence.test.ts` separately verifies completion history through a fresh extension host. Add E2E coverage only when a lifecycle change crosses one of those runtime boundaries or introduces a new one. For example, #1453 persistence-readiness semantics require a controlled fresh-host test, and #369/#372 fan-out requires scheduler permit, live-parent routing, orphan cleanup, and task-scoping E2E. Do not add E2E cases solely to replay reducer orderings already exhausted here; they increase fixture and timing cost without strengthening the proof claim. + +## Run-state kernel submodel + +`scripts/check-run-state-kernel.ts` explores the pure `nextRunState` in `src/core/task/run-state/runState.ts`. It is an independent submodel. It does not use `taskLifecycle.ts` and does not share state with the persisted graph. + +The checker applies every event in every reachable state, within the generation and ask-timestamp caps. The budget is 20,000 states and the depth bound is 20. The stream generation and the ask timestamp are each at most 3. At each step it checks: + +- the result of the kernel against a transcription of the guard and effect table ([Transitions](task-lifecycle-target-architecture.md#transitions)), including that a rejection leaves the state unchanged; +- the seven design-doc invariants (same document, Invariants); +- the kernel does not mutate its state or its event, and the transition checks that hold without the table (latches, generation, phase, stream, and ask rules). + +The checker also requires: + +- the event tags in `runState.ts` to equal the event tags in the table (once, at startup, as a text scan of `case` labels); +- every event to be reachable, with the `askSettledStale` no-op, a settle for an ask that a newer pending ask replaced, as its own action; +- every guard clause to be the only failing clause in some rejected event; +- named witnesses, each tied to an event and the states around it: abort during a pending approval in a live stream, an ask after the stream ends, abandon after the stream ends, dispose after completion, a new stream after a failure cleanup without abort, a stream that starts without user input, an approval before the first stream, and a rejected late completion accept after a newer ask; +- every ordered pair of the stream, ask, cancel, and dispose event sources to follow each other, where both events change the state; +- no unseen successor at the depth bound; +- thirty-one known-unsafe kernels, each of which the checker must reject. They include a throwing kernel, a kernel that returns nothing, and one kernel for each check that runs at every search step without the table; and +- self-tests that a small depth bound and a small state budget each fail the checker. + +The kernel has no resume event. A resumed task is `initialized` followed by `streamStarted` or `askStarted`. The witnesses `first-stream-without-ask` and `approval-before-first-stream` cover those paths. + +The mapping from kernel events and latches to the cleanup protocol model is in [The verification boundary](task-lifecycle-target-architecture.md#the-verification-boundary). diff --git a/docs/architecture/task-lifecycle-target-architecture.md b/docs/architecture/task-lifecycle-target-architecture.md index 571413f43a..285f0c08b3 100644 --- a/docs/architecture/task-lifecycle-target-architecture.md +++ b/docs/architecture/task-lifecycle-target-architecture.md @@ -423,7 +423,7 @@ Replace the process-global tool state with keyed state. Key partial-call state b Draw one line. The checked core is verified. The periphery is ordinary code. -1. Checked core: the seven baseline checkers in `pnpm lifecycle:model-check` (task lifecycle, task store concurrency, provider handoff and scheduler, cleanup protocol, parser scope, completion persistence, and delegated mode readers), plus the pure `nextRunState`. Bounded model-checkers explore these exhaustively within the state budget. This is the "conservative abstraction that keeps only state-relevant information" pattern that AWS uses with formal methods ([How Amazon Web Services uses formal methods](https://cacm.acm.org/research/how-amazon-web-services-uses-formal-methods/)). +1. Checked core: the baseline checkers in `pnpm lifecycle:model-check` (task lifecycle, task store concurrency, provider handoff and scheduler, cleanup protocol, parser scope, completion persistence, delegated mode readers, tool-turn continuation, and the run-state kernel `nextRunState`). Bounded model-checkers explore these exhaustively within the state budget. This is the "conservative abstraction that keeps only state-relevant information" pattern that AWS uses with formal methods ([How Amazon Web Services uses formal methods](https://cacm.acm.org/research/how-amazon-web-services-uses-formal-methods/)). 2. Periphery: streaming, filesystem, webview, and provider I/O. Focused tests and E2E cover these. No exhaustive proof is claimed. The run-state kernel is a new in-memory submodel. It is not a persisted lifecycle mutation, so it does not live in the shared reducers that `AGENTS.md` pins to `taskLifecycle.ts`. It uses the independent-submodel pattern (`task-lifecycle-model.md`, Extending the model). Add a boundary mapping before any claim spans two submodels (`task-lifecycle-model.md`, introduction). The abort and dispose ordering is already model-checked in `task-cleanup-protocol-model.md`. Map the kernel latches to that model rather than re-proving them: diff --git a/package.json b/package.json index f438decdbd..eca5d69733 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "test:agent-evals:unit": "node --import tsx --test scripts/agent-evals/*.test.ts", "agent-evals:check-types": "tsc -p scripts/agent-evals/tsconfig.json", "test:mutation-ci": "node --test scripts/stryker-diff.test.mjs", - "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-file-replacement-restart.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check && tsx scripts/check-queue-ask-landing.ts", + "lifecycle:model-check": "tsx scripts/check-task-lifecycle.ts && tsx scripts/check-task-store-concurrency.ts && tsx scripts/check-provider-handoff-scheduler.ts && pnpm cleanup-protocol:model-check && pnpm parser-scope:model-check && tsx scripts/check-completion-persistence.ts && tsx scripts/check-file-replacement-restart.ts && tsx scripts/check-delegated-mode-readers.ts && pnpm tool-turn:model-check && tsx scripts/check-queue-ask-landing.ts && tsx scripts/check-run-state-kernel.ts", "webview-recovery:model-check": "node --import tsx scripts/check-webview-recovery.ts", "webview-recovery:check-types": "tsc -p scripts/tsconfig.webview-recovery.json", "fanout-protocol:model-check": "tsx scripts/check-task-fanout-protocol.ts", diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts new file mode 100644 index 0000000000..d4dd9b4ddd --- /dev/null +++ b/scripts/check-run-state-kernel.ts @@ -0,0 +1,835 @@ +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { fileURLToPath } from "node:url" + +import { nextRunState, initialRunState } from "../src/core/task/run-state/runState" +import type { AbortReason, Ask, RunEvent, RunState, Transition } from "../src/core/task/run-state/runState" + +type Source = "init" | "stream" | "ask" | "cancel" | "dispose" +type EventTag = RunEvent["tag"] +type StartedAsk = Exclude +type Step = (state: RunState, event: RunEvent) => Transition | undefined + +interface ModelState { + run: RunState + askCount: number + completionAsks: number[] + lastEndedStreamCleaned: boolean +} + +interface ModelEvent { + name: string + source: Source + event: RunEvent +} + +interface TraceStep { + action: string + source: Source | "initial" + state: ModelState +} + +interface Options { + step: Step + useTable: boolean + maxDepth: number + maxStates: number +} + +interface Applied { + next: ModelState + accepted: boolean + problems: string[] +} + +interface WitnessContext { + prev: ModelState + next: ModelState + event: RunEvent +} + +const MAX_DEPTH = 20 +const MAX_STATES = 20_000 +const MAX_GENERATION = 3 +const MAX_ASK_TS = 3 +const KERNEL_PATH = fileURLToPath(new URL("../src/core/task/run-state/runState.ts", import.meta.url)) + +const abortReasons: Record = { user_cancelled: true, streaming_failed: true } +const askBuilders: Record StartedAsk> = { + approval: (askTs) => ({ tag: "approval", askTs, toolCallId: `tool-${askTs}`, approvalId: `approval-${askTs}` }), + approvalWithoutIds: (askTs) => ({ tag: "approval", askTs }), + completion: (askTs) => ({ tag: "completion", askTs }), + other: (askTs) => ({ tag: "other", askTs }), +} +const eventSources = ["stream", "ask", "cancel", "dispose"] as const + +// One field per cell of the state described in the target-architecture doc, so rows read like the doc table. +interface Flat { + phase: RunState["phase"] + generation: number + streamLive: boolean + streamGeneration: number | null + cleanupFinished: boolean | null + askTag: Ask["tag"] + askTs: number | null + toolCallId: string | null + approvalId: string | null + abort: boolean + abandoned: boolean + disposed: boolean + reason: AbortReason | null + lastCompletionAskTs: number | null +} + +function flatten(run: RunState): Flat { + const ask = run.ask + return { + phase: run.phase, + generation: run.generation, + streamLive: run.stream.tag === "live", + streamGeneration: run.stream.tag === "live" ? run.stream.generation : null, + cleanupFinished: run.stream.tag === "live" ? run.stream.cleanupFinished : null, + askTag: ask.tag, + askTs: ask.tag === "none" ? null : ask.askTs, + toolCallId: ("toolCallId" in ask && ask.toolCallId) || null, + approvalId: ("approvalId" in ask && ask.approvalId) || null, + abort: run.latches.abort, + abandoned: run.latches.abandoned, + disposed: run.latches.disposed, + reason: run.latches.reason ?? null, + lastCompletionAskTs: run.lastCompletionAskTs ?? null, + } +} + +interface Row { + guard?: Record boolean> + effect: (f: Flat, e: E) => void +} + +type Rows = { [T in EventTag]: Row> } + +const clearAsk = (f: Flat) => { + f.askTag = "none" + f.askTs = f.toolCallId = f.approvalId = null +} + +// Transcribes the Guard and Effect columns of task-lifecycle-target-architecture.md (Transitions). +const rows: Rows = { + initialized: { + guard: { uninitialized: (f) => f.phase === "uninitialized" }, + effect: (f) => void (f.phase = "running"), + }, + streamStarted: { + guard: { running: (f) => f.phase === "running", "no live stream": (f) => !f.streamLive }, + effect: (f) => { + f.generation += 1 + f.streamLive = true + f.streamGeneration = f.generation + f.cleanupFinished = false + f.lastCompletionAskTs = null + }, + }, + streamCleanupFinished: { + guard: { "live stream": (f) => f.streamLive }, + effect: (f) => void (f.cleanupFinished = true), + }, + streamEnded: { + guard: { "live stream": (f) => f.streamLive }, + effect: (f) => { + f.streamLive = false + f.streamGeneration = f.cleanupFinished = null + }, + }, + askStarted: { + guard: { running: (f) => f.phase === "running", "not aborted": (f) => !f.abort }, + effect: (f, e) => { + f.askTag = e.ask.tag + f.askTs = e.ask.askTs + f.toolCallId = (e.ask.tag === "approval" && e.ask.toolCallId) || null + f.approvalId = (e.ask.tag === "approval" && e.ask.approvalId) || null + f.lastCompletionAskTs = e.ask.tag === "completion" ? e.ask.askTs : null + }, + }, + askSettled: { + effect: (f, e) => { + if (f.askTs === e.askTs) clearAsk(f) + }, + }, + completionAccepted: { + guard: { + running: (f) => f.phase === "running", + "not aborted": (f) => !f.abort, + "latest completion ask": (f, e) => e.askTs === f.lastCompletionAskTs, + }, + effect: (f) => { + f.phase = "completed" + clearAsk(f) + }, + }, + reasonSet: { + effect: (f, e) => void (f.reason ??= e.reason), + }, + abortRequested: { + effect: (f, e) => { + f.abort = true + if (e.abandoned) f.abandoned = true + }, + }, + abandonRequested: { + guard: { aborted: (f) => f.abort }, + effect: (f) => void (f.abandoned = true), + }, + disposeRequested: { + effect: (f) => { + f.abort = true + f.disposed = true + }, + }, +} + +const eventTags = Object.keys(rows) as EventTag[] +const guardClauses = eventTags.flatMap((tag) => + Object.keys((rows[tag] as Row).guard ?? {}).map((clause) => `${tag}: ${clause}`), +) +const expectedActions = [...eventTags, "askSettledStale"] + +function referenceResult(run: RunState, event: RunEvent): { accepted: boolean; flat: Flat; failedClauses: string[] } { + const flat = flatten(run) + const row = rows[event.tag] as Row + const failedClauses = Object.entries(row.guard ?? {}) + .filter(([, holds]) => !holds(flat, event)) + .map(([clause]) => `${event.tag}: ${clause}`) + const accepted = failedClauses.length === 0 + if (accepted) row.effect(flat, event) + return { accepted, flat, failedClauses } +} + +function kernelEventTags(): string[] { + const source = readFileSync(KERNEL_PATH, "utf8") + return [...source.matchAll(/case "(\w+)":/g)].map((match) => match[1]!) +} + +const rejectionWitnesses = new Set(["late-completion-accept-rejected-after-new-ask"]) +const semanticWitnesses = { + "abort-during-pending-approval-in-live-stream": ({ prev, next, event }: WitnessContext) => + event.tag === "abortRequested" && + prev.run.stream.tag === "live" && + prev.run.ask.tag === "approval" && + !prev.run.latches.abort && + next.run.latches.abort, + "ask-after-stream-ended": ({ prev, next, event }: WitnessContext) => + event.tag === "askStarted" && + prev.run.stream.tag === "none" && + prev.run.generation > 0 && + next.run.ask.tag !== "none", + "abandon-after-stream-ended": ({ prev, next, event }: WitnessContext) => + (event.tag === "abandonRequested" || (event.tag === "abortRequested" && event.abandoned)) && + prev.run.stream.tag === "none" && + prev.run.generation > 0 && + !prev.run.latches.abandoned && + next.run.latches.abandoned, + "dispose-after-completion": ({ prev, next, event }: WitnessContext) => + event.tag === "disposeRequested" && prev.run.phase === "completed" && next.run.latches.disposed, + "stream-after-failure-cleanup": ({ prev, next, event }: WitnessContext) => + event.tag === "streamStarted" && + prev.lastEndedStreamCleaned && + !prev.run.latches.abort && + next.run.stream.tag === "live" && + !next.run.stream.cleanupFinished && + next.run.generation >= 2, + "first-stream-without-ask": ({ prev, next, event }: WitnessContext) => + event.tag === "streamStarted" && + prev.run.phase === "running" && + prev.run.generation === 0 && + prev.run.ask.tag === "none" && + next.run.stream.tag === "live", + "approval-before-first-stream": ({ prev, next, event }: WitnessContext) => + event.tag === "askStarted" && + prev.run.phase === "running" && + prev.run.generation === 0 && + next.run.ask.tag === "approval", + "late-completion-accept-rejected-after-new-ask": ({ prev, event }: WitnessContext) => + event.tag === "completionAccepted" && + prev.completionAsks.includes(event.askTs) && + prev.askCount > event.askTs && + prev.run.lastCompletionAskTs !== event.askTs && + prev.run.phase === "running" && + !prev.run.latches.abort, +} satisfies Record boolean> + +function initialState(): ModelState { + return { run: structuredClone(initialRunState), askCount: 0, completionAsks: [], lastEndedStreamCleaned: false } +} + +function stable(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stable) + if (typeof value !== "object" || value === null) return value + return Object.fromEntries( + Object.entries(value) + .filter(([, child]) => child !== undefined) + .sort(([a], [b]) => (a < b ? -1 : 1)) + .map(([key, child]) => [key, stable(child)]), + ) +} + +function canonical(value: unknown): string { + return JSON.stringify(stable(value)) +} + +function eventsFor({ run, askCount }: ModelState): ModelEvent[] { + const events: ModelEvent[] = [{ name: "initialized()", source: "init", event: { tag: "initialized" } }] + if (run.generation < MAX_GENERATION) { + events.push({ name: "streamStarted()", source: "stream", event: { tag: "streamStarted" } }) + } + events.push( + { name: "streamCleanupFinished()", source: "stream", event: { tag: "streamCleanupFinished" } }, + { name: "streamEnded()", source: "stream", event: { tag: "streamEnded" } }, + ) + if (askCount < MAX_ASK_TS) { + for (const [kind, build] of Object.entries(askBuilders)) { + const ask = build(askCount + 1) + events.push({ name: `askStarted(${kind}, ${ask.askTs})`, source: "ask", event: { tag: "askStarted", ask } }) + } + } + for (let askTs = 1; askTs <= MAX_ASK_TS; askTs++) { + const supersededByNewerAsk = run.ask.tag !== "none" && run.ask.askTs > askTs + events.push( + { + name: `${supersededByNewerAsk ? "askSettledStale" : "askSettled"}(${askTs})`, + source: "ask", + event: { tag: "askSettled", askTs }, + }, + { name: `completionAccepted(${askTs})`, source: "ask", event: { tag: "completionAccepted", askTs } }, + ) + } + for (const reason of Object.keys(abortReasons) as AbortReason[]) { + events.push({ name: `reasonSet(${reason})`, source: "cancel", event: { tag: "reasonSet", reason } }) + } + for (const abandoned of [false, true]) { + events.push({ + name: `abortRequested(${abandoned})`, + source: "cancel", + event: { tag: "abortRequested", abandoned }, + }) + } + events.push( + { name: "abandonRequested()", source: "cancel", event: { tag: "abandonRequested" } }, + { name: "disposeRequested()", source: "dispose", event: { tag: "disposeRequested" } }, + ) + return events +} + +const phaseOrder = { uninitialized: 0, running: 1, completed: 2 } as const + +function stateViolations({ run, askCount }: ModelState): string[] { + const violations: string[] = [] + if (run.latches.abandoned && !run.latches.abort) violations.push("abandoned without abort") + if (run.latches.disposed && !run.latches.abort) violations.push("disposed without abort") + if (run.phase === "completed" && run.ask.tag !== "none") violations.push("completed task has a pending ask") + if (run.ask.tag !== "none" && run.ask.askTs !== askCount) violations.push("pending ask is not the newest ask") + if (run.stream.tag === "live" && run.stream.generation !== run.generation) { + violations.push("live stream generation differs from run generation") + } + return violations +} + +function transitionViolations(before: RunState, after: RunState): string[] { + const violations: string[] = [] + for (const latch of ["abort", "abandoned", "disposed"] as const) { + if (before.latches[latch] && !after.latches[latch]) violations.push(`latch ${latch} returned to false`) + } + if (before.latches.reason !== undefined && after.latches.reason !== before.latches.reason) { + violations.push("abort reason changed after the first write") + } + if (after.generation < before.generation) violations.push("generation decreased") + if (phaseOrder[after.phase] < phaseOrder[before.phase]) violations.push("phase moved backwards") + const askChanged = canonical(after.ask) !== canonical(before.ask) + if (after.latches.abort && after.ask.tag !== "none" && askChanged) violations.push("an ask started after abort") + if (before.phase === "completed" && before.stream.tag === "none" && after.stream.tag === "live") { + violations.push("a stream started after completion") + } + if ( + after.phase === "completed" && + before.phase !== "completed" && + before.stream.tag === "none" && + after.stream.tag === "live" + ) { + violations.push("a task completed and started a stream at once") + } + if ( + before.stream.tag === "live" && + after.stream.tag === "live" && + after.stream.generation !== before.stream.generation + ) { + violations.push("a live stream was replaced") + } + if (before.stream.tag === "none" && after.stream.tag === "live" && after.stream.cleanupFinished) { + violations.push("a new stream started with cleanupFinished true") + } + return violations +} + +function apply(options: Options, state: ModelState, { event }: ModelEvent): Applied { + const input = structuredClone(state.run) + const eventInput = structuredClone(event) + let result: Transition | undefined + try { + result = options.step(input, eventInput) + } catch (error) { + return { next: state, accepted: false, problems: [`${event.tag} threw: ${String(error)}`] } + } + if (result === undefined || !("ok" in result || "rejected" in result)) { + return { next: state, accepted: false, problems: [`${event.tag} returned neither ok nor rejected`] } + } + + const problems: string[] = [] + if (canonical(input) !== canonical(state.run)) problems.push(`${event.tag} mutated its state`) + if (canonical(eventInput) !== canonical(event)) problems.push(`${event.tag} mutated its event`) + const accepted = "ok" in result + const actual = "ok" in result ? result.ok : result.rejected + if (options.useTable) { + const wanted = referenceResult(state.run, event) + if (accepted !== wanted.accepted) + problems.push(`${event.tag} ${accepted ? "accepted" : "rejected"} against the table`) + if (canonical(flatten(actual)) !== canonical(wanted.flat)) + problems.push(`${event.tag} result differs from the table`) + } + if (!accepted && canonical(actual) !== canonical(state.run)) problems.push(`${event.tag} rejection changed state`) + if (!accepted) return { next: state, accepted, problems } + + problems.push(...transitionViolations(state.run, actual)) + const next: ModelState = { ...state, run: actual } + if (event.tag === "askStarted") { + next.askCount = event.ask.askTs + if (event.ask.tag === "completion") next.completionAsks = [...state.completionAsks, event.ask.askTs] + } + if (event.tag === "streamEnded" && state.run.stream.tag === "live") { + next.lastEndedStreamCleaned = state.run.stream.cleanupFinished + } + return { next, accepted, problems } +} + +function formatCounterexample(message: string, trace: TraceStep[], options: Options): string { + const steps = trace.map( + (step, index) => + `${index}. ${step.action}\n${JSON.stringify(step.state, null, 2) + .split("\n") + .map((line) => ` ${line}`) + .join("\n")}`, + ) + return [ + `Run-state kernel invariant failed: ${message}`, + `Bounds: depth=${options.maxDepth}, states=${options.maxStates}, generations=${MAX_GENERATION}, asks=${MAX_ASK_TS}`, + ...steps, + ].join("\n") +} + +interface CheckResult { + states: number + deepest: number +} + +function runModelCheck(options: Options): CheckResult { + const start = initialState() + const startTrace: TraceStep[] = [{ action: "initial", source: "initial", state: start }] + const startProblems = stateViolations(start) + if (startProblems.length) throw new Error(formatCounterexample(startProblems.join("; "), startTrace, options)) + + const queue = [{ state: start, trace: startTrace }] + const visited = new Set([canonical(start)]) + const reachedActions = new Set() + const reachedRejections = new Set() + const reachedWitnesses = new Set() + const reachedInterleavings = new Set() + let deepest = 0 + + for (let index = 0; index < queue.length; index++) { + const node = queue[index]! + const depth = node.trace.length - 1 + deepest = Math.max(deepest, depth) + const previousStep = node.trace[depth]! + const previousChangedState = + depth > 0 && canonical(previousStep.state.run) !== canonical(node.trace[depth - 1]!.state.run) + + for (const modelEvent of eventsFor(node.state)) { + const applied = apply(options, node.state, modelEvent) + const trace = [...node.trace, { action: modelEvent.name, source: modelEvent.source, state: applied.next }] + const problems = [...applied.problems, ...(applied.accepted ? stateViolations(applied.next) : [])] + if (problems.length) throw new Error(formatCounterexample(problems.join("; "), trace, options)) + + for (const [name, matches] of Object.entries(semanticWitnesses)) { + if (applied.accepted === rejectionWitnesses.has(name)) continue + if (matches({ prev: node.state, next: applied.next, event: modelEvent.event })) + reachedWitnesses.add(name) + } + if (!applied.accepted) { + const { failedClauses } = referenceResult(node.state.run, modelEvent.event) + if (failedClauses.length === 1) reachedRejections.add(failedClauses[0]!) + continue + } + reachedActions.add(modelEvent.name.slice(0, modelEvent.name.indexOf("("))) + const changesState = canonical(applied.next.run) !== canonical(node.state.run) + if (previousChangedState && changesState && previousStep.source !== modelEvent.source) { + reachedInterleavings.add(`${previousStep.source}->${modelEvent.source}`) + } + + const key = canonical(applied.next) + if (visited.has(key)) continue + if (depth === options.maxDepth) { + throw new Error( + `Run-state kernel exploration reached depth ${options.maxDepth} with an unseen successor (${modelEvent.name}); increase the depth bound`, + ) + } + visited.add(key) + queue.push({ state: applied.next, trace }) + if (visited.size > options.maxStates) { + throw new Error( + `Run-state kernel exploration exceeded its ${options.maxStates}-state budget; increase or reduce bounds`, + ) + } + } + } + + const assertNoneMissing = (label: string, names: string[]) => { + if (names.length) throw new Error(`Run-state kernel model has ${label}: ${names.join(", ")}`) + } + assertNoneMissing( + "unreachable actions", + expectedActions.filter((action) => !reachedActions.has(action)), + ) + assertNoneMissing( + "guard clauses that are never the only reason for a rejection", + guardClauses.filter((clause) => !reachedRejections.has(clause)), + ) + assertNoneMissing( + "unreachable semantic witnesses", + Object.keys(semanticWitnesses).filter((name) => !reachedWitnesses.has(name)), + ) + assertNoneMissing( + "missing event-source interleavings between state changes", + eventSources + .flatMap((from) => eventSources.filter((to) => to !== from).map((to) => `${from}->${to}`)) + .filter((pair) => !reachedInterleavings.has(pair)), + ) + return { states: visited.size, deepest } +} + +function assertKernelHandlesTableEvents(): void { + const handled = new Set(kernelEventTags()) + const missingFromKernel = eventTags.filter((tag) => !handled.has(tag)) + const missingFromTable = [...handled].filter((tag) => !(tag in rows)) + assert.deepEqual( + { missingFromKernel, missingFromTable }, + { missingFromKernel: [], missingFromTable: [] }, + `the event tags in ${KERNEL_PATH} and in the checker table differ`, + ) +} + +const freshLive = (generation: number): RunState["stream"] => ({ tag: "live", generation, cleanupFinished: false }) + +interface UnsafeKernel { + step: Step + caughtBy: RegExp + useTable: boolean +} + +const differsFor = (tag: EventTag) => + new RegExp(`${tag} (result differs from the table|(accepted|rejected) against the table)`) +const unsafeKernels: Record = { + "second-stream-replaces-live-stream": { + step: (state, event) => + event.tag === "streamStarted" && state.phase === "running" && state.stream.tag === "live" + ? { ok: { ...state, generation: state.generation + 1, stream: freshLive(state.generation + 1) } } + : nextRunState(state, event), + caughtBy: /live stream was replaced/, + useTable: false, + }, + "stream-end-clears-ask": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "streamEnded" && "ok" in result + ? { ok: { ...result.ok, ask: { tag: "none" } } } + : result + }, + caughtBy: differsFor("streamEnded"), + useTable: true, + }, + "ask-starts-after-abort": { + step: (state, event) => + event.tag === "askStarted" && state.phase === "running" && state.latches.abort + ? { ok: { ...state, ask: event.ask } } + : nextRunState(state, event), + caughtBy: /ask started after abort/, + useTable: false, + }, + "stale-settle-clears-newer-ask": { + step: (state, event) => + event.tag === "askSettled" ? { ok: { ...state, ask: { tag: "none" } } } : nextRunState(state, event), + caughtBy: differsFor("askSettled"), + useTable: true, + }, + "abandon-without-abort": { + step: (state, event) => + event.tag === "abandonRequested" + ? { ok: { ...state, latches: { ...state.latches, abandoned: true } } } + : nextRunState(state, event), + caughtBy: /abandoned without abort/, + useTable: false, + }, + "later-reason-overwrites-first": { + step: (state, event) => + event.tag === "reasonSet" + ? { ok: { ...state, latches: { ...state.latches, reason: event.reason } } } + : nextRunState(state, event), + caughtBy: /abort reason changed/, + useTable: false, + }, + "dispose-without-abort": { + step: (state, event) => + event.tag === "disposeRequested" + ? { ok: { ...state, latches: { ...state.latches, disposed: true } } } + : nextRunState(state, event), + caughtBy: /disposed without abort/, + useTable: false, + }, + "stream-starts-after-completion": { + step: (state, event) => + event.tag === "streamStarted" && state.phase === "completed" + ? { ok: { ...state, stream: freshLive(state.generation) } } + : nextRunState(state, event), + caughtBy: /stream started after completion/, + useTable: false, + }, + "completion-starts-a-stream": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "completionAccepted" && "ok" in result && result.ok.stream.tag === "none" + ? { ok: { ...result.ok, stream: freshLive(result.ok.generation) } } + : result + }, + caughtBy: /completed and started a stream/, + useTable: false, + }, + "abort-drops-abandon": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "abortRequested" && "ok" in result + ? { ok: { ...result.ok, latches: { ...result.ok.latches, abandoned: state.latches.abandoned } } } + : result + }, + caughtBy: differsFor("abortRequested"), + useTable: true, + }, + "cleanup-flag-survives-new-stream": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "streamStarted" && "ok" in result && result.ok.stream.tag === "live" + ? { ok: { ...result.ok, stream: { ...result.ok.stream, cleanupFinished: true } } } + : result + }, + caughtBy: /cleanupFinished true/, + useTable: false, + }, + "approval-drops-opaque-ids": { + step: (state, event) => + event.tag === "askStarted" && event.ask.tag === "approval" + ? nextRunState(state, { ...event, ask: { tag: "approval", askTs: event.ask.askTs } }) + : nextRunState(state, event), + caughtBy: differsFor("askStarted"), + useTable: true, + }, + "ask-merges-previous-ask": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "askStarted" && "ok" in result + ? { ok: { ...result.ok, ask: { ...state.ask, ...event.ask } } } + : result + }, + caughtBy: differsFor("askStarted"), + useTable: true, + }, + "late-completion-accepted": { + step: (state, event) => + event.tag === "completionAccepted" && state.phase === "running" && !state.latches.abort + ? { ok: { ...state, phase: "completed", ask: { tag: "none" } } } + : nextRunState(state, event), + caughtBy: differsFor("completionAccepted"), + useTable: true, + }, + "reason-set-clears-abort": { + step: (state, event) => + event.tag === "reasonSet" && state.latches.abort + ? { ok: { ...state, latches: { ...state.latches, abort: false } } } + : nextRunState(state, event), + caughtBy: /latch abort returned to false/, + useTable: false, + }, + "reason-set-clears-abandoned": { + step: (state, event) => + event.tag === "reasonSet" && state.latches.abandoned + ? { ok: { ...state, latches: { ...state.latches, abandoned: false } } } + : nextRunState(state, event), + caughtBy: /latch abandoned returned to false/, + useTable: false, + }, + "reason-set-clears-disposed": { + step: (state, event) => + event.tag === "reasonSet" && state.latches.disposed + ? { ok: { ...state, latches: { ...state.latches, disposed: false } } } + : nextRunState(state, event), + caughtBy: /latch disposed returned to false/, + useTable: false, + }, + "repeated-reason-rejected": { + step: (state, event) => + event.tag === "reasonSet" && state.latches.reason !== undefined + ? { rejected: state } + : nextRunState(state, event), + caughtBy: /reasonSet rejected against the table/, + useTable: true, + }, + "stream-start-guard-dropped": { + step: (state, event) => + event.tag === "streamStarted" && state.phase !== "running" && state.stream.tag === "none" + ? { ok: state } + : nextRunState(state, event), + caughtBy: /never the only reason for a rejection: streamStarted: running$/, + useTable: false, + }, + "abandon-guard-dropped": { + step: (state, event) => + event.tag === "abandonRequested" && !state.latches.abort ? { ok: state } : nextRunState(state, event), + caughtBy: /never the only reason for a rejection: abandonRequested: aborted$/, + useTable: false, + }, + "dispose-always-rejected": { + step: (state, event) => (event.tag === "disposeRequested" ? { rejected: state } : nextRunState(state, event)), + caughtBy: /unreachable actions: disposeRequested$/, + useTable: false, + }, + "completion-keeps-ask": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "completionAccepted" && "ok" in result + ? { ok: { ...result.ok, ask: state.ask } } + : result + }, + caughtBy: /completed task has a pending ask/, + useTable: false, + }, + "stream-start-lowers-generation": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "streamStarted" && "ok" in result + ? { ok: { ...result.ok, generation: state.generation - 1 } } + : result + }, + caughtBy: /generation decreased/, + useTable: false, + }, + "ask-timestamp-skips-ahead": { + step: (state, event) => + event.tag === "askStarted" && state.phase === "running" && !state.latches.abort + ? { ok: { ...state, ask: { ...event.ask, askTs: event.ask.askTs + 1 } } } + : nextRunState(state, event), + caughtBy: /pending ask is not the newest ask/, + useTable: false, + }, + "abort-reopens-completed-task": { + step: (state, event) => + event.tag === "abortRequested" && state.phase === "completed" + ? { ok: { ...state, phase: "running", latches: { ...state.latches, abort: true } } } + : nextRunState(state, event), + caughtBy: /phase moved backwards/, + useTable: false, + }, + "stream-start-skews-live-generation": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "streamStarted" && "ok" in result + ? { ok: { ...result.ok, stream: freshLive(result.ok.generation + 1) } } + : result + }, + caughtBy: /live stream generation differs/, + useTable: false, + }, + "rejection-bumps-generation": { + step: (state, event) => { + const result = nextRunState(state, event) + return "rejected" in result + ? { rejected: { ...result.rejected, generation: state.generation + 1 } } + : result + }, + caughtBy: /rejection changed state/, + useTable: false, + }, + "mutates-its-state": { + step: (state, event) => { + if (event.tag === "abortRequested") state.latches.abort = true + return nextRunState(state, event) + }, + caughtBy: /abortRequested mutated its state/, + useTable: false, + }, + "mutates-its-event": { + step: (state, event) => { + if (event.tag === "askSettled" && state.ask.tag !== "none") event.askTs = state.ask.askTs + return nextRunState(state, event) + }, + caughtBy: /askSettled mutated its event/, + useTable: false, + }, + "returns-nothing": { + step: (state, event) => (event.tag === "streamEnded" ? undefined : nextRunState(state, event)), + caughtBy: /returned neither/, + useTable: false, + }, + "throws-on-stale-settle": { + step: (state, event) => { + if (event.tag === "askSettled" && state.ask.tag !== "none" && state.ask.askTs !== event.askTs) { + throw new Error("stale settle") + } + return nextRunState(state, event) + }, + caughtBy: /threw/, + useTable: false, + }, +} + +function runUnsafeKernels(): void { + for (const [name, { step, caughtBy, useTable }] of Object.entries(unsafeKernels)) { + const options = { step, useTable, maxDepth: MAX_DEPTH, maxStates: MAX_STATES } + assert.throws(() => runModelCheck(options), caughtBy, `unsafe kernel not caught as expected: ${name}`) + } +} + +function runBoundSelfTests(): void { + const base = { step: nextRunState, useTable: true, maxDepth: MAX_DEPTH, maxStates: MAX_STATES } + assert.throws(() => runModelCheck({ ...base, maxDepth: 3 }), /unseen successor/) + assert.throws(() => runModelCheck({ ...base, maxStates: 50 }), /state budget/) +} + +function runRepresentativeScenarios(): void { + const running = nextRunState(structuredClone(initialRunState), { tag: "initialized" }) + assert.ok("ok" in running) + const asked = nextRunState(running.ok, { tag: "askStarted", ask: { tag: "approval", askTs: 1 } }) + assert.ok("ok" in asked) + const aborted = nextRunState(asked.ok, { tag: "abortRequested", abandoned: false }) + assert.ok("ok" in aborted) + assert.ok("rejected" in nextRunState(aborted.ok, { tag: "askStarted", ask: { tag: "other", askTs: 2 } })) + assert.deepEqual(nextRunState(asked.ok, { tag: "askSettled", askTs: 2 }), { ok: asked.ok }) +} + +assertKernelHandlesTableEvents() +runRepresentativeScenarios() +const { states, deepest } = runModelCheck({ + step: nextRunState, + useTable: true, + maxDepth: MAX_DEPTH, + maxStates: MAX_STATES, +}) +runUnsafeKernels() +runBoundSelfTests() +console.log( + `Run-state kernel model check passed: ${states} reachable states, ${expectedActions.length}/${expectedActions.length} actions reachable, ${guardClauses.length}/${guardClauses.length} guard clauses isolated, ${Object.keys(semanticWitnesses).length}/${Object.keys(semanticWitnesses).length} witnesses reached, ${eventSources.length * (eventSources.length - 1)} source interleavings, ${Object.keys(unsafeKernels).length} unsafe kernels caught, fixed point at depth ${deepest} (bound ${MAX_DEPTH}), generations <= ${MAX_GENERATION}, asks <= ${MAX_ASK_TS}`, +) From 8f999b6eb24c5571aec77481579775cd18826494 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 15:35:29 -0400 Subject: [PATCH 2/9] test(run-state): cover the turn events in the kernel model checker --- docs/architecture/task-lifecycle-model.md | 5 +- scripts/check-run-state-kernel.ts | 115 +++++++++++++++++++++- 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index 377d41b8c8..dcf52199d8 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -285,7 +285,7 @@ Add E2E coverage only when a lifecycle change crosses one of those runtime bound `scripts/check-run-state-kernel.ts` explores the pure `nextRunState` in `src/core/task/run-state/runState.ts`. It is an independent submodel. It does not use `taskLifecycle.ts` and does not share state with the persisted graph. -The checker applies every event in every reachable state, within the generation and ask-timestamp caps. The budget is 20,000 states and the depth bound is 20. The stream generation and the ask timestamp are each at most 3. At each step it checks: +The checker applies every event except the seven turn events in every reachable state, within the generation and ask-timestamp caps. The budget is 20,000 states and the depth bound is 20. The stream generation and the ask timestamp are each at most 3. At each step it checks: - the result of the kernel against a transcription of the guard and effect table ([Transitions](task-lifecycle-target-architecture.md#transitions)), including that a rejection leaves the state unchanged; - the seven design-doc invariants (same document, Invariants); @@ -293,13 +293,14 @@ The checker applies every event in every reachable state, within the generation The checker also requires: +- the seven turn events (`turnStarted`, `firstChunkAwaitStarted`, `firstChunkAwaitEnded`, `toolRejected`, `toolUsed`, `toolFailed`, `streamReadCompleted`) to be accepted and to match the table in each of the 32 turn states, and in one state of every distinct shape of the reachable states. Every other event is also applied to the same shape states with all five turn fields true, so no other event may change the turn. The search does not include turn events, because they would multiply the state count by 32; - the event tags in `runState.ts` to equal the event tags in the table (once, at startup, as a text scan of `case` labels); - every event to be reachable, with the `askSettledStale` no-op, a settle for an ask that a newer pending ask replaced, as its own action; - every guard clause to be the only failing clause in some rejected event; - named witnesses, each tied to an event and the states around it: abort during a pending approval in a live stream, an ask after the stream ends, abandon after the stream ends, dispose after completion, a new stream after a failure cleanup without abort, a stream that starts without user input, an approval before the first stream, and a rejected late completion accept after a newer ask; - every ordered pair of the stream, ask, cancel, and dispose event sources to follow each other, where both events change the state; - no unseen successor at the depth bound; -- thirty-one known-unsafe kernels, each of which the checker must reject. They include a throwing kernel, a kernel that returns nothing, and one kernel for each check that runs at every search step without the table; and +- thirty-three known-unsafe kernels, each of which the checker must reject. They include a throwing kernel, a kernel that returns nothing, and one kernel for each check that runs at every search step without the table; and - self-tests that a small depth bound and a small state budget each fail the checker. The kernel has no resume event. A resumed task is `initialized` followed by `streamStarted` or `askStarted`. The witnesses `first-stream-without-ask` and `approval-before-first-stream` cover those paths. diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts index d4dd9b4ddd..30f609830b 100644 --- a/scripts/check-run-state-kernel.ts +++ b/scripts/check-run-state-kernel.ts @@ -5,7 +5,7 @@ import { fileURLToPath } from "node:url" import { nextRunState, initialRunState } from "../src/core/task/run-state/runState" import type { AbortReason, Ask, RunEvent, RunState, Transition } from "../src/core/task/run-state/runState" -type Source = "init" | "stream" | "ask" | "cancel" | "dispose" +type Source = "init" | "stream" | "ask" | "cancel" | "dispose" | "turn" type EventTag = RunEvent["tag"] type StartedAsk = Exclude type Step = (state: RunState, event: RunEvent) => Transition | undefined @@ -62,6 +62,20 @@ const askBuilders: Record StartedAsk> = { other: (askTs) => ({ tag: "other", askTs }), } const eventSources = ["stream", "ask", "cancel", "dispose"] as const +const turnTags = [ + "turnStarted", + "firstChunkAwaitStarted", + "firstChunkAwaitEnded", + "toolRejected", + "toolUsed", + "toolFailed", + "streamReadCompleted", +] as const +const turnEvent = (tag: (typeof turnTags)[number]): ModelEvent => ({ + name: `${tag}()`, + source: "turn", + event: { tag }, +}) // One field per cell of the state described in the target-architecture doc, so rows read like the doc table. interface Flat { @@ -70,6 +84,11 @@ interface Flat { streamLive: boolean streamGeneration: number | null cleanupFinished: boolean | null + waitingForFirstChunk: boolean + toolRejected: boolean + toolUsed: boolean + toolFailed: boolean + streamReadComplete: boolean askTag: Ask["tag"] askTs: number | null toolCallId: string | null @@ -89,6 +108,7 @@ function flatten(run: RunState): Flat { streamLive: run.stream.tag === "live", streamGeneration: run.stream.tag === "live" ? run.stream.generation : null, cleanupFinished: run.stream.tag === "live" ? run.stream.cleanupFinished : null, + ...run.turn, askTag: ask.tag, askTs: ask.tag === "none" ? null : ask.askTs, toolCallId: ("toolCallId" in ask && ask.toolCallId) || null, @@ -119,6 +139,17 @@ const rows: Rows = { guard: { uninitialized: (f) => f.phase === "uninitialized" }, effect: (f) => void (f.phase = "running"), }, + turnStarted: { + effect: (f) => { + f.toolRejected = f.toolUsed = f.toolFailed = f.streamReadComplete = false + }, + }, + firstChunkAwaitStarted: { effect: (f) => void (f.waitingForFirstChunk = true) }, + firstChunkAwaitEnded: { effect: (f) => void (f.waitingForFirstChunk = false) }, + toolRejected: { effect: (f) => void (f.toolRejected = true) }, + toolUsed: { effect: (f) => void (f.toolUsed = true) }, + toolFailed: { effect: (f) => void (f.toolFailed = true) }, + streamReadCompleted: { effect: (f) => void (f.streamReadComplete = true) }, streamStarted: { guard: { running: (f) => f.phase === "running", "no live stream": (f) => !f.streamLive }, effect: (f) => { @@ -424,6 +455,62 @@ function formatCounterexample(message: string, trace: TraceStep[], options: Opti ].join("\n") } +const dirtyTurn = { + waitingForFirstChunk: true, + toolRejected: true, + toolUsed: true, + toolFailed: true, + streamReadComplete: true, +} + +const shapeOf = ({ run }: ModelState) => + canonical({ + phase: run.phase, + generation: run.generation > 0, + stream: run.stream.tag === "live" ? run.stream.cleanupFinished : "none", + ask: run.ask.tag, + latches: run.latches, + lastCompletionAskTs: run.lastCompletionAskTs !== undefined, + }) + +function checkTurnRegion(options: Options, searched: ModelState[], reachedActions: Set): void { + const shapes = new Map(searched.map((state) => [shapeOf(state), state])) + const cases = [...shapes.values()].flatMap((state) => [ + { state, events: [...eventsFor(state), ...turnTags.map(turnEvent)] }, + { state: { ...state, run: { ...state.run, turn: dirtyTurn } }, events: eventsFor(state) }, + ]) + const everyTurn = Array.from({ length: 32 }, (_, bits) => ({ + waitingForFirstChunk: Boolean(bits & 1), + toolRejected: Boolean(bits & 2), + toolUsed: Boolean(bits & 4), + toolFailed: Boolean(bits & 8), + streamReadComplete: Boolean(bits & 16), + })) + for (const turn of everyTurn) { + const state = initialState() + cases.push({ state: { ...state, run: { ...state.run, turn } }, events: turnTags.map(turnEvent) }) + } + + for (const { state, events } of cases) { + for (const modelEvent of events) { + const applied = apply(options, state, modelEvent) + const turnEventRejected = modelEvent.source === "turn" && !applied.accepted + const problems = [ + ...applied.problems, + ...(turnEventRejected ? [`${modelEvent.event.tag} was rejected`] : []), + ] + if (problems.length) { + const trace = [ + { action: "initial", source: "initial" as const, state }, + { action: modelEvent.name, source: modelEvent.source, state: applied.next }, + ] + throw new Error(formatCounterexample(problems.join("; "), trace, options)) + } + if (modelEvent.source === "turn") reachedActions.add(modelEvent.event.tag) + } + } +} + interface CheckResult { states: number deepest: number @@ -490,6 +577,12 @@ function runModelCheck(options: Options): CheckResult { } } + checkTurnRegion( + options, + queue.map((node) => node.state), + reachedActions, + ) + const assertNoneMissing = (label: string, names: string[]) => { if (names.length) throw new Error(`Run-state kernel model has ${label}: ${names.join(", ")}`) } @@ -707,6 +800,26 @@ const unsafeKernels: Record = { caughtBy: /unreachable actions: disposeRequested$/, useTable: false, }, + "turn-start-clears-first-chunk-wait": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "turnStarted" && "ok" in result + ? { ok: { ...result.ok, turn: { ...result.ok.turn, waitingForFirstChunk: false } } } + : result + }, + caughtBy: differsFor("turnStarted"), + useTable: true, + }, + "stream-start-resets-turn": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "streamStarted" && "ok" in result + ? { ok: { ...result.ok, turn: { ...result.ok.turn, toolUsed: false } } } + : result + }, + caughtBy: differsFor("streamStarted"), + useTable: true, + }, "completion-keeps-ask": { step: (state, event) => { const result = nextRunState(state, event) From 5b22e9171c9cb924e074b1a9121e9b6bcbd116d3 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sat, 31 Oct 2026 16:27:57 -0400 Subject: [PATCH 3/9] test(run-state): require a recorded stream failure in the failure-cleanup witness --- scripts/check-run-state-kernel.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts index 30f609830b..ba4daf9433 100644 --- a/scripts/check-run-state-kernel.ts +++ b/scripts/check-run-state-kernel.ts @@ -14,7 +14,7 @@ interface ModelState { run: RunState askCount: number completionAsks: number[] - lastEndedStreamCleaned: boolean + lastEndedStreamFailedAndCleaned: boolean } interface ModelEvent { @@ -263,7 +263,7 @@ const semanticWitnesses = { event.tag === "disposeRequested" && prev.run.phase === "completed" && next.run.latches.disposed, "stream-after-failure-cleanup": ({ prev, next, event }: WitnessContext) => event.tag === "streamStarted" && - prev.lastEndedStreamCleaned && + prev.lastEndedStreamFailedAndCleaned && !prev.run.latches.abort && next.run.stream.tag === "live" && !next.run.stream.cleanupFinished && @@ -289,7 +289,12 @@ const semanticWitnesses = { } satisfies Record boolean> function initialState(): ModelState { - return { run: structuredClone(initialRunState), askCount: 0, completionAsks: [], lastEndedStreamCleaned: false } + return { + run: structuredClone(initialRunState), + askCount: 0, + completionAsks: [], + lastEndedStreamFailedAndCleaned: false, + } } function stable(value: unknown): unknown { @@ -435,7 +440,8 @@ function apply(options: Options, state: ModelState, { event }: ModelEvent): Appl if (event.ask.tag === "completion") next.completionAsks = [...state.completionAsks, event.ask.askTs] } if (event.tag === "streamEnded" && state.run.stream.tag === "live") { - next.lastEndedStreamCleaned = state.run.stream.cleanupFinished + next.lastEndedStreamFailedAndCleaned = + state.run.stream.cleanupFinished && state.run.latches.reason === "streaming_failed" } return { next, accepted, problems } } From a342217fe6559ccb9115f596b868a99970039025 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 2 Nov 2026 20:48:35 -0500 Subject: [PATCH 4/9] test(run-state): track the stream failure per stream in the checker --- scripts/check-run-state-kernel.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts index ba4daf9433..bce13476dd 100644 --- a/scripts/check-run-state-kernel.ts +++ b/scripts/check-run-state-kernel.ts @@ -15,6 +15,7 @@ interface ModelState { askCount: number completionAsks: number[] lastEndedStreamFailedAndCleaned: boolean + currentStreamFailed: boolean } interface ModelEvent { @@ -294,6 +295,7 @@ function initialState(): ModelState { askCount: 0, completionAsks: [], lastEndedStreamFailedAndCleaned: false, + currentStreamFailed: false, } } @@ -439,9 +441,12 @@ function apply(options: Options, state: ModelState, { event }: ModelEvent): Appl next.askCount = event.ask.askTs if (event.ask.tag === "completion") next.completionAsks = [...state.completionAsks, event.ask.askTs] } + if (event.tag === "reasonSet" && event.reason === "streaming_failed" && state.run.stream.tag === "live") { + next.currentStreamFailed = state.currentStreamFailed || state.run.latches.reason === undefined + } if (event.tag === "streamEnded" && state.run.stream.tag === "live") { - next.lastEndedStreamFailedAndCleaned = - state.run.stream.cleanupFinished && state.run.latches.reason === "streaming_failed" + next.lastEndedStreamFailedAndCleaned = state.run.stream.cleanupFinished && state.currentStreamFailed + next.currentStreamFailed = false } return { next, accepted, problems } } From b9135127005b0a250a12adb250b3073579ffe4ce Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Wed, 4 Nov 2026 20:24:19 -0500 Subject: [PATCH 5/9] test(run-state): add an unsafe kernel for a rejected turn event --- docs/architecture/task-lifecycle-model.md | 2 +- scripts/check-run-state-kernel.ts | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index dcf52199d8..f12d84c473 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -300,7 +300,7 @@ The checker also requires: - named witnesses, each tied to an event and the states around it: abort during a pending approval in a live stream, an ask after the stream ends, abandon after the stream ends, dispose after completion, a new stream after a failure cleanup without abort, a stream that starts without user input, an approval before the first stream, and a rejected late completion accept after a newer ask; - every ordered pair of the stream, ask, cancel, and dispose event sources to follow each other, where both events change the state; - no unseen successor at the depth bound; -- thirty-three known-unsafe kernels, each of which the checker must reject. They include a throwing kernel, a kernel that returns nothing, and one kernel for each check that runs at every search step without the table; and +- thirty-four known-unsafe kernels, each of which the checker must reject. They include a throwing kernel, a kernel that returns nothing, and one kernel for each check that runs at every search step without the table; and - self-tests that a small depth bound and a small state budget each fail the checker. The kernel has no resume event. A resumed task is `initialized` followed by `streamStarted` or `askStarted`. The witnesses `first-stream-without-ask` and `approval-before-first-stream` cover those paths. diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts index bce13476dd..23cc156ef0 100644 --- a/scripts/check-run-state-kernel.ts +++ b/scripts/check-run-state-kernel.ts @@ -821,6 +821,11 @@ const unsafeKernels: Record = { caughtBy: differsFor("turnStarted"), useTable: true, }, + "turn-event-rejected": { + step: (state, event) => (event.tag === "toolUsed" ? { rejected: state } : nextRunState(state, event)), + caughtBy: /toolUsed was rejected/, + useTable: false, + }, "stream-start-resets-turn": { step: (state, event) => { const result = nextRunState(state, event) From b200020fd4002286b73f760d0589f635d7e5ed19 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Sun, 8 Nov 2026 12:01:07 -0500 Subject: [PATCH 6/9] test(run-state): check turn effects of asks and later streams --- docs/architecture/task-lifecycle-model.md | 4 +- scripts/check-run-state-kernel.ts | 104 ++++++++++++++++------ 2 files changed, 78 insertions(+), 30 deletions(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index f12d84c473..3057acca15 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -293,14 +293,14 @@ The checker applies every event except the seven turn events in every reachable The checker also requires: -- the seven turn events (`turnStarted`, `firstChunkAwaitStarted`, `firstChunkAwaitEnded`, `toolRejected`, `toolUsed`, `toolFailed`, `streamReadCompleted`) to be accepted and to match the table in each of the 32 turn states, and in one state of every distinct shape of the reachable states. Every other event is also applied to the same shape states with all five turn fields true, so no other event may change the turn. The search does not include turn events, because they would multiply the state count by 32; +- the seven turn events (`turnStarted`, `firstChunkAwaitStarted`, `firstChunkAwaitEnded`, `toolRejected`, `toolUsed`, `toolFailed`, `streamReadCompleted`) to be accepted and to match the table in each of the 32 turn states. A shape is a group of reachable states with the same phase, stream, ask, latches and completion marker. The states in one shape also agree on whether a stream started, and whether the generation and the ask count are below their caps. The checker takes one state of each shape and gives it each of the 32 turn states. It applies every event to each of these states. Thus every other event must change the turn only as the table allows. The known-unsafe kernel runs use only two turn states: all false and all true. The search does not include the turn events, because they would make the state count 32 times larger; - the event tags in `runState.ts` to equal the event tags in the table (once, at startup, as a text scan of `case` labels); - every event to be reachable, with the `askSettledStale` no-op, a settle for an ask that a newer pending ask replaced, as its own action; - every guard clause to be the only failing clause in some rejected event; - named witnesses, each tied to an event and the states around it: abort during a pending approval in a live stream, an ask after the stream ends, abandon after the stream ends, dispose after completion, a new stream after a failure cleanup without abort, a stream that starts without user input, an approval before the first stream, and a rejected late completion accept after a newer ask; - every ordered pair of the stream, ask, cancel, and dispose event sources to follow each other, where both events change the state; - no unseen successor at the depth bound; -- thirty-four known-unsafe kernels, each of which the checker must reject. They include a throwing kernel, a kernel that returns nothing, and one kernel for each check that runs at every search step without the table; and +- thirty-nine known-unsafe kernels. The checker must reject each of them. They include a throwing kernel, a kernel that returns nothing and a kernel that returns both `ok` and `rejected`. There is also one kernel for each check that runs at every search step without the table; and - self-tests that a small depth bound and a small state budget each fail the checker. The kernel has no resume event. A resumed task is `initialized` followed by `streamStarted` or `askStarted`. The witnesses `first-stream-without-ask` and `approval-before-first-stream` cover those paths. diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts index 23cc156ef0..0e04ccc7b3 100644 --- a/scripts/check-run-state-kernel.ts +++ b/scripts/check-run-state-kernel.ts @@ -35,6 +35,7 @@ interface Options { useTable: boolean maxDepth: number maxStates: number + allTurnStates: boolean } interface Applied { @@ -316,7 +317,8 @@ function canonical(value: unknown): string { function eventsFor({ run, askCount }: ModelState): ModelEvent[] { const events: ModelEvent[] = [{ name: "initialized()", source: "init", event: { tag: "initialized" } }] - if (run.generation < MAX_GENERATION) { + const streamStartRejects = run.phase !== "running" || run.stream.tag !== "none" + if (run.generation < MAX_GENERATION || streamStartRejects) { events.push({ name: "streamStarted()", source: "stream", event: { tag: "streamStarted" } }) } events.push( @@ -419,6 +421,9 @@ function apply(options: Options, state: ModelState, { event }: ModelEvent): Appl if (result === undefined || !("ok" in result || "rejected" in result)) { return { next: state, accepted: false, problems: [`${event.tag} returned neither ok nor rejected`] } } + if ("ok" in result && "rejected" in result) { + return { next: state, accepted: false, problems: [`${event.tag} returned both ok and rejected`] } + } const problems: string[] = [] if (canonical(input) !== canonical(state.run)) problems.push(`${event.tag} mutated its state`) @@ -442,7 +447,8 @@ function apply(options: Options, state: ModelState, { event }: ModelEvent): Appl if (event.ask.tag === "completion") next.completionAsks = [...state.completionAsks, event.ask.askTs] } if (event.tag === "reasonSet" && event.reason === "streaming_failed" && state.run.stream.tag === "live") { - next.currentStreamFailed = state.currentStreamFailed || state.run.latches.reason === undefined + const firstReason = state.run.latches.reason === undefined + next.currentStreamFailed = state.currentStreamFailed || firstReason } if (event.tag === "streamEnded" && state.run.stream.tag === "live") { next.lastEndedStreamFailedAndCleaned = state.run.stream.cleanupFinished && state.currentStreamFailed @@ -466,41 +472,37 @@ function formatCounterexample(message: string, trace: TraceStep[], options: Opti ].join("\n") } -const dirtyTurn = { - waitingForFirstChunk: true, - toolRejected: true, - toolUsed: true, - toolFailed: true, - streamReadComplete: true, -} - -const shapeOf = ({ run }: ModelState) => +const shapeOf = ({ run, askCount }: ModelState) => canonical({ phase: run.phase, generation: run.generation > 0, + belowGenerationCap: run.generation < MAX_GENERATION, + belowAskCap: askCount < MAX_ASK_TS, stream: run.stream.tag === "live" ? run.stream.cleanupFinished : "none", ask: run.ask.tag, latches: run.latches, lastCompletionAskTs: run.lastCompletionAskTs !== undefined, }) +const everyTurn = Array.from({ length: 32 }, (_, bits) => ({ + waitingForFirstChunk: Boolean(bits & 1), + toolRejected: Boolean(bits & 2), + toolUsed: Boolean(bits & 4), + toolFailed: Boolean(bits & 8), + streamReadComplete: Boolean(bits & 16), +})) +const allFalseTurn = everyTurn[0]! +const allTrueTurn = everyTurn[31]! + function checkTurnRegion(options: Options, searched: ModelState[], reachedActions: Set): void { const shapes = new Map(searched.map((state) => [shapeOf(state), state])) - const cases = [...shapes.values()].flatMap((state) => [ - { state, events: [...eventsFor(state), ...turnTags.map(turnEvent)] }, - { state: { ...state, run: { ...state.run, turn: dirtyTurn } }, events: eventsFor(state) }, - ]) - const everyTurn = Array.from({ length: 32 }, (_, bits) => ({ - waitingForFirstChunk: Boolean(bits & 1), - toolRejected: Boolean(bits & 2), - toolUsed: Boolean(bits & 4), - toolFailed: Boolean(bits & 8), - streamReadComplete: Boolean(bits & 16), - })) - for (const turn of everyTurn) { - const state = initialState() - cases.push({ state: { ...state, run: { ...state.run, turn } }, events: turnTags.map(turnEvent) }) - } + const turns = options.allTurnStates ? everyTurn : [allFalseTurn, allTrueTurn] + const cases = [...shapes.values()].flatMap((state) => + turns.map((turn) => ({ + state: { ...state, run: { ...state.run, turn } }, + events: [...eventsFor(state), ...turnTags.map(turnEvent)], + })), + ) for (const { state, events } of cases) { for (const modelEvent of events) { @@ -508,6 +510,7 @@ function checkTurnRegion(options: Options, searched: ModelState[], reachedAction const turnEventRejected = modelEvent.source === "turn" && !applied.accepted const problems = [ ...applied.problems, + ...(applied.accepted ? stateViolations(applied.next) : []), ...(turnEventRejected ? [`${modelEvent.event.tag} was rejected`] : []), ] if (problems.length) { @@ -836,6 +839,42 @@ const unsafeKernels: Record = { caughtBy: differsFor("streamStarted"), useTable: true, }, + "ask-start-resets-turn": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "askStarted" && "ok" in result + ? { ok: { ...result.ok, turn: { ...result.ok.turn, toolUsed: false } } } + : result + }, + caughtBy: differsFor("askStarted"), + useTable: true, + }, + "later-stream-start-resets-turn": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "streamStarted" && "ok" in result && state.generation > 0 + ? { ok: { ...result.ok, turn: { ...result.ok.turn, toolUsed: false } } } + : result + }, + caughtBy: differsFor("streamStarted"), + useTable: true, + }, + "capped-stream-start-resets-turn": { + step: (state, event) => + event.tag === "streamStarted" && state.stream.tag === "live" && state.generation === MAX_GENERATION + ? { rejected: { ...state, turn: allFalseTurn } } + : nextRunState(state, event), + caughtBy: /streamStarted rejection changed state/, + useTable: false, + }, + "stream-start-rejected-after-failure": { + step: (state, event) => + event.tag === "streamStarted" && state.latches.reason === "streaming_failed" + ? { rejected: state } + : nextRunState(state, event), + caughtBy: /unreachable semantic witnesses: stream-after-failure-cleanup/, + useTable: false, + }, "completion-keeps-ask": { step: (state, event) => { const result = nextRunState(state, event) @@ -908,6 +947,14 @@ const unsafeKernels: Record = { caughtBy: /askSettled mutated its event/, useTable: false, }, + "returns-both": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "streamEnded" && "ok" in result ? { ok: result.ok, rejected: state } : result + }, + caughtBy: /returned both ok and rejected/, + useTable: false, + }, "returns-nothing": { step: (state, event) => (event.tag === "streamEnded" ? undefined : nextRunState(state, event)), caughtBy: /returned neither/, @@ -927,13 +974,13 @@ const unsafeKernels: Record = { function runUnsafeKernels(): void { for (const [name, { step, caughtBy, useTable }] of Object.entries(unsafeKernels)) { - const options = { step, useTable, maxDepth: MAX_DEPTH, maxStates: MAX_STATES } + const options = { step, useTable, maxDepth: MAX_DEPTH, maxStates: MAX_STATES, allTurnStates: false } assert.throws(() => runModelCheck(options), caughtBy, `unsafe kernel not caught as expected: ${name}`) } } function runBoundSelfTests(): void { - const base = { step: nextRunState, useTable: true, maxDepth: MAX_DEPTH, maxStates: MAX_STATES } + const base = { step: nextRunState, useTable: true, maxDepth: MAX_DEPTH, maxStates: MAX_STATES, allTurnStates: true } assert.throws(() => runModelCheck({ ...base, maxDepth: 3 }), /unseen successor/) assert.throws(() => runModelCheck({ ...base, maxStates: 50 }), /state budget/) } @@ -956,6 +1003,7 @@ const { states, deepest } = runModelCheck({ useTable: true, maxDepth: MAX_DEPTH, maxStates: MAX_STATES, + allTurnStates: true, }) runUnsafeKernels() runBoundSelfTests() From 970a375660710d7a0e87fbd614a36a4763b0bfd3 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Mon, 9 Nov 2026 23:36:47 +0000 Subject: [PATCH 7/9] test(run-state): cover mixed turns and approval timestamps --- docs/architecture/task-lifecycle-model.md | 4 +-- scripts/check-run-state-kernel.ts | 38 +++++++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index 3057acca15..e916c43fdc 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -293,14 +293,14 @@ The checker applies every event except the seven turn events in every reachable The checker also requires: -- the seven turn events (`turnStarted`, `firstChunkAwaitStarted`, `firstChunkAwaitEnded`, `toolRejected`, `toolUsed`, `toolFailed`, `streamReadCompleted`) to be accepted and to match the table in each of the 32 turn states. A shape is a group of reachable states with the same phase, stream, ask, latches and completion marker. The states in one shape also agree on whether a stream started, and whether the generation and the ask count are below their caps. The checker takes one state of each shape and gives it each of the 32 turn states. It applies every event to each of these states. Thus every other event must change the turn only as the table allows. The known-unsafe kernel runs use only two turn states: all false and all true. The search does not include the turn events, because they would make the state count 32 times larger; +- the seven turn events (`turnStarted`, `firstChunkAwaitStarted`, `firstChunkAwaitEnded`, `toolRejected`, `toolUsed`, `toolFailed`, `streamReadCompleted`) to be accepted and to match the table in each of the 32 turn states. A shape is a group of reachable states with the same phase, stream, ask tag and timestamp, latches and completion marker. The states in one shape also agree on whether a stream started, and whether the generation and the ask count are below their caps. The checker takes one state of each shape and gives it each of the 32 turn states. It applies every event to each of these states. Thus every other event must change the turn only as the table allows. The known-unsafe kernel runs use only two turn states, all false and all true, except for a mixed-turn-only mutant that requires all 32. Separate mutants check that starting an ask preserves the turn after pending approvals with timestamps 1 and 2, before any stream or cancellation. The search does not include the turn events, because they would make the state count 32 times larger; - the event tags in `runState.ts` to equal the event tags in the table (once, at startup, as a text scan of `case` labels); - every event to be reachable, with the `askSettledStale` no-op, a settle for an ask that a newer pending ask replaced, as its own action; - every guard clause to be the only failing clause in some rejected event; - named witnesses, each tied to an event and the states around it: abort during a pending approval in a live stream, an ask after the stream ends, abandon after the stream ends, dispose after completion, a new stream after a failure cleanup without abort, a stream that starts without user input, an approval before the first stream, and a rejected late completion accept after a newer ask; - every ordered pair of the stream, ask, cancel, and dispose event sources to follow each other, where both events change the state; - no unseen successor at the depth bound; -- thirty-nine known-unsafe kernels. The checker must reject each of them. They include a throwing kernel, a kernel that returns nothing and a kernel that returns both `ok` and `rejected`. There is also one kernel for each check that runs at every search step without the table; and +- forty-two known-unsafe kernels. The checker must reject each of them. They include a throwing kernel, a kernel that returns nothing and a kernel that returns both `ok` and `rejected`. There is also one kernel for each check that runs at every search step without the table; and - self-tests that a small depth bound and a small state budget each fail the checker. The kernel has no resume event. A resumed task is `initialized` followed by `streamStarted` or `askStarted`. The witnesses `first-stream-without-ask` and `approval-before-first-stream` cover those paths. diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts index 0e04ccc7b3..4df826b9b6 100644 --- a/scripts/check-run-state-kernel.ts +++ b/scripts/check-run-state-kernel.ts @@ -480,6 +480,7 @@ const shapeOf = ({ run, askCount }: ModelState) => belowAskCap: askCount < MAX_ASK_TS, stream: run.stream.tag === "live" ? run.stream.cleanupFinished : "none", ask: run.ask.tag, + askTs: run.ask.tag === "none" ? undefined : run.ask.askTs, latches: run.latches, lastCompletionAskTs: run.lastCompletionAskTs !== undefined, }) @@ -638,6 +639,7 @@ interface UnsafeKernel { step: Step caughtBy: RegExp useTable: boolean + allTurnStates?: boolean } const differsFor = (tag: EventTag) => @@ -849,6 +851,38 @@ const unsafeKernels: Record = { caughtBy: differsFor("askStarted"), useTable: true, }, + "ask-start-resets-mixed-turn": { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "askStarted" && "ok" in result && state.turn.toolUsed && !state.turn.toolFailed + ? { ok: { ...result.ok, turn: { ...result.ok.turn, toolUsed: false } } } + : result + }, + caughtBy: differsFor("askStarted"), + useTable: true, + allTurnStates: true, + }, + ...Object.fromEntries( + [1, 2].map((askTs): [string, UnsafeKernel] => [ + `ask-start-resets-turn-after-approval-${askTs}`, + { + step: (state, event) => { + const result = nextRunState(state, event) + return event.tag === "askStarted" && + "ok" in result && + state.ask.tag === "approval" && + state.ask.askTs === askTs && + state.generation === 0 && + !state.latches.abort && + state.latches.reason === undefined + ? { ok: { ...result.ok, turn: { ...result.ok.turn, toolUsed: false } } } + : result + }, + caughtBy: differsFor("askStarted"), + useTable: true, + }, + ]), + ), "later-stream-start-resets-turn": { step: (state, event) => { const result = nextRunState(state, event) @@ -973,8 +1007,8 @@ const unsafeKernels: Record = { } function runUnsafeKernels(): void { - for (const [name, { step, caughtBy, useTable }] of Object.entries(unsafeKernels)) { - const options = { step, useTable, maxDepth: MAX_DEPTH, maxStates: MAX_STATES, allTurnStates: false } + for (const [name, { step, caughtBy, useTable, allTurnStates = false }] of Object.entries(unsafeKernels)) { + const options = { step, useTable, maxDepth: MAX_DEPTH, maxStates: MAX_STATES, allTurnStates } assert.throws(() => runModelCheck(options), caughtBy, `unsafe kernel not caught as expected: ${name}`) } } From 0c808b53ac040ac6696a9413e26b720f68f0862a Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Thu, 19 Nov 2026 13:45:20 -0500 Subject: [PATCH 8/9] test(run-state): add TLA+ spec of the run-state kernel --- docs/architecture/task-lifecycle-model.md | 4 +- .../tla/RunStateKernel.AbandonNoGuard.cfg | 9 ++ .../tla/RunStateKernel.AcceptKeepsAsk.cfg | 9 ++ .../tla/RunStateKernel.AcceptNoTs.cfg | 9 ++ .../tla/RunStateKernel.DisposeNoAbort.cfg | 9 ++ .../tla/RunStateKernel.EndResetsGen.cfg | 9 ++ .../tla/RunStateKernel.NoAskAbortGuard.cfg | 9 ++ .../tla/RunStateKernel.NoStreamPhaseGuard.cfg | 9 ++ .../tla/RunStateKernel.ReasonOverwrite.cfg | 9 ++ .../RunStateKernel.ResumeResetsLatches.cfg | 9 ++ .../architecture/tla/RunStateKernel.Reuse.cfg | 9 ++ .../tla/RunStateKernel.SettleAny.cfg | 9 ++ docs/architecture/tla/RunStateKernel.cfg | 9 ++ docs/architecture/tla/RunStateKernel.tla | 140 ++++++++++++++++++ scripts/check-run-state-kernel.ts | 6 + 15 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 docs/architecture/tla/RunStateKernel.AbandonNoGuard.cfg create mode 100644 docs/architecture/tla/RunStateKernel.AcceptKeepsAsk.cfg create mode 100644 docs/architecture/tla/RunStateKernel.AcceptNoTs.cfg create mode 100644 docs/architecture/tla/RunStateKernel.DisposeNoAbort.cfg create mode 100644 docs/architecture/tla/RunStateKernel.EndResetsGen.cfg create mode 100644 docs/architecture/tla/RunStateKernel.NoAskAbortGuard.cfg create mode 100644 docs/architecture/tla/RunStateKernel.NoStreamPhaseGuard.cfg create mode 100644 docs/architecture/tla/RunStateKernel.ReasonOverwrite.cfg create mode 100644 docs/architecture/tla/RunStateKernel.ResumeResetsLatches.cfg create mode 100644 docs/architecture/tla/RunStateKernel.Reuse.cfg create mode 100644 docs/architecture/tla/RunStateKernel.SettleAny.cfg create mode 100644 docs/architecture/tla/RunStateKernel.cfg create mode 100644 docs/architecture/tla/RunStateKernel.tla diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index e916c43fdc..45ecf89dfa 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -206,7 +206,7 @@ These are safety claims within the documented bounds. The checks do not claim li | Cleanup | Abstract bounded universal plus adapter tests | Abort, disposal, settlement, rejection, and provider shutdown are modeled as protocol/environment actions. | No direct execution of all production cleanup methods, filesystem/editor promises, timing liveness, fairness, or arbitrary task counts. | | Parser request scope | Production-backed bounded schedule replay | The checker executes production parser APIs across 924 order-preserving schedules for two scopes. | Assumes callers stop invoking a finalized scope; transport behavior, arbitrary request counts, indices, and malformed histories are outside the claim. | | Completion persistence | Abstract bounded universal plus production tests and one fresh-host E2E path | The model abstracts persistence as a durable phase with at most two write starts; production guards and retry paths are tested separately. | Production permits more retries; no power-loss/filesystem proof, fairness, arbitrary retry count, complete delegated fallback, provider status metadata, or downstream event-consumer model. | -| Run-state kernel | Production-backed bounded universal | The explorer calls the production `nextRunState` and compares each result with a transcription of the design-doc guard and effect table. | Models one task's run state only. Excludes persistence, the actor mailbox, and the wiring of `Task.ts` to the kernel. Witness and source-pair checks have no self-test. | +| Run-state kernel | Production-backed bounded universal | The explorer calls the production `nextRunState` and compares each result with a transcription of the design-doc guard and effect table. | Models one task's run state only. Excludes persistence, the actor mailbox, and the wiring of `Task.ts` to the kernel. `tla/RunStateKernel.tla` checks the design table itself, not the code. | The production mapping above names primary lifecycle transitions, not every mutation or consumer. Generic store upserts, reconciliation, repair replay, migrations, tool entry points, webview/public API abandonment, provider status updates, and public `TaskCompleted` re-emission remain outside the persisted reducer graph unless explicitly named by a submodel or focused test. For task-local mode, the consumers named under #921/#1623 are a confirmed set, not an exhaustive repository-wide inventory; other mode-sensitive tools must be audited before claiming universal reader isolation. @@ -273,7 +273,7 @@ Completion-readiness changes belong in `scripts/check-completion-persistence.ts` Parser request scoping is one such independent bounded submodel within the umbrella suite. Extend `scripts/check-native-tool-call-parser-scoping.ts` and its focused architecture document instead of adding parser state or transitions to `taskLifecycle.ts` or the persisted lifecycle state graph. -Run-state kernel changes belong in `scripts/check-run-state-kernel.ts`. Update the guard and effect table in the checker together with the Transitions table in the target-architecture document, and add a known-unsafe kernel for each new check. +Run-state kernel changes belong in `scripts/check-run-state-kernel.ts`. Update the guard and effect table in the checker together with the Transitions table in the target-architecture document, and add a known-unsafe kernel for each new check. Add the same check to `docs/architecture/tla/RunStateKernel.tla` and run `check.sh RunStateKernel`. ## Test layering diff --git a/docs/architecture/tla/RunStateKernel.AbandonNoGuard.cfg b/docs/architecture/tla/RunStateKernel.AbandonNoGuard.cfg new file mode 100644 index 0000000000..638f782d54 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.AbandonNoGuard.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "AbandonNoGuard" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.AcceptKeepsAsk.cfg b/docs/architecture/tla/RunStateKernel.AcceptKeepsAsk.cfg new file mode 100644 index 0000000000..587d3033fa --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.AcceptKeepsAsk.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "AcceptKeepsAsk" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.AcceptNoTs.cfg b/docs/architecture/tla/RunStateKernel.AcceptNoTs.cfg new file mode 100644 index 0000000000..38b27a1db4 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.AcceptNoTs.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "AcceptNoTs" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.DisposeNoAbort.cfg b/docs/architecture/tla/RunStateKernel.DisposeNoAbort.cfg new file mode 100644 index 0000000000..84f017bd66 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.DisposeNoAbort.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "DisposeNoAbort" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.EndResetsGen.cfg b/docs/architecture/tla/RunStateKernel.EndResetsGen.cfg new file mode 100644 index 0000000000..98edb24686 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.EndResetsGen.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "EndResetsGen" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.NoAskAbortGuard.cfg b/docs/architecture/tla/RunStateKernel.NoAskAbortGuard.cfg new file mode 100644 index 0000000000..a2cb0e1590 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.NoAskAbortGuard.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "NoAskAbortGuard" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.NoStreamPhaseGuard.cfg b/docs/architecture/tla/RunStateKernel.NoStreamPhaseGuard.cfg new file mode 100644 index 0000000000..4f6e943d74 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.NoStreamPhaseGuard.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "NoStreamPhaseGuard" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.ReasonOverwrite.cfg b/docs/architecture/tla/RunStateKernel.ReasonOverwrite.cfg new file mode 100644 index 0000000000..fac9795b59 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.ReasonOverwrite.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "ReasonOverwrite" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.ResumeResetsLatches.cfg b/docs/architecture/tla/RunStateKernel.ResumeResetsLatches.cfg new file mode 100644 index 0000000000..d0c1e80837 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.ResumeResetsLatches.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "ResumeResetsLatches" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.Reuse.cfg b/docs/architecture/tla/RunStateKernel.Reuse.cfg new file mode 100644 index 0000000000..ce755a2d50 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.Reuse.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "Kernel" + Ts = "Reuse" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.SettleAny.cfg b/docs/architecture/tla/RunStateKernel.SettleAny.cfg new file mode 100644 index 0000000000..c084670df5 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.SettleAny.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "SettleAny" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.cfg b/docs/architecture/tla/RunStateKernel.cfg new file mode 100644 index 0000000000..bd7d046988 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.cfg @@ -0,0 +1,9 @@ +SPECIFICATION Spec +CONSTANTS + Mode = "Kernel" + Ts = "Unique" + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.tla b/docs/architecture/tla/RunStateKernel.tla new file mode 100644 index 0000000000..c83c864496 --- /dev/null +++ b/docs/architecture/tla/RunStateKernel.tla @@ -0,0 +1,140 @@ +---- MODULE RunStateKernel ---- +\* Models nextRunState in src/core/task/run-state/runState.ts, as the Transitions table in task-lifecycle-target-architecture.md gives it. +\* Source: the events that Task and ClineProvider send. Sink: the run state that the getters read. +\* Failure 1: a task is abandoned or disposed without abort. +\* Failure 2: a latch or the first abort reason changes back. +\* Failure 3: an ask starts after abort. +\* Failure 4: a completed task starts a stream or keeps a pending ask. +\* Failure 5: the generation decreases. +\* Failure 6: a stale settle hides the newest ask while that ask still waits. +\* Failure 7: a late accept completes the task after a newer ask or stream. +\* Invariants: AbandonImpliesAbort, DisposeImpliesAbort, CompletedQuiet, PendingIsNewest, NewestShown, AcceptCurrent. +\* Properties: LatchesMonotone, NoAskAfterAbort, CompletedNoStream, GenerationMonotone. +\* A rejected event is a stutter, so the spec does not model it. The turn region has no guard and no reader here. +\* Expected before the run: Kernel holds. Each sabotage config fails the invariant of its failure. Reuse fails NewestShown. +\* Result: RunStateKernel.cfg holds (5160 states). Each of the 11 other configs fails, 10 of them on the expected invariant. +\* NoStreamPhaseGuard fails AcceptCurrent first. CompletedNoStream alone also fails on it, so it is not vacuous. +\* Null result: the Kernel config found no new bug in the table. +\* Review ratchet, PR 2244 (findings before this spec): +\* covered: CodeRabbit 4498864158 (a stale settle must come from a replaced ask). Settle(k) needs k open, so a stale settle is a replaced ask. +\* added: taltas 4499201597 (a reused askTs). Ts = "Reuse" shows a stale settle that hides the newest ask (issue 2365). +\* gap: taltas 4499201605 (a result with both ok and rejected). The result shape is TypeScript, and the spec has no result. +\* gap: taltas 4499201579 (cross-region effects). The spec writes each action, so it cannot see an effect that the code adds. +\* gap: CodeRabbit 4519561241 (no negative test for source pairs). TLC tries every order, so the spec needs no pair rule. +\* The TS checker got the test instead: the settle-ignored-after-dispose unsafe kernel. +\* Counts: covered 1, added 1, gap 3. Findings about the spec files: 0. +\* Code check (step 10): scripts/check-run-state-kernel.ts runs the same invariants on the real nextRunState. +\* Cost: TLC takes under 10 seconds per config. Agent time is not measured. + +EXTENDS Naturals + +CONSTANTS Mode, Ts, MaxAsks, MaxGen + +Reasons == {"user_cancelled", "streaming_failed"} +AskIds == 1..MaxAsks +NoAsk == [id |-> 0, tag |-> "none", ts |-> 0] + +\* ask.id, tsOf, open, n, fresh and done are ghost data. The kernel reads only ask.tag and ask.ts. +\* n: the asks that started. open: the asks that have not exited. fresh: the newest completion ask with no stream after it. +VARIABLES phase, gen, stream, ask, lastComp, abort, abandoned, disposed, reason, n, tsOf, open, fresh, done +vars == <> +ghosts == <> +latches == <> + +Init == /\ phase = "uninitialized" /\ gen = 0 /\ stream = "none" /\ ask = NoAsk /\ lastComp = 0 + /\ abort = FALSE /\ abandoned = FALSE /\ disposed = FALSE /\ reason = "" + /\ n = 0 /\ tsOf = [k \in AskIds |-> 0] /\ open = {} /\ fresh = 0 /\ done = 0 + +\* Sabotage ResumeResetsLatches: the first start clears the latches, as resumeAfterDelegation did before. +Initialized == /\ phase = "uninitialized" + /\ phase' = "running" + /\ IF Mode = "ResumeResetsLatches" + THEN abort' = FALSE /\ abandoned' = FALSE /\ disposed' = FALSE /\ UNCHANGED reason + ELSE UNCHANGED latches + /\ UNCHANGED <> + +StreamStarted == /\ phase = "running" \/ Mode = "NoStreamPhaseGuard" + /\ stream = "none" /\ gen < MaxGen + /\ gen' = gen + 1 /\ stream' = "live" /\ lastComp' = 0 /\ fresh' = 0 + /\ UNCHANGED <> + +\* "clean" is a live stream with cleanupFinished true. +StreamCleanupFinished == /\ stream # "none" /\ stream' = "clean" + /\ UNCHANGED <> + +StreamEnded == /\ stream # "none" /\ stream' = "none" + /\ gen' = IF Mode = "EndResetsGen" THEN 0 ELSE gen + /\ UNCHANGED <> + +\* Approval and other asks have the same effect, so the spec merges them. +\* Ts = "Unique": each ask gets a new askTs. Ts = "Reuse": an ask can carry the askTs of an earlier ask. +AskStarted(tag, t) == + /\ n < MaxAsks /\ phase = "running" /\ (~abort \/ Mode = "NoAskAbortGuard") + /\ IF Ts = "Unique" THEN t = n + 1 ELSE TRUE + /\ n' = n + 1 /\ tsOf' = [tsOf EXCEPT ![n + 1] = t] /\ open' = open \cup {n + 1} + /\ ask' = [id |-> n + 1, tag |-> tag, ts |-> t] + /\ lastComp' = IF tag = "completion" THEN t ELSE 0 + /\ fresh' = IF tag = "completion" THEN n + 1 ELSE 0 + /\ UNCHANGED <> + +\* Each started ask exits once: the abort throw, superseded or the return. Each exit sends askSettled. +AskSettled(k) == /\ k \in open + /\ open' = open \ {k} + /\ ask' = IF ask.tag # "none" /\ (ask.ts = tsOf[k] \/ Mode = "SettleAny") THEN NoAsk ELSE ask + /\ UNCHANGED <> + +\* The accept carries the askTs of an ask that started. The guard decides if it is current. +CompletionAccepted(k) == + /\ k \in 1..n /\ phase = "running" /\ ~abort + /\ lastComp = tsOf[k] \/ Mode = "AcceptNoTs" + /\ phase' = "completed" /\ done' = k + /\ ask' = IF Mode = "AcceptKeepsAsk" THEN ask ELSE NoAsk + /\ UNCHANGED <> + +ReasonSet(r) == /\ reason = "" \/ Mode = "ReasonOverwrite" + /\ reason' = r + /\ UNCHANGED <> + +AbortRequested(b) == /\ abort' = TRUE /\ abandoned' = (abandoned \/ b) + /\ UNCHANGED <> + +AbandonRequested == /\ abort \/ Mode = "AbandonNoGuard" + /\ abandoned' = TRUE + /\ UNCHANGED <> + +DisposeRequested == /\ abort' = (abort \/ Mode # "DisposeNoAbort") /\ disposed' = TRUE + /\ UNCHANGED <> + +Next == \/ Initialized \/ StreamStarted \/ StreamCleanupFinished \/ StreamEnded + \/ \E tag \in {"completion", "other"}, t \in AskIds : AskStarted(tag, t) + \/ \E k \in AskIds : AskSettled(k) \/ CompletionAccepted(k) + \/ \E r \in Reasons : ReasonSet(r) + \/ \E b \in BOOLEAN : AbortRequested(b) + \/ AbandonRequested \/ DisposeRequested + +Spec == Init /\ [][Next]_vars + +TypeOK == /\ phase \in {"uninitialized", "running", "completed"} /\ gen \in 0..MaxGen + /\ stream \in {"none", "live", "clean"} /\ lastComp \in 0..MaxAsks + /\ reason \in Reasons \cup {""} /\ n \in 0..MaxAsks /\ open \subseteq 1..n + +\* Design-doc invariants 1, 2, 5 and 7. +AbandonImpliesAbort == abandoned => abort +DisposeImpliesAbort == disposed => abort +CompletedQuiet == phase = "completed" => ask.tag = "none" +PendingIsNewest == ask.tag # "none" => ask.id = n +\* The newest ask still waits, so the view must show it (race R-6). +NewestShown == (n \in open /\ phase # "completed") => ask.id = n +\* PR-6: only an accept of the newest completion ask, with no stream after it, completes the task. +AcceptCurrent == phase = "completed" => (done # 0 /\ done = fresh) + +\* Design-doc invariants 3, 4, 5 and 6. PR-3: the first reason wins. +LatchesMonotone == [][/\ abort => abort' + /\ abandoned => abandoned' + /\ disposed => disposed' + /\ reason # "" => reason' = reason]_vars +NoAskAfterAbort == [][n' > n => ~abort]_vars +CompletedNoStream == [][phase = "completed" => gen' = gen]_vars +GenerationMonotone == [][gen' >= gen]_vars + +==== diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts index 4df826b9b6..e18c7cce44 100644 --- a/scripts/check-run-state-kernel.ts +++ b/scripts/check-run-state-kernel.ts @@ -965,6 +965,12 @@ const unsafeKernels: Record = { caughtBy: /rejection changed state/, useTable: false, }, + "settle-ignored-after-dispose": { + step: (state, event) => + event.tag === "askSettled" && state.latches.disposed ? { ok: state } : nextRunState(state, event), + caughtBy: /missing event-source interleavings between state changes: dispose->ask$/, + useTable: false, + }, "mutates-its-state": { step: (state, event) => { if (event.tag === "abortRequested") state.latches.abort = true From f69e2ec630d3646b6862cfd45463ce8657650a03 Mon Sep 17 00:00:00 2001 From: Elliott de Launay Date: Thu, 19 Nov 2026 14:29:53 -0500 Subject: [PATCH 9/9] test(run-state): address the second CodeRabbit round --- docs/architecture/task-lifecycle-model.md | 4 +- .../tla/RunStateKernel.AcceptKeepsAsk.cfg | 9 --- .../tla/RunStateKernel.DisposeNoAbort.cfg | 9 --- .../tla/RunStateKernel.NoAskAbortGuard.cfg | 9 --- .../tla/RunStateKernel.NoStreamPhaseGuard.cfg | 9 --- .../tla/RunStateKernel.ReasonOverwrite.cfg | 9 --- .../RunStateKernel.ResumeResetsLatches.cfg | 9 --- docs/architecture/tla/RunStateKernel.tla | 71 +++++++++-------- ...fg => RunStateKernelMC.AbandonNoGuard.cfg} | 3 +- ...fg => RunStateKernelMC.AcceptKeepsAsk.cfg} | 3 +- ...ny.cfg => RunStateKernelMC.AcceptNoTs.cfg} | 3 +- ...fg => RunStateKernelMC.DisposeNoAbort.cfg} | 3 +- .../tla/RunStateKernelMC.EndResetsGen.cfg | 8 ++ .../tla/RunStateKernelMC.NoAskAbortGuard.cfg | 8 ++ ...> RunStateKernelMC.NoStreamPhaseGuard.cfg} | 3 +- .../tla/RunStateKernelMC.ReasonOverwrite.cfg | 8 ++ ... RunStateKernelMC.ResumeResetsLatches.cfg} | 3 +- .../tla/RunStateKernelMC.Reuse.cfg | 8 ++ .../tla/RunStateKernelMC.SettleAny.cfg | 8 ++ docs/architecture/tla/RunStateKernelMC.cfg | 7 ++ docs/architecture/tla/RunStateKernelMC.tla | 19 +++++ scripts/check-run-state-kernel.ts | 76 ++++++++++++++----- 22 files changed, 170 insertions(+), 119 deletions(-) delete mode 100644 docs/architecture/tla/RunStateKernel.AcceptKeepsAsk.cfg delete mode 100644 docs/architecture/tla/RunStateKernel.DisposeNoAbort.cfg delete mode 100644 docs/architecture/tla/RunStateKernel.NoAskAbortGuard.cfg delete mode 100644 docs/architecture/tla/RunStateKernel.NoStreamPhaseGuard.cfg delete mode 100644 docs/architecture/tla/RunStateKernel.ReasonOverwrite.cfg delete mode 100644 docs/architecture/tla/RunStateKernel.ResumeResetsLatches.cfg rename docs/architecture/tla/{RunStateKernel.Reuse.cfg => RunStateKernelMC.AbandonNoGuard.cfg} (88%) rename docs/architecture/tla/{RunStateKernel.cfg => RunStateKernelMC.AcceptKeepsAsk.cfg} (87%) rename docs/architecture/tla/{RunStateKernel.SettleAny.cfg => RunStateKernelMC.AcceptNoTs.cfg} (87%) rename docs/architecture/tla/{RunStateKernel.AcceptNoTs.cfg => RunStateKernelMC.DisposeNoAbort.cfg} (86%) create mode 100644 docs/architecture/tla/RunStateKernelMC.EndResetsGen.cfg create mode 100644 docs/architecture/tla/RunStateKernelMC.NoAskAbortGuard.cfg rename docs/architecture/tla/{RunStateKernel.EndResetsGen.cfg => RunStateKernelMC.NoStreamPhaseGuard.cfg} (86%) create mode 100644 docs/architecture/tla/RunStateKernelMC.ReasonOverwrite.cfg rename docs/architecture/tla/{RunStateKernel.AbandonNoGuard.cfg => RunStateKernelMC.ResumeResetsLatches.cfg} (84%) create mode 100644 docs/architecture/tla/RunStateKernelMC.Reuse.cfg create mode 100644 docs/architecture/tla/RunStateKernelMC.SettleAny.cfg create mode 100644 docs/architecture/tla/RunStateKernelMC.cfg create mode 100644 docs/architecture/tla/RunStateKernelMC.tla diff --git a/docs/architecture/task-lifecycle-model.md b/docs/architecture/task-lifecycle-model.md index 45ecf89dfa..18175e7c5d 100644 --- a/docs/architecture/task-lifecycle-model.md +++ b/docs/architecture/task-lifecycle-model.md @@ -273,7 +273,7 @@ Completion-readiness changes belong in `scripts/check-completion-persistence.ts` Parser request scoping is one such independent bounded submodel within the umbrella suite. Extend `scripts/check-native-tool-call-parser-scoping.ts` and its focused architecture document instead of adding parser state or transitions to `taskLifecycle.ts` or the persisted lifecycle state graph. -Run-state kernel changes belong in `scripts/check-run-state-kernel.ts`. Update the guard and effect table in the checker together with the Transitions table in the target-architecture document, and add a known-unsafe kernel for each new check. Add the same check to `docs/architecture/tla/RunStateKernel.tla` and run `check.sh RunStateKernel`. +Run-state kernel changes belong in `scripts/check-run-state-kernel.ts`. Update the guard and effect table in the checker together with the Transitions table in the target-architecture document, and add a known-unsafe kernel for each new check. Add the same check to `docs/architecture/tla/RunStateKernel.tla` and run `check.sh RunStateKernelMC`. ## Test layering @@ -300,7 +300,7 @@ The checker also requires: - named witnesses, each tied to an event and the states around it: abort during a pending approval in a live stream, an ask after the stream ends, abandon after the stream ends, dispose after completion, a new stream after a failure cleanup without abort, a stream that starts without user input, an approval before the first stream, and a rejected late completion accept after a newer ask; - every ordered pair of the stream, ask, cancel, and dispose event sources to follow each other, where both events change the state; - no unseen successor at the depth bound; -- forty-two known-unsafe kernels. The checker must reject each of them. They include a throwing kernel, a kernel that returns nothing and a kernel that returns both `ok` and `rejected`. There is also one kernel for each check that runs at every search step without the table; and +- known-unsafe kernels. The checker must reject each of them. They include a throwing kernel, a kernel that returns nothing and a kernel that returns both `ok` and `rejected`. There is also one kernel for each check that runs at every search step without the table; and - self-tests that a small depth bound and a small state budget each fail the checker. The kernel has no resume event. A resumed task is `initialized` followed by `streamStarted` or `askStarted`. The witnesses `first-stream-without-ask` and `approval-before-first-stream` cover those paths. diff --git a/docs/architecture/tla/RunStateKernel.AcceptKeepsAsk.cfg b/docs/architecture/tla/RunStateKernel.AcceptKeepsAsk.cfg deleted file mode 100644 index 587d3033fa..0000000000 --- a/docs/architecture/tla/RunStateKernel.AcceptKeepsAsk.cfg +++ /dev/null @@ -1,9 +0,0 @@ -SPECIFICATION Spec -CONSTANTS - Mode = "AcceptKeepsAsk" - Ts = "Unique" - MaxAsks = 3 - MaxGen = 2 -INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent -PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.DisposeNoAbort.cfg b/docs/architecture/tla/RunStateKernel.DisposeNoAbort.cfg deleted file mode 100644 index 84f017bd66..0000000000 --- a/docs/architecture/tla/RunStateKernel.DisposeNoAbort.cfg +++ /dev/null @@ -1,9 +0,0 @@ -SPECIFICATION Spec -CONSTANTS - Mode = "DisposeNoAbort" - Ts = "Unique" - MaxAsks = 3 - MaxGen = 2 -INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent -PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.NoAskAbortGuard.cfg b/docs/architecture/tla/RunStateKernel.NoAskAbortGuard.cfg deleted file mode 100644 index a2cb0e1590..0000000000 --- a/docs/architecture/tla/RunStateKernel.NoAskAbortGuard.cfg +++ /dev/null @@ -1,9 +0,0 @@ -SPECIFICATION Spec -CONSTANTS - Mode = "NoAskAbortGuard" - Ts = "Unique" - MaxAsks = 3 - MaxGen = 2 -INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent -PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.NoStreamPhaseGuard.cfg b/docs/architecture/tla/RunStateKernel.NoStreamPhaseGuard.cfg deleted file mode 100644 index 4f6e943d74..0000000000 --- a/docs/architecture/tla/RunStateKernel.NoStreamPhaseGuard.cfg +++ /dev/null @@ -1,9 +0,0 @@ -SPECIFICATION Spec -CONSTANTS - Mode = "NoStreamPhaseGuard" - Ts = "Unique" - MaxAsks = 3 - MaxGen = 2 -INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent -PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.ReasonOverwrite.cfg b/docs/architecture/tla/RunStateKernel.ReasonOverwrite.cfg deleted file mode 100644 index fac9795b59..0000000000 --- a/docs/architecture/tla/RunStateKernel.ReasonOverwrite.cfg +++ /dev/null @@ -1,9 +0,0 @@ -SPECIFICATION Spec -CONSTANTS - Mode = "ReasonOverwrite" - Ts = "Unique" - MaxAsks = 3 - MaxGen = 2 -INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent -PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.ResumeResetsLatches.cfg b/docs/architecture/tla/RunStateKernel.ResumeResetsLatches.cfg deleted file mode 100644 index d0c1e80837..0000000000 --- a/docs/architecture/tla/RunStateKernel.ResumeResetsLatches.cfg +++ /dev/null @@ -1,9 +0,0 @@ -SPECIFICATION Spec -CONSTANTS - Mode = "ResumeResetsLatches" - Ts = "Unique" - MaxAsks = 3 - MaxGen = 2 -INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent -PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone -CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.tla b/docs/architecture/tla/RunStateKernel.tla index c83c864496..3d35a086c3 100644 --- a/docs/architecture/tla/RunStateKernel.tla +++ b/docs/architecture/tla/RunStateKernel.tla @@ -12,23 +12,29 @@ \* Properties: LatchesMonotone, NoAskAfterAbort, CompletedNoStream, GenerationMonotone. \* A rejected event is a stutter, so the spec does not model it. The turn region has no guard and no reader here. \* Expected before the run: Kernel holds. Each sabotage config fails the invariant of its failure. Reuse fails NewestShown. -\* Result: RunStateKernel.cfg holds (5160 states). Each of the 11 other configs fails, 10 of them on the expected invariant. +\* Result: RunStateKernelMC.cfg holds (5160 states). Each of the 11 other configs fails, 10 of them on the expected invariant. \* NoStreamPhaseGuard fails AcceptCurrent first. CompletedNoStream alone also fails on it, so it is not vacuous. \* Null result: the Kernel config found no new bug in the table. -\* Review ratchet, PR 2244 (findings before this spec): +\* Run each config on RunStateKernelMC.tla. The MC module holds the sabotage overrides of the named guards and effects. +\* Review ratchet, PR 2244, round 1 (findings before this spec): \* covered: CodeRabbit 4498864158 (a stale settle must come from a replaced ask). Settle(k) needs k open, so a stale settle is a replaced ask. -\* added: taltas 4499201597 (a reused askTs). Ts = "Reuse" shows a stale settle that hides the newest ask (issue 2365). +\* added: taltas 4499201597 (a reused askTs). The Reuse config shows a stale settle that hides the newest ask (issue 2365). \* gap: taltas 4499201605 (a result with both ok and rejected). The result shape is TypeScript, and the spec has no result. \* gap: taltas 4499201579 (cross-region effects). The spec writes each action, so it cannot see an effect that the code adds. \* gap: CodeRabbit 4519561241 (no negative test for source pairs). TLC tries every order, so the spec needs no pair rule. \* The TS checker got the test instead: the settle-ignored-after-dispose unsafe kernel. -\* Counts: covered 1, added 1, gap 3. Findings about the spec files: 0. +\* Round 1 counts: covered 1, added 1, gap 3. Findings about the spec files: 0. +\* Round 2 (CodeRabbit on 0c808b53a): 7 findings. 1 is about the spec files: 4633780176 (move the sabotage to an MC module). Done. +\* gap: 4633780180, 4633780186, 4633780209 and 4633780232 are about the coverage and the traces of the TS explorer. +\* gap: 4633780197 (an own property with an undefined value). The spec has no JavaScript objects. +\* gap: 4633780165 (a stale count in task-lifecycle-model.md). It is prose. +\* Round 2 counts: covered 0, added 0, gap 6. Findings about the spec files: 1. \* Code check (step 10): scripts/check-run-state-kernel.ts runs the same invariants on the real nextRunState. \* Cost: TLC takes under 10 seconds per config. Agent time is not measured. EXTENDS Naturals -CONSTANTS Mode, Ts, MaxAsks, MaxGen +CONSTANTS MaxAsks, MaxGen Reasons == {"user_cancelled", "streaming_failed"} AskIds == 1..MaxAsks @@ -41,20 +47,29 @@ vars == <> latches == <> +\* The guards and effects of the Transitions table. RunStateKernelMC.tla overrides one of them in each sabotage config. +\* Input rule: Task gives each ask a new askTs. Issue 2365 tracks that Task does not make sure of this. +AskTsFresh(t) == t = n + 1 +InitializedLatches == UNCHANGED <> +StreamStartGuard == phase = "running" +StreamEndGen == gen +AskStartGuard == phase = "running" /\ ~abort +SettleMatches(k) == ask.ts = tsOf[k] +AcceptGuard(k) == phase = "running" /\ ~abort /\ lastComp = tsOf[k] +AcceptAsk == NoAsk +ReasonGuard == reason = "" +AbandonGuard == abort +DisposeAbort == TRUE + Init == /\ phase = "uninitialized" /\ gen = 0 /\ stream = "none" /\ ask = NoAsk /\ lastComp = 0 /\ abort = FALSE /\ abandoned = FALSE /\ disposed = FALSE /\ reason = "" /\ n = 0 /\ tsOf = [k \in AskIds |-> 0] /\ open = {} /\ fresh = 0 /\ done = 0 -\* Sabotage ResumeResetsLatches: the first start clears the latches, as resumeAfterDelegation did before. -Initialized == /\ phase = "uninitialized" - /\ phase' = "running" - /\ IF Mode = "ResumeResetsLatches" - THEN abort' = FALSE /\ abandoned' = FALSE /\ disposed' = FALSE /\ UNCHANGED reason - ELSE UNCHANGED latches - /\ UNCHANGED <> +Initialized == /\ phase = "uninitialized" /\ phase' = "running" + /\ InitializedLatches + /\ UNCHANGED <> -StreamStarted == /\ phase = "running" \/ Mode = "NoStreamPhaseGuard" - /\ stream = "none" /\ gen < MaxGen +StreamStarted == /\ StreamStartGuard /\ stream = "none" /\ gen < MaxGen /\ gen' = gen + 1 /\ stream' = "live" /\ lastComp' = 0 /\ fresh' = 0 /\ UNCHANGED <> @@ -62,15 +77,12 @@ StreamStarted == /\ phase = "running" \/ Mode = "NoStreamPhaseGuard" StreamCleanupFinished == /\ stream # "none" /\ stream' = "clean" /\ UNCHANGED <> -StreamEnded == /\ stream # "none" /\ stream' = "none" - /\ gen' = IF Mode = "EndResetsGen" THEN 0 ELSE gen +StreamEnded == /\ stream # "none" /\ stream' = "none" /\ gen' = StreamEndGen /\ UNCHANGED <> \* Approval and other asks have the same effect, so the spec merges them. -\* Ts = "Unique": each ask gets a new askTs. Ts = "Reuse": an ask can carry the askTs of an earlier ask. AskStarted(tag, t) == - /\ n < MaxAsks /\ phase = "running" /\ (~abort \/ Mode = "NoAskAbortGuard") - /\ IF Ts = "Unique" THEN t = n + 1 ELSE TRUE + /\ n < MaxAsks /\ AskStartGuard /\ AskTsFresh(t) /\ n' = n + 1 /\ tsOf' = [tsOf EXCEPT ![n + 1] = t] /\ open' = open \cup {n + 1} /\ ask' = [id |-> n + 1, tag |-> tag, ts |-> t] /\ lastComp' = IF tag = "completion" THEN t ELSE 0 @@ -80,29 +92,24 @@ AskStarted(tag, t) == \* Each started ask exits once: the abort throw, superseded or the return. Each exit sends askSettled. AskSettled(k) == /\ k \in open /\ open' = open \ {k} - /\ ask' = IF ask.tag # "none" /\ (ask.ts = tsOf[k] \/ Mode = "SettleAny") THEN NoAsk ELSE ask + /\ ask' = IF ask.tag # "none" /\ SettleMatches(k) THEN NoAsk ELSE ask /\ UNCHANGED <> \* The accept carries the askTs of an ask that started. The guard decides if it is current. -CompletionAccepted(k) == - /\ k \in 1..n /\ phase = "running" /\ ~abort - /\ lastComp = tsOf[k] \/ Mode = "AcceptNoTs" - /\ phase' = "completed" /\ done' = k - /\ ask' = IF Mode = "AcceptKeepsAsk" THEN ask ELSE NoAsk - /\ UNCHANGED <> - -ReasonSet(r) == /\ reason = "" \/ Mode = "ReasonOverwrite" - /\ reason' = r +CompletionAccepted(k) == /\ k \in 1..n /\ AcceptGuard(k) + /\ phase' = "completed" /\ done' = k /\ ask' = AcceptAsk + /\ UNCHANGED <> + +ReasonSet(r) == /\ ReasonGuard /\ reason' = r /\ UNCHANGED <> AbortRequested(b) == /\ abort' = TRUE /\ abandoned' = (abandoned \/ b) /\ UNCHANGED <> -AbandonRequested == /\ abort \/ Mode = "AbandonNoGuard" - /\ abandoned' = TRUE +AbandonRequested == /\ AbandonGuard /\ abandoned' = TRUE /\ UNCHANGED <> -DisposeRequested == /\ abort' = (abort \/ Mode # "DisposeNoAbort") /\ disposed' = TRUE +DisposeRequested == /\ abort' = DisposeAbort /\ disposed' = TRUE /\ UNCHANGED <> Next == \/ Initialized \/ StreamStarted \/ StreamCleanupFinished \/ StreamEnded diff --git a/docs/architecture/tla/RunStateKernel.Reuse.cfg b/docs/architecture/tla/RunStateKernelMC.AbandonNoGuard.cfg similarity index 88% rename from docs/architecture/tla/RunStateKernel.Reuse.cfg rename to docs/architecture/tla/RunStateKernelMC.AbandonNoGuard.cfg index ce755a2d50..6cc24db5b6 100644 --- a/docs/architecture/tla/RunStateKernel.Reuse.cfg +++ b/docs/architecture/tla/RunStateKernelMC.AbandonNoGuard.cfg @@ -1,9 +1,8 @@ SPECIFICATION Spec CONSTANTS - Mode = "Kernel" - Ts = "Reuse" MaxAsks = 3 MaxGen = 2 + AbandonGuard <- AbandonAnyTime INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.cfg b/docs/architecture/tla/RunStateKernelMC.AcceptKeepsAsk.cfg similarity index 87% rename from docs/architecture/tla/RunStateKernel.cfg rename to docs/architecture/tla/RunStateKernelMC.AcceptKeepsAsk.cfg index bd7d046988..c2c80f2878 100644 --- a/docs/architecture/tla/RunStateKernel.cfg +++ b/docs/architecture/tla/RunStateKernelMC.AcceptKeepsAsk.cfg @@ -1,9 +1,8 @@ SPECIFICATION Spec CONSTANTS - Mode = "Kernel" - Ts = "Unique" MaxAsks = 3 MaxGen = 2 + AcceptAsk <- AcceptKeepsAsk INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.SettleAny.cfg b/docs/architecture/tla/RunStateKernelMC.AcceptNoTs.cfg similarity index 87% rename from docs/architecture/tla/RunStateKernel.SettleAny.cfg rename to docs/architecture/tla/RunStateKernelMC.AcceptNoTs.cfg index c084670df5..23b9284f86 100644 --- a/docs/architecture/tla/RunStateKernel.SettleAny.cfg +++ b/docs/architecture/tla/RunStateKernelMC.AcceptNoTs.cfg @@ -1,9 +1,8 @@ SPECIFICATION Spec CONSTANTS - Mode = "SettleAny" - Ts = "Unique" MaxAsks = 3 MaxGen = 2 + AcceptGuard <- AcceptIgnoresTs INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.AcceptNoTs.cfg b/docs/architecture/tla/RunStateKernelMC.DisposeNoAbort.cfg similarity index 86% rename from docs/architecture/tla/RunStateKernel.AcceptNoTs.cfg rename to docs/architecture/tla/RunStateKernelMC.DisposeNoAbort.cfg index 38b27a1db4..4530f8efed 100644 --- a/docs/architecture/tla/RunStateKernel.AcceptNoTs.cfg +++ b/docs/architecture/tla/RunStateKernelMC.DisposeNoAbort.cfg @@ -1,9 +1,8 @@ SPECIFICATION Spec CONSTANTS - Mode = "AcceptNoTs" - Ts = "Unique" MaxAsks = 3 MaxGen = 2 + DisposeAbort <- DisposeKeepsAbort INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernelMC.EndResetsGen.cfg b/docs/architecture/tla/RunStateKernelMC.EndResetsGen.cfg new file mode 100644 index 0000000000..dc9089214b --- /dev/null +++ b/docs/architecture/tla/RunStateKernelMC.EndResetsGen.cfg @@ -0,0 +1,8 @@ +SPECIFICATION Spec +CONSTANTS + MaxAsks = 3 + MaxGen = 2 + StreamEndGen <- StreamEndResetsGen +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernelMC.NoAskAbortGuard.cfg b/docs/architecture/tla/RunStateKernelMC.NoAskAbortGuard.cfg new file mode 100644 index 0000000000..7a2efa254d --- /dev/null +++ b/docs/architecture/tla/RunStateKernelMC.NoAskAbortGuard.cfg @@ -0,0 +1,8 @@ +SPECIFICATION Spec +CONSTANTS + MaxAsks = 3 + MaxGen = 2 + AskStartGuard <- AskStartIgnoresAbort +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.EndResetsGen.cfg b/docs/architecture/tla/RunStateKernelMC.NoStreamPhaseGuard.cfg similarity index 86% rename from docs/architecture/tla/RunStateKernel.EndResetsGen.cfg rename to docs/architecture/tla/RunStateKernelMC.NoStreamPhaseGuard.cfg index 98edb24686..83e7311092 100644 --- a/docs/architecture/tla/RunStateKernel.EndResetsGen.cfg +++ b/docs/architecture/tla/RunStateKernelMC.NoStreamPhaseGuard.cfg @@ -1,9 +1,8 @@ SPECIFICATION Spec CONSTANTS - Mode = "EndResetsGen" - Ts = "Unique" MaxAsks = 3 MaxGen = 2 + StreamStartGuard <- StreamStartAnyPhase INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernelMC.ReasonOverwrite.cfg b/docs/architecture/tla/RunStateKernelMC.ReasonOverwrite.cfg new file mode 100644 index 0000000000..c60787cd65 --- /dev/null +++ b/docs/architecture/tla/RunStateKernelMC.ReasonOverwrite.cfg @@ -0,0 +1,8 @@ +SPECIFICATION Spec +CONSTANTS + MaxAsks = 3 + MaxGen = 2 + ReasonGuard <- ReasonOverwrites +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernel.AbandonNoGuard.cfg b/docs/architecture/tla/RunStateKernelMC.ResumeResetsLatches.cfg similarity index 84% rename from docs/architecture/tla/RunStateKernel.AbandonNoGuard.cfg rename to docs/architecture/tla/RunStateKernelMC.ResumeResetsLatches.cfg index 638f782d54..3386189ba9 100644 --- a/docs/architecture/tla/RunStateKernel.AbandonNoGuard.cfg +++ b/docs/architecture/tla/RunStateKernelMC.ResumeResetsLatches.cfg @@ -1,9 +1,8 @@ SPECIFICATION Spec CONSTANTS - Mode = "AbandonNoGuard" - Ts = "Unique" MaxAsks = 3 MaxGen = 2 + InitializedLatches <- InitializedResetsLatches INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernelMC.Reuse.cfg b/docs/architecture/tla/RunStateKernelMC.Reuse.cfg new file mode 100644 index 0000000000..40bf091378 --- /dev/null +++ b/docs/architecture/tla/RunStateKernelMC.Reuse.cfg @@ -0,0 +1,8 @@ +SPECIFICATION Spec +CONSTANTS + MaxAsks = 3 + MaxGen = 2 + AskTsFresh <- AskTsAny +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernelMC.SettleAny.cfg b/docs/architecture/tla/RunStateKernelMC.SettleAny.cfg new file mode 100644 index 0000000000..ef52ba8ca3 --- /dev/null +++ b/docs/architecture/tla/RunStateKernelMC.SettleAny.cfg @@ -0,0 +1,8 @@ +SPECIFICATION Spec +CONSTANTS + MaxAsks = 3 + MaxGen = 2 + SettleMatches <- SettleAny +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernelMC.cfg b/docs/architecture/tla/RunStateKernelMC.cfg new file mode 100644 index 0000000000..2f99f48ec4 --- /dev/null +++ b/docs/architecture/tla/RunStateKernelMC.cfg @@ -0,0 +1,7 @@ +SPECIFICATION Spec +CONSTANTS + MaxAsks = 3 + MaxGen = 2 +INVARIANTS TypeOK AbandonImpliesAbort DisposeImpliesAbort CompletedQuiet PendingIsNewest NewestShown AcceptCurrent +PROPERTIES LatchesMonotone NoAskAfterAbort CompletedNoStream GenerationMonotone +CHECK_DEADLOCK FALSE diff --git a/docs/architecture/tla/RunStateKernelMC.tla b/docs/architecture/tla/RunStateKernelMC.tla new file mode 100644 index 0000000000..5bba75d7c1 --- /dev/null +++ b/docs/architecture/tla/RunStateKernelMC.tla @@ -0,0 +1,19 @@ +---- MODULE RunStateKernelMC ---- +\* Model-checking settings for RunStateKernel. Each operator replaces one guard or effect in one sabotage config. +EXTENDS RunStateKernel + +\* Reuse: an ask can carry the askTs of an earlier ask (issue 2365). +AskTsAny(t) == TRUE +\* ResumeResetsLatches: the first start clears the latches, as resumeAfterDelegation did before. +InitializedResetsLatches == abort' = FALSE /\ abandoned' = FALSE /\ disposed' = FALSE +StreamStartAnyPhase == TRUE +StreamEndResetsGen == 0 +AskStartIgnoresAbort == phase = "running" +SettleAny(k) == TRUE +AcceptIgnoresTs(k) == phase = "running" /\ ~abort +AcceptKeepsAsk == ask +ReasonOverwrites == TRUE +AbandonAnyTime == TRUE +DisposeKeepsAbort == abort + +==== diff --git a/scripts/check-run-state-kernel.ts b/scripts/check-run-state-kernel.ts index e18c7cce44..e7a2054199 100644 --- a/scripts/check-run-state-kernel.ts +++ b/scripts/check-run-state-kernel.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict" import { readFileSync } from "node:fs" +import { isDeepStrictEqual } from "node:util" import { fileURLToPath } from "node:url" import { nextRunState, initialRunState } from "../src/core/task/run-state/runState" @@ -38,6 +39,11 @@ interface Options { allTurnStates: boolean } +interface SearchNode { + state: ModelState + trace: TraceStep[] +} + interface Applied { next: ModelState accepted: boolean @@ -274,7 +280,7 @@ const semanticWitnesses = { event.tag === "streamStarted" && prev.run.phase === "running" && prev.run.generation === 0 && - prev.run.ask.tag === "none" && + prev.askCount === 0 && next.run.stream.tag === "live", "approval-before-first-stream": ({ prev, next, event }: WitnessContext) => event.tag === "askStarted" && @@ -325,9 +331,10 @@ function eventsFor({ run, askCount }: ModelState): ModelEvent[] { { name: "streamCleanupFinished()", source: "stream", event: { tag: "streamCleanupFinished" } }, { name: "streamEnded()", source: "stream", event: { tag: "streamEnded" } }, ) - if (askCount < MAX_ASK_TS) { + const askStartRejects = run.phase !== "running" || run.latches.abort + if (askCount < MAX_ASK_TS || askStartRejects) { for (const [kind, build] of Object.entries(askBuilders)) { - const ask = build(askCount + 1) + const ask = build(Math.min(askCount + 1, MAX_ASK_TS)) events.push({ name: `askStarted(${kind}, ${ask.askTs})`, source: "ask", event: { tag: "askStarted", ask } }) } } @@ -426,8 +433,8 @@ function apply(options: Options, state: ModelState, { event }: ModelEvent): Appl } const problems: string[] = [] - if (canonical(input) !== canonical(state.run)) problems.push(`${event.tag} mutated its state`) - if (canonical(eventInput) !== canonical(event)) problems.push(`${event.tag} mutated its event`) + if (!isDeepStrictEqual(input, state.run)) problems.push(`${event.tag} mutated its state`) + if (!isDeepStrictEqual(eventInput, event)) problems.push(`${event.tag} mutated its event`) const accepted = "ok" in result const actual = "ok" in result ? result.ok : result.rejected if (options.useTable) { @@ -482,7 +489,7 @@ const shapeOf = ({ run, askCount }: ModelState) => ask: run.ask.tag, askTs: run.ask.tag === "none" ? undefined : run.ask.askTs, latches: run.latches, - lastCompletionAskTs: run.lastCompletionAskTs !== undefined, + lastCompletionAskTs: run.lastCompletionAskTs ?? null, }) const everyTurn = Array.from({ length: 32 }, (_, bits) => ({ @@ -495,17 +502,18 @@ const everyTurn = Array.from({ length: 32 }, (_, bits) => ({ const allFalseTurn = everyTurn[0]! const allTrueTurn = everyTurn[31]! -function checkTurnRegion(options: Options, searched: ModelState[], reachedActions: Set): void { - const shapes = new Map(searched.map((state) => [shapeOf(state), state])) +function checkTurnRegion(options: Options, searched: SearchNode[], reachedActions: Set): void { + const shapes = new Map(searched.map((node) => [shapeOf(node.state), node])) const turns = options.allTurnStates ? everyTurn : [allFalseTurn, allTrueTurn] - const cases = [...shapes.values()].flatMap((state) => + const cases = [...shapes.values()].flatMap((node) => turns.map((turn) => ({ - state: { ...state, run: { ...state.run, turn } }, - events: [...eventsFor(state), ...turnTags.map(turnEvent)], + trace: node.trace, + state: { ...node.state, run: { ...node.state.run, turn } }, + events: [...eventsFor(node.state), ...turnTags.map(turnEvent)], })), ) - for (const { state, events } of cases) { + for (const { trace: reached, state, events } of cases) { for (const modelEvent of events) { const applied = apply(options, state, modelEvent) const turnEventRejected = modelEvent.source === "turn" && !applied.accepted @@ -516,7 +524,8 @@ function checkTurnRegion(options: Options, searched: ModelState[], reachedAction ] if (problems.length) { const trace = [ - { action: "initial", source: "initial" as const, state }, + ...reached, + { action: `fixture: turn = ${JSON.stringify(state.run.turn)}`, source: "turn" as const, state }, { action: modelEvent.name, source: modelEvent.source, state: applied.next }, ] throw new Error(formatCounterexample(problems.join("; "), trace, options)) @@ -537,7 +546,7 @@ function runModelCheck(options: Options): CheckResult { const startProblems = stateViolations(start) if (startProblems.length) throw new Error(formatCounterexample(startProblems.join("; "), startTrace, options)) - const queue = [{ state: start, trace: startTrace }] + const queue: SearchNode[] = [{ state: start, trace: startTrace }] const visited = new Set([canonical(start)]) const reachedActions = new Set() const reachedRejections = new Set() @@ -592,11 +601,7 @@ function runModelCheck(options: Options): CheckResult { } } - checkTurnRegion( - options, - queue.map((node) => node.state), - reachedActions, - ) + checkTurnRegion(options, queue, reachedActions) const assertNoneMissing = (label: string, names: string[]) => { if (names.length) throw new Error(`Run-state kernel model has ${label}: ${names.join(", ")}`) @@ -979,6 +984,39 @@ const unsafeKernels: Record = { caughtBy: /abortRequested mutated its state/, useTable: false, }, + "adds-undefined-approval-id": { + step: (state, event) => { + if (event.tag === "askStarted" && event.ask.tag === "approval" && !("approvalId" in event.ask)) { + event.ask.approvalId = undefined + } + return nextRunState(state, event) + }, + caughtBy: /askStarted mutated its event/, + useTable: false, + }, + "ask-after-abort-at-ask-cap": { + step: (state, event) => + event.tag === "askStarted" && + state.latches.abort && + state.ask.tag !== "none" && + state.ask.askTs === MAX_ASK_TS + ? { ok: { ...state, ask: event.ask } } + : nextRunState(state, event), + caughtBy: differsFor("askStarted"), + useTable: true, + }, + "settled-accept-rejected-with-tool-used-at-marker-1": { + step: (state, event) => + event.tag === "completionAccepted" && + event.askTs === 1 && + state.lastCompletionAskTs === 1 && + state.turn.toolUsed && + state.ask.tag === "none" + ? { rejected: state } + : nextRunState(state, event), + caughtBy: differsFor("completionAccepted"), + useTable: true, + }, "mutates-its-event": { step: (state, event) => { if (event.tag === "askSettled" && state.ask.tag !== "none") event.askTs = state.ask.askTs