From 8b5dbde0209f80ececeb3b0743fc911bae18931f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 6 Sep 2026 05:15:07 +0900 Subject: [PATCH] fix(lab): keep producer deadline failures authoritative until close --- .../034_verification_followup.md | 3 + scripts/test-layout/layout.json | 1 + src/lab/fabric/producer-isolate.ts | 130 ++++-- tests/fixtures/test-layout-expected.json | 1 + .../lab/lab-fabric-producer-deadline.test.ts | 423 ++++++++++++++++++ 5 files changed, 512 insertions(+), 46 deletions(-) create mode 100644 devlog/_plan/260906_lane_b_catalog_stack/034_verification_followup.md create mode 100644 tests/lab/lab-fabric-producer-deadline.test.ts diff --git a/devlog/_plan/260906_lane_b_catalog_stack/034_verification_followup.md b/devlog/_plan/260906_lane_b_catalog_stack/034_verification_followup.md new file mode 100644 index 0000000000..4b220a619d --- /dev/null +++ b/devlog/_plan/260906_lane_b_catalog_stack/034_verification_followup.md @@ -0,0 +1,3 @@ +# Verification follow-up + +The ordering CI run reported an unrelated Lab supervision test failure. A bounded verification prerequisite is reviewed separately from the catalog change. Detailed pre-publication analysis and the implementation plan remain in ignored scratch under the repository security-working-note policy. Product limits and existing assertions are not relaxed. The original ordering branch and failed outputs remain preserved; no success is claimed at this planning checkpoint. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index 045707d947..ce7d767d65 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -718,6 +718,7 @@ "lab-evidence-sanitization.test.ts": "lab", "lab-fabric-outcome-validation.test.ts": "lab", "lab-fabric-persistence-boundary.test.ts": "lab", + "lab-fabric-producer-deadline.test.ts": "lab", "lab-fabric-task.test.ts": "lab", "lab-installation-salt-cache.test.ts": "lab", "lab-ledger-mutation-lock.test.ts": "lab", diff --git a/src/lab/fabric/producer-isolate.ts b/src/lab/fabric/producer-isolate.ts index 3ab672b9bd..70291adffc 100644 --- a/src/lab/fabric/producer-isolate.ts +++ b/src/lab/fabric/producer-isolate.ts @@ -84,6 +84,11 @@ function killChild(child: ChildProcess): void { export async function runIsolatedFabricProducer(request: IsolateRequest): Promise { const now = request.now ?? (() => Date.now()); let lastActivityAt = now(); + // Budget enforcement must not follow wall-clock adjustments; telemetry still does. + const budgetNow = request.now ?? (() => performance.now()); + const startedAt = request.now ? lastActivityAt : budgetNow(); + const totalDeadline = startedAt + request.totalTimeoutMs; + let inactivityDeadline = startedAt + request.inactivityTimeoutMs; return await new Promise((resolve, reject) => { let child: ChildProcess; @@ -104,48 +109,74 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis let stdoutBuffer = ""; let stderrBytes = 0; let settled = false; + let childClosed = false; let receivedResult: SyntheticPatchV1 | undefined; let killReason: FabricTaskError | undefined; const finish = (fn: () => void) => { - if (settled) return; + // A latched failure owns settlement, but scratch cleanup must wait for close. + if (settled || (killReason && !childClosed)) return; settled = true; clearTimeout(totalTimer); clearTimeout(inactivityTimer); - fn(); + if (killReason) reject(killReason); + else fn(); }; const settleTimeout = (error: FabricTaskError) => { - if (settled) return; + if (settled || killReason) return; killReason = error; - killChild(child); + if (childClosed) finish(() => reject(error)); + else killChild(child); + }; + + const expiredDeadline = (at: number): FabricTaskError | undefined => { + // Choose the earliest deadline, regardless of which timer/data callback ran first. + if (at >= inactivityDeadline && inactivityDeadline <= totalDeadline) { + return new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment"); + } + if (at >= totalDeadline) { + return new FabricTaskError("total timeout exceeded", "timeout", "environment"); + } + return undefined; + }; + + const onInactivityTimeout = () => { + settleTimeout(expiredDeadline(budgetNow()) + ?? new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); }; const armInactivity = () => { clearTimeout(inactivityTimer); - inactivityTimer = setTimeout(() => { - settleTimeout(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); - }, request.inactivityTimeoutMs); + inactivityTimer = setTimeout(onInactivityTimeout, request.inactivityTimeoutMs); }; - let inactivityTimer: ReturnType = setTimeout(() => { - settleTimeout(new FabricTaskError("inactivity timeout exceeded", "inactivity_timeout", "environment")); - }, request.inactivityTimeoutMs); + let inactivityTimer: ReturnType = setTimeout(onInactivityTimeout, request.inactivityTimeoutMs); const totalTimer = setTimeout(() => { - settleTimeout(new FabricTaskError("total timeout exceeded", "timeout", "environment")); + settleTimeout(expiredDeadline(budgetNow()) + ?? new FabricTaskError("total timeout exceeded", "timeout", "environment")); }, request.totalTimeoutMs); const handleProtocolLine = (line: string) => { + if (settled || killReason) return; try { const message = parseProducerProtocolLine(line); - if (message.type === "activity") { - lastActivityAt = now(); - armInactivity(); - return; + if (message.type === "activity" || message.type === "result") { + const at = budgetNow(); + const expired = expiredDeadline(at); + if (expired) { + settleTimeout(expired); + return; + } + if (message.type === "activity") { + lastActivityAt = request.now ? at : now(); + inactivityDeadline = at + request.inactivityTimeoutMs; + armInactivity(); + return; + } } if (message.type === "result") { - if (settled) return; receivedResult = message.patch; finish(() => resolve({ patch: message.patch, lastActivityAt })); return; @@ -176,6 +207,7 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis }; const consumeStdout = (chunk: string) => { + if (settled || killReason) return; stdoutBuffer += chunk; if (Buffer.byteLength(stdoutBuffer, "utf8") > FABRIC_PRODUCER_PROTOCOL_MAX_BYTES) { settleTimeout(new FabricTaskError("producer protocol output exceeded limit", "budget_exhausted", "environment")); @@ -207,16 +239,50 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis } }); + child.stderr?.on("error", (error) => { + settleTimeout(new FabricTaskError(error.message, "harness_failure", "harness")); + }); + child.on("error", (error) => { finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); }); child.stdin?.on("error", (error: NodeJS.ErrnoException) => { - if (settled || error.code === "EPIPE") return; + if (settled || killReason || error.code === "EPIPE") return; killChild(child); finish(() => reject(new FabricTaskError(error.message, "harness_failure", "harness"))); }); + child.on("close", (code, signal) => { + childClosed = true; + if (settled) return; + if (killReason) { + finish(() => reject(killReason!)); + return; + } + if (receivedResult) { + finish(() => resolve({ patch: receivedResult!, lastActivityAt })); + return; + } + if (stdoutBuffer.trim()) { + try { + handleProtocolLine(stdoutBuffer.trim()); + if (settled) return; + } catch { + /* fall through */ + } + } + if (signal === "SIGKILL") { + finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); + return; + } + finish(() => reject(new FabricTaskError( + code === 0 ? "isolated producer returned no result" : `isolated producer exited (${code ?? signal ?? "unknown"})`, + "harness_failure", + "harness", + ))); + }); + const payload = JSON.stringify({ harnessKind: request.harnessKind, executorModulePath: request.executorModulePath, @@ -242,6 +308,7 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis child.stdin?.write(payload); child.stdin?.end(); } catch (error) { + if (killReason) return; killChild(child); finish(() => reject(new FabricTaskError( error instanceof Error ? error.message : String(error), @@ -250,35 +317,6 @@ export async function runIsolatedFabricProducer(request: IsolateRequest): Promis ))); return; } - - child.on("close", (code, signal) => { - if (settled) return; - if (killReason) { - finish(() => reject(killReason!)); - return; - } - if (receivedResult) { - finish(() => resolve({ patch: receivedResult!, lastActivityAt })); - return; - } - if (stdoutBuffer.trim()) { - try { - handleProtocolLine(stdoutBuffer.trim()); - if (receivedResult) return; - } catch { - /* fall through */ - } - } - if (signal === "SIGKILL") { - finish(() => reject(new FabricTaskError("total timeout exceeded", "timeout", "environment"))); - return; - } - finish(() => reject(new FabricTaskError( - code === 0 ? "isolated producer returned no result" : `isolated producer exited (${code ?? signal ?? "unknown"})`, - "harness_failure", - "harness", - ))); - }); }); } diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 7e7dd7d126..ad415d8a6f 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -555,6 +555,7 @@ "lab-evidence-sanitization.test.ts": "lab", "lab-fabric-outcome-validation.test.ts": "lab", "lab-fabric-persistence-boundary.test.ts": "lab", + "lab-fabric-producer-deadline.test.ts": "lab", "lab-fabric-task.test.ts": "lab", "lab-installation-salt-cache.test.ts": "lab", "lab-ledger-mutation-lock.test.ts": "lab", diff --git a/tests/lab/lab-fabric-producer-deadline.test.ts b/tests/lab/lab-fabric-producer-deadline.test.ts new file mode 100644 index 0000000000..f4895b90cf --- /dev/null +++ b/tests/lab/lab-fabric-producer-deadline.test.ts @@ -0,0 +1,423 @@ +import { describe, expect, spyOn, test } from "bun:test"; +import * as childProcess from "node:child_process"; +import { EventEmitter } from "node:events"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PassThrough } from "node:stream"; +import { setImmediate as nextTurn } from "node:timers"; +import { runIsolatedFabricProducer } from "../../src/lab/fabric/producer-isolate"; +import type { IsolatedProducerResult } from "../../src/lab/fabric/producer-protocol"; +import { FabricTaskError, type FabricTaskRunResult, type SyntheticPatchV1 } from "../../src/lab/fabric/types"; +import { runFabricSyntheticPatchTaskForRoute } from "../../src/lab/fabric/executor"; +import { createLabDestination } from "../../src/lab/live/destination"; +import { fabricCorrectPatchExecutor, fabricMockRoute } from "../helpers/fabric-task-test"; + +// Hand-written valid fixture: an always-reject supervisor must fail the controls. +const PATCH: SyntheticPatchV1 = { + schemaVersion: 1, + operations: [{ op: "replace", path: "src/value.txt", contentUtf8: "after\n" }], +}; +const RESULT = JSON.stringify({ type: "result", patch: PATCH }); +const ACTIVITY = '{"type":"activity"}\n'; +const START = 1_000; +const IDLE_MS = 100; +const TOTAL_MS = 250; + +class DeadlineChild extends EventEmitter { + readonly stdin = new PassThrough(); + readonly stdout = new PassThrough(); + readonly stderr = new PassThrough(); + readonly signals: Array = []; + closed = false; + + kill(signal?: NodeJS.Signals | number): boolean { + this.signals.push(signal); + return true; // Buffered data can arrive after kill; only the test emits close. + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.emit("close", 0, null); + } +} + +type CapturedTimer = { callback: () => void; delay: number; cleared: boolean }; +type Outcome = + | { status: "pending" } + | { status: "resolved"; value: T } + | { status: "rejected"; error: unknown }; + +// Drain promise adoption and stream nextTicks, without sleeping or advancing time. +const drain = () => new Promise((resolve) => nextTurn(resolve)); + +function installTimers(restorers: Array<() => void>) { + const timers: CapturedTimer[] = []; + const handles = new Map, CapturedTimer>(); + const setSpy = spyOn(globalThis, "setTimeout").mockImplementation(((callback: () => void, delay: number) => { + const timer = { callback, delay, cleared: false }; + // Only timer identity/unref are consumed by this supervisor; no real handle. + const handle = { unref() { return this; } } as unknown as ReturnType; + timers.push(timer); + handles.set(handle, timer); + return handle; + }) as typeof setTimeout); + restorers.push(() => setSpy.mockRestore()); + const clearSpy = spyOn(globalThis, "clearTimeout").mockImplementation((handle) => { + const timer = handles.get(handle as ReturnType); + if (timer) timer.cleared = true; + }); + restorers.push(() => clearSpy.mockRestore()); + return timers; +} + +type ExpectedFailure = + | [code: "inactivity_timeout" | "timeout"] + | [code: "harness_failure", attribution: "harness", message: string]; + +type Harness = { + child: DeadlineChild; + timers: CapturedTimer[]; + at: (time: number) => void; + result: (newline?: boolean) => void; + pending: () => Promise; + failure: (...expected: ExpectedFailure) => Promise; + success: (lastActivityAt?: number) => Promise; +}; + +async function withProducer(body: (h: Harness) => Promise, totalTimeoutMs = TOTAL_MS) { + const scratchRoot = mkdtempSync(join(tmpdir(), "ocx-fabric-deadline-")); + const child = new DeadlineChild(); + const originals = { spawn: childProcess.spawn, set: globalThis.setTimeout, clear: globalThis.clearTimeout }; + const restorers: Array<() => void> = []; + let time = START; + let outcome: Outcome = { status: "pending" }; + try { + // Repository namespace-spy precedent; never delegates to the original spawn. + const spawnSpy = spyOn(childProcess, "spawn").mockImplementation(() => child as unknown as childProcess.ChildProcess); + restorers.push(() => spawnSpy.mockRestore()); + const timers = installTimers(restorers); + void runIsolatedFabricProducer({ + scratchRoot, harnessKind: "deterministic_correct", totalTimeoutMs, + inactivityTimeoutMs: IDLE_MS, now: () => time, + }).then( + (value) => { outcome = { status: "resolved", value }; }, + (error: unknown) => { outcome = { status: "rejected", error }; }, + ); + expect(spawnSpy).toHaveBeenCalledTimes(1); + expect(spawnSpy.mock.results[0]?.value).toBe(child); + expect(child.stdout.listenerCount("data")).toBe(1); + expect(child.listenerCount("close")).toBe(1); + expect(timers.map(({ delay }) => delay).sort((a, b) => a - b)).toEqual([IDLE_MS, totalTimeoutMs]); + await body({ + child, timers, at: (value) => { time = value; }, + result: (newline = true) => { child.stdout.write(RESULT + (newline ? "\n" : "")); }, + pending: async () => { await drain(); expect(outcome.status).toBe("pending"); }, + failure: async (...expected) => { + const [code] = expected; + const attribution = code === "harness_failure" ? expected[1] : "environment"; + const message = code === "harness_failure" ? expected[2] + : code === "inactivity_timeout" ? "inactivity timeout exceeded" : "total timeout exceeded"; + child.close(); + await drain(); + expect(outcome.status).toBe("rejected"); + if (outcome.status !== "rejected") throw new Error("producer did not reject after close"); + expect(outcome.error).toBeInstanceOf(FabricTaskError); + expect(outcome.error).toMatchObject({ code, attribution, message }); + expect(timers.every(({ cleared }) => cleared)).toBe(true); + }, + success: async (lastActivityAt = START) => { + await drain(); + expect(outcome).toEqual({ status: "resolved", value: { patch: PATCH, lastActivityAt } }); + expect(child.signals).toEqual([]); + expect(timers.every(({ cleared }) => cleared)).toBe(true); + }, + }); + expect(spawnSpy).toHaveBeenCalledTimes(1); + } finally { + // Always reap the fake before removing its scratch, including failed assertions. + try { + child.close(); + child.stdin.destroy(); + child.stdout.destroy(); + child.stderr.destroy(); + } finally { + for (const restore of restorers.reverse()) restore(); + rmSync(scratchRoot, { recursive: true, force: true }); + expect(childProcess.spawn).toBe(originals.spawn); + expect(globalThis.setTimeout).toBe(originals.set); + expect(globalThis.clearTimeout).toBe(originals.clear); + } + } +} + +describe("isolated fabric producer deadline admission", () => { + test("idle timer then buffered result cannot settle before child close", async () => { + await withProducer(async (h) => { + h.at(1_100); + h.timers[0]!.callback(); + expect(h.child.signals).toEqual(["SIGKILL"]); + await h.pending(); + h.result(); + await h.pending(); + await h.failure("inactivity_timeout"); + }); + }); + + for (const time of [1_100, 1_101]) { + test(`result at ${time} rejects even when no timer callback ran`, async () => { + await withProducer(async (h) => { + h.at(time); + h.result(); + await h.pending(); + expect(h.child.signals).toEqual(["SIGKILL"]); + await h.failure("inactivity_timeout"); + }); + }); + } + + test("late activity and result in the same chunk cannot renew idle", async () => { + await withProducer(async (h) => { + h.at(1_101); + h.child.stdout.write(ACTIVITY + RESULT + "\n"); + await h.pending(); + expect(h.timers).toHaveLength(2); + expect(h.child.signals).toEqual(["SIGKILL"]); + await h.failure("inactivity_timeout"); + }); + }); + + test("total deadline is fixed despite accepted activity", async () => { + await withProducer(async (h) => { + for (const time of [1_090, 1_180]) { + h.at(time); + h.child.stdout.write(ACTIVITY); + await h.pending(); + expect(h.child.signals).toEqual([]); + } + h.at(1_250); + h.result(); + await h.pending(); + expect(h.child.signals).toEqual(["SIGKILL"]); + await h.failure("timeout"); + }); + }); + + test("a delayed total callback chooses the earlier elapsed idle deadline", async () => { + await withProducer(async (h) => { + h.at(1_251); + h.timers[1]!.callback(); + await h.pending(); + await h.failure("inactivity_timeout"); + }); + }); + + test("inactivity wins an exact deadline tie even if total callback runs first", async () => { + await withProducer(async (h) => { + for (const time of [1_090, 1_150]) { + h.at(time); + h.child.stdout.write(ACTIVITY); + await h.pending(); + } + h.at(1_250); + h.timers[1]!.callback(); + await h.pending(); + await h.failure("inactivity_timeout"); + }); + }); + + test("first timeout survives later timers, protocol and process/stream errors", async () => { + await withProducer(async (h) => { + h.at(1_100); + h.timers[0]!.callback(); + await h.pending(); + h.at(1_251); + const laterEvents = [ + () => h.timers[1]!.callback(), + () => h.child.stdout.write('{"type":"error","code":"sandbox_violation","message":"later protocol error","attribution":"harness"}\n'), + () => h.child.stdout.write("not-json\n"), + () => h.child.stdout.emit("error", new Error("later stdout error")), + () => h.child.stderr.emit("error", new Error("later stderr error")), + () => h.child.stdin.emit("error", new Error("later stdin error")), + () => h.child.emit("error", new Error("later child error")), + () => h.child.stdout.write(ACTIVITY), + () => h.result(), + () => h.timers[0]!.callback(), + ]; + for (const event of laterEvents) { + event(); + await h.pending(); + expect(h.child.signals).toEqual(["SIGKILL"]); + } + expect(h.timers).toHaveLength(2); + await h.failure("inactivity_timeout"); + }); + }); + + test("valid result just before idle boundary succeeds", async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(); + await h.success(); + }); + }); + + test("stderr failure stays authoritative until close across later data, errors and timers", async () => { + await withProducer(async (h) => { + h.child.stderr.emit("error", new Error("first stderr read failure")); + await h.pending(); + expect(h.child.signals).toEqual(["SIGKILL"]); + h.at(1_251); + const laterEvents = [ + () => h.child.stdout.write(ACTIVITY + RESULT + "\n"), + () => h.child.stdout.write('{"type":"error","code":"sandbox_violation","message":"later protocol error","attribution":"harness"}\n'), + () => h.child.stderr.emit("error", new Error("second stderr error")), + () => h.child.stdout.emit("error", new Error("later stdout error")), + () => h.child.stdin.emit("error", new Error("later stdin error")), + () => h.child.emit("error", new Error("later child error")), + () => h.timers[0]!.callback(), + () => h.timers[1]!.callback(), + ]; + for (const event of laterEvents) { + event(); + await h.pending(); + expect(h.child.signals).toEqual(["SIGKILL"]); + } + expect(h.timers).toHaveLength(2); + await h.failure("harness_failure", "harness", "first stderr read failure"); + }); + }); + + test("valid activity renews idle and reports its accepted timestamp", async () => { + await withProducer(async (h) => { + h.at(1_090); + h.child.stdout.write(ACTIVITY); + await h.pending(); + expect(h.timers).toHaveLength(3); + expect(h.timers[0]!.cleared).toBe(true); + expect(h.timers[1]!.cleared).toBe(false); + h.at(1_189); + h.result(); + await h.success(1_090); + }); + }); + + test("valid result just before the fixed total deadline succeeds", async () => { + await withProducer(async (h) => { + for (const time of [1_090, 1_180]) { + h.at(time); + h.child.stdout.write(ACTIVITY); + await h.pending(); + } + h.at(1_249); + h.result(); + await h.success(1_180); + }); + }); + + for (const closeAt of [1_099, 1_100, 1_101]) { + test(`unterminated result is admitted at close time ${closeAt}`, async () => { + await withProducer(async (h) => { + h.at(1_099); + h.result(false); + await h.pending(); + h.at(closeAt); + h.child.close(); + if (closeAt < 1_100) await h.success(); + else await h.failure("inactivity_timeout"); + }); + }); + } +}); + +test("trusted route keeps scratch until stderr-failed child closes, then cleans it", async () => { + const configDir = mkdtempSync(join(tmpdir(), "ocx-fabric-consumer-deadline-")); + const child = new DeadlineChild(); + const originals = { spawn: childProcess.spawn, set: globalThis.setTimeout, clear: globalThis.clearTimeout }; + const restorers: Array<() => void> = []; + const proxyNames = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy"]; + const proxyEnv = proxyNames.map((name) => [name, process.env[name]] as const); + const outer: { outcome: Outcome } = { outcome: { status: "pending" } }; + try { + for (const name of proxyNames) delete process.env[name]; + // Resolve through the existing destination contract before capturing producer timers. + const destination = await createLabDestination({ + baseUrl: "https://api.example.com/v1", labRunApproval: true, configDir, + resolve: async () => [{ address: "93.184.216.34", family: 4 }], + }); + const spawnSpy = spyOn(childProcess, "spawn").mockImplementation(() => child as unknown as childProcess.ChildProcess); + restorers.push(() => spawnSpy.mockRestore()); + const timers = installTimers(restorers); + void runFabricSyntheticPatchTaskForRoute({ + routeContext: fabricMockRoute(), destination, configDir, now: () => START, + patchExecutor: fabricCorrectPatchExecutor(), + }).then( + (value) => { outer.outcome = { status: "resolved", value }; }, + (error: unknown) => { outer.outcome = { status: "rejected", error }; }, + ); + expect(spawnSpy).toHaveBeenCalledTimes(1); + expect(spawnSpy.mock.results[0]?.value).toBe(child); + const scratchRoot = spawnSpy.mock.calls[0]?.[2]?.env?.OCX_FABRIC_SCRATCH_ROOT; + expect(typeof scratchRoot).toBe("string"); + if (!scratchRoot) throw new Error("producer spawn omitted its scratch root"); + expect(child.listenerCount("close")).toBe(1); + expect(timers).toHaveLength(2); + expect(existsSync(scratchRoot)).toBe(true); + await drain(); + expect(outer.outcome.status).toBe("pending"); + + child.stderr.emit("error", new Error("consumer stderr failure")); + const assertPendingScratch = async () => { + await drain(); + expect(outer.outcome.status).toBe("pending"); + expect(child.closed).toBe(false); + expect(child.signals).toEqual(["SIGKILL"]); + expect(existsSync(scratchRoot)).toBe(true); + expect(readFileSync(join(scratchRoot, "src/value.txt"), "utf8")).toBe("before\n"); + }; + await assertPendingScratch(); + const afterFailure = [ + () => child.stdout.write(ACTIVITY + RESULT + "\n"), + () => child.stderr.emit("error", new Error("later stderr failure")), + () => timers[0]!.callback(), + () => timers[1]!.callback(), + ]; + for (const event of afterFailure) { + event(); + await assertPendingScratch(); + } + child.close(); + await drain(); + expect(outer.outcome.status).toBe("resolved"); + if (outer.outcome.status !== "resolved") throw new Error("route did not settle after child close"); + expect(outer.outcome.value).toMatchObject({ + executionAuthority: "trusted_route", + outcome: { + outcome: "inconclusive", + failure: { class: "harness_failure", code: "harness_failure", attribution: "harness", retryable: false }, + verifier: { passed: false, reason: "harness_failure" }, + usage: { outputBytes: 0, patchOperations: 0, filesTouched: 0 }, + }, + }); + expect(existsSync(scratchRoot)).toBe(false); + expect(timers.every(({ cleared }) => cleared)).toBe(true); + expect(spawnSpy).toHaveBeenCalledTimes(1); + } finally { + try { + child.close(); + await drain(); + child.stdin.destroy(); child.stdout.destroy(); child.stderr.destroy(); + } finally { + for (const restore of restorers.reverse()) restore(); + for (const [name, value] of proxyEnv) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + rmSync(configDir, { recursive: true, force: true }); + expect(childProcess.spawn).toBe(originals.spawn); + expect(globalThis.setTimeout).toBe(originals.set); + expect(globalThis.clearTimeout).toBe(originals.clear); + } + } +});