From 9a8a5cdfad3765f28665299b71d2eb6b364a43a6 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:43:29 -0500 Subject: [PATCH 001/102] docs(project): fix create URL flag example (#42) --- DOCUMENTATION.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 7e50c36..a0fa9a4 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -319,10 +319,10 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--target-url` against local addresses for fast feedback. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. ```bash -testsprite project create --name "Checkout" --target-url https://staging.example.com +testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` From 7928b5ce2f026f397c107bf4172469a1bba2f4b0 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:43:33 -0500 Subject: [PATCH 002/102] fix(help): include request-timeout in global flag guidance (#41) --- src/lib/output.ts | 4 +-- src/lib/render-error.test.ts | 12 ++++++++ src/lib/render-error.ts | 4 ++- test/__snapshots__/help.snapshot.test.ts.snap | 30 +++++++++---------- 4 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/lib/output.ts b/src/lib/output.ts index 2b2db39..79b3f11 100644 --- a/src/lib/output.ts +++ b/src/lib/output.ts @@ -3,10 +3,10 @@ export type OutputMode = 'json' | 'text'; /** * Help-text footer pointing at the global options surface so users * looking at any subcommand `--help` don't miss `--dry-run`, `--output`, - * `--profile`, `--endpoint-url`, `--debug`. + * `--profile`, `--endpoint-url`, `--request-timeout`, `--debug`. */ export const GLOBAL_OPTS_HINT = - '\nGlobal options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug):' + + '\nGlobal options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug):' + '\n testsprite --help'; export function isOutputMode(value: unknown): value is OutputMode { diff --git a/src/lib/render-error.test.ts b/src/lib/render-error.test.ts index 2b1ced7..9635f2b 100644 --- a/src/lib/render-error.test.ts +++ b/src/lib/render-error.test.ts @@ -37,6 +37,12 @@ describe('rephraseUnknownOption', () => { expect(result).toContain('--endpoint-url'); }); + it('rephrases --request-timeout placed after subcommand', () => { + const result = rephraseUnknownOption("error: unknown option '--request-timeout'"); + expect(result).not.toBeNull(); + expect(result).toContain('--request-timeout'); + }); + it('rephrases --debug placed after subcommand', () => { const result = rephraseUnknownOption("error: unknown option '--debug'"); expect(result).not.toBeNull(); @@ -87,6 +93,12 @@ describe('rephraseUnknownOption', () => { expect(result).not.toBeNull(); expect(result).toContain('testsprite --endpoint-url '); }); + + it('value flag (--request-timeout) example DOES include a placeholder', () => { + const result = rephraseUnknownOption("error: unknown option '--request-timeout'"); + expect(result).not.toBeNull(); + expect(result).toContain('testsprite --request-timeout '); + }); }); describe('renderCommanderError', () => { diff --git a/src/lib/render-error.ts b/src/lib/render-error.ts index ebb4208..1898acd 100644 --- a/src/lib/render-error.ts +++ b/src/lib/render-error.ts @@ -11,13 +11,15 @@ import type { OutputMode } from './output.js'; * * `boolean` flags (--dry-run, --debug, --verbose) take no value; emit * example without a placeholder. `value` flags (--output, --profile, - * --endpoint-url) take a single argument; emit example with ``. + * --endpoint-url, --request-timeout) take a single argument; emit example + * with ``. */ const GLOBAL_FLAG_ARITY: Record = { 'dry-run': 'boolean', output: 'value', profile: 'value', 'endpoint-url': 'value', + 'request-timeout': 'value', debug: 'boolean', verbose: 'boolean', }; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6f1f158..6fee8af 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -36,7 +36,7 @@ Options: destroyed. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -49,7 +49,7 @@ List supported agent targets and skills, their status, and landing paths Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -86,7 +86,7 @@ exports[`--help snapshots > auth logout 1`] = ` Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -97,7 +97,7 @@ exports[`--help snapshots > auth whoami 1`] = ` Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -153,7 +153,7 @@ Get a project by id Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -175,7 +175,7 @@ Options: --max-items stop after this many items across auto-paged pages -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -280,7 +280,7 @@ Options: source body; json mode: wire envelope) -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -315,7 +315,7 @@ Options: per invocation; pin one yourself for safe retries. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -331,7 +331,7 @@ Options: --failed-only Keep only the failed step plus its immediate neighbors (±1) -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -344,7 +344,7 @@ Get a test by id Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -369,7 +369,7 @@ Options: --max-items stop after this many items across auto-paged pages -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -433,7 +433,7 @@ Dry-run shape notes: omit \`closure\` (or return it as null) since there is no dependency expansion. • \`autoHeal\` defaults true for FE reruns; BE reruns ignore the field entirely. -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -465,7 +465,7 @@ Options: --cursor with --history: opaque cursor from a prior page -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -516,7 +516,7 @@ Dependency-aware fresh run (M4): BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -539,7 +539,7 @@ Options: excluded when this flag is set. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; From 5f7dba1ca26d1bd3b45d1a6eaeabc0f5c4622e80 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:43:37 -0500 Subject: [PATCH 003/102] fix(auth): honor request timeout during configure (#52) --- src/commands/auth.test.ts | 44 +++++++++++++++++++++++++++++++++++++++ src/commands/auth.ts | 1 + 2 files changed, 45 insertions(+) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index d65e6cc..b360d2a 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -110,6 +110,50 @@ describe('runConfigure', () => { ); }); + it('uses requestTimeoutMs for the pre-write key validation ping', async () => { + const { deps } = makeCapture(); + let sawAbort = false; + const fetchImpl = vi.fn( + async (_input: string | URL | Request, init?: RequestInit) => + new Promise((_resolve, reject) => { + const signal = init?.signal; + const timeout = setTimeout(() => { + reject(new Error('requestTimeoutMs was not applied to the validation ping')); + }, 50); + signal?.addEventListener( + 'abort', + () => { + sawAbort = true; + clearTimeout(timeout); + reject(new DOMException('The operation timed out.', 'TimeoutError')); + }, + { once: true }, + ); + }), + ) as unknown as typeof fetch; + + await expect( + runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + requestTimeoutMs: 1, + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl, + }, + ), + ).rejects.toBeInstanceOf(CLIError); + + expect(sawAbort).toBe(true); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + it('throws VALIDATION_ERROR when --from-env is set but key is missing', async () => { const { deps } = makeCapture(); await expect( diff --git a/src/commands/auth.ts b/src/commands/auth.ts index de1eda4..18cd622 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -145,6 +145,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): baseUrl: facadeBaseUrl(apiUrl), apiKey, fetchImpl: deps.fetchImpl, + requestTimeoutMs: opts.requestTimeoutMs, }); try { // Tag the validation call with the originating command (when provided) so From 547b5be45799f5c8287638eb0ebc6a8bdef4cecb Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:43:42 -0500 Subject: [PATCH 004/102] fix(history): reject fractional page sizes (#55) --- src/commands/test.result.history.spec.ts | 26 ++++++++++++++++++++++++ src/commands/test.ts | 5 ++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index e3fb963..6f6cdd2 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -535,6 +535,32 @@ describe('runResultHistory — pagination', () => { expect(capturedUrl).toContain('pageSize=5'); }); + + it('rejects fractional --page-size before making a request', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => { + throw new Error('should not be called'); + }); + + await expect( + runResultHistory( + { + output: 'json', + testId: 'test_abc', + pageSize: 1.5, + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'page-size' }), + }); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/test.ts b/src/commands/test.ts index 08e4fe2..c61e84b 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3836,7 +3836,10 @@ export async function runResultHistory( // configured (codex round-2), matching validatePaginationFlags ordering // in `test list` / `project list`. if (opts.pageSize !== undefined) { - if (!Number.isFinite(opts.pageSize) || opts.pageSize < 1 || opts.pageSize > 100) { + if (!Number.isFinite(opts.pageSize) || !Number.isInteger(opts.pageSize)) { + throw localValidationError('page-size', 'must be an integer between 1 and 100'); + } + if (opts.pageSize < 1 || opts.pageSize > 100) { throw localValidationError('page-size', 'must be between 1 and 100'); } } From 91024301832cbba0e4beb2ef34ad72a8d85712cd Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:43:54 +0400 Subject: [PATCH 005/102] fix(target-url): treat trailing-dot hostnames as loopback in SSRF guard (#37) assertNotLocal lowercased the hostname but did not strip a trailing dot, so http://localhost. (the FQDN form of localhost, RFC 6761) and http://localhost%2e bypassed the host === 'localhost' loopback check. IP literals are already dot-normalized by the WHATWG URL parser, so only named hosts were affected. Strips one trailing dot before the comparison. Adds 4 regression tests (3 blocked variants + 1 public-FQDN no-false-positive). --- src/lib/target-url.spec.ts | 25 +++++++++++++++++++++++++ src/lib/target-url.ts | 9 ++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/lib/target-url.spec.ts b/src/lib/target-url.spec.ts index 96f705b..4998bc3 100644 --- a/src/lib/target-url.spec.ts +++ b/src/lib/target-url.spec.ts @@ -210,6 +210,31 @@ describe('assertNotLocal — IPv6 hardening (SSRF bypass guard)', () => { }); }); +describe('assertNotLocal — trailing-dot FQDN normalization (SSRF bypass guard)', () => { + // `localhost.` is the fully-qualified form of `localhost` (RFC 6761 reserves + // both to resolve to loopback). It previously bypassed the + // `host === 'localhost'` check because the WHATWG URL parser keeps the + // trailing dot on named hosts (IP literals are dot-normalized, named hosts + // are not). + it('blocks http://localhost. (trailing-dot loopback)', () => { + expectBlocked('http://localhost.'); + }); + + it('blocks http://localhost.:8080 (trailing-dot loopback with port)', () => { + expectBlocked('http://localhost.:8080'); + }); + + it('blocks http://localhost%2e (percent-encoded trailing dot)', () => { + expectBlocked('http://localhost%2e'); + }); + + // A legitimate public FQDN with a trailing dot must still be allowed + // (no false positive from the dot strip). + it('allows https://example.com. (public FQDN with trailing dot)', () => { + expectAllowed('https://example.com.'); + }); +}); + describe('assertNotLocal — allowed public URLs', () => { it('allows https://example.com', () => { expectAllowed('https://example.com'); diff --git a/src/lib/target-url.ts b/src/lib/target-url.ts index 030fc93..895c0f0 100644 --- a/src/lib/target-url.ts +++ b/src/lib/target-url.ts @@ -41,7 +41,14 @@ export function assertNotLocal(rawUrl: string): void { throw localTargetError('target-url', 'must use http or https scheme'); } - const host = parsed.hostname.toLowerCase(); + // Normalize a single trailing dot in the hostname. `localhost.` is the + // fully-qualified form of `localhost` (RFC 6761 reserves both to resolve to + // loopback), so `http://localhost.` must be rejected just like + // `http://localhost`. Without this strip, the trailing-dot form (also + // reachable via `localhost%2e`) slips past the `host === 'localhost'` check. + // IP literals are already dot-normalized by the WHATWG URL parser, so this + // only affects named hosts. + const host = parsed.hostname.toLowerCase().replace(/\.$/, ''); // Loopback / unspecified. if (host === 'localhost' || host === '0.0.0.0') { From 33b78482203a15fc7c2fb9473d6be9f50994befd Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:43:59 +0400 Subject: [PATCH 006/102] fix(auth): preserve typed API error envelope when key verification fails (#38) --- src/commands/auth.test.ts | 40 +++++++++++++++++++++++++++++++++++++++ src/commands/auth.ts | 19 ++++++++++++++----- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index b360d2a..5207085 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -312,6 +312,46 @@ describe('runConfigure', () => { expect(capture.stderr.join('\n')).toContain('profile NOT updated'); }); + it('key-rejected error preserves the typed ApiError envelope (JSON contract)', async () => { + const { deps } = makeCapture(); + const rejectedFetch: AuthDeps['fetchImpl'] = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'API key is invalid or revoked.', + nextAction: 'Rotate your key.', + requestId: 'req_reject', + details: { reason: 'malformed' }, + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ), + ) as unknown as AuthDeps['fetchImpl']; + + // The thrown error must be an ApiError (with code, nextAction, requestId) + // — not a CLIError wrapper that drops those fields. Under --output json, + // index.ts renders ApiError as the full typed envelope; CLIError would + // render only {"error":"...string..."}, violating the JSON contract. + await expect( + runConfigure( + { profile: 'default', output: 'json', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-bad' }, + credentialsPath, + fetchImpl: rejectedFetch, + }, + ), + ).rejects.toMatchObject({ + code: 'AUTH_INVALID', + exitCode: 3, + nextAction: 'Rotate your key.', + requestId: 'req_reject', + }); + }); + // The old "run `testsprite agent install`" self-bootstrap tip was removed with // the setup consolidation — runConfigure now runs ONLY as part of `setup`, // which installs the skill itself. These guard that the tip stays gone. diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 18cd622..e3b3867 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -159,13 +159,22 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): } catch (err) { const message = err instanceof Error ? err.message : String(err); stderr(`API key rejected by ${apiUrl}: ${message} — profile NOT updated`); - const exitCode = err instanceof ApiError ? err.exitCode : 3; - // Include the resolved endpoint in the thrown message so the user knows - // which host rejected the key. This prevents the "invalid or revoked" - // message from being ambiguous when the key is valid for a different env. + // When the verification call returned a typed API error (AUTH_INVALID, + // AUTH_FORBIDDEN, etc.), re-throw it directly so `index.ts` renders the + // full typed envelope under `--output json` (code, nextAction, requestId, + // details). Previously wrapping it in CLIError discarded those fields and + // emitted a bare `{"error":"...string..."}` — violating the JSON contract. + // Augment the message with the endpoint context so text-mode users still + // see which host rejected the key. + if (err instanceof ApiError) { + err.message = `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`; + throw err; + } + // Non-ApiError (truly unexpected throws like a TypeError from a + // misconfigured fetchImpl). Exit 3 (auth family). throw new CLIError( `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`, - exitCode, + 3, ); } From 2fd8f4cb78ca1cc100b634880383a8a4270ea64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EB=8F=84=ED=9B=88?= Date: Fri, 3 Jul 2026 05:44:03 +0900 Subject: [PATCH 007/102] fix(skill-nudge): require complete Codex managed section (#90) * fix(skill-nudge): require complete Codex managed section * docs(skill-nudge): document helper contracts --------- Co-authored-by: ahndohun <19940813+ahndohun@users.noreply.github.com> --- src/lib/skill-nudge.test.ts | 11 +++++++++-- src/lib/skill-nudge.ts | 17 +++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index 34f043b..c15b09d 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { MANAGED_SECTION_BEGIN, TARGETS } from './agent-targets.js'; +import { MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, TARGETS } from './agent-targets.js'; import type { OutputMode } from './output.js'; import { SKILL_NUDGE_COMMANDS, @@ -36,10 +36,17 @@ describe('isVerifySkillInstalled', () => { it('true when AGENTS.md exists AND carries our BEGIN sentinel', () => { const existsSync = (p: string) => p.endsWith('AGENTS.md'); - const readFileSync = () => `# project\n${MANAGED_SECTION_BEGIN}\n...skill...\n`; + const readFileSync = () => + `# project\n${MANAGED_SECTION_BEGIN}\n...skill...\n${MANAGED_SECTION_END}\n`; expect(isVerifySkillInstalled('/proj', { existsSync, readFileSync })).toBe(true); }); + it('false when AGENTS.md has only the BEGIN sentinel without a complete managed section', () => { + const existsSync = (p: string) => p.endsWith('AGENTS.md'); + const readFileSync = () => `# project\n${MANAGED_SECTION_BEGIN}\n...partial skill...\n`; + expect(isVerifySkillInstalled('/proj', { existsSync, readFileSync })).toBe(false); + }); + it('false when only a bare AGENTS.md (no sentinel) exists', () => { const existsSync = (p: string) => p.endsWith('AGENTS.md'); const readFileSync = () => '# my project\nNothing TestSprite here.\n'; diff --git a/src/lib/skill-nudge.ts b/src/lib/skill-nudge.ts index 1afc863..0f67572 100644 --- a/src/lib/skill-nudge.ts +++ b/src/lib/skill-nudge.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { MANAGED_SECTION_BEGIN, TARGETS } from './agent-targets.js'; +import { MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, TARGETS } from './agent-targets.js'; import { defaultCredentialsPath, readProfile } from './credentials.js'; import type { OutputMode } from './output.js'; @@ -44,8 +44,8 @@ export interface SkillPresenceDeps { * True if the `testsprite-verify` skill is installed for ANY supported agent in * `dir`. own-file targets (claude/cursor/cline/antigravity): the landing file * exists. managed-section target (codex / AGENTS.md): the file exists AND - * carries our BEGIN sentinel — a user-authored AGENTS.md without the sentinel - * does NOT count as our skill. + * carries one complete managed section — a user-authored AGENTS.md without the + * sentinels, or with a truncated section, does NOT count as our skill. * * The TARGETS table is the single source of truth for landing paths, so this * stays in lockstep with `agent install` without re-listing paths. Best-effort: @@ -59,7 +59,7 @@ export function isVerifySkillInstalled(dir: string, deps: SkillPresenceDeps = {} if (!exists(full)) continue; if (spec.mode === 'managed-section') { try { - if (read(full).includes(MANAGED_SECTION_BEGIN)) return true; + if (hasCompleteManagedSection(read(full))) return true; } catch { // unreadable AGENTS.md → treat this target as absent, keep checking } @@ -70,6 +70,14 @@ export function isVerifySkillInstalled(dir: string, deps: SkillPresenceDeps = {} return false; } +/** True when a managed-section file contains an ordered TestSprite BEGIN/END pair. */ +function hasCompleteManagedSection(content: string): boolean { + const begin = content.indexOf(MANAGED_SECTION_BEGIN); + if (begin === -1) return false; + const end = content.indexOf(MANAGED_SECTION_END, begin + MANAGED_SECTION_BEGIN.length); + return end !== -1; +} + export interface SkillNudgeContext { /** Full command path, e.g. "test run" / "auth whoami". */ commandPath: string; @@ -133,6 +141,7 @@ export function maybeEmitSkillNudge(ctx: SkillNudgeContext): void { } } +/** Interpret common env-var spellings for an enabled opt-out flag. */ function isTruthyEnv(v: string | undefined): boolean { if (v === undefined) return false; const s = v.trim().toLowerCase(); From a524702e75b24d563b351b9a1c6c9aa70d45796b Mon Sep 17 00:00:00 2001 From: nopp Date: Fri, 3 Jul 2026 03:44:08 +0700 Subject: [PATCH 008/102] fix: align documented Node engine floor (#84) --- CONTRIBUTING.md | 2 +- DOCUMENTATION.md | 2 +- README.md | 6 +++--- package-lock.json | 2 +- package.json | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b995b9..2110f0b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ the thread or in Discord is welcome. ## Prerequisites -- Node 20 or newer (development happens on Node 22). +- Node 20.19+, Node 22.13+, or Node 24+ (development happens on Node 22). ## Build from source diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index a0fa9a4..1806d02 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -35,7 +35,7 @@ Or run it without installing: npx @testsprite/testsprite-cli --version ``` -Requires **Node.js ≥ 20**. +Requires **Node.js 20.19+**, **22.13+**, or **24+**. Confirm the binary works **without** configuring an API key: diff --git a/README.md b/README.md index 2d90012..067a620 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ AI ships code in minutes — verifying it hasn't. `testsprite` opens your live a

npm version npm downloads - Node >= 20 + Node 20.19+, 22.13+, or 24+ License Apache 2.0 CI

@@ -54,7 +54,7 @@ If you find `testsprite` useful, a GitHub Star ⭐️ would be greatly appreciat ## Quickstart -Requires **Node.js ≥ 20**. (No global install? `npx @testsprite/testsprite-cli` works too.) +Requires **Node.js 20.19+**, **22.13+**, or **24+**. (No global install? `npx @testsprite/testsprite-cli` works too.) ```bash npm install -g @testsprite/testsprite-cli @@ -173,7 +173,7 @@ That's the point of all of this: you no longer need the biggest, most expensive ## Contributing -Contributions are welcome — the CLI is plain TypeScript/Node (≥ 20), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: +Contributions are welcome — the CLI is plain TypeScript/Node (20.19+, 22.13+, or 24+), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: ```bash git clone https://github.com/TestSprite/testsprite-cli.git diff --git a/package-lock.json b/package-lock.json index 4ea700c..456c498 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,7 +29,7 @@ "vitest": "^2.1.4" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@ampproject/remapping": { diff --git a/package.json b/package.json index 2037d26..63c5aac 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "test:e2e": "npm run build && vitest run -c vitest.e2e.config.ts" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "publishConfig": { "access": "public" From 497bf62e3e444f8e0d8e4eda15a8b9a275eb2809 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:45:06 -0500 Subject: [PATCH 009/102] fix(pagination): reject fractional pagination flags (#40) --- src/lib/pagination.test.ts | 8 ++++++++ src/lib/pagination.ts | 10 +++++----- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/lib/pagination.test.ts b/src/lib/pagination.test.ts index 33fd735..a11ea33 100644 --- a/src/lib/pagination.test.ts +++ b/src/lib/pagination.test.ts @@ -43,10 +43,18 @@ describe('validatePaginationFlags', () => { expect(() => validatePaginationFlags({ pageSize: Number.NaN })).toThrow(ApiError); }); + it('rejects fractional pageSize values', () => { + expect(() => validatePaginationFlags({ pageSize: 1.5 })).toThrow(ApiError); + }); + it('rejects maxItems=0', () => { expect(() => validatePaginationFlags({ maxItems: 0 })).toThrow(ApiError); }); + it('rejects fractional maxItems values', () => { + expect(() => validatePaginationFlags({ maxItems: 2.5 })).toThrow(ApiError); + }); + it('accepts pageSize=100 (the hard cap)', () => { expect(() => validatePaginationFlags({ pageSize: 100 })).not.toThrow(); }); diff --git a/src/lib/pagination.ts b/src/lib/pagination.ts index 59dbc12..ba63f9e 100644 --- a/src/lib/pagination.ts +++ b/src/lib/pagination.ts @@ -27,9 +27,9 @@ const DEFAULT_PAGE_SIZE = 25; * Validates and normalizes pagination flags. Per the CLI OpenAPI spec * §components.parameters.PageSize the hard cap is 100. Values above the * cap are now rejected with exit 5 rather than silently clamped, giving - * callers fast feedback that their flag value is out of range. Sub-1 / - * NaN values also throw. `maxItems` is validated but not capped (it is a - * client-side cursor, not a server parameter). + * callers fast feedback that their flag value is out of range. Fractional, + * sub-1, and NaN values also throw. `maxItems` is validated but not capped + * (it is a client-side cursor, not a server parameter). * * NOTE: `runResultHistory` previously did its own silent clamp via * `Math.min(Math.max(1, n), 100)` — that was unified to this path by the @@ -38,7 +38,7 @@ const DEFAULT_PAGE_SIZE = 25; export function validatePaginationFlags(flags: PaginationFlags): PaginationFlags { const out: PaginationFlags = { ...flags }; if (out.pageSize !== undefined) { - if (!Number.isFinite(out.pageSize) || out.pageSize < 1) { + if (!Number.isFinite(out.pageSize) || !Number.isInteger(out.pageSize) || out.pageSize < 1) { throw localValidationError( 'page-size', `must be a positive integer between 1 and ${HARD_PAGE_SIZE_CAP}`, @@ -52,7 +52,7 @@ export function validatePaginationFlags(flags: PaginationFlags): PaginationFlags } } if (out.maxItems !== undefined) { - if (!Number.isFinite(out.maxItems) || out.maxItems < 1) { + if (!Number.isFinite(out.maxItems) || !Number.isInteger(out.maxItems) || out.maxItems < 1) { throw localValidationError('maxItems', 'must be a positive integer'); } } From 9aa7ba9d6767b64234b28de044cec76bf15aee6e Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:45:13 +0400 Subject: [PATCH 010/102] fix(project): reject empty/whitespace-only --name in create and update (#36) project create/update validated --name with the action handler's if (!name) check, which a whitespace-only string passes (a non-empty string is truthy). The blank name was then sent verbatim, creating a junk-named project. The sibling est create already rejects this via the requireString whitespace guard (dogfood P1 fix #1); this aligns project create/update with that behavior. Adds 2 regression tests. --- src/commands/project.test.ts | 52 ++++++++++++++++++++++++++++++++++++ src/commands/project.ts | 12 +++++++++ 2 files changed, 64 insertions(+) diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 057296a..c928a86 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -569,6 +569,33 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + + it('rejects a whitespace-only --name with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network — validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: ' ', + targetUrl: 'https://example.com', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); }); // --------------------------------------------------------------------------- @@ -646,6 +673,31 @@ describe('runUpdate', () => { expect(fetchImpl).not.toHaveBeenCalled(); }); + it('rejects a whitespace-only --name with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + name: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('P7 — dry-run returns canned shape without network call', async () => { resetDryRunBannerForTesting(); const { credentialsPath } = makeCreds(); diff --git a/src/commands/project.ts b/src/commands/project.ts index 41abab1..74d621e 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -141,6 +141,15 @@ export async function runCreate( // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. assertIdempotencyKey(opts.idempotencyKey); + // Reject empty / whitespace-only names so a junk record never reaches the + // backend — matches the `requireString` whitespace guard `test create` uses + // (dogfood P1 fix #1). Without this, `--name " "` passes the action + // handler's `if (!name)` check (a non-empty string is truthy) and is sent + // verbatim, creating a blank-named project. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); + } + // P1-3: client-side length checks matching server limits. if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); @@ -251,6 +260,9 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } From 21dc760f1bcc898d300ba5c4d873ad807cdde7b8 Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:45:24 +0100 Subject: [PATCH 011/102] fix(cli): validate --output uniformly across all command groups (#14) Only `test` and `project` validated the global `--output` flag. The `auth`, `usage`, `agent`, and `init` command groups resolved it with `globals.output ?? 'text'`, so an unrecognised value (e.g. a typo like `--output josn`) was silently coerced to text mode instead of being rejected. A coding agent that asked for `--output json` then received a human-readable text payload and failed to parse it as JSON, with no signal as to why. Extract the validation into a shared `resolveOutputMode` helper in `lib/output.js` and route every command group's `resolveCommonOptions` through it. Invalid values now throw a typed VALIDATION_ERROR (exit 5) with an actionable message everywhere. This also unifies the error wording, which previously differed between `test` ("Flag `--output` is invalid: must be one of: json, text.") and `project` ("--output must be one of: json, text"). --- src/commands/agent.ts | 4 ++-- src/commands/auth.ts | 4 ++-- src/commands/init.ts | 4 ++-- src/commands/project.ts | 8 ++------ src/commands/test.ts | 8 ++------ src/commands/usage.ts | 4 ++-- src/lib/output.test.ts | 33 ++++++++++++++++++++++++++++++++- src/lib/output.ts | 22 ++++++++++++++++++++++ test/cli.subprocess.test.ts | 27 +++++++++++++++++++++++++++ 9 files changed, 93 insertions(+), 21 deletions(-) diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 3347453..0c6c6c1 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -4,7 +4,7 @@ import { Command } from 'commander'; import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; import { CLIError, localValidationError } from '../lib/errors.js'; import type { OutputMode } from '../lib/output.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptText } from '../lib/prompt.js'; import { type AgentTarget, @@ -852,7 +852,7 @@ function resolveCommonOptions(command: Command): CommonOptions { const globals = command.optsWithGlobals() as Partial; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/auth.ts b/src/commands/auth.ts index e3b3867..bd6dc13 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -18,7 +18,7 @@ import { import { loadConfig } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptSecret } from '../lib/prompt.js'; export interface MeResponse { @@ -328,7 +328,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/init.ts b/src/commands/init.ts index 6a432c2..7615a44 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -14,7 +14,7 @@ import { Command } from 'commander'; import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import type { AuthDeps, MeResponse } from './auth.js'; import { runConfigure, runWhoami } from './auth.js'; import type { AgentDeps, AgentFs, InstallResult } from './agent.js'; @@ -424,7 +424,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/project.ts b/src/commands/project.ts index 74d621e..2eaa416 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -9,7 +9,7 @@ import { import { ApiError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { assertNotLocal } from '../lib/target-url.js'; import { assertIdempotencyKey } from '../lib/validate.js'; import { @@ -506,13 +506,9 @@ function resolveCommonOptions(command: Command): CommonOptions { requestTimeout?: string; }; // P2-8: validate --output before allowing silent fallback to 'text'. - const rawOutput = globals.output; - if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { - throw localValidationError('--output must be one of: json, text'); - } return { profile: globals.profile ?? 'default', - output: (globals.output as OutputMode | undefined) ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/test.ts b/src/commands/test.ts index c61e84b..62a14ec 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -34,7 +34,7 @@ import { import { REQUEST_TIMEOUT_DEFAULT_MS, REQUEST_TIMEOUT_MAX_MS } from '../lib/http.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { fetchSinglePage, paginate, @@ -7809,13 +7809,9 @@ function resolveCommonOptions(command: Command): CommonOptions { // P2-8: validate --output before allowing silent fallback to 'text'. // An invalid value (e.g. `--output yaml`) must exit 5 with a clear error // rather than silently treating the request as text mode. - const rawOutput = globals.output; - if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { - throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); - } return { profile: globals.profile ?? 'default', - output: (globals.output as OutputMode | undefined) ?? 'text', + output: resolveOutputMode(globals.output), dryRun: globals.dryRun ?? false, endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, diff --git a/src/commands/usage.ts b/src/commands/usage.ts index d8a0f21..a539261 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -22,7 +22,7 @@ import { import { loadConfig } from '../lib/config.js'; import { resolvePortalBase } from '../lib/facade.js'; import type { FetchImpl } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; /** * Usage/balance response from `/me` (when the backend supplies it) or a future @@ -216,7 +216,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/lib/output.test.ts b/src/lib/output.test.ts index b572bda..bab3b53 100644 --- a/src/lib/output.test.ts +++ b/src/lib/output.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { Output, isOutputMode } from './output.js'; +import { Output, isOutputMode, resolveOutputMode } from './output.js'; +import { ApiError } from './errors.js'; describe('isOutputMode', () => { it('accepts json and text', () => { @@ -15,6 +16,36 @@ describe('isOutputMode', () => { }); }); +describe('resolveOutputMode', () => { + it('returns the mode verbatim for valid values', () => { + expect(resolveOutputMode('json')).toBe('json'); + expect(resolveOutputMode('text')).toBe('text'); + }); + + it('defaults to text when the flag is omitted (undefined)', () => { + expect(resolveOutputMode(undefined)).toBe('text'); + }); + + it('throws a typed VALIDATION_ERROR (exit 5) instead of silently falling back to text', () => { + // The footgun this guards against: an agent that asks for `--output json` + // but mistypes it would otherwise receive a text payload and fail to parse + // it as JSON with no signal. Every command group must reject, not coerce. + for (const bad of ['josn', 'yaml', 'JSON', 'Text', '']) { + let caught: unknown; + try { + resolveOutputMode(bad); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('must be one of: json, text'); + } + }); +}); + describe('Output', () => { let logSpy: ReturnType; let errorSpy: ReturnType; diff --git a/src/lib/output.ts b/src/lib/output.ts index 79b3f11..4d59cca 100644 --- a/src/lib/output.ts +++ b/src/lib/output.ts @@ -1,3 +1,5 @@ +import { localValidationError } from './errors.js'; + export type OutputMode = 'json' | 'text'; /** @@ -13,6 +15,26 @@ export function isOutputMode(value: unknown): value is OutputMode { return value === 'json' || value === 'text'; } +/** + * Resolve a raw `--output` flag value to a concrete {@link OutputMode}. + * + * `undefined` (flag omitted) resolves to the default `'text'`. Any other + * value that is not `'json'` or `'text'` throws a typed VALIDATION_ERROR + * (exit 5) with an actionable message. + * + * The alternative — silently falling back to `'text'` — is a footgun for the + * CLI's primary consumer (coding agents): a caller that asks for + * `--output json` but mistypes it (`--output josn`) would otherwise receive a + * human-readable text payload and fail to parse it as JSON, with no signal as + * to why. Every command group routes its global-option resolution through this + * helper so the validation is uniform. + */ +export function resolveOutputMode(raw: unknown): OutputMode { + if (raw === undefined) return 'text'; + if (isOutputMode(raw)) return raw; + throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); +} + export interface OutputStreams { /** * Line-oriented stdout writer. Each call is one logical line; the diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 21d7cfe..0f05bf7 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -530,6 +530,33 @@ describe('malformed --endpoint-url is rejected (exit 5), not retried as a networ }, 30_000); }); +describe('invalid --output is rejected uniformly (exit 5)', () => { + // Regression: previously only `test` and `project` validated `--output`; + // `auth`, `usage`, `agent`, and `init` silently coerced an unknown value to + // text mode. An agent that asked for `--output json` but mistyped it then + // received a text payload it could not parse, with no signal as to why. Every + // command group now routes through resolveOutputMode (exit 5 on bad input). + + // Note: when `--output` itself is the invalid value, the requested mode is + // unusable for the error envelope, so it is rendered in text mode (the catch + // block in index.ts falls back to text for an unrecognised --output). + + it('agent list --output josn exits 5 with an actionable message (offline command)', async () => { + const result = await runCli(['--output', 'josn', 'agent', 'list'], {}); + expect(result.exitCode).toBe(5); + expect(result.stderr).toContain('must be one of: json, text'); + }, 30_000); + + it('auth status --output yaml exits 5 before any network call', async () => { + const result = await runCli(['--output', 'yaml', 'auth', 'status'], { + TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_URL: baseUrl, + }); + expect(result.exitCode).toBe(5); + expect(result.stderr).toContain('must be one of: json, text'); + }, 30_000); +}); + describe('a malformed --profile is rejected (exit 5), not silently corrupting credentials', () => { // A profile name becomes an INI section header (`[name]`). `prod]` would // serialise to `[prod]]`, which the parser cannot read back — `setup` would From 000c196375883e2df773a55c53df79ff7f67412f Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:45:32 +0400 Subject: [PATCH 012/102] ci: add Node 20 to test and build matrix (#133) --- .github/workflows/ci.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 890f9c7..ad3ef6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,15 +48,18 @@ jobs: - run: npm run typecheck test: - name: Unit Tests + name: Unit Tests (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: - node-version: 22 + node-version: ${{ matrix.node-version }} cache: 'npm' - run: npm ci @@ -65,15 +68,18 @@ jobs: CI: true build: - name: Build + name: Build (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: - node-version: 22 + node-version: ${{ matrix.node-version }} cache: 'npm' - run: npm ci From 5e0f7c4db423885eb2b442e9627643bd6f8c6199 Mon Sep 17 00:00:00 2001 From: Sahil Rakhaiya <144577420+SahilRakhaiya05@users.noreply.github.com> Date: Fri, 3 Jul 2026 02:15:39 +0530 Subject: [PATCH 013/102] fix(test): reject empty code get --out and strip code-file BOM (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(test): reject empty code get --out and strip code-file BOM When test code get --out receives an empty inline body, closeOutputFile left a zero-byte file with exit 0 — scripts and agents treated that as a successful download. Abort the sink, unlink any artifact, and surface VALIDATION_ERROR instead. Plan/steps JSON already strip a leading UTF-8 BOM from PowerShell 5.1 files; apply the same strip to --code-file reads so uploaded test source is not corrupted by an invisible U+FEFF prefix. * fix(test): rebase onto v0.2.0 and harden empty-code --out cleanup - Resolve rebase conflicts: keep atomic temp-file --out writes from main while rejecting empty inline code with VALIDATION_ERROR (exit 5) - abortOutputFile: wait for stream close before unlinking tmpPath - BOM test: use .py code-file (assertPythonCodeFile from v0.2.0) - CI: build before test + fileParallelism false (dist/ race flake) --- .github/workflows/ci.yml | 1 + src/commands/test.test.ts | 80 ++++++++++++++++++++++++++++++++++++--- src/commands/test.ts | 34 ++++++++++++++--- vitest.config.ts | 3 ++ 4 files changed, 107 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad3ef6e..edde076 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,6 +63,7 @@ jobs: cache: 'npm' - run: npm ci + - run: npm run build - run: npm test env: CI: true diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 4d205e0..49127bd 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -1591,22 +1591,66 @@ describe('runCodeGet', () => { expect(leftovers).toEqual([]); }); - // Regression: the "no code generated yet" branch writes nothing but - // previously still closed (and thus truncated) the opened file. + it('--out (text mode) rejects empty inline code with VALIDATION_ERROR and leaves no artifact', async () => { + const { credentialsPath } = makeCreds(); + const emptyCode: CliTestCode = { ...TEST_CODE_INLINE, code: '' }; + const fetchImpl = makeFetch(() => ({ body: emptyCode })); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-empty-out-')); + const target = join(dir, 'empty.ts'); + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: target, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'out' }), + }); + expect(existsSync(target)).toBe(false); + }); + + // Regression: empty inline code with --out must reject (exit 5) without + // truncating or replacing a pre-existing destination file. it('--out: "no code generated yet" leaves a pre-existing file untouched', async () => { const { credentialsPath } = makeCreds(); const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-empty-')); const target = join(dir, 'existing.ts'); writeFileSync(target, 'PRE-EXISTING CONTENT', 'utf8'); const fetchImpl = makeFetch(() => ({ body: { ...TEST_CODE_INLINE, code: '' } })); - await runCodeGet( - { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: target }, - { credentialsPath, fetchImpl, stderr: () => undefined }, - ); + await expect( + runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: target }, + { credentialsPath, fetchImpl, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'out' }), + }); expect(readFileSync(target, 'utf-8')).toBe('PRE-EXISTING CONTENT'); const leftovers = readdirSync(dir).filter(f => f !== 'existing.ts'); expect(leftovers).toEqual([]); }); + + it('text mode without --out still hints on stderr when inline code is empty', async () => { + const { credentialsPath } = makeCreds(); + const emptyCode: CliTestCode = { ...TEST_CODE_INLINE, code: '' }; + const fetchImpl = makeFetch(() => ({ body: emptyCode })); + const stderr: string[] = []; + const got = await runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stderr: line => stderr.push(line) }, + ); + expect(got.code).toBe(''); + expect(stderr.join('\n')).toContain('no code generated yet'); + }); }); describe('runCodePut', () => { @@ -1661,6 +1705,30 @@ describe('runCodePut', () => { expect(sent.headers.get('content-type')).toBe('application/json'); }); + it('strips a UTF-8 BOM from --code-file before uploading (Windows PowerShell 5.1 default)', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p4-bom-')); + const codeFile = join(dir, 'updated.py'); + writeFileSync(codeFile, '\uFEFF' + 'updated body', 'utf8'); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenBody).toEqual({ code: 'updated body' }); + }); + it('forwards --language in the body when set', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('print("hi")'); diff --git a/src/commands/test.ts b/src/commands/test.ts index 62a14ec..7e6c94f 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -911,7 +911,7 @@ function readCodeFileGuarded(path: string): string { function readCodeFile(path: string): string { try { - return readFileSync(resolveAbsolute(path), 'utf8'); + return stripBom(readFileSync(resolveAbsolute(path), 'utf8')); } catch (err) { const code = (err as NodeJS.ErrnoException).code; if (code === 'ENOENT') { @@ -3292,7 +3292,7 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro return code; } - const fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; + let fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; const out = fileSink ? makeFileOutput(opts.output, fileSink) : makeOutput(opts.output, deps); const client = makeClient(opts, deps); @@ -3317,9 +3317,20 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro } else if (code.code === '' || code.code === null) { // P2-10: draft test with no code yet — empty body would produce // silent empty stdout. Print a friendly hint to stderr instead so - // the operator knows what happened, and keep exit 0. Nothing was - // written, so the temp file is discarded below without touching - // a pre-existing `--out` file. + // the operator knows what happened, and keep exit 0 when no `--out`. + // + // With `--out`, refuse to leave a zero-byte artifact behind: agents + // and scripts that check file size would otherwise treat exit 0 as + // a successful download. Discard the temp sink without touching a + // pre-existing destination file. + if (fileSink) { + await abortOutputFile(fileSink); + fileSink = null; + throw localValidationError( + 'out', + 'test has no generated code yet — run the test first (refusing to write an empty --out file)', + ); + } const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); stderrFn('(no code generated yet — run the test first)'); } else { @@ -7997,6 +8008,19 @@ async function closeOutputFile(sink: FileSink, commit: boolean): Promise { await rename(sink.tmpPath, sink.path); } +/** Tear down an opened `--out` sink without leaving a zero-byte artifact. */ +async function abortOutputFile(sink: FileSink): Promise { + await new Promise(resolve => { + if (sink.stream.destroyed) { + resolve(); + return; + } + sink.stream.once('close', () => resolve()); + sink.stream.destroy(); + }); + await unlink(sink.tmpPath).catch(() => undefined); +} + /** A presigned `code` body is any `https://` URL — never anything else. */ export function isPresignedCodeUrl(code: string): boolean { return code.startsWith('https://'); diff --git a/vitest.config.ts b/vitest.config.ts index 18ea00f..add8f0e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,9 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], + // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel + // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). + fileParallelism: false, coverage: { provider: 'v8', reporter: ['text', 'text-summary', 'json-summary', 'html'], From dfdee5a3e051987c0ce090512c6d197532f4deb1 Mon Sep 17 00:00:00 2001 From: Chara Date: Thu, 2 Jul 2026 22:45:45 +0200 Subject: [PATCH 014/102] fix(prompt): preserve buffered input between prompts (#118) Co-authored-by: cmdr-chara <249489759+cmdr-chara@users.noreply.github.com> --- src/lib/prompt.test.ts | 35 ++++++++++++++++++++++++++++ src/lib/prompt.ts | 53 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/src/lib/prompt.test.ts b/src/lib/prompt.test.ts index a552882..45275e3 100644 --- a/src/lib/prompt.test.ts +++ b/src/lib/prompt.test.ts @@ -39,6 +39,31 @@ describe('promptText', () => { expect(await promptText('? ', { input, output })).toBe('ok'); }); + it('preserves buffered answers for sequential prompts on the same stream', async () => { + const input = Readable.from(['first\nsecond\nthird\n']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + await expect(promptText('Three: ', { input, output })).resolves.toBe('third'); + }); + + it('uses buffered tail input at EOF for a following prompt', async () => { + const input = Readable.from(['first\nsecond']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + }); + + it('preserves buffered CRLF answers for sequential prompts', async () => { + const input = Readable.from(['first\r\nsecond\r\n']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + }); + it('returns the buffered input on stream end without newline', async () => { const input = Readable.from(['eof-no-newline']); const output = new CaptureStream(); @@ -70,6 +95,16 @@ describe('promptSecret (non-TTY behavior)', () => { expect(await promptSecret('? ', { input, output })).toBe('abd'); }); + it('preserves buffered secret answers for sequential prompts', async () => { + const input = Readable.from(['sk-one\nsk-two\n']); + const output = new CaptureStream(); + + await expect(promptSecret('First key: ', { input, output })).resolves.toBe('sk-one'); + await expect(promptSecret('Second key: ', { input, output })).resolves.toBe('sk-two'); + expect(output.text()).not.toContain('sk-one'); + expect(output.text()).not.toContain('sk-two'); + }); + it('rejects on Ctrl-C input', async () => { const ETX = String.fromCharCode(0x03); const input = Readable.from([`abc${ETX}`]); diff --git a/src/lib/prompt.ts b/src/lib/prompt.ts index 96fca29..3561b0a 100644 --- a/src/lib/prompt.ts +++ b/src/lib/prompt.ts @@ -11,6 +11,8 @@ interface RawModeCapable { resume?: () => unknown; } +const pendingPromptInput = new WeakMap(); + export async function promptText(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; const output = streams.output ?? process.stdout; @@ -41,18 +43,29 @@ function readLine( return new Promise((resolve, reject) => { let buffer = ''; let resolved = false; + let listening = false; const onData = (chunk: Buffer | string): void => { const str = typeof chunk === 'string' ? chunk : chunk.toString('utf-8'); - for (const ch of str) { - const code = ch.charCodeAt(0); + processText(str); + }; + + const processText = (str: string): void => { + for (let i = 0; i < str.length; i += 1) { + const code = str.charCodeAt(i); // Enter (CR or LF) if (code === 13 || code === 10) { + let nextIndex = i + 1; + if (code === 13 && str.charCodeAt(nextIndex) === 10) { + nextIndex += 1; + } + savePendingInput(str.slice(nextIndex)); finish(); return; } // Ctrl-C if (code === 3) { + savePendingInput(''); cleanup(); output.write('\n'); if (!resolved) { @@ -71,7 +84,7 @@ function readLine( } // Drop other control chars if (code < 32) continue; - buffer += ch; + buffer += str[i]; if (mask) output.write('*'); } }; @@ -89,9 +102,12 @@ function readLine( }; const cleanup = (): void => { - input.off('data', onData); - input.off('end', onEnd); - input.off('error', onError); + if (listening) { + input.off('data', onData); + input.off('end', onEnd); + input.off('error', onError); + listening = false; + } const pausable = input as { pause?: () => unknown }; if (typeof pausable.pause === 'function') pausable.pause(); }; @@ -105,12 +121,37 @@ function readLine( } }; + const savePendingInput = (text: string): void => { + if (text.length > 0) { + pendingPromptInput.set(input, text); + } else { + pendingPromptInput.delete(input); + } + }; + + const pending = pendingPromptInput.get(input); + if (pending !== undefined) { + pendingPromptInput.delete(input); + processText(pending); + if (resolved) return; + if (isInputEnded(input)) { + finish(); + return; + } + } + input.on('data', onData); input.on('end', onEnd); input.on('error', onError); + listening = true; const resumable = input as { resume?: () => unknown }; if (typeof resumable.resume === 'function') resumable.resume(); }); } +function isInputEnded(input: NodeJS.ReadableStream): boolean { + const state = input as { readableEnded?: boolean; destroyed?: boolean; closed?: boolean }; + return state.readableEnded === true || state.destroyed === true || state.closed === true; +} + export type { Writable }; From be9784a639931fb602b00cb00624793db35f6557 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EB=8F=84=ED=9B=88?= Date: Fri, 3 Jul 2026 05:45:52 +0900 Subject: [PATCH 015/102] fix(test): emit auto-minted idempotency-key under --output json in rerun and run --all (#128) test run, test create, create-batch, plan put, code put, update, and delete all print the auto-minted idempotency key to stderr under --output json (as well as --verbose / --debug) so JSON-mode automation can capture the key and replay a retry safely. test rerun and test run --all minted a key but only echoed it under --debug / --verbose, so CI flows using --output json silently lost it. Align both paths with the shared guard used by every other minting site, and cover the JSON-mode emission (and the text-mode silence) with regression tests. Co-authored-by: ahndohun <19940813+ahndohun@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- DOCUMENTATION.md | 4 +- src/commands/test.rerun.spec.ts | 68 +++++++++++++++++++++++++++++++++ src/commands/test.run.spec.ts | 33 ++++++++++++++++ src/commands/test.ts | 10 +---- 4 files changed, 105 insertions(+), 10 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 1806d02..700671b 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -346,7 +346,7 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ testsprite test run test_xxxxxxxx --dry-run --output json ``` -`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr at `--verbose`); pass `--idempotency-key ` to control it explicitly. +`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. #### `testsprite test rerun [test-id...]` @@ -376,7 +376,7 @@ Flags: - `--auto-heal` / `--no-auto-heal` — frontend AI heal-on-drift, **on by default** for FE reruns; opt out with `--no-auto-heal`. Verbatim-replay passes are free; a heal engage costs a small amount of credit. Ignored for backend tests. - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. -- `--idempotency-key ` — auto-minted when omitted. +- `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 86c8e89..acd5b02 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -1637,6 +1637,74 @@ describe('--idempotency-key passthrough', () => { expect(receivedKey).toBe('my-custom-key-abc'); }); + + it('emits the auto-minted idempotency-key on stderr in JSON output mode (parity with test run)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + + it('does NOT emit an idempotency-key line in default text mode', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'text', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines.some(l => l.includes('idempotency-key:'))).toBe(false); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 7e2b55e..2de9f9c 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -3470,6 +3470,39 @@ describe('dashboardUrl on run completion', () => { ); }); + it('run --all: emits the auto-minted idempotency-key on stderr in JSON output mode (parity with test run)', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_f_01', enqueuedAt: '2026-06-10T10:00:00.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const stderrLines: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: batchResp })), + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + it('run --all --wait (prod endpoint): summary items carry dashboardUrl + stderr Dashboard line', async () => { const { credentialsPath } = makeCreds('sk-user-test', PROD_API); const batchResp: BatchRunFreshResponse = { diff --git a/src/commands/test.ts b/src/commands/test.ts index 7e6c94f..a9ff9e3 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5150,12 +5150,9 @@ export async function runTestRunAll( }; const idempotencyKey = opts.idempotencyKey ?? `cli-batch-run-fresh-${randomUUID()}`; - if (opts.idempotencyKey === undefined && opts.debug) { + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderrFn(`idempotency-key: ${idempotencyKey}`); } - if (opts.idempotencyKey === undefined && opts.verbose) { - stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); - } // Resolve testIds: fetch all BE tests in the project, apply --filter. let testIds: string[] | undefined; @@ -5718,12 +5715,9 @@ export async function runTestRerun( // slow rerun trigger / long-poll under load isn't cut at the 120s default. const client = makeClient({ ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs(opts) }, deps); const idempotencyKey = opts.idempotencyKey ?? `cli-rerun-${randomUUID()}`; - if (opts.idempotencyKey === undefined && opts.debug) { + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderrFn(`idempotency-key: ${idempotencyKey}`); } - if (opts.idempotencyKey === undefined && opts.verbose) { - stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); - } // ------------------------------------------------------------------------- // Single rerun path From 03c4f11ae8ef14251299863d521e49d1e4f10a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EB=8F=84=ED=9B=88?= Date: Fri, 3 Jul 2026 05:45:59 +0900 Subject: [PATCH 016/102] fix(agent): apply symlink fail-close guard to own-file --dry-run installs (#129) The codex managed-section branch already runs inspectTargetPath during --dry-run ([P2]) so a planted symlink is refused the same way the real install refuses it. The own-file targets (claude, cursor, cline, antigravity) skipped that guard in dry-run and reported the write as successful, so 'agent install --dry-run' could claim success for an install that would actually exit 5. Run the same guard in the own-file dry-run branch and cover both the symlinked-target and symlinked-parent cases with regression tests. Co-authored-by: ahndohun <19940813+ahndohun@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- src/commands/agent.test.ts | 44 ++++++++++++++++++++++++++++++++++++++ src/commands/agent.ts | 11 ++++++++++ 2 files changed, 55 insertions(+) diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 523c5fc..9bdbdf1 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -1311,6 +1311,50 @@ describe('runInstall — symlink safety', () => { expect(writeCalls.length).toBe(0); // never wrote a .bak nor through the link }); + it('dry-run: refuses (exit 5) when the target file is a symlink (parity with real install)', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + // Same planted SKILL.md symlink as the real-install case above: dry-run + // must report the same refusal the real install would, not a success. + seedSymlink(path.resolve(CWD, TARGETS.claude.path)); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false, dryRun: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + + it('dry-run: refuses (exit 5) when a parent path component is a symlink', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + seedSymlink(path.resolve(CWD, '.claude')); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false, dryRun: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + it('does not write through a symlinked .bak slot — backs up to a numbered slot', async () => { const { store, fs: agentFs, seedFile, seedSymlink } = makeMemFs(); const abs = path.resolve(CWD, TARGETS.claude.path); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 0c6c6c1..ebde317 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -638,6 +638,17 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr const content = renderForTarget(t, skill, bodyForSkill(skill)).content; if (opts.dryRun) { + // Apply the SAME symlink fail-close guard as the real install path + // below (the codex managed-section branch already does this). Without + // it, dry-run reports success for a planted symlink that the real + // install would refuse with exit 5. + const dryRunSt = await inspectTargetPath(agentFs, root, relPath); + if (dryRunSt !== null && !dryRunSt.isFile) { + throw new CLIError( + `${relPath} exists but is not a regular file — remove it and re-run.`, + 5, + ); + } const bytes = Buffer.byteLength(content, 'utf8'); dryRunLines.push({ abs, bytes, note: '' }); results.push({ target: t, path: relPath, action: 'dry-run', skills: [skill] }); From 03251d744a0d5324eac177f2c42a1eb7e7f3277b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=95=88=EB=8F=84=ED=9B=88?= Date: Fri, 3 Jul 2026 05:46:05 +0900 Subject: [PATCH 017/102] fix(test): stop run --all --wait from polling queued runs past the shared deadline (#130) runTestRunAll computes each member's poll budget as Math.max(1, ceil(batchDeadlineMs - now)), so a run whose turn arrives after the shared --timeout deadline still gets a fresh >=1s poll. With --max-concurrency bounding the fan-out, a batch could overshoot the documented shared deadline and report a late 'passed' for a member that should have been reported as 'timeout' (exit 7). Guard the poll helper the same way the create-batch --run --wait path already does: if the shared deadline is exhausted before a member's poll starts, return the existing timeout-shaped member result without calling pollRunUntilTerminal. Regression test drives the clock past the deadline during the first member's poll and asserts the second member is never polled and reports 'timeout'. Co-authored-by: ahndohun <19940813+ahndohun@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- src/commands/test.run.spec.ts | 55 +++++++++++++++++++++++++++++++++++ src/commands/test.ts | 15 +++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 2de9f9c..0a10df5 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -2487,6 +2487,61 @@ describe('runTestRunAll — batch fresh run', () => { expect(payload.accepted.every(r => r.status === 'passed')).toBe(true); }); + it('run --all --wait: does not start a fresh poll for a queued run after the shared deadline expired', async () => { + const { credentialsPath } = makeCreds(); + const baseNow = new Date('2026-06-09T10:00:00.000Z').getTime(); + let nowMs = baseNow; + const runFetches: string[] = []; + const stdoutLines: string[] = []; + let caughtError: unknown; + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => nowMs); + + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'POST') return { body: BATCH_FRESH_RESP }; + + const runId = url.split('/runs/')[1]?.split('?')[0] ?? 'run_unknown'; + runFetches.push(runId); + if (runId === 'run_fresh_01') { + nowMs = baseNow + 2000; + return { body: makePassedRun(runId, 'test_be_01') }; + } + return { body: makePassedRun(runId, 'test_be_02') }; + }); + + try { + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 1, + maxConcurrency: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + } catch (err) { + caughtError = err; + } finally { + nowSpy.mockRestore(); + } + + const payload = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ runId: string; status: string }>; + }; + expect(runFetches).toEqual(['run_fresh_01']); + expect(payload.accepted.find(r => r.runId === 'run_fresh_02')?.status).toBe('timeout'); + expect((caughtError as { exitCode?: number } | undefined)?.exitCode).toBe(7); + }); + it('--wait with a failed run → exit 1', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch((url, init) => { diff --git a/src/commands/test.ts b/src/commands/test.ts index a9ff9e3..8489a01 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5432,7 +5432,20 @@ export async function runTestRunAll( async function pollFreshAccepted(entry: BatchRunFreshAccepted): Promise { const runId = entry.runId; - const remainingSeconds = Math.max(1, Math.ceil((batchDeadlineMs - Date.now()) / 1000)); + const remainingMs = batchDeadlineMs - Date.now(); + if (remainingMs <= 0) { + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${opts.timeoutSeconds}s`, + exitCode: 7, + }, + }; + } + const remainingSeconds = Math.ceil(remainingMs / 1000); const resolveAlternate = makeBackendWaitFallback({ client, resolveTestId: () => entry.testId, From 7b7d80db786068b50e4f44b7f200ed3556e31a1f Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:46:12 +0400 Subject: [PATCH 018/102] fix(credentials): strip CR/LF from values to prevent INI injection (#131) --- src/lib/credentials.test.ts | 32 ++++++++++++++++++++++++++++++++ src/lib/credentials.ts | 10 +++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index ac038f3..896d057 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -88,6 +88,38 @@ describe('serializeCredentials', () => { expect(text).toContain('api_key = sk'); expect(text).not.toContain('api_url'); }); + + it('strips newline characters from values to prevent INI injection', () => { + // A malicious apiUrl with embedded newlines could inject new key-value + // pairs or section headers into the credentials file. The serializer + // must strip \n and \r so the written file has exactly one value per + // field and no injected content parsed as separate keys/sections. + const malicious = 'https://evil.com\napi_key = sk-HIJACKED\n[admin]\napi_key = sk-admin'; + const text = serializeCredentials({ default: { apiKey: 'sk-real', apiUrl: malicious } }); + // The output must NOT contain a standalone [admin] section header + // (it would be on its own line if injection succeeded) + const lines = text.split('\n'); + // Only one section header exists: [default] + const sectionHeaders = lines.filter(l => /^\[.+\]$/.test(l.trim())); + expect(sectionHeaders).toEqual(['[default]']); + // Only one api_key line exists (the real one, not an injected duplicate) + const apiKeyLines = lines.filter(l => l.trim().startsWith('api_key')); + expect(apiKeyLines).toHaveLength(1); + expect(apiKeyLines[0]).toContain('sk-real'); + // Round-trip: reading back must return only the real key, not the injected one + const parsed = parseCredentials(text); + expect(parsed['default']?.apiKey).toBe('sk-real'); + expect(parsed['admin']).toBeUndefined(); + }); + + it('strips \\r\\n (CRLF) injection from values', () => { + const text = serializeCredentials({ default: { apiUrl: 'https://x.com\r\napi_key = pwned' } }); + const parsed = parseCredentials(text); + // The injected api_key must NOT be parsed as a real key + expect(parsed['default']?.apiKey).toBeUndefined(); + // The api_url value is on one line (newlines stripped) + expect(parsed['default']?.apiUrl).toContain('https://x.com'); + }); }); describe('readCredentialsFile / readProfile', () => { diff --git a/src/lib/credentials.ts b/src/lib/credentials.ts index c885c66..21e2ac0 100644 --- a/src/lib/credentials.ts +++ b/src/lib/credentials.ts @@ -116,7 +116,15 @@ export function serializeCredentials(file: CredentialsFile): string { for (const field of fields) { const value = entry[field]; if (value === undefined || value === '') continue; - lines.push(`${FIELD_TO_FILE_KEY[field]} = ${value}`); + // Guard against INI injection: a value containing newline characters + // would be serialized across multiple lines, allowing an attacker to + // inject arbitrary key-value pairs (or new section headers) into the + // credentials file. A valid API key or URL never contains \n or \r. + // Strip them so a compromised env var or MITM'd backend response + // cannot override the stored api_key on subsequent reads. + const sanitized = value.replace(/[\r\n]/g, ''); + if (sanitized === '') continue; + lines.push(`${FIELD_TO_FILE_KEY[field]} = ${sanitized}`); } lines.push(''); } From 76fff29c91c95f0900ba3ea2dd14f75e69396042 Mon Sep 17 00:00:00 2001 From: Resque Date: Fri, 3 Jul 2026 00:48:23 +0400 Subject: [PATCH 019/102] fix(target-url): treat trailing-dot hostnames as loopback in SSRF guard (#37) assertNotLocal lowercased the hostname but did not strip a trailing dot, so http://localhost. (the FQDN form of localhost, RFC 6761) and http://localhost%2e bypassed the host === 'localhost' loopback check. IP literals are already dot-normalized by the WHATWG URL parser, so only named hosts were affected. Strips one trailing dot before the comparison. Adds 4 regression tests (3 blocked variants + 1 public-FQDN no-false-positive). From 8da07fb4b11bf003e9bc62c5d479424c663350b2 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:48:27 -0500 Subject: [PATCH 020/102] fix(project): avoid password file reads in dry-run update (#54) --- src/commands/project.test.ts | 27 ++++++++++++++++++++ src/commands/project.ts | 48 +++++++++++++++++++++++------------- test/cli.subprocess.test.ts | 16 ++++++++++++ 3 files changed, 74 insertions(+), 17 deletions(-) diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index c928a86..1fcbd99 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -729,6 +729,33 @@ describe('runUpdate', () => { expect(err).toContain(DRY_RUN_BANNER); }); + it('P7 — dry-run with --password-file does not read the filesystem', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'proj_dry', + passwordFile: '/tmp/definitely-not-here-testsprite', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(result.id).toBe('proj_dry'); + expect(result.updatedFields).toContain('password'); + }); + it('P7 — renders text mode with updatedFields and updatedAt', async () => { const { credentialsPath } = makeCreds(); const updateResponse: CliUpdateProjectResponse = { diff --git a/src/commands/project.ts b/src/commands/project.ts index 2eaa416..fd34b47 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -270,27 +270,24 @@ export async function runUpdate( throw localValidationError('--description must be at most 2000 characters'); } - // Resolve password - let password = opts.password; - if (password === undefined && opts.passwordFile !== undefined) { - password = readFileSync(opts.passwordFile, 'utf8').trim(); - } - // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); } - const mutableFields: Record = { - name: opts.name, - targetUrl: opts.targetUrl, - username: opts.username, - password, - description: opts.description, - instruction: opts.instruction, + const passwordSupplied = opts.password !== undefined || opts.passwordFile !== undefined; + const mutableFields: Record = { + name: opts.name !== undefined, + targetUrl: opts.targetUrl !== undefined, + username: opts.username !== undefined, + password: passwordSupplied, + description: opts.description !== undefined, + instruction: opts.instruction !== undefined, }; - const presentFields = Object.entries(mutableFields).filter(([, v]) => v !== undefined); - if (presentFields.length === 0) { + const presentFieldNames = Object.entries(mutableFields) + .filter(([, present]) => present) + .map(([field]) => field); + if (presentFieldNames.length === 0) { throw localValidationError( 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', ); @@ -308,19 +305,36 @@ export async function runUpdate( } const sample: CliUpdateProjectResponse = { id: opts.projectId, - updatedFields: presentFields.map(([k]) => k), + updatedFields: presentFieldNames, updatedAt: '2026-05-16T00:00:00.000Z', }; out.print(sample, data => renderUpdateText(data as CliUpdateProjectResponse)); return sample; } + // Resolve password only on the real path. Dry-run must not touch the + // filesystem, even when --password-file is present. + let password = opts.password; + if (password === undefined && opts.passwordFile !== undefined) { + password = readFileSync(opts.passwordFile, 'utf8').trim(); + } + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-update-${randomUUID()}`; if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderr(`idempotency-key: ${idempotencyKey}`); } - const body = Object.fromEntries(presentFields) as Record; + const bodyFields: Record = { + name: opts.name, + targetUrl: opts.targetUrl, + username: opts.username, + password, + description: opts.description, + instruction: opts.instruction, + }; + const body = Object.fromEntries( + Object.entries(bodyFields).filter(([, v]) => v !== undefined), + ) as Record; const client = makeClient(opts, deps); const updated = await client.patch( `/projects/${encodeURIComponent(opts.projectId)}`, diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 0f05bf7..95d4a35 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -923,6 +923,22 @@ describe('--dry-run subprocess smoke', () => { expect(parsed.id).toBeTruthy(); }, 30_000); + it('project update --dry-run does not read a missing --password-file', async () => { + const result = await runCli([ + 'project', + 'update', + 'proj_anything', + '--password-file', + '/tmp/definitely-not-here-testsprite', + '--dry-run', + '--output', + 'json', + ]); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout) as { updatedFields: string[] }; + expect(parsed.updatedFields).toContain('password'); + }, 30_000); + it('test list --dry-run returns canned TestList', async () => { const result = await runCli([ 'test', From e53257d00887b636a05f0aad09828c13eb28e83c Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Thu, 2 Jul 2026 21:48:31 +0100 Subject: [PATCH 021/102] fix(cli): validate --request-timeout flag and de-duplicate its parser (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `parseRequestTimeoutFlag` was copy-pasted byte-for-byte into five command files (auth, project, usage, init, test). Every copy silently returned `undefined` for an invalid value, so an explicit `--request-timeout 30s` (a natural "30 seconds" typo) resolved to undefined and the command ran with the default 120s deadline — the operator believed they had set a timeout but had not, with no signal. Hoist a single definition into client-factory.ts (next to resolveRequestTimeoutMs and the REQUEST_TIMEOUT_* constants) and make the flag strict: a non-numeric, zero, or negative value now throws a typed VALIDATION_ERROR (exit 5), consistent with every other validated flag (--page-size, --output, --type). Positive out-of-range values are still accepted and clamped by resolveRequestTimeoutMs, and the TESTSPRITE_REQUEST_TIMEOUT_MS env-var path stays lenient by design (a stray global env var should not hard-fail every command). Adds unit coverage for parseRequestTimeoutFlag and a subprocess regression that `--request-timeout 30s` exits 5 instead of falling back to 120s. --- src/commands/auth.ts | 14 +----------- src/commands/init.ts | 12 ++++------ src/commands/project.ts | 13 +---------- src/commands/test.ts | 13 +---------- src/commands/usage.ts | 8 +------ src/lib/client-factory.test.ts | 40 ++++++++++++++++++++++++++++++++++ src/lib/client-factory.ts | 37 +++++++++++++++++++++++++++++++ test/cli.subprocess.test.ts | 18 +++++++++++++++ 8 files changed, 103 insertions(+), 52 deletions(-) diff --git a/src/commands/auth.ts b/src/commands/auth.ts index bd6dc13..5c1b482 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -2,6 +2,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import type { ErrorCode } from '../lib/errors.js'; @@ -337,19 +338,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values — the - * factory applies the same clamp so there is no double-clamp risk. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - function makeOutput(mode: OutputMode, deps: AuthDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } diff --git a/src/commands/init.ts b/src/commands/init.ts index 7615a44..e01215a 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -11,7 +11,10 @@ */ import { Command } from 'commander'; -import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; +import { + parseRequestTimeoutFlag, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; @@ -433,13 +436,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); -} - const SETUP_DESCRIPTION = 'Set up TestSprite: configure your API key and install the TestSprite agent skills for your coding agent'; diff --git a/src/commands/project.ts b/src/commands/project.ts index fd34b47..2af8664 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -4,6 +4,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { ApiError } from '../lib/errors.js'; @@ -531,18 +532,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - function makeClient(opts: CommonOptions, deps: ProjectDeps): HttpClient { return makeHttpClient(opts, { env: deps.env, diff --git a/src/commands/test.ts b/src/commands/test.ts index 8489a01..c6e6c31 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -6,6 +6,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { @@ -7838,18 +7839,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - /** D4: headroom added on top of `--timeout` when deriving the per-request window under `--wait`. */ const WAIT_REQUEST_TIMEOUT_CUSHION_MS = 5_000; diff --git a/src/commands/usage.ts b/src/commands/usage.ts index a539261..d479cab 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -17,6 +17,7 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { loadConfig } from '../lib/config.js'; @@ -225,13 +226,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); -} - function makeOutput(mode: OutputMode, deps: UsageDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index f9fdc44..61ffe4b 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -5,6 +5,7 @@ import { assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, resetDryRunBannerForTesting, resolveRequestTimeoutMs, } from './client-factory.js'; @@ -213,6 +214,45 @@ describe('resolveRequestTimeoutMs', () => { }); }); +// --------------------------------------------------------------------------- +// parseRequestTimeoutFlag — strict flag parsing (seconds → ms) +// --------------------------------------------------------------------------- + +describe('parseRequestTimeoutFlag', () => { + it('returns undefined when the flag is omitted (factory falls back to env/default)', () => { + expect(parseRequestTimeoutFlag(undefined)).toBeUndefined(); + }); + + it('converts a positive number of seconds to milliseconds', () => { + expect(parseRequestTimeoutFlag('30')).toBe(30_000); + expect(parseRequestTimeoutFlag('1')).toBe(1_000); + expect(parseRequestTimeoutFlag('2.5')).toBe(2_500); + }); + + it('does NOT reject positive out-of-range values — resolveRequestTimeoutMs clamps them', () => { + // 700s is above the 600s cap, but parsing succeeds; the clamp lives in + // resolveRequestTimeoutMs so a large script-supplied value still works. + expect(parseRequestTimeoutFlag('700')).toBe(700_000); + }); + + it.each(['abc', '30s', '0', '-5', 'NaN', 'Infinity', ''])( + 'throws a VALIDATION_ERROR (exit 5) on the invalid flag value %j', + bad => { + let caught: unknown; + try { + parseRequestTimeoutFlag(bad); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('request-timeout'); + }, + ); +}); + // --------------------------------------------------------------------------- // makeHttpClient — requestTimeoutMs propagation // --------------------------------------------------------------------------- diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index fa1f8dc..8f0cc9b 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -180,6 +180,43 @@ export function assertValidEndpointUrl(rawUrl: string): void { } } +/** + * Parse the `--request-timeout ` flag value into milliseconds. + * + * Returns `undefined` when the flag was omitted (the factory then falls back to + * the `TESTSPRITE_REQUEST_TIMEOUT_MS` env var, else the 120s default). + * + * A supplied-but-invalid value (non-numeric, zero, or negative) throws a typed + * VALIDATION_ERROR (exit 5) rather than being silently dropped. An explicit + * `--request-timeout 30s` typo previously resolved to `undefined` and the + * command ran with the default 120s deadline — the operator believed they had + * set a timeout but had not, with no signal. Failing loudly here is consistent + * with every other validated flag (`--page-size`, `--output`, `--type`). + * + * Out-of-range but positive values are intentionally NOT rejected — they flow + * through to {@link resolveRequestTimeoutMs}, which clamps to + * `[REQUEST_TIMEOUT_MIN_MS, REQUEST_TIMEOUT_MAX_MS]`. The env-var path stays + * lenient by design (a stray global env var should not hard-fail every + * command); only the explicit per-invocation flag is strict. + * + * This single definition replaces five byte-identical copies that previously + * lived in `auth`, `project`, `usage`, `init`, and `test` — drift between them + * would have silently changed timeout behaviour depending on the command. + */ +export function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) { + // Surface the offending value in the message (same as assertValidEndpointUrl) + // so the operator sees exactly what they typed. + throw localValidationError( + 'request-timeout', + `"${raw}" is not valid — must be a positive number of seconds`, + ); + } + return Math.round(n * 1000); // seconds → milliseconds +} + export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {}): HttpClient { const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const env = deps.env ?? process.env; diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 95d4a35..b0537b1 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -499,6 +499,24 @@ describe('project list subprocess', () => { const parsed = JSON.parse(result.stderr) as { error: { code: string } }; expect(parsed.error.code).toBe('VALIDATION_ERROR'); }, 30_000); + + it('--request-timeout 30s exits 5 (VALIDATION_ERROR), not a silent fallback to 120s', async () => { + // Previously an invalid flag value resolved to `undefined` and the command + // silently ran with the default 120s deadline — the operator believed they + // had set a timeout but had not. Now the explicit flag is validated like + // every other flag. + const result = await runCli( + ['--output', 'json', '--request-timeout', '30s', 'project', 'list'], + { + TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_URL: baseUrl, + }, + ); + expect(result.exitCode).toBe(5); + const parsed = JSON.parse(result.stderr) as { error: { code: string; nextAction: string } }; + expect(parsed.error.code).toBe('VALIDATION_ERROR'); + expect(parsed.error.nextAction).toContain('request-timeout'); + }, 30_000); }); describe('malformed --endpoint-url is rejected (exit 5), not retried as a network error', () => { From cff19aed2e3cd1cc2242c036f0410bcf906f025e Mon Sep 17 00:00:00 2001 From: Contributor Date: Fri, 3 Jul 2026 04:23:57 +0200 Subject: [PATCH 022/102] fix(rerun): emit partial stdout on TimeoutError in single-FE rerun --wait Apply the fix in src/commands/test.ts. When the overall --timeout polling deadline is exceeded on a single FE rerun, emit {runId, status:"running"} to stdout before exit 7. Co-authored-by: Cursor --- src/commands/test.rerun.spec.ts | 75 +++++++++++++++++++++++++++++++++ src/commands/test.ts | 12 ++++++ 2 files changed, 87 insertions(+) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 86c8e89..3645c12 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4630,3 +4630,78 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); + +// --------------------------------------------------------------------------- +// TimeoutError on single FE rerun --wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + let fetchCallCount = 0; + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + fetchCallCount++; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + const runningRun: RunResponse = { + ...makeTerminalRun(rerunResp.runId, 'passed'), + status: 'running', + finishedAt: null, + }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 0, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch(e => e); + + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(parsed.runId).toBe(rerunResp.runId); + expect(parsed.status).toBe('running'); + + void fetchCallCount; + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index 08e4fe2..69239ca 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -6099,6 +6099,18 @@ export async function runTestRerun( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${rerunResp.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(timeoutPartial, data => { + const p = data as typeof timeoutPartial; + return [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', From 6e7f2b3f0aafa61328c01daf276497b3030b192a Mon Sep 17 00:00:00 2001 From: Resque Date: Sun, 5 Jul 2026 23:27:55 +0400 Subject: [PATCH 023/102] fix(test): reject whitespace-only --name in test update (parity with test create) (#39) * fix(test): reject whitespace-only --name in test update (parity with test create) * style: run prettier on src/commands/test.ts --- src/commands/test.test.ts | 25 +++++++++++++++++++++++++ src/commands/test.ts | 6 ++++++ 2 files changed, 31 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 49127bd..f5ec62a 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -5777,6 +5777,31 @@ describe('runUpdate', () => { expect(called).toBe(0); }); + it('rejects a whitespace-only --name before sending (parity with test create)', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: ' ', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'name' }), + }); + expect(called).toBe(0); + }); + it('renders text mode with one line per updated field', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); diff --git a/src/commands/test.ts b/src/commands/test.ts index c6e6c31..d001c6c 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1243,6 +1243,12 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); requireNonEmpty('test-id', opts.testId); // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError( + 'name', + 'must be a non-empty string (whitespace-only is not allowed)', + ); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('name', 'must be at most 200 characters'); } From dc4d3375dc6ff0cff0a9104bbc1776898f14ed5d Mon Sep 17 00:00:00 2001 From: Rahul Joshi <186129212+crypticsaiyan@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:58:51 +0530 Subject: [PATCH 024/102] fix(test): guard parseDuration --since against overflow with VALIDATION_ERROR (#27) --- src/commands/test.result.history.spec.ts | 18 ++++++++++++++++++ src/commands/test.ts | 12 ++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index 6f6cdd2..a2d2018 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -169,6 +169,24 @@ describe('parseDuration', () => { it('case-insensitive day suffix', () => { expect(parseDuration('7D', NOW)).toBe('2026-05-27T12:00:00.000Z'); }); + + it('overflow hours throws VALIDATION_ERROR instead of crashing', () => { + expect(() => parseDuration('99999999999h', NOW)).toThrow(); + try { + parseDuration('99999999999h', NOW); + } catch (err: unknown) { + expect((err as { code?: string }).code).toBe('VALIDATION_ERROR'); + } + }); + + it('overflow days throws VALIDATION_ERROR instead of crashing', () => { + expect(() => parseDuration('99999999999d', NOW)).toThrow(); + try { + parseDuration('99999999999d', NOW); + } catch (err: unknown) { + expect((err as { code?: string }).code).toBe('VALIDATION_ERROR'); + } + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/test.ts b/src/commands/test.ts index d001c6c..07e4e29 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3809,12 +3809,20 @@ export function parseDuration(raw: string, now: Date = new Date()): string { const hourMatch = /^(\d+)h$/i.exec(raw); if (hourMatch) { const hours = Number(hourMatch[1]); - return new Date(now.getTime() - hours * 60 * 60 * 1000).toISOString(); + const result = new Date(now.getTime() - hours * 60 * 60 * 1000); + if (!Number.isFinite(result.getTime())) { + throw localValidationError('since', 'duration is too large; maximum is ~1141552511h'); + } + return result.toISOString(); } const dayMatch = /^(\d+)d$/i.exec(raw); if (dayMatch) { const days = Number(dayMatch[1]); - return new Date(now.getTime() - days * 24 * 60 * 60 * 1000).toISOString(); + const result = new Date(now.getTime() - days * 24 * 60 * 60 * 1000); + if (!Number.isFinite(result.getTime())) { + throw localValidationError('since', 'duration is too large; maximum is ~47564688d'); + } + return result.toISOString(); } // Pass-through: ISO timestamp or epoch value — server validates. return raw; From 6b90ff405006ff0731c944bec961ae7d120c2250 Mon Sep 17 00:00:00 2001 From: Resque Date: Sun, 5 Jul 2026 23:29:07 +0400 Subject: [PATCH 025/102] feat(cli): respect NO_COLOR environment variable per no-color.org (#12) * feat(cli): respect NO_COLOR environment variable per no-color.org * fix(ticker): treat empty NO_COLOR as color-enabled per no-color.org --- DOCUMENTATION.md | 13 +++++---- src/lib/ticker.spec.ts | 65 +++++++++++++++++++++++++++++++++++++++++- src/lib/ticker.ts | 35 +++++++++++++++++++++++ 3 files changed, 106 insertions(+), 7 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 700671b..461fc63 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -426,12 +426,13 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ------------------------------- | --------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| Variable | Purpose | +| ------------------------------- | --------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | ### Scopes diff --git a/src/lib/ticker.spec.ts b/src/lib/ticker.spec.ts index d449900..72760a9 100644 --- a/src/lib/ticker.spec.ts +++ b/src/lib/ticker.spec.ts @@ -3,7 +3,7 @@ */ import { describe, expect, it, vi } from 'vitest'; -import { createTicker } from './ticker.js'; +import { createTicker, isNoColor } from './ticker.js'; describe('createTicker — non-TTY (CI mode)', () => { it('update is a no-op (no writes)', () => { @@ -222,3 +222,66 @@ describe('createTicker — spy on process.stderr', () => { } }); }); + +describe('createTicker — NO_COLOR support', () => { + it('suppresses ANSI escape sequences when noColor=true on TTY', () => { + const lines: string[] = []; + const raw: string[] = []; + const ticker = createTicker( + line => lines.push(line), + true, // isTTY = true + text => raw.push(text), + true, // noColor = true + ); + ticker.update('progress'); + // Should use stderrWrite (line-oriented) instead of rawWrite with ANSI + expect(raw).toHaveLength(0); + expect(lines).toHaveLength(1); + expect(lines[0]!).not.toContain('\x1b[2K'); + expect(lines[0]!).not.toContain('\r'); + expect(lines[0]!).toContain('progress'); + }); + + it('finalize emits plain text without ANSI when noColor=true on TTY', () => { + const lines: string[] = []; + const raw: string[] = []; + const ticker = createTicker( + line => lines.push(line), + true, + text => raw.push(text), + true, // noColor = true + ); + ticker.finalize('done'); + expect(raw).toHaveLength(0); + expect(lines).toHaveLength(1); + expect(lines[0]!).not.toContain('\x1b[2K'); + expect(lines[0]!).toContain('done'); + }); + + it('normal ANSI output when noColor=false on TTY', () => { + const raw: string[] = []; + const ticker = createTicker( + () => {}, + true, + text => raw.push(text), + false, // noColor = false + ); + ticker.update('progress'); + expect(raw).toHaveLength(1); + expect(raw[0]!).toContain('\x1b[2K\r'); + }); +}); + +describe('isNoColor', () => { + it('returns true when NO_COLOR is set to a non-empty value', () => { + expect(isNoColor({ NO_COLOR: '1' })).toBe(true); + expect(isNoColor({ NO_COLOR: 'true' })).toBe(true); + }); + + it('returns false when NO_COLOR is absent or empty', () => { + expect(isNoColor({})).toBe(false); + expect(isNoColor({ OTHER_VAR: '1' })).toBe(false); + // Per https://no-color.org/, an empty NO_COLOR does NOT disable color. + expect(isNoColor({ NO_COLOR: '' })).toBe(false); + }); +}); diff --git a/src/lib/ticker.ts b/src/lib/ticker.ts index 4cae6b1..d9eaed2 100644 --- a/src/lib/ticker.ts +++ b/src/lib/ticker.ts @@ -8,6 +8,9 @@ * - Uses `\r` + ANSI clear-line to overwrite in place on TTY * - On terminal, emits one final line + newline then prints the result * - `--output json` disables the ticker (caller doesn't create one) + * - Respects the NO_COLOR env var (https://no-color.org/): when set, + * ANSI escape sequences are suppressed and updates are emitted as + * plain lines instead of in-place overwrites. * * Overhead: <2ms per update (no syscalls beyond a single write). * @@ -25,6 +28,15 @@ export interface Ticker { finalize(line?: string): void; } +/** + * Returns true when NO_COLOR is present in the environment and is not + * an empty string, per https://no-color.org/. + */ +export function isNoColor(env: NodeJS.ProcessEnv = process.env): boolean { + const value = env.NO_COLOR; + return typeof value === 'string' && value.length > 0; +} + /** * Create a ticker bound to the given stderr writer. Respects * `isTTY` to silently no-op in CI environments. @@ -35,11 +47,14 @@ export interface Ticker { * @param stderrRaw - optional raw writer (no \n appended); used for * the carriage-return + clear-line trick. Defaults to * `process.stderr.write.bind(process.stderr)`. + * @param noColor - whether to suppress ANSI escape sequences. + * Defaults to checking `NO_COLOR` env var per https://no-color.org/. */ export function createTicker( stderrWrite: (line: string) => void, isTTY?: boolean, stderrRaw?: (text: string) => void, + noColor?: boolean, ): Ticker { const tty = isTTY ?? (typeof process !== 'undefined' ? process.stderr.isTTY === true : false); const rawWrite = @@ -47,6 +62,7 @@ export function createTicker( (typeof process !== 'undefined' ? (text: string) => process.stderr.write(text) : (_text: string) => undefined); + const suppressAnsi = noColor ?? isNoColor(); let lastLength = 0; @@ -58,6 +74,25 @@ export function createTicker( }; } + if (suppressAnsi) { + // TTY but NO_COLOR: emit plain-text lines without ANSI escape sequences. + return { + update(line: string): void { + const stamped = `${new Date().toISOString()} ${line}`; + stderrWrite(stamped); + lastLength = stamped.length; + }, + finalize(line?: string): void { + if (line !== undefined) { + const stamped = `${new Date().toISOString()} ${line}`; + stderrWrite(stamped); + lastLength = stamped.length; + } + void stderrWrite; + }, + }; + } + return { update(line: string): void { // ANSI ESC[2K clears the entire line; \r moves to column 0. From 205b1fb37f2778bf7a3ddaf7c1ab458dec9ae119 Mon Sep 17 00:00:00 2001 From: Aldo Rizona Date: Mon, 6 Jul 2026 02:29:22 +0700 Subject: [PATCH 026/102] fix(paginate): bound cursor loops without dropping empty pages (#48) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(paginate): break loop on empty items with non-null nextToken paginate() in pagination.ts entered an infinite loop when the API returned { items: [], nextToken: 'cursor' }. The loop only checked nextToken !== null, never whether items was empty. Any filtered list command returning zero results would hang indefinitely. Add an early-exit guard: if the page contains zero items, break regardless of nextToken. Includes 5 unit tests covering the bug reproduction, maxItems cap, single-page, and consecutive empty pages. Closes #35 Signed-off-by: Aldo Rizona * style(test): reflow pagination.test.ts to satisfy prettier format:check gate Pure mechanical prettier reflow of two vi.fn(async () => ({...})) callbacks in test/pagination.test.ts. No logic change — only line wrapping to satisfy the format:check CI gate (npm run format:check). All other gates already pass. Coverage remains >=80% on all 4 metrics (lines/statements/functions/branches). * fix(paginate): bound cursor loops without dropping empty pages --------- Signed-off-by: Aldo Rizona --- src/lib/pagination.test.ts | 59 +++++++++++++++++++++++++++++++++++++- src/lib/pagination.ts | 31 +++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/lib/pagination.test.ts b/src/lib/pagination.test.ts index a11ea33..8a20295 100644 --- a/src/lib/pagination.test.ts +++ b/src/lib/pagination.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest'; import { ApiError } from './errors.js'; -import { paginate, validatePaginationFlags, type FetchPage, type Page } from './pagination.js'; +import { + MAX_AUTO_PAGES, + paginate, + validatePaginationFlags, + type FetchPage, + type Page, +} from './pagination.js'; function makePages(pages: Page[]): { fetchPage: FetchPage; @@ -121,4 +127,55 @@ describe('paginate', () => { await expect(paginate(fetchPage, { pageSize: 0 })).rejects.toBeInstanceOf(ApiError); expect(calls).toHaveLength(0); }); + + it('continues through empty cursor pages until later data', async () => { + const { fetchPage, calls } = makePages([ + { items: [], nextToken: 'cursor-1' }, + { items: [1], nextToken: null }, + ]); + + const page = await paginate(fetchPage); + + expect(page.items).toEqual([1]); + expect(page.nextToken).toBeNull(); + expect(calls).toHaveLength(2); + expect(calls[1]!.cursor).toBe('cursor-1'); + }); + + it('rejects when API repeats a non-null cursor without making progress', async () => { + const { fetchPage, calls } = makePages([{ items: [], nextToken: 'cursor-1' }]); + + await expect(paginate(fetchPage)).rejects.toMatchObject({ + code: 'UNAVAILABLE', + details: expect.objectContaining({ reason: 'repeated_next_token' }), + }); + + expect(calls).toHaveLength(2); + }); + + it('rejects when auto-pagination exceeds the page safety cap', async () => { + const calls: Array<{ pageSize: number; cursor: string | undefined }> = []; + const fetchPage: FetchPage = async args => { + calls.push(args); + return { + items: [], + nextToken: + args.cursor === undefined ? 'cursor-1' : `cursor-${Number(args.cursor.slice(7)) + 1}`, + }; + }; + + let thrown: unknown; + try { + await paginate(fetchPage); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(ApiError); + expect(thrown).toMatchObject({ + code: 'UNAVAILABLE', + details: expect.objectContaining({ reason: 'max_pages_exceeded' }), + }); + expect(calls).toHaveLength(MAX_AUTO_PAGES); + }); }); diff --git a/src/lib/pagination.ts b/src/lib/pagination.ts index ba63f9e..cae7581 100644 --- a/src/lib/pagination.ts +++ b/src/lib/pagination.ts @@ -1,5 +1,5 @@ import type { HttpClient } from './http.js'; -import { localValidationError } from './errors.js'; +import { ApiError, localValidationError } from './errors.js'; /** * Page shape returned by every list endpoint per @@ -22,6 +22,7 @@ export interface PaginationFlags { const HARD_PAGE_SIZE_CAP = 100; const DEFAULT_PAGE_SIZE = 25; +export const MAX_AUTO_PAGES = 1000; /** * Validates and normalizes pagination flags. Per the CLI OpenAPI spec @@ -91,14 +92,24 @@ export async function paginate( const items: T[] = []; let cursor: string | undefined = flags.startingToken; let lastNextToken: string | null = null; + let pagesFetched = 0; + const seenNextTokens = new Set(); + if (cursor !== undefined) seenNextTokens.add(cursor); while (true) { const remaining = maxItems !== undefined ? maxItems - items.length : Infinity; if (remaining <= 0) break; + if (pagesFetched >= MAX_AUTO_PAGES) { + throw paginationSafetyError('max_pages_exceeded', { + maxPages: MAX_AUTO_PAGES, + lastCursor: cursor ?? null, + }); + } const callPageSize = Number.isFinite(remaining) ? Math.min(pageSize, remaining) : pageSize; const page = await fetchPage({ pageSize: callPageSize, cursor }); + pagesFetched += 1; lastNextToken = page.nextToken; for (const item of page.items) { @@ -107,12 +118,30 @@ export async function paginate( } if (page.nextToken === null) break; + if (seenNextTokens.has(page.nextToken)) { + throw paginationSafetyError('repeated_next_token', { + cursor: page.nextToken, + pagesFetched, + }); + } + seenNextTokens.add(page.nextToken); cursor = page.nextToken; } return { items, nextToken: lastNextToken }; } +function paginationSafetyError(reason: string, details: Record): ApiError { + return ApiError.fromEnvelope({ + code: 'UNAVAILABLE', + message: 'Pagination did not make progress safely.', + nextAction: + 'Retry later. If the problem continues, contact TestSprite support with the cursor details.', + requestId: 'local', + details: { reason, ...details }, + }); +} + /** * Drop-in helper for commands that take a single page and surface * the cursor verbatim (no auto-follow). Used when the caller passed From 06d2c4deb5c2e51123ca6358673af782f2fb56a6 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:29:38 -0500 Subject: [PATCH 027/102] fix(project): validate list flags before client setup (#150) --- src/commands/project.test.ts | 41 ++++++++++++++++++++++++++++++++++++ src/commands/project.ts | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 1fcbd99..268df16 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -199,6 +199,47 @@ describe('runList', () => { }); }); + it('rejects invalid pagination before requiring credentials', async () => { + const credentialsPath = join(mkdtempSync(join(tmpdir(), 'cli-p2-no-creds-')), 'credentials'); + const fetchImpl = vi.fn(); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, pageSize: 1.5 }, + { credentialsPath, fetchImpl: fetchImpl as unknown as typeof globalThis.fetch }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('rejects invalid dry-run pagination before emitting the dry-run banner', async () => { + const stderr: string[] = []; + const fetchImpl = vi.fn(); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, dryRun: true, pageSize: 1.5 }, + { + credentialsPath: join(mkdtempSync(join(tmpdir(), 'cli-p2-dryrun-')), 'credentials'), + fetchImpl: fetchImpl as unknown as typeof globalThis.fetch, + stderr: line => stderr.push(line), + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(stderr.join('\n')).not.toContain(DRY_RUN_BANNER); + }); + it('rejects pageSize=101 with VALIDATION_ERROR exit 5 (Fix 7 — upper-bound enforced client-side)', async () => { // Previously silently clamped to 100; now rejected so callers get fast feedback. const { credentialsPath } = makeCreds(); diff --git a/src/commands/project.ts b/src/commands/project.ts index 2af8664..2510672 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -51,13 +51,13 @@ export async function runList( deps: ProjectDeps = {}, ): Promise> { const out = makeOutput(opts.output, deps); - const client = makeClient(opts, deps); const paginationFlags: PaginationFlags = validatePaginationFlags({ pageSize: opts.pageSize, startingToken: opts.startingToken, maxItems: opts.maxItems, }); + const client = makeClient(opts, deps); // When the user explicitly passed a page-size flag and did NOT ask // for --max-items, treat that as a "give me one page and the cursor" From 495db2f9f098b30cbb9db3bd7fe809332150cf7b Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:29:54 -0500 Subject: [PATCH 028/102] fix(test): validate list status before client setup (#152) --- src/commands/test.test.ts | 28 ++++++++++++++++++++++++++++ src/commands/test.ts | 12 ++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index f5ec62a..dc0527e 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -475,6 +475,34 @@ describe('runList', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', details: { field: 'page-size' } }); }); + it('rejects invalid --status before requiring credentials', async () => { + const credentialsPath = join(mkdtempSync(join(tmpdir(), 'cli-list-status-')), 'credentials'); + const fetchImpl = vi.fn(); + + await expect( + runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + status: 'notastatus', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => undefined, + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'status' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('forwards a server-side VALIDATION_ERROR envelope as ApiError exit 5', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ diff --git a/src/commands/test.ts b/src/commands/test.ts index 07e4e29..1644f85 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -452,6 +452,12 @@ export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise

= { projectId: opts.projectId, type: opts.type, From 25ed285cbadb7c1a7a56cac1fd2ec25f8cdb437f Mon Sep 17 00:00:00 2001 From: nopp Date: Mon, 6 Jul 2026 02:30:10 +0700 Subject: [PATCH 029/102] fix: reject blank project passwords (#139) --- src/commands/project.test.ts | 51 ++++++++++++++++++++++++++++++++++++ src/commands/project.ts | 6 +++++ 2 files changed, 57 insertions(+) diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 268df16..f63928c 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -637,6 +637,33 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + it('rejects a whitespace-only --password with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network - validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Password Guard Project', + targetUrl: 'https://example.com', + password: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); }); // --------------------------------------------------------------------------- @@ -739,6 +766,30 @@ describe('runUpdate', () => { expect(fetchImpl).not.toHaveBeenCalled(); }); + it('rejects a whitespace-only --password with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + password: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); it('P7 — dry-run returns canned shape without network call', async () => { resetDryRunBannerForTesting(); const { credentialsPath } = makeCreds(); diff --git a/src/commands/project.ts b/src/commands/project.ts index 2510672..ef4d5e8 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -150,6 +150,9 @@ export async function runCreate( if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } + if (opts.password !== undefined && opts.password.trim().length === 0) { + throw localValidationError('--password must not be empty or whitespace-only'); + } // P1-3: client-side length checks matching server limits. if (opts.name !== undefined && opts.name.length > 200) { @@ -264,6 +267,9 @@ export async function runUpdate( if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } + if (opts.password !== undefined && opts.password.trim().length === 0) { + throw localValidationError('--password must not be empty or whitespace-only'); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } From 87a6dff24e5f9744a1c5f6c9335100e60961e000 Mon Sep 17 00:00:00 2001 From: nopp Date: Mon, 6 Jul 2026 02:30:27 +0700 Subject: [PATCH 030/102] fix: reject directory code output paths (#140) --- src/commands/test.test.ts | 25 +++++++++++++++++++++++++ src/commands/test.ts | 11 +++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index dc0527e..d3b45ca 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -1550,6 +1550,31 @@ describe('runCodeGet', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('--out rejects an existing directory path with VALIDATION_ERROR (exit 5) before any network I/O', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-dir-')); + let fetchCalls = 0; + const fetchImpl = (() => { + fetchCalls += 1; + return Promise.resolve(new Response('{}')); + }) as typeof globalThis.fetch; + + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: dir, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchCalls).toBe(0); + expect(readdirSync(dir)).toEqual([]); + }); + // Regression: a parent dir that doesn't exist used to surface as exit 1 // (TRANSPORT_ERROR) — `createWriteStream` opens lazily and ENOENT fires // mid-write. Synchronous parent stat keeps every `--out` user-input diff --git a/src/commands/test.ts b/src/commands/test.ts index 1644f85..5d6eb80 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -7953,6 +7953,17 @@ function openOutputFile(rawPath: string): FileSink { if (!parentStat.isDirectory()) { throw localValidationError('out', `parent path is not a directory: ${parent}`); } + let targetStat; + try { + targetStat = statSync(resolved); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + throw localValidationError('out', `cannot stat output path: ${resolved}`); + } + } + if (targetStat?.isDirectory()) { + throw localValidationError('out', `must point to a file, not a directory: ${resolved}`); + } const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); const sink: FileSink = { stream, path: resolved, tmpPath, error: null }; From b5879de422891208a9b5a57fec8d45d04267c587 Mon Sep 17 00:00:00 2001 From: nopp Date: Mon, 6 Jul 2026 02:30:42 +0700 Subject: [PATCH 031/102] fix: reject malformed api keys (#141) --- src/lib/client-factory.test.ts | 55 +++++++++++++++++++++++++++++++++- src/lib/client-factory.ts | 17 +++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index 61ffe4b..ab5fd99 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -1,7 +1,8 @@ -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { DRY_RUN_API_KEY, DRY_RUN_BANNER, + assertValidApiKeyHeaderValue, assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, @@ -331,6 +332,58 @@ describe('makeHttpClient — real path (regression)', () => { // assertValidEndpointUrl — endpoint syntax guard (NOT an SSRF guard) // --------------------------------------------------------------------------- +describe('makeHttpClient - API key validation', () => { + it.each([ + ['newline', 'sk-user-abc\ndef'], + ['carriage return', 'sk-user-abc\rdef'], + ['smart dash', 'sk-user-abc\u2013def'], + ['smart quote', 'sk-user-\u201cabc\u201d'], + ['emoji', 'sk-user-abc\u{1f600}'], + ['whitespace-only', ' '], + ])('rejects a malformed configured API key with %s before fetch/retry', (_label, apiKey) => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: apiKey } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }); +}); + +describe('assertValidApiKeyHeaderValue', () => { + it('accepts a normal ASCII API key value', () => { + expect(() => assertValidApiKeyHeaderValue('sk-user-abc-def_123')).not.toThrow(); + }); + + it('throws a VALIDATION_ERROR (exit 5) for a key that cannot be sent as x-api-key', () => { + let caught: unknown; + try { + assertValidApiKeyHeaderValue('sk-user-abc\u2013def'); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }); +}); + describe('assertValidEndpointUrl', () => { it('accepts http(s) URLs, including private / localhost hosts (self-hosted, dev, mock)', () => { for (const url of [ diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index 8f0cc9b..829410d 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -180,6 +180,22 @@ export function assertValidEndpointUrl(rawUrl: string): void { } } +export function assertValidApiKeyHeaderValue(apiKey: string): void { + const reason = + 'must be a non-empty HTTP header value; paste the raw key without smart punctuation, emoji, or line breaks'; + + if (apiKey.trim().length === 0) { + throw localValidationError('api-key', reason, undefined, 'field'); + } + + for (let i = 0; i < apiKey.length; i += 1) { + const code = apiKey.charCodeAt(i); + if (code < 0x20 || code === 0x7f || code > 0xff) { + throw localValidationError('api-key', reason, undefined, 'field'); + } + } +} + /** * Parse the `--request-timeout ` flag value into milliseconds. * @@ -251,6 +267,7 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} // VALIDATION_ERROR rather than an opaque URL throw or a retried "fetch failed". assertValidEndpointUrl(config.apiUrl); if (!config.apiKey) throw ApiError.authRequired(); + assertValidApiKeyHeaderValue(config.apiKey); return new HttpClient({ baseUrl: facadeBaseUrl(config.apiUrl), apiKey: config.apiKey, From c0b52f852394e0199e663ddf02262ad2feb9fe51 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Sun, 5 Jul 2026 14:30:59 -0500 Subject: [PATCH 032/102] fix(setup): validate endpoint before key check (#149) --- src/commands/auth.test.ts | 94 +++++++++++++++++++++++++++++++++++++++ src/commands/auth.ts | 5 ++- src/commands/init.test.ts | 30 +++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 5207085..8a7ae8b 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -164,6 +164,100 @@ describe('runConfigure', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('rejects a malformed endpoint before key validation fetch', async () => { + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: true, + endpointUrl: 'not-a-url', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(capture.stderr.join('\n')).not.toContain('API key rejected'); + }); + + it('rejects a non-http endpoint before key validation fetch', async () => { + const { deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: true, + endpointUrl: 'ftp://example.com', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + + it('rejects a malformed dry-run endpoint before emitting dry-run output', async () => { + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: false, + dryRun: true, + endpointUrl: 'not-a-url', + }, + { + ...deps, + env: {}, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(capture.stderr.join('\n')).not.toContain('[dry-run]'); + expect(capture.stdout).toEqual([]); + }); + it('prompts only for the API key (never the endpoint) and defaults to prod', async () => { const { capture, deps } = makeCapture(); // Prompt object exposes ONLY `secret`. If runConfigure tried to prompt for diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 5c1b482..1ec3e40 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -1,5 +1,6 @@ import { Command } from 'commander'; import { + assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, parseRequestTimeoutFlag, @@ -87,8 +88,9 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // Print the canned success shape so an agent sees exactly the JSON it // would get on a real configure (modulo the endpoint string). if (opts.dryRun) { - emitDryRunBanner(stderr); const apiUrl = opts.endpointUrl ?? envApiUrl ?? DEFAULT_API_URL; + assertValidEndpointUrl(apiUrl); + emitDryRunBanner(stderr); stderr(`[dry-run] would write credentials for profile="${opts.profile}" to ${credentialsPath}`); out.print({ profile: opts.profile, apiUrl, status: 'configured' }, data => { const d = data as { profile: string; apiUrl: string }; @@ -114,6 +116,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // api_url doesn't silently validate a new key against the default endpoint. const resolvedFromProfile = existingProfile?.apiUrl; const apiUrl = opts.endpointUrl ?? envApiUrl ?? resolvedFromProfile ?? DEFAULT_API_URL; + assertValidEndpointUrl(apiUrl); if (opts.fromEnv) { apiKey = env.TESTSPRITE_API_KEY?.trim(); diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index be19095..75fdff0 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -542,6 +542,36 @@ describe('runInit — codex-review hardening', () => { expect(fetchImpl).toHaveBeenCalled(); }); + it('rejects malformed --endpoint-url before setup key verification', async () => { + const { captured, deps } = makeCapture(); + const fetchImpl = makeOkFetch(); + + await expect( + runInit( + makeBaseOpts({ + fromEnv: true, + endpointUrl: 'not-a-url', + noAgent: true, + output: 'json', + }), + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + fetchImpl, + credentialsPath, + isTTY: false, + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(captured.stderr.join('\n')).not.toContain('API key rejected'); + }); + it('whoami banner uses --api-key, not a stale TESTSPRITE_API_KEY in env (E2E 2026-06-09)', async () => { const { captured, deps } = makeCapture(); // Key-aware fetch: only the real key gets a 200 + identity; the stale env key 401s. From 0de372c51d1282d61ccf11b1c3c7b1f1a0d22398 Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:31:15 +0100 Subject: [PATCH 033/102] fix(http): map non-JSON 200 responses to a typed error envelope (#166) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A successful (200) response whose body is not JSON — a misconfigured endpoint, a proxy / captive-portal / login page returning HTML with a success status, or an empty body — caused the OK path to call raw `response.json()`, whose SyntaxError escaped to the top-level handler. That produced an opaque exit 1 and, under --output json, a bare `{"error":""}` that breaks the typed-envelope contract every other error honors (the non-OK path already reads defensively via safeReadJson). Wrap the OK-path parse failure in a typed INTERNAL ApiError (exit 1 unchanged) that carries the requestId, names the likely cause, and points the operator at their endpoint config; details include the HTTP status, content-type, and underlying parse message. Abort/timeout errors mid-read keep their existing classification. Fixes #94 --- src/lib/http.test.ts | 47 +++++++++++++++++++++++++++++++++++++ src/lib/http.ts | 55 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 3387eac..0320367 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -140,6 +140,53 @@ describe('HttpClient happy path', () => { }); }); +describe('HttpClient — 200 response with a non-JSON body', () => { + function htmlResponse(status = 200): Response { + return new Response('Login', { + status, + headers: { 'content-type': 'text/html' }, + }); + } + + it('throws a typed INTERNAL ApiError (not a raw SyntaxError) with the requestId and details', async () => { + const fetchImpl = vi.fn(async () => htmlResponse()); + const client = makeClient(fetchImpl as unknown as typeof fetch); + let caught: unknown; + try { + await client.get('/me'); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('INTERNAL'); + expect(apiErr.exitCode).toBe(1); + expect(apiErr.requestId).toMatch(/^cli_/); + expect(apiErr.message).toContain('non-JSON response'); + expect(apiErr.nextAction).toContain('TESTSPRITE_API_URL'); + expect(apiErr.details).toMatchObject({ httpStatus: 200, contentType: 'text/html' }); + expect(String(apiErr.details.parseError)).toContain('JSON'); + }); + + it('does not retry a malformed 200 body (single fetch call)', async () => { + const fetchImpl = vi.fn(async () => htmlResponse()); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.get('/me')).rejects.toBeInstanceOf(ApiError); + // A non-JSON success body is a hard config error, not a transient transport + // failure — it must not burn the retry budget. + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('handles an empty 200 body the same way', async () => { + const fetchImpl = vi.fn( + async () => + new Response('', { status: 200, headers: { 'content-type': 'application/json' } }), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.get('/me')).rejects.toMatchObject({ code: 'INTERNAL', exitCode: 1 }); + }); +}); + describe('HttpClient error mapping', () => { it('does not retry AUTH_INVALID and exits 3', async () => { const fetchImpl = vi.fn(async () => errorEnvelopeResponse(401, 'AUTH_INVALID')); diff --git a/src/lib/http.ts b/src/lib/http.ts index 83ca0f1..b5dabb1 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -498,7 +498,13 @@ export class HttpClient { } catch (err) { // A timeout/abort can fire mid-body-read (headers received, stream stalls). this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - throw err; + // Otherwise the successful response body was not valid JSON — a + // misconfigured endpoint, a proxy / captive-portal / login page that + // returns HTML with a 200 status, or an empty body. Surface a typed + // error carrying the requestId instead of letting the raw SyntaxError + // escape to index.ts, where it would print a bare `{"error":"..."}` + // and break the --output json envelope contract. + throw malformedResponseError(response, requestId, err); } } @@ -692,6 +698,53 @@ async function safeReadJson(response: Response): Promise { } } +/** + * Build a typed error for a successful response whose body could not be parsed + * as JSON. + * + * The CLI expects every API response to be a JSON envelope. When a `200 OK` + * carries a non-JSON body — a misconfigured endpoint, a proxy / captive-portal + * / SSO login page returning HTML with a success status, or an empty body — + * `response.json()` throws a raw `SyntaxError`. Left unhandled it escapes to + * the top-level handler in `index.ts`, which prints a bare + * `{"error":""}` under `--output json` (breaking the + * typed-envelope contract every other error honors) and gives the operator no + * actionable context. + * + * This wraps it in a typed `INTERNAL` `ApiError` (exit 1, unchanged) that + * carries the `requestId`, names the likely cause, and points the operator at + * their endpoint configuration. `details` includes the HTTP status, the + * response `content-type` (when present), and the underlying parse message. + */ +export function malformedResponseError( + response: Response, + requestId: string, + cause: unknown, +): ApiError { + const contentType = response.headers.get('content-type') ?? undefined; + const parseError = cause instanceof Error ? cause.message : String(cause); + const contentTypeNote = contentType ? ` (content-type: ${contentType})` : ''; + return new ApiError( + { + code: 'INTERNAL', + message: + `The server returned a non-JSON response${contentTypeNote} for an HTTP ${response.status}. ` + + `This usually means the endpoint is not the TestSprite API — a proxy, captive portal, or ` + + `login page can return HTML with a success status.`, + nextAction: + 'Check that --endpoint-url / TESTSPRITE_API_URL points at the TestSprite API ' + + '(default https://api.testsprite.com), then retry.', + requestId, + details: { + httpStatus: response.status, + ...(contentType ? { contentType } : {}), + parseError, + }, + }, + response.status, + ); +} + export function parseRetryAfter(headerValue: string | null): number | undefined { if (!headerValue) return undefined; const numeric = Number(headerValue); From b675630b852e73b1f474d905a060cbb9eeb40b0a Mon Sep 17 00:00:00 2001 From: Sahil Rakhaiya <144577420+SahilRakhaiya05@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:01:37 +0530 Subject: [PATCH 034/102] feat(test): JUnit XML report export for batch --wait runs (#96) * feat(test): JUnit XML report export for batch --wait runs * fix(ci): prettier formatting and help snapshot alignment for report flags * fix(test): address CodeRabbit review on JUnit report export * fix(test): add projectId to CliBatchRunFreshResult for JUnit inference --- CHANGELOG.md | 4 + DOCUMENTATION.md | 21 ++ src/commands/test.run.spec.ts | 220 +++++++++++- src/commands/test.ts | 141 +++++++- src/lib/dry-run/samples.test.ts | 20 +- src/lib/dry-run/samples.ts | 32 ++ src/lib/junit-report.test.ts | 330 ++++++++++++++++++ src/lib/junit-report.ts | 267 ++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 102 +++--- 9 files changed, 1087 insertions(+), 50 deletions(-) create mode 100644 src/lib/junit-report.test.ts create mode 100644 src/lib/junit-report.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e0d49..a4ab9b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +### Added + +- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. + ## [0.2.0] - 2026-06-29 ### Added diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 461fc63..a1ac967 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -344,8 +344,18 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json + +# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json ``` +Batch `--report` flags apply only to `test run --all --wait` (and batch `test rerun --wait`). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. Optional `--report-suite-name ` overrides the default `testsprite:` suite name. + `--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. #### `testsprite test rerun [test-id...]` @@ -365,10 +375,20 @@ testsprite test rerun test_be_xxxx --skip-dependencies --output json # Rerun every test in a project (batch) testsprite test rerun --all --project proj_xxxxxxxx --wait --max-concurrency 4 --output json +# Batch rerun with JUnit XML for CI +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json + # Several specific tests testsprite test rerun test_aaaa test_bbbb --wait --output json ``` +Batch `--report` flags apply only to batch `--wait` reruns (`--all` or multiple test ids). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. When `--project` is omitted, the CLI infers `projectId` from polled run rows for classname / default suite naming; if inference fails, pass `--project ` explicitly (required under `--dry-run`). + Flags: - `--all` — rerun every test in the resolved project; requires `--project `. @@ -377,6 +397,7 @@ Flags: - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. - `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). +- `--report junit --report-file ` — with batch `--wait`, write a JUnit XML sidecar after polling (atomic write). Optional `--report-suite-name ` overrides the default `testsprite:` suite name. Requires `--wait`; not available on single-test reruns. A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 0a10df5..f33d08a 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -5,7 +5,7 @@ * sleep injection is wired through `TestDeps.sleep` to avoid real delays. */ -import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; @@ -3624,3 +3624,221 @@ describe('dashboardUrl on run completion', () => { ).toBe(true); }); }); + +describe('runTestRunAll — JUnit report export', () => { + const BATCH_FRESH_RESP: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + + function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { + return { + runId, + testId, + projectId: 'project_be', + userId: 'user_1', + status: status as RunResponse['status'], + source: 'cli', + createdAt: '2026-06-09T10:00:00.000Z', + startedAt: '2026-06-09T10:00:01.000Z', + finishedAt: '2026-06-09T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://api.example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 3, + completed: 3, + passedCount: status === 'passed' ? 3 : 0, + failedCount: 0, + }, + }; + } + + it('--wait --report junit writes XML after polling', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'junit-run-all-')); + const reportPath = join(dir, 'results.xml'); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_fresh_01') + return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; + if (runId === 'run_fresh_02') + return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'passed') }; + return errorBody('NOT_FOUND'); + }); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain(' { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'junit-run-fail-')); + const reportPath = join(dir, 'results.xml'); + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; + const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; + if (runId === 'run_fresh_01') + return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; + if (runId === 'run_fresh_02') + return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'failed') }; + return errorBody('NOT_FOUND'); + }); + + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + credentialsPath, + fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 1 }); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('failures="1"'); + expect(xml).toContain('name="test_be_02"'); + }); + + it('rejects --report without --wait', async () => { + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: './results.xml', + }, + {}, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('rejects --report-suite-name without --report', async () => { + await expect( + runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + reportSuiteName: 'orphan-suite', + }, + {}, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('--dry-run --report junit writes canned sample XML', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-')); + const reportPath = join(dir, 'results.xml'); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + }, + { + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('name="test_fresh_wave_01"'); + expect(xml).toContain('failures="1"'); + }); + + it('--dry-run --report junit --report-suite-name overrides canned suite name', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-suite-')); + const reportPath = join(dir, 'results.xml'); + + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + report: 'junit', + reportFile: reportPath, + reportSuiteName: 'ci-checkout-suite', + }, + { + stdout: () => undefined, + stderr: () => undefined, + }, + ); + + const xml = readFileSync(reportPath, 'utf8'); + expect(xml).toContain('. */ + reportSuiteName?: string; } /** @@ -5085,6 +5100,32 @@ interface RunTestRunAllOptions extends CommonOptions { maxConcurrency: number; /** Caller-supplied idempotency token; auto-minted if absent. */ idempotencyKey?: string; + /** --report junit: write a JUnit XML sidecar after batch --wait completes. */ + report?: JUnitReportFormat; + /** --report-file: destination path for the JUnit XML artifact. */ + reportFile?: string; + /** --report-suite-name: optional override for the JUnit . */ + reportSuiteName?: string; +} + +async function writeBatchJUnitReportIfRequested( + opts: { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + projectId?: string; + }, + results: readonly JUnitTestResult[], +): Promise { + if (opts.report !== 'junit' || opts.reportFile === undefined) return; + const projectId = resolveBatchReportProjectId(opts, results); + const suiteName = opts.reportSuiteName ?? `testsprite:${projectId}`; + const xml = buildJUnitReport({ + suiteName, + classname: projectId, + results, + }); + await writeJUnitReportFile(opts.reportFile, xml); } /** @@ -5093,6 +5134,8 @@ interface RunTestRunAllOptions extends CommonOptions { interface CliBatchRunFreshResult { testId: string; runId: string | undefined; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; status: string; error?: { code: string; message: string; exitCode: number }; /** CLIENT-synthesized Portal deep link (projectId from opts, testId per item). */ @@ -5119,6 +5162,13 @@ export async function runTestRunAll( ) { throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); } + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: true, + }); const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const out = makeOutput(opts.output, deps); @@ -5142,6 +5192,12 @@ export async function runTestRunAll( idempotencyKey, ...(opts.wait ? { thenPoll: '/api/cli/v1/runs/?waitSeconds=25' } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(opts.projectId, opts.reportSuiteName), + ); + } out.print(batchRunSample ?? envelope); return undefined; } @@ -5481,7 +5537,12 @@ export async function runTestRunAll( }, resolveAlternate, }); - return { testId: entry.testId, runId, status: finalRun.status }; + return { + testId: entry.testId, + runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -5563,6 +5624,7 @@ export async function runTestRunAll( total: pollable.length, }, }; + await writeBatchJUnitReportIfRequested(opts, freshRunResults); out.print(jsonPayload); // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7), @@ -5625,6 +5687,8 @@ export async function runTestRunAll( interface CliRerunResult { testId: string; runId: string; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; /** Terminal status, or 'timeout' for per-run deadline exceeded. */ status: string; /** Set when the test is a closure member (not the user's named test). */ @@ -5694,6 +5758,13 @@ export async function runTestRerun( } const isSingle = !opts.all && opts.testIds.length === 1; + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: !isSingle, + }); // ------------------------------------------------------------------------- // Pre-flight: auto-heal + Free-tier hint (best-effort, non-blocking) @@ -5733,6 +5804,13 @@ export async function runTestRerun( idempotencyKey, ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + const projectKey = resolveBatchReportProjectId(opts, []); + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(projectKey, opts.reportSuiteName), + ); + } out.print(findSample('POST', '/api/cli/v1/tests/batch/rerun')?.body() ?? envelope); } void client; @@ -6630,7 +6708,12 @@ export async function runTestRerun( }, resolveAlternate, }); - return { testId: entry.testId, runId: entry.runId, status: finalRun.status }; + return { + testId: entry.testId, + runId: entry.runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -6718,6 +6801,7 @@ export async function runTestRerun( total: accepted.length, }, }; + await writeBatchJUnitReportIfRequested(opts, rerunResults); out.print(jsonPayload); // Determine exit code: timeout (deferred or any timeout) → 7; any fail → 1; all pass → 0 @@ -7442,11 +7526,21 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--max-concurrency ', `with --all --wait, max in-flight polls at once (1-100, default: ${DEFAULT_BATCH_RUN_CONCURRENCY})`, ) + .option( + '--report ', + 'with --all --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + ' testsprite test run --all --project run all BE tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + '(see `testsprite test create --help` for details).', ) @@ -7476,6 +7570,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all --project .', ); } + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isAll, + }); if (isAll) { // --all path: wave-ordered fresh batch run. @@ -7506,6 +7608,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); @@ -7613,6 +7718,15 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--idempotency-key ', 'opaque key for safe retries (1–256 chars). Printed to stderr at --verbose if auto-generated.', ) + .option( + '--report ', + 'with batch --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nNotes:\n' + @@ -7638,10 +7752,20 @@ export function createTestCommand(deps: TestDeps = {}): Command { // `--no-auto-heal`. There is no explicit `--auto-heal` flag, so // autoHealExplicit is always false in this design — the default-on value // is never a deliberate user choice to opt in. + const testIds = testIdsArg ?? []; + const isBatch = cmdOpts.all === true || testIds.length !== 1; + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isBatch, + }); await runTestRerun( { ...resolveCommonOptions(command), - testIds: testIdsArg ?? [], + testIds, all: cmdOpts.all === true, projectId: cmdOpts.project, skipTerminal: cmdOpts.skipTerminal === true, @@ -7656,6 +7780,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); @@ -7679,6 +7806,9 @@ interface RunFlagOpts { project?: string; filter?: string; maxConcurrency?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface WaitFlagOpts { @@ -7697,6 +7827,9 @@ interface RerunFlagOpts { skipDependencies?: boolean; maxConcurrency?: string; idempotencyKey?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface UpdateFlagOpts { diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index e1bf8ff..a1838d6 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -1,5 +1,23 @@ import { describe, expect, it } from 'vitest'; -import { DRY_RUN_SAMPLE_ENTRIES, findSample } from './samples.js'; +import { DRY_RUN_SAMPLE_ENTRIES, findSample, sampleJUnitReportXml } from './samples.js'; + +describe('sampleJUnitReportXml', () => { + it('returns well-formed JUnit XML with canned batch ids', () => { + const xml = sampleJUnitReportXml('proj_dry'); + expect(xml).toContain(''); + expect(xml).toContain(' { + const xml = sampleJUnitReportXml('proj_dry', 'custom-ci-suite'); + expect(xml).toContain(' { it('resolves /me', () => { diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 28fa1bb..4c7cbb0 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -27,6 +27,7 @@ import type { CliTestStep, } from '../../commands/test.js'; import type { MeResponse } from '../../commands/auth.js'; +import { buildJUnitReport } from '../junit-report.js'; import type { Page } from '../pagination.js'; import type { TriggerRunResponse, @@ -67,6 +68,37 @@ const SAMPLE_REQUEST_ID = 'req_dry-run'; export const SAMPLE_DRY_RUN_REQUEST_ID = SAMPLE_REQUEST_ID; +/** + * Canned JUnit XML for batch `--wait --report junit --dry-run`. Mirrors the + * fresh batch-run sample ids so agents can learn the sidecar shape offline. + */ +export function sampleJUnitReportXml( + projectId: string = SAMPLE_PROJECT_ID, + reportSuiteName?: string, +): string { + return buildJUnitReport({ + suiteName: reportSuiteName ?? `testsprite:${projectId}`, + classname: projectId, + results: [ + { + testId: SAMPLE_TEST_ID_FRESH_1, + runId: SAMPLE_BATCH_FRESH_RUN_ID_1, + status: 'passed', + }, + { + testId: SAMPLE_TEST_ID_FRESH_2, + runId: SAMPLE_BATCH_FRESH_RUN_ID_2, + status: 'failed', + error: { + code: 'ASSERTION', + message: 'Expected checkout heading to be visible', + exitCode: 1, + }, + }, + ], + }); +} + const me: MeResponse = { userId: SAMPLE_USER_ID, keyId: SAMPLE_KEY_ID, diff --git a/src/lib/junit-report.test.ts b/src/lib/junit-report.test.ts new file mode 100644 index 0000000..d504dce --- /dev/null +++ b/src/lib/junit-report.test.ts @@ -0,0 +1,330 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError } from './errors.js'; +import { + assertJUnitReportOptions, + buildJUnitReport, + escapeXml, + parseJUnitReportFormat, + resolveBatchReportProjectId, + writeJUnitReportFile, + type JUnitTestResult, +} from './junit-report.js'; + +function makeResult(overrides: Partial & { testId: string }): JUnitTestResult { + return { + status: 'passed', + ...overrides, + }; +} + +describe('escapeXml', () => { + it('escapes XML special characters', () => { + expect(escapeXml(`a&bd"e'f`)).toBe('a&b<c>d"e'f'); + }); + + it('leaves plain text unchanged', () => { + expect(escapeXml('test_abc123')).toBe('test_abc123'); + }); +}); + +describe('parseJUnitReportFormat', () => { + it('accepts junit', () => { + expect(parseJUnitReportFormat('junit')).toBe('junit'); + }); + + it('returns undefined for absent value', () => { + expect(parseJUnitReportFormat(undefined)).toBeUndefined(); + expect(parseJUnitReportFormat('')).toBeUndefined(); + }); + + it('rejects unknown formats', () => { + expect(() => parseJUnitReportFormat('html')).toThrowError(ApiError); + try { + parseJUnitReportFormat('html'); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('assertJUnitReportOptions', () => { + it('allows absent report flags', () => { + expect(() => assertJUnitReportOptions({ wait: false, batchPath: true })).not.toThrow(); + }); + + it('rejects report-file without report', () => { + expect(() => + assertJUnitReportOptions({ reportFile: './out.xml', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); + + it('rejects report-suite-name without report', () => { + expect(() => + assertJUnitReportOptions({ + reportSuiteName: 'my-suite', + wait: true, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report on non-batch paths', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: true, + batchPath: false, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without wait', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: false, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without report-file', () => { + expect(() => + assertJUnitReportOptions({ report: 'junit', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); +}); + +describe('resolveBatchReportProjectId', () => { + it('prefers explicit projectId', () => { + expect(resolveBatchReportProjectId({ projectId: 'proj_a' }, [])).toBe('proj_a'); + }); + + it('infers from polled run rows', () => { + expect(resolveBatchReportProjectId({}, [{ projectId: 'proj_from_run' }])).toBe('proj_from_run'); + }); + + it('requires --project when the project cannot be inferred', () => { + expect(() => resolveBatchReportProjectId({}, [])).toThrowError(ApiError); + try { + resolveBatchReportProjectId({}, []); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('buildJUnitReport', () => { + it('renders an empty suite', () => { + const xml = buildJUnitReport({ + suiteName: 'Dry suite', + classname: 'proj_empty', + results: [], + }); + expect(xml).toContain( + ''); + }); + + it('counts passed tests without child elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [makeResult({ testId: 'test_a', status: 'passed' })], + }); + expect(xml).toContain(''); + expect(xml).not.toContain(' { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 'test_fail', + status: 'failed', + runId: 'run_1', + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('runId: run_1'); + expect(xml).toContain('failures="1"'); + }); + + it('maps blocked and cancelled to failures', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ testId: 't_blocked', status: 'blocked' }), + makeResult({ testId: 't_cancelled', status: 'cancelled' }), + ], + }); + expect(xml).toContain('failures="2"'); + expect(xml).toContain('type="blocked"'); + expect(xml).toContain('type="cancelled"'); + }); + + it('maps timeout to failure', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_timeout', + status: 'timeout', + error: { code: 'UNSUPPORTED', message: 'Timed out', exitCode: 7 }, + }), + ], + }); + expect(xml).toContain(''); + }); + + it('maps API error status to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_err', + status: 'error', + error: { code: 'NOT_FOUND', message: 'Run missing', exitCode: 4 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('errors="1"'); + }); + + it('maps auth failures to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_auth', + status: 'failed', + error: { code: 'AUTH_INVALID', message: 'Bad key', exitCode: 3 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('failures="0" errors="1"'); + }); + + it('escapes special characters in testcase names and messages', () => { + const xml = buildJUnitReport({ + suiteName: 'Suite "A"', + classname: 'proj<&>', + results: [ + makeResult({ + testId: 'test<1>', + status: 'failed', + error: { code: 'ASSERT', message: 'expected & "ok"', exitCode: 1 }, + }), + ], + }); + expect(xml).toContain('name="test<1>"'); + expect(xml).toContain('classname="proj<&>"'); + expect(xml).toContain('message="expected <true> & "ok""'); + }); + + it('aggregates mixed outcomes', () => { + const xml = buildJUnitReport({ + suiteName: 'Mixed', + classname: 'proj_mix', + results: [ + makeResult({ testId: 'p', status: 'passed' }), + makeResult({ testId: 'f', status: 'failed' }), + makeResult({ + testId: 'e', + status: 'error', + error: { code: 'INTERNAL', message: 'boom', exitCode: 10 }, + }), + ], + }); + expect(xml).toContain('tests="3" failures="1" errors="1" skipped="0"'); + }); +}); + +describe('writeJUnitReportFile', () => { + it('writes XML atomically to the target path', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-')); + const target = join(dir, 'results.xml'); + const xml = buildJUnitReport({ + suiteName: 'Suite', + classname: 'proj_write', + results: [makeResult({ testId: 't1', status: 'passed' })], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects a directory target', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-dir-')); + await expect(writeJUnitReportFile(dir, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects missing parent directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-parent-')); + const missing = join(dir, 'missing', 'out.xml'); + await expect(writeJUnitReportFile(missing, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('overwrites an existing file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-overwrite-')); + const target = join(dir, 'results.xml'); + writeFileSync(target, 'old', 'utf8'); + const xml = buildJUnitReport({ + suiteName: 'New', + classname: 'proj_new', + results: [], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects empty path', async () => { + await expect(writeJUnitReportFile('', '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + }); + + it('rejects parent that is a file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-file-parent-')); + const parentFile = join(dir, 'not-a-dir'); + writeFileSync(parentFile, 'x', 'utf8'); + const target = join(parentFile, 'out.xml'); + await expect(writeJUnitReportFile(target, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/src/lib/junit-report.ts b/src/lib/junit-report.ts new file mode 100644 index 0000000..3b11442 --- /dev/null +++ b/src/lib/junit-report.ts @@ -0,0 +1,267 @@ +import { createWriteStream } from 'node:fs'; +import { rename, stat, unlink } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { localValidationError, TransportError } from './errors.js'; + +export type JUnitReportFormat = 'junit'; + +/** Minimal testcase input shared by batch run and batch rerun poll results. */ +export interface JUnitTestResult { + testId: string; + runId?: string; + status: string; + /** Observed on polled runs; used for classname when --project is omitted. */ + projectId?: string; + error?: { code: string; message: string; exitCode?: number }; +} + +export interface JUnitReportBuildOptions { + suiteName: string; + classname: string; + results: readonly JUnitTestResult[]; +} + +export interface JUnitReportFlagOptions { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + wait: boolean; + /** True when the invocation is a batch path (run --all or rerun batch). */ + batchPath: boolean; +} + +const XML_DECL = ''; + +/** + * Parse `--report `. Only `junit` is accepted in v1. + */ +export function parseJUnitReportFormat(raw: string | undefined): JUnitReportFormat | undefined { + if (raw === undefined || raw === '') return undefined; + if (raw === 'junit') return 'junit'; + throw localValidationError('report', `unsupported report format "${raw}" — accepted: junit`); +} + +/** + * Validate `--report` / `--report-file` / `--report-suite-name` combinations. + * Report export is a sidecar artifact for batch `--wait` runs only. + */ +export function assertJUnitReportOptions(opts: JUnitReportFlagOptions): void { + if (opts.report === undefined) { + if (opts.reportFile !== undefined && opts.reportFile !== '') { + throw localValidationError('report-file', '--report-file requires --report junit'); + } + if (opts.reportSuiteName !== undefined && opts.reportSuiteName !== '') { + throw localValidationError( + 'report-suite-name', + '--report-suite-name requires --report junit', + ); + } + return; + } + + if (!opts.batchPath) { + throw localValidationError( + 'report', + '--report junit only applies to batch --wait runs (test run --all, or test rerun --all / multiple test ids)', + ); + } + if (!opts.wait) { + throw localValidationError( + 'report', + '--report junit requires --wait (the report is written after batch polling completes)', + ); + } + if (opts.reportFile === undefined || opts.reportFile === '') { + throw localValidationError('report-file', '--report junit requires --report-file '); + } +} + +/** + * Resolve the project id used for JUnit classname / default suite naming. + * Prefer explicit `--project`, then ids observed on polled run rows. + */ +export function resolveBatchReportProjectId( + opts: { projectId?: string }, + results: ReadonlyArray<{ projectId?: string }>, +): string { + if (opts.projectId) return opts.projectId; + const fromPoll = results.map(r => r.projectId).find((id): id is string => !!id); + if (fromPoll) return fromPoll; + throw localValidationError( + 'project', + '--report junit requires --project when the project cannot be inferred from run results', + ); +} + +/** + * Escape text for inclusion in XML element bodies and double-quoted attributes. + */ +export function escapeXml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +type JUnitOutcome = 'passed' | 'failure' | 'error' | 'skipped'; + +function classifyJUnitOutcome(status: string, error?: JUnitTestResult['error']): JUnitOutcome { + if (status === 'passed') return 'passed'; + if (status === 'skipped') return 'skipped'; + if (status === 'error' || error?.exitCode === 3) return 'error'; + return 'failure'; +} + +function failureMessage(result: JUnitTestResult): string { + if (result.error?.message) return result.error.message; + if (result.error?.code) return result.error.code; + return result.status; +} + +function renderTestcase(result: JUnitTestResult, classname: string): string { + const outcome = classifyJUnitOutcome(result.status, result.error); + const name = escapeXml(result.testId); + const cls = escapeXml(classname); + const lines = [` `]; + + if (outcome === 'failure') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.status); + const body = escapeXml( + [ + `status: ${result.status}`, + result.runId ? `runId: ${result.runId}` : undefined, + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'error') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.error?.code ?? result.status); + const body = escapeXml( + [ + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + result.runId ? `runId: ${result.runId}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'skipped') { + lines.push(` `); + } + + lines.push(' '); + return lines.join('\n'); +} + +/** + * Build a JUnit XML document from batch poll results. Duration is `0` in v1 + * because batch poll envelopes do not carry per-run timing. + */ +export function buildJUnitReport(opts: JUnitReportBuildOptions): string { + const results = opts.results; + let failures = 0; + let errors = 0; + let skipped = 0; + + for (const result of results) { + const outcome = classifyJUnitOutcome(result.status, result.error); + if (outcome === 'failure') failures++; + else if (outcome === 'error') errors++; + else if (outcome === 'skipped') skipped++; + } + + const suiteName = escapeXml(opts.suiteName); + const testcases = results.map(r => renderTestcase(r, opts.classname)).join('\n'); + + return [ + XML_DECL, + '', + ` `, + testcases, + ' ', + '', + '', + ].join('\n'); +} + +async function assertReportFileParent(rawPath: string): Promise { + if (typeof rawPath !== 'string' || rawPath.length === 0) { + throw localValidationError('report-file', 'must be a non-empty file path'); + } + const resolved = isAbsolute(rawPath) ? rawPath : resolve(process.cwd(), rawPath); + if (resolved.endsWith('/') || resolved.endsWith('\\')) { + throw localValidationError('report-file', 'must point to a file, not a directory'); + } + + const parent = dirname(resolved); + let parentStat; + try { + parentStat = await stat(parent); + } catch { + throw localValidationError('report-file', `parent directory does not exist: ${parent}`); + } + if (!parentStat.isDirectory()) { + throw localValidationError('report-file', `parent path is not a directory: ${parent}`); + } + + let targetStat; + try { + targetStat = await stat(resolved); + } catch { + return resolved; + } + if (targetStat.isDirectory()) { + throw localValidationError('report-file', `must point to a file, not a directory: ${resolved}`); + } + return resolved; +} + +/** + * Atomically write JUnit XML to `--report-file` (temp sibling + rename). + */ +export async function writeJUnitReportFile(rawPath: string, xml: string): Promise { + const resolved = await assertReportFileParent(rawPath); + const parent = dirname(resolved); + const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); + + await new Promise((resolvePromise, reject) => { + const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); + let streamError: Error | null = null; + stream.on('error', err => { + streamError = err instanceof Error ? err : new Error(String(err)); + }); + stream.write(xml, err => { + if (err) { + streamError = err instanceof Error ? err : new Error(String(err)); + } + stream.end(() => { + if (streamError) { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError(`Failed to write --report-file ${resolved}: ${streamError.message}`), + ); + return; + } + rename(tmpPath, resolved) + .then(() => resolvePromise()) + .catch(renameErr => { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError( + `Failed to write --report-file ${resolved}: ${renameErr instanceof Error ? renameErr.message : String(renameErr)}`, + ), + ); + }); + }); + }); + }); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6fee8af..9e5e0d9 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -392,32 +392,40 @@ Exit codes: On failure/blocked/cancelled, run: testsprite test artifact get Options: - --all rerun all tests in the resolved project (requires - --project) (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --skip-terminal with --all: skip tests already in a terminal status - (passed|failed|blocked|cancelled) (default: false) - --status with --all: only dispatch tests whose status matches - one of these values (comma-separated; accepted: - draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) - --filter with --all: only rerun tests whose name contains - this substring (case-insensitive) - --wait block until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --no-auto-heal opt out of AI heal-on-drift for this FE rerun - (default: auto-heal is ON). Costs 0.2 credits per - engage when a step has drifted. Ignored for backend - tests. - --skip-dependencies BE only: rerun only the named test without expanding - the producer/teardown closure (default: false) - --max-concurrency with --wait, max in-flight polls at once (1-100, - default: 50) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --verbose if auto-generated. - -h, --help display help for command + --all rerun all tests in the resolved project (requires + --project) (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --skip-terminal with --all: skip tests already in a terminal + status (passed|failed|blocked|cancelled) + (default: false) + --status with --all: only dispatch tests whose status + matches one of these values (comma-separated; + accepted: + draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) + --filter with --all: only rerun tests whose name contains + this substring (case-insensitive) + --wait block until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --no-auto-heal opt out of AI heal-on-drift for this FE rerun + (default: auto-heal is ON). Costs 0.2 credits per + engage when a step has drifted. Ignored for + backend tests. + --skip-dependencies BE only: rerun only the named test without + expanding the producer/teardown closure (default: + false) + --max-concurrency with --wait, max in-flight polls at once (1-100, + default: 50) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --verbose if auto-generated. + --report with batch --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Notes: • rerun replays a saved run/script and is MORE LENIENT than a fresh \`test run\` @@ -490,28 +498,34 @@ Exit codes: On failure/blocked/cancelled, run: testsprite test artifact get Options: - --target-url override the project default env URL for this run - (http/https only, no localhost/private IPs) - --wait poll until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --debug if auto-generated. - --all run all BE tests in the project (wave-ordered fresh - run; requires --project). Mutually exclusive with - . (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --filter with --all: only run tests whose name contains this - substring (case-insensitive) - --max-concurrency with --all --wait, max in-flight polls at once - (1-100, default: 50) - -h, --help display help for command + --target-url override the project default env URL for this run + (http/https only, no localhost/private IPs) + --wait poll until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --debug if auto-generated. + --all run all BE tests in the project (wave-ordered + fresh run; requires --project). Mutually + exclusive with . (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --filter with --all: only run tests whose name contains + this substring (case-insensitive) + --max-concurrency with --all --wait, max in-flight polls at once + (1-100, default: 50) + --report with --all --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Dependency-aware fresh run (M4): testsprite test run --all --project run all BE tests in wave order testsprite test run --all --project --filter name-glob subset + testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). From 38e0f190666267e2c9b5d526424f7b5a9682db0e Mon Sep 17 00:00:00 2001 From: Rahul Joshi <186129212+crypticsaiyan@users.noreply.github.com> Date: Mon, 6 Jul 2026 01:02:06 +0530 Subject: [PATCH 035/102] fix(rerun): preserve exit code and escalate auth errors in batch rerun (#33) pollAccepted in runTestRerun hardcoded exitCode:1 on ApiError, causing the auth-escalation find(r => r.error?.exitCode === 3) to always return undefined -- auth failures silently exited 1 instead of 3. - preserve err.exitCode in pollAccepted (mirrors runTestRunAll fix) - add auth escalation block before generic exit-1 throw - bound initial chunk idempotency key to <=256 chars (mirrors retry path) --- src/commands/test.rerun.spec.ts | 482 ++++++++++++++++++++++++++++++++ src/commands/test.ts | 26 +- 2 files changed, 506 insertions(+), 2 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index acd5b02..d36a233 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4698,3 +4698,485 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); + +// --------------------------------------------------------------------------- +// [fix-exitcode] pollAccepted preserves ApiError exit codes (not hardcoded 1) +// --------------------------------------------------------------------------- + +describe('[fix-exitcode] polling error exit codes preserved in batch rerun results', () => { + it('AUTH_REQUIRED during polling → batch escalates to exitCode 3', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a1', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_auth_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_auth_fail')) return errorBody('AUTH_REQUIRED'); + if (url.includes('/runs/run_pass_a1')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e as { exitCode?: number; message?: string }); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + }); + + it('RATE_LIMITED during polling → non-auth, batch exits 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a2', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_rl', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_rl')) return errorBody('RATE_LIMITED'); + if (url.includes('/runs/run_pass_a2')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); + + it('NOT_FOUND during run polling → non-auth, batch exits 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_pass_a3', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_nf', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_a3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_nf')) return errorBody('NOT_FOUND'); + if (url.includes('/runs/run_pass_a3')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// [fix-auth-escalation] batch auth failure escalates to exit 3 +// --------------------------------------------------------------------------- + +describe('[fix-auth-escalation] auth error in batch rerun polling escalates to exit 3', () => { + it('auth failure in batch poll → batch exits 3, not 1', async () => { + const creds = makeCreds(); + const passRun = makeTerminalRun('run_other', 'passed'); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_auth', runId: 'run_auth', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_other', runId: 'run_other', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_auth')) return errorBody('AUTH_REQUIRED'); + if (url.includes('/runs/run_other')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_auth', 'test_other'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + }); + + it('mixed batch: one pass, one auth failure → exits 3 (auth wins)', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_pass', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_auth2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const passRun = makeTerminalRun('run_pass', 'passed'); + passRun.testId = 'test_1'; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_pass')) return { body: passRun }; + if (url.includes('/runs/run_auth2')) return errorBody('AUTH_REQUIRED'); + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(3); + expect((err as { message?: string }).message).toMatch(/auth error/i); + }); + + it('non-auth failure → exits 1 (no escalation)', async () => { + const creds = makeCreds(); + const failRun = makeTerminalRun('run_fail', 'failed'); + failRun.testId = 'test_1'; + const passRun = makeTerminalRun('run_pass_c3', 'passed'); + passRun.testId = 'test_2'; + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_pass_c3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/run_fail')) return { body: failRun }; + if (url.includes('/runs/run_pass_c3')) return { body: passRun }; + return errorBody('NOT_FOUND'); + }); + + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 10, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 5, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + + expect((err as { exitCode?: number }).exitCode).toBe(1); + }); +}); + +// --------------------------------------------------------------------------- +// [fix-D4] initial chunk idempotency key bounded to ≤256 chars +// --------------------------------------------------------------------------- + +describe('[fix-D4] initial chunk dispatch idempotency key bounded to 256 chars', () => { + it('short key with multiple chunks passes through unchanged', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // 51 test IDs forces 2 chunks (MAX_BATCH_RERUN_IDS = 50) + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + const key = h.get('idempotency-key') ?? ''; + receivedKeys.push(key); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: 'short-key', + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + expect(receivedKeys[0]).toBe('short-key:chunk0'); + expect(receivedKeys[1]).toBe('short-key:chunk1'); + expect(receivedKeys[0]!.length).toBeLessThanOrEqual(256); + expect(receivedKeys[1]!.length).toBeLessThanOrEqual(256); + }); + + it('249-char key + :chunk0 suffix would exceed 256 → key truncated to keep total ≤256', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // key is 249 chars; `:chunk0` is 7 chars → 256 total (edge case, fits exactly) + const longKey = 'k'.repeat(249); + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + receivedKeys.push(h.get('idempotency-key') ?? ''); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: longKey, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + for (const key of receivedKeys) { + expect(key.length).toBeLessThanOrEqual(256); + } + // suffix must be preserved + expect(receivedKeys[0]).toMatch(/:chunk0$/); + expect(receivedKeys[1]).toMatch(/:chunk1$/); + }); + + it('256-char key + :chunk0 suffix → base truncated so total is exactly 256', async () => { + const creds = makeCreds(); + const receivedKeys: string[] = []; + + // Max-length user key: 256 chars. `:chunk0` = 7 chars → need to truncate base to 249. + const maxKey = 'x'.repeat(256); + const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); + const batchResp: BatchRerunResponse = { + accepted: testIds.slice(0, 50).map(id => ({ + testId: id, + runId: `run_${id}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + })), + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + const batchResp2: BatchRerunResponse = { + accepted: [ + { + testId: testIds[50]!, + runId: `run_${testIds[50]}`, + enqueuedAt: '2026-06-03T10:00:00.000Z', + }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + let callCount = 0; + + const fetchImpl = makeFetch((url, init) => { + if (url.includes('/tests/batch/rerun')) { + const h = new Headers(init.headers ?? {}); + receivedKeys.push(h.get('idempotency-key') ?? ''); + callCount++; + return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds, + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + idempotencyKey: maxKey, + }, + { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(receivedKeys).toHaveLength(2); + for (const key of receivedKeys) { + expect(key.length).toBeLessThanOrEqual(256); + } + expect(receivedKeys[0]).toMatch(/:chunk0$/); + expect(receivedKeys[1]).toMatch(/:chunk1$/); + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index 6c7fe65..a48812e 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -6377,7 +6377,15 @@ export async function runTestRerun( chunkResponses = []; for (let idx = 0; idx < chunks.length; idx++) { const chunk = chunks[idx]!; - const chunkKey = chunks.length === 1 ? idempotencyKey : `${idempotencyKey}:chunk${idx}`; + // Bound the per-chunk idempotency key to <=256 chars (mirrors the retry + // path). A long base key plus the `:chunkN` suffix could otherwise exceed + // the server cap and be rejected or truncated inconsistently. + const chunkSuffix = chunks.length === 1 ? '' : `:chunk${idx}`; + const chunkBase = + chunkSuffix.length > 0 && idempotencyKey.length + chunkSuffix.length > 256 + ? idempotencyKey.slice(0, 256 - chunkSuffix.length) + : idempotencyKey; + const chunkKey = `${chunkBase}${chunkSuffix}`; const chunkResp = await client.triggerBatchRerun( { source: 'cli', @@ -6728,11 +6736,14 @@ export async function runTestRerun( }; } if (err instanceof ApiError) { + // Preserve the real exit code (AUTH_INVALID=3, RATE_LIMITED=11, …) so the + // batch exit-code aggregator can escalate auth failures correctly. Mirroring + // the identical fix already applied to runTestRunAll's pollFreshAccepted. return { testId: entry.testId, runId: entry.runId, status: 'error', - error: { code: err.code, message: err.message, exitCode: 1 }, + error: { code: err.code, message: err.message, exitCode: err.exitCode }, }; } throw err; @@ -6834,6 +6845,17 @@ export async function runTestRerun( } if (failed > 0) { + // Auth failure on any member is a batch-wide condition — the credential is + // bad, not the test. Propagate exit 3 so the operator fixes auth rather than + // chasing a "rerun failed" (exit 1). Mirrors the identical logic already + // applied to runTestRunAll lines 5462-5468. + const authErr = rerunResults.find(r => r.error?.exitCode === 3); + if (authErr) { + throw new CLIError( + `${failed} rerun${failed !== 1 ? 's' : ''} failed — auth error (${authErr.error?.code}): ${authErr.error?.message}`, + 3, + ); + } throw new CLIError(`${failed} rerun${failed !== 1 ? 's' : ''} failed.`, 1); } From e1d00f1816cd87fd2f64337708f2ed8849456efa Mon Sep 17 00:00:00 2001 From: Resque Date: Sun, 5 Jul 2026 23:32:23 +0400 Subject: [PATCH 036/102] feat(agent): add kiro as an install target (#10) * feat(agent): add kiro as an install target Adds kiro as an own-file agent target on the current multi-skill model: AgentTarget union, a pathFor('kiro') case landing at .kiro/skills//SKILL.md, and a TARGETS entry (experimental, own-file, wrapSkill frontmatter like claude/antigravity). Kiro installs both default skills (testsprite-verify + testsprite-onboard). Updates the --target help string, README/DOCUMENTATION target lists and counts, unit + command tests (six keys, list 12 rows, five own-file targets, 10 dry-run would-write lines, renderForTarget + content-integrity coverage), and regenerates the agent/setup help snapshots. Rebuilt on current main. * test(e2e): include kiro in target matrix guards and multi-target install The Local E2E Tests CI job failed because two matrix-coverage guards hardcoded the target set (claude, antigravity, cursor, cline, codex) and did not include the new kiro target. Add kiro (own-file, between cline and codex to match TARGETS order) to both guards, and add kiro to the multi-target install e2e so the target is actually exercised end-to-end. --- DOCUMENTATION.md | 7 ++--- README.md | 2 +- src/commands/agent.test.ts | 25 +++++++++-------- src/commands/agent.ts | 2 +- src/lib/agent-targets.test.ts | 27 ++++++++++++++++--- src/lib/agent-targets.ts | 12 ++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 7 ++--- test/e2e/agent-install.e2e.test.ts | 15 ++++++++--- test/e2e/setup.e2e.test.ts | 9 ++++++- 9 files changed, 77 insertions(+), 29 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index a1ac967..7a89216 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -113,14 +113,15 @@ testsprite agent install codex # install into AGENTS.md for Codex (managed- testsprite agent install cursor # .cursor/rules/testsprite-verify.mdc testsprite agent install cline # .clinerules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md -testsprite agent list # list all 5 targets with status + mode + path +testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md +testsprite agent list # list all 6 targets with status + mode + path ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental). +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental). The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro) backs up the existing file to `.bak` first. ## Command reference diff --git a/README.md b/README.md index 067a620..1e20a1d 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- | | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | | | `test wait` | Block on a `runId` until terminal | | | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 9bdbdf1..493cc4e 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -741,6 +741,7 @@ describe('runList', () => { expect(out).toContain('cursor'); expect(out).toContain('cline'); expect(out).toContain('antigravity'); + expect(out).toContain('kiro'); expect(out).toContain('codex'); expect(out).toContain('ga'); expect(out).toContain('experimental'); @@ -749,6 +750,7 @@ describe('runList', () => { expect(out).toContain(TARGETS.cursor.path); expect(out).toContain(TARGETS.cline.path); expect(out).toContain(TARGETS.antigravity.path); + expect(out).toContain(TARGETS.kiro.path); expect(out).toContain(TARGETS.codex.path); }); @@ -759,13 +761,14 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 5 targets × 2 default skills = 10 rows - expect(json).toHaveLength(10); + // 6 targets × 2 default skills = 12 rows + expect(json).toHaveLength(12); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); expect(targets).toContain('antigravity'); + expect(targets).toContain('kiro'); expect(targets).toContain('codex'); // skill field present on each row const skills = json.map(r => r.skill); @@ -905,11 +908,11 @@ describe('createAgentCommand wiring', () => { }); // --------------------------------------------------------------------------- -// All four own-file targets installed at once +// All five own-file targets installed at once // --------------------------------------------------------------------------- -describe('runInstall — all four own-file targets', () => { - it('installs all four own-file targets in one invocation', async () => { +describe('runInstall — all five own-file targets', () => { + it('installs all five own-file targets in one invocation', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -919,7 +922,7 @@ describe('runInstall — all four own-file targets', () => { output: 'text', debug: false, dryRun: false, - target: ['claude', 'cursor', 'cline', 'antigravity'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro'], skills: ['testsprite-verify'], force: false, }, @@ -937,11 +940,11 @@ describe('runInstall — all four own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all four own-file targets +// Dry-run for all five own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the four own-file targets (default 2 skills = 8 would-write lines)', async () => { + it('writes nothing for any of the five own-file targets (default 2 skills = 10 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -951,7 +954,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -961,9 +964,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 4 targets × 2 default skills = 8 would-write lines + // 5 targets × 2 default skills = 10 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(8); + expect(wouldWriteLines.length).toBe(10); }); }); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index ebde317..8213755 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -809,7 +809,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, codex (comma-separated or repeated)', collect, [], ) diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index b049711..8ef0e36 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -74,19 +74,20 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all five required keys', () => { + it('has all six required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor']); + expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro']); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, antigravity, and codex are experimental', () => { + it('cursor, cline, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); + expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); }); @@ -102,6 +103,7 @@ describe('TARGETS', () => { expect(TARGETS.antigravity.mode).toBe('own-file'); expect(TARGETS.cursor.mode).toBe('own-file'); expect(TARGETS.cline.mode).toBe('own-file'); + expect(TARGETS.kiro.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -200,6 +202,22 @@ describe('renderForTarget("antigravity")', () => { }); }); +describe('renderForTarget("kiro")', () => { + const result = renderForTarget('kiro', 'testsprite-verify', STUB_BODY); + + it('returns the correct path', () => { + expect(result.path).toBe('.kiro/skills/testsprite-verify/SKILL.md'); + }); + + it('frontmatter contains name: testsprite-verify', () => { + expect(result.content).toContain('name: testsprite-verify'); + }); + + it('frontmatter contains description:', () => { + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); +}); + describe('renderForTarget("claude") vs renderForTarget("antigravity")', () => { it('produce the same frontmatter lines (name + description)', () => { const claude = renderForTarget('claude', 'testsprite-verify', STUB_BODY); @@ -274,11 +292,12 @@ describe('renderForTarget("cline")', () => { // --------------------------------------------------------------------------- describe('content integrity — own-file targets', () => { - const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity'> = [ + const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity' | 'kiro'> = [ 'claude', 'cursor', 'cline', 'antigravity', + 'kiro', ]; // Use the real body for these checks, since we're guarding against trimming. diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 2547daf..663a998 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -1,6 +1,6 @@ import { readFileSync } from 'node:fs'; -export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex'; +export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex' | 'kiro'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -140,6 +140,8 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.cursor/rules/${skill}.mdc`; case 'cline': return `.clinerules/${skill}.md`; + case 'kiro': + return `.kiro/skills/${skill}/SKILL.md`; case 'codex': return 'AGENTS.md'; } @@ -170,6 +172,14 @@ export const TARGETS: Record = { mode: 'own-file', wrap: (_name, _description, body) => body, }, + kiro: { + status: 'experimental', + path: pathFor('kiro', SKILL_NAME), + mode: 'own-file', + // kiro reads SKILL.md files with name/description frontmatter, same as + // claude/antigravity, so it shares the wrapSkill wrapper. + wrap: wrapSkill, + }, /** * codex target — managed-section mode. * diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 9e5e0d9..03cb551 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -25,8 +25,8 @@ Write the TestSprite agent skills (verification loop + first-run onboarding) into a project for a coding agent Options: - --target Agent target(s): claude, cursor, cline, antigravity, codex - (comma-separated or repeated) (default: []) + --target Agent target(s): claude, cursor, cline, antigravity, kiro, + codex (comma-separated or repeated) (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -112,7 +112,8 @@ Options: --from-env Read TESTSPRITE_API_KEY from the environment instead of prompting (default: false) --agent Coding-agent target to install: claude, antigravity, - cursor, cline, codex (default: claude) (default: "claude") + cursor, cline, kiro, codex (default: claude) (default: + "claude") --no-agent Skip the agent skill install (configure credentials only) --force Overwrite an existing skill file (a .bak backup is kept) --dir Project root for the skill install (default: current diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 1cbf4fc..0513b8b 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -430,13 +430,13 @@ describe('dry-run', () => { // --------------------------------------------------------------------------- describe('multi-target install', () => { - it('--target=claude,cursor,cline,antigravity,codex writes all targets + skills, exit 0', () => { + it('--target=claude,cursor,cline,antigravity,kiro,codex writes all targets + skills, exit 0', () => { const tmpDir = freshTmpDir(); const result = runCli([ 'agent', 'install', - '--target=claude,cursor,cline,antigravity,codex', + '--target=claude,cursor,cline,antigravity,kiro,codex', '--dir', tmpDir, '--output', @@ -449,7 +449,7 @@ describe('multi-target install', () => { action: string; path: string; }>; - const allTargets: AgentTarget[] = ['claude', 'cursor', 'cline', 'antigravity', 'codex']; + const allTargets: AgentTarget[] = ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'codex']; for (const target of allTargets) { if (TARGETS[target].mode === 'managed-section') { @@ -819,7 +819,14 @@ describe('agent list', () => { // --------------------------------------------------------------------------- describe('matrix coverage guard', () => { it('TARGETS matches the documented, e2e-covered set (update this list when adding a target)', () => { - expect(Object.keys(TARGETS)).toEqual(['claude', 'antigravity', 'cursor', 'cline', 'codex']); + expect(Object.keys(TARGETS)).toEqual([ + 'claude', + 'antigravity', + 'cursor', + 'cline', + 'kiro', + 'codex', + ]); }); it('SKILLS matches the documented, e2e-covered set (update this list when adding a skill)', () => { diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index b5d3963..a30c3fb 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -222,7 +222,14 @@ describe('deprecated `init` alias', () => { describe('matrix coverage guard', () => { it('TARGETS matches the documented set (update this list when adding a target)', () => { - expect(Object.keys(TARGETS)).toEqual(['claude', 'antigravity', 'cursor', 'cline', 'codex']); + expect(Object.keys(TARGETS)).toEqual([ + 'claude', + 'antigravity', + 'cursor', + 'cline', + 'kiro', + 'codex', + ]); }); }); From 3fd703ffac59a38b6b5c227befdaf2510497b308 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:32:44 +0200 Subject: [PATCH 037/102] fix(poll): emit partial run to stdout on TimeoutError in run --wait and test wait (#153) Apply the fix in src/commands/ (the compiled CLI path). When the overall --timeout polling deadline is exceeded, emit {runId, status:"running"} to stdout before exit 7 so JSON agents can chain into test wait. Co-authored-by: Contributor Co-authored-by: Cursor --- src/commands/test.run.spec.ts | 68 ++++++++++++++++++++++++++++++++++ src/commands/test.ts | 32 ++++++++++++++++ src/commands/test.wait.spec.ts | 63 +++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+) diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index f33d08a..2f2ea9d 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -1935,6 +1935,74 @@ describe('runTestRun --wait: Fix 3 — RequestTimeoutError writes partial JSON t }); }); +// --------------------------------------------------------------------------- +// TimeoutError on --wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestRun --wait: TimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const { credentialsPath } = makeCreds(); + let dateCallCount = 0; + let fetchCallCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (++dateCallCount > 6 ? base + 2000 : base); + + try { + const fetchImpl: typeof globalThis.fetch = async () => { + ++fetchCallCount; + if (fetchCallCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + const runningRun: RunResponse = { ...makePassedRun(), status: 'running' }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + targetUrl: string; + }; + expect(stdoutJson.runId).toBe(TRIGGER_RESP.runId); + expect(stdoutJson.status).toBe('running'); + expect(stdoutJson.targetUrl).toBe(TRIGGER_RESP.targetUrl); + } finally { + Date.now = realDateNow; + } + }); +}); + // --------------------------------------------------------------------------- // Fix 5 — B2(c): --timeout hint fires on default, not on explicit timeout // --------------------------------------------------------------------------- diff --git a/src/commands/test.ts b/src/commands/test.ts index a48812e..29cffd5 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -4861,6 +4861,26 @@ export async function runTestRun( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${triggerResponse.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, timeoutPartial, opts.createContext, data => { + const p = data as typeof timeoutPartial; + const lines = [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + ]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + return lines.join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md @@ -5021,6 +5041,18 @@ export async function runTestWait( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${opts.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { runId: opts.runId, status: 'running' as const }; + out.print(timeoutPartial, data => { + const p = data as typeof timeoutPartial; + return [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md diff --git a/src/commands/test.wait.spec.ts b/src/commands/test.wait.spec.ts index 68a4c99..c83195d 100644 --- a/src/commands/test.wait.spec.ts +++ b/src/commands/test.wait.spec.ts @@ -1021,6 +1021,69 @@ describe('runTestWait: Fix 3 — RequestTimeoutError writes partial JSON to stdo }); }); +// --------------------------------------------------------------------------- +// TimeoutError on test wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestWait: TimeoutError writes partial JSON to stdout', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('running') })); + + let callCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (callCount++ > 4 ? base + 2000 : base); + + try { + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + } finally { + Date.now = realDateNow; + } + }); +}); + // --------------------------------------------------------------------------- // FIX 4 — D5-UX: text mode shows the `error` string for failed/blocked runs // --------------------------------------------------------------------------- From dcbcbee16c868e8fb96dc84824d0596e6f1c2703 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:33:11 -0400 Subject: [PATCH 038/102] fix(steps): surface run-scoped per-step error text and stepType (#167) --- src/commands/test.test.ts | 59 +++++++++++++++++++++++++++++++++++++++ src/commands/test.ts | 44 +++++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index d3b45ca..501ed50 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -2603,6 +2603,65 @@ describe('runSteps', () => { expect(step2.outcomeContributesToFailure).toBe(true); }); + it('--run-id carries the per-step error text and stepType through to JSON (no silent drop)', async () => { + // Regression lock for the "steps discard RunStepDto.error" gap: the wire + // already returns the failure text in the same response; it must survive + // the CliTestStep mapping instead of forcing an artifact-bundle download. + const { credentialsPath } = makeCreds(); + const runWithStepError = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + steps: [ + RUN_WITH_STEPS.steps[0]!, + { + ...RUN_WITH_STEPS.steps[1]!, + status: 'failed', + error: 'Expected heading "Order confirmed" to be visible, got hidden', + }, + ], + }; + const fetchImpl = makeFetch(() => ({ body: runWithStepError })); + const page = await runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + const passing = page.items.find(s => s.stepIndex === 1)!; + const failing = page.items.find(s => s.stepIndex === 2)!; + expect(failing.error).toBe('Expected heading "Order confirmed" to be visible, got hidden'); + expect(failing.stepType).toBe('assertion'); + expect(passing.error).toBeNull(); + expect(passing.stepType).toBe('action'); + }); + + it('--run-id text mode prints an indented error: sub-line under the failed row only', async () => { + const { credentialsPath } = makeCreds(); + const runWithStepError = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + steps: [ + RUN_WITH_STEPS.steps[0]!, + { + ...RUN_WITH_STEPS.steps[1]!, + status: 'failed', + error: 'Locator resolved to hidden element\n at assert heading', + }, + ], + }; + const fetchImpl = makeFetch(() => ({ body: runWithStepError })); + const out: string[] = []; + await runSteps( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + // Newlines in the wire error collapse to one displayable line. + expect(block).toContain('error: Locator resolved to hidden element at assert heading'); + // Exactly one sub-line: the passing step must not grow one. + expect(block.match(/error: /g)).toHaveLength(1); + }); + it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => { const { credentialsPath } = makeCreds(); // The run-scoped endpoint returns a run whose testId differs from the diff --git a/src/commands/test.ts b/src/commands/test.ts index 29cffd5..ede6376 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -186,6 +186,19 @@ export interface CliTestStep { * that don't emit the field still type-check. */ outcomeContributesToFailure?: boolean | null; + /** + * Per-step failure text, carried from `RunStepDto.error` on the run-scoped + * endpoint (`GET /runs/{id}?includeSteps=true`). Only present on `--run-id` + * responses; the cumulative `/tests/{id}/steps` rows do not carry it, so the + * field stays optional (additive, non-breaking for existing consumers). + */ + error?: string | null; + /** + * Wire step kind from `RunStepDto.type` on the run-scoped endpoint. Same + * availability rules as `error`. Named `stepType` to avoid colliding with + * the free-form `action` label above. + */ + stepType?: 'action' | 'assertion'; } /** @@ -3642,6 +3655,12 @@ function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestSte // non-contributors. (Per the CliTestStep contract: null ≠ false.) outcomeContributesToFailure: run.failedStepIndex === null ? null : numericIndex === run.failedStepIndex, + // Carry the per-step failure text and the wire step kind through instead + // of dropping them: the agent asking "why did this step fail?" would + // otherwise have to download the whole artifact bundle to read a string + // this very response already contained. + error: step.error, + stepType: step.type, }; } @@ -3970,6 +3989,14 @@ const RUN_HISTORY_TABLE_COL_WIDTHS = { */ const DESC_COL_MAX = 60; +/** + * Cap, in chars, for the one-line `error:` sub-line under a failed step row + * in `renderStepsText`. Long enough for a full assertion message, short + * enough that a stack-trace blob can't flood the table. Full text is in + * `--output json`. + */ +const ERROR_SUBLINE_MAX = 200; + /** Max chars to show in the TARGETURL sub-line (excess truncated with …). */ const HISTORY_TARGET_URL_MAX = 80; @@ -8433,9 +8460,9 @@ function renderStepsText(page: Page): string { ' ' + 'UPDATED'; - const rows = page.items.map(s => { + const rows = page.items.flatMap(s => { const marker = s.outcomeContributesToFailure === true ? '* ' : ' '; - return [ + const row = [ marker, pad(String(s.stepIndex), indexWidth), pad(s.action, actionWidth), @@ -8443,6 +8470,19 @@ function renderStepsText(page: Page): string { pad(descOf(s), descWidth), s.updatedAt, ].join(' '); + // Run-scoped rows carry the per-step failure text; surface it as an + // indented sub-line under failed rows (mirrors the history table's + // `targetUrl:` sub-line). Collapsed to one line and capped so a huge + // stack blob can't wreck the table; full text ships in --output json. + if (s.status === 'failed' && typeof s.error === 'string' && s.error.length > 0) { + const oneLine = s.error.replace(/\s+/g, ' ').trim(); + const shown = + oneLine.length > ERROR_SUBLINE_MAX + ? `${oneLine.slice(0, ERROR_SUBLINE_MAX - 1)}…` + : oneLine; + return [row, ` error: ${shown}`]; + } + return [row]; }); const lines: string[] = [header, ...rows, '']; From 9457583f27b8c4e57541933db9ed45e554dc9ad2 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:33:26 -0400 Subject: [PATCH 039/102] feat(test): add "test diff " to isolate run-to-run regressions (#168) * feat(test): add "test diff " to isolate run-to-run regressions * test(diff): cover runDiff --dry-run branch (offline canned sample) --- src/commands/test.test.ts | 127 +++++++++++ src/commands/test.ts | 200 ++++++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 5 + 3 files changed, 332 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 501ed50..38c16e5 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -27,6 +27,7 @@ import { runCreateBatch, runCreateFromPlan, runDelete, + runDiff, runFailureGet, runFailureSummary, runGet, @@ -121,6 +122,7 @@ describe('createTestCommand — surface', () => { 'create-batch', 'delete', 'delete-batch', + 'diff', 'failure', 'get', 'list', @@ -2833,6 +2835,131 @@ describe('runSteps', () => { }); }); +describe('runDiff', () => { + const baseRun = { + testId: 'test_fe', + projectId: 'project_alice', + userId: 'u1', + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + targetUrl: 'https://example.com', + createdFrom: null, + error: null, + videoUrl: null, + stepSummary: { total: 2, completed: 2, passedCount: 2, failedCount: 0 }, + }; + const makeStep = (index: string, status: string, error: string | null = null) => ({ + stepIndex: index, + type: 'action', + action: `step ${index}`, + status, + description: `Step ${index}`, + error, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-06-01T10:00:05.000Z', + }); + const RUN_GREEN = { + ...baseRun, + runId: 'run_green', + status: 'passed', + codeVersion: 'v1', + failedStepIndex: null, + failureKind: null, + steps: [makeStep('0001', 'passed'), makeStep('0002', 'passed')], + }; + const RUN_RED = { + ...baseRun, + runId: 'run_red', + status: 'failed', + codeVersion: 'v2', + failedStepIndex: 2, + failureKind: 'assertion', + steps: [makeStep('0001', 'passed'), makeStep('0002', 'failed', 'heading not visible')], + }; + const fetchForRuns = () => + makeFetch(url => ({ body: url.includes('run_green') ? RUN_GREEN : RUN_RED })); + + it('reports the verdict flip, the changed step with its error, and code drift, then exits 1', async () => { + const { credentialsPath } = makeCreds(); + const out: string[] = []; + const rejection = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' }, + { credentialsPath, fetchImpl: fetchForRuns(), stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const printed = JSON.parse(out.join('')) as { + verdictChanged: boolean; + codeVersionChanged: boolean; + crossTest: boolean; + changedSteps: Array<{ stepIndex: number; statusA: string; statusB: string; errorB?: string }>; + }; + expect(printed.verdictChanged).toBe(true); + expect(printed.codeVersionChanged).toBe(true); + expect(printed.crossTest).toBe(false); + expect(printed.changedSteps).toHaveLength(1); + expect(printed.changedSteps[0]).toMatchObject({ + stepIndex: 2, + statusA: 'passed', + statusB: 'failed', + errorB: 'heading not visible', + }); + }); + + it('identical verdicts resolve with exit 0 and no step changes', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RUN_GREEN })); + const diff = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_green' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(diff.verdictChanged).toBe(false); + expect(diff.changedSteps).toHaveLength(0); + }); + + it('warns (not fails) when the runs belong to different tests', async () => { + const { credentialsPath } = makeCreds(); + const OTHER = { ...RUN_GREEN, runId: 'run_red', testId: 'test_other' }; + const fetchImpl = makeFetch(url => ({ + body: url.includes('run_green') ? RUN_GREEN : OTHER, + })); + const errs: string[] = []; + const diff = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) }, + ); + expect(diff.crossTest).toBe(true); + expect(errs.join('\n')).toContain('different tests'); + }); + + it('--dry-run returns the canned sample fully offline (no credentials, no fetch)', async () => { + // Dry-run must not require credentials or hit the network — it returns a + // canned CliRunDiff so `--dry-run` shows the shape offline. + const diff = await runDiff( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runA: 'run_aaa', + runB: 'run_bbb', + }, + { stdout: () => undefined, stderr: () => undefined }, + ); + expect(diff.runA.runId).toBe('run_aaa'); + expect(diff.runB.runId).toBe('run_bbb'); + expect(diff.verdictChanged).toBe(true); + expect(diff.changedSteps).toHaveLength(1); + expect(diff.changedSteps[0]).toMatchObject({ + stepIndex: 2, + statusA: 'passed', + statusB: 'failed', + }); + }); +}); + describe('runResult', () => { it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index ede6376..7fe985d 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3664,6 +3664,196 @@ function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestSte }; } +export interface DiffOptions extends CommonOptions { + runA: string; + runB: string; +} + +/** One step whose status flipped between the two compared runs. */ +export interface CliDiffStep { + stepIndex: number; + statusA: string; + statusB: string; + /** First divergent failing side's error text, when the wire carried one. */ + errorA?: string | null; + errorB?: string | null; +} + +export interface CliRunDiff { + runA: { + runId: string; + testId: string; + status: string; + failureKind: string | null; + failedStepIndex: number | null; + codeVersion: string | null; + }; + runB: { + runId: string; + testId: string; + status: string; + failureKind: string | null; + failedStepIndex: number | null; + codeVersion: string | null; + }; + verdictChanged: boolean; + failedStepIndexChanged: boolean; + failureKindChanged: boolean; + codeVersionChanged: boolean; + /** True when the two runs belong to DIFFERENT tests (deltas may be meaningless). */ + crossTest: boolean; + changedSteps: CliDiffStep[]; +} + +/** + * `test diff ` (issue #124): isolate what regressed between two + * runs, the first question when CI goes red ("what changed since the last + * green run?"). Pure client-side composition of the existing per-run read + * (`GET /runs/{id}?includeSteps=true`); the endpoint accepts any two run-ids, + * so a cross-test pair is a WARNING, not an error. Exit 0 when the verdicts + * match, exit 1 when they differ, so the command is CI-scriptable. + */ +export async function runDiff(opts: DiffOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const sample: CliRunDiff = { + runA: { + runId: opts.runA, + testId: 'test_dryrun', + status: 'passed', + failureKind: null, + failedStepIndex: null, + codeVersion: 'v1', + }, + runB: { + runId: opts.runB, + testId: 'test_dryrun', + status: 'failed', + failureKind: 'assertion', + failedStepIndex: 2, + codeVersion: 'v1', + }, + verdictChanged: true, + failedStepIndexChanged: true, + failureKindChanged: true, + codeVersionChanged: false, + crossTest: false, + changedSteps: [{ stepIndex: 2, statusA: 'passed', statusB: 'failed' }], + }; + out.print(sample, () => renderRunDiffText(sample)); + return sample; + } + + const client = makeClient(opts, deps); + const [runA, runB] = await Promise.all([ + client.getRun(opts.runA, { includeSteps: true }), + client.getRun(opts.runB, { includeSteps: true }), + ]); + + const crossTest = runA.testId !== runB.testId; + if (crossTest) { + stderrFn( + `⚠ the two runs belong to different tests (${runA.testId} vs ${runB.testId}) — step deltas may be meaningless`, + ); + } + + const stepsByIndex = ( + run: RunResponse, + ): Map => { + const map = new Map(); + for (const step of run.steps ?? []) { + const index = parseInt(step.stepIndex, 10); + if (Number.isInteger(index)) + map.set(index, { status: step.status ?? 'unknown', error: step.error }); + } + return map; + }; + const stepsA = stepsByIndex(runA); + const stepsB = stepsByIndex(runB); + const allIndexes = [...new Set([...stepsA.keys(), ...stepsB.keys()])].sort( + (left, right) => left - right, + ); + const changedSteps: CliDiffStep[] = []; + for (const index of allIndexes) { + const sideA = stepsA.get(index); + const sideB = stepsB.get(index); + const statusA = sideA?.status ?? 'absent'; + const statusB = sideB?.status ?? 'absent'; + if (statusA === statusB) continue; + changedSteps.push({ + stepIndex: index, + statusA, + statusB, + ...(sideA?.error ? { errorA: sideA.error } : {}), + ...(sideB?.error ? { errorB: sideB.error } : {}), + }); + } + + const summarize = (run: RunResponse) => ({ + runId: run.runId, + testId: run.testId, + status: run.status, + failureKind: run.failureKind ?? null, + failedStepIndex: run.failedStepIndex, + codeVersion: run.codeVersion ?? null, + }); + const diff: CliRunDiff = { + runA: summarize(runA), + runB: summarize(runB), + verdictChanged: runA.status !== runB.status, + failedStepIndexChanged: runA.failedStepIndex !== runB.failedStepIndex, + failureKindChanged: (runA.failureKind ?? null) !== (runB.failureKind ?? null), + codeVersionChanged: (runA.codeVersion ?? null) !== (runB.codeVersion ?? null), + crossTest, + changedSteps, + }; + out.print(diff, () => renderRunDiffText(diff)); + + if (diff.verdictChanged) { + // Result already printed; the typed exit makes `test diff` a CI gate. + throw new CLIError( + `verdicts differ: ${runA.runId}=${runA.status} vs ${runB.runId}=${runB.status}`, + 1, + ); + } + return diff; +} + +function renderRunDiffText(diff: CliRunDiff): string { + const lines: string[] = []; + lines.push(`runA: ${diff.runA.runId} ${diff.runA.status} (test ${diff.runA.testId})`); + lines.push(`runB: ${diff.runB.runId} ${diff.runB.status} (test ${diff.runB.testId})`); + lines.push( + `verdict: ${diff.verdictChanged ? `${diff.runA.status} -> ${diff.runB.status}` : `unchanged (${diff.runA.status})`}`, + ); + if (diff.failureKindChanged) + lines.push( + `failureKind: ${diff.runA.failureKind ?? '(none)'} -> ${diff.runB.failureKind ?? '(none)'}`, + ); + if (diff.failedStepIndexChanged) + lines.push( + `failedStepIndex: ${diff.runA.failedStepIndex ?? '(none)'} -> ${diff.runB.failedStepIndex ?? '(none)'}`, + ); + lines.push( + `codeVersion: ${diff.codeVersionChanged ? `${diff.runA.codeVersion ?? '(none)'} -> ${diff.runB.codeVersion ?? '(none)'} (code drift)` : 'unchanged'}`, + ); + if (diff.changedSteps.length === 0) { + lines.push('steps: no per-step status changes'); + } else { + lines.push(`steps changed: ${diff.changedSteps.length}`); + for (const step of diff.changedSteps) { + lines.push(` #${step.stepIndex} ${step.statusA} -> ${step.statusB}`); + if (step.errorB) lines.push(` error(B): ${step.errorB.replace(/\s+/g, ' ').trim()}`); + else if (step.errorA) + lines.push(` error(A): ${step.errorA.replace(/\s+/g, ' ').trim()}`); + } + } + return lines.join('\n'); +} + export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -7413,6 +7603,16 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('diff ') + .description( + 'Compare two runs and print what regressed: verdict, failureKind, failedStepIndex, per-step status flips, codeVersion drift. Exit 0 when verdicts match, 1 when they differ.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (runA: string, runB: string, _cmdOpts: unknown, command: Command) => { + await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps); + }); + test .command('result ') .description( diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 03cb551..f8e9fa2 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -200,6 +200,11 @@ Commands: steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) + diff Compare two runs and print what + regressed: verdict, failureKind, + failedStepIndex, per-step status flips, + codeVersion drift. Exit 0 when verdicts + match, 1 when they differ. result [options] Get the latest result for a test (default) or list prior runs (--history). --output json shape differs by mode: From 011208c46f3d132e98cb3138b0b36770eb05e639 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 15:34:11 -0400 Subject: [PATCH 040/102] feat(agent): stamp installed skills with a version/hash marker and add "agent status" (#177) --- src/commands/agent.test.ts | 129 ++++++++- src/commands/agent.ts | 265 +++++++++++++++++- src/lib/agent-targets.test.ts | 113 ++++++++ src/lib/agent-targets.ts | 146 +++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 3 + 5 files changed, 645 insertions(+), 11 deletions(-) diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 493cc4e..f8ed59f 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -9,13 +9,21 @@ import { MANAGED_SECTION_END, ONBOARD_CODEX_LINE, SKILLS, + buildSkillMarker, pathFor, renderForTarget, + renderOwnFileWithMarker, TARGETS, type AgentTarget, } from '../lib/agent-targets.js'; -import type { AgentDeps, AgentFs, InstallResult, ListResult } from './agent.js'; -import { AGENTS_MD_CODEX_BUDGET_BYTES, createAgentCommand, runInstall, runList } from './agent.js'; +import type { AgentDeps, AgentFs, InstallResult, ListResult, StatusResult } from './agent.js'; +import { + AGENTS_MD_CODEX_BUDGET_BYTES, + createAgentCommand, + runInstall, + runList, + runStatus, +} from './agent.js'; // --------------------------------------------------------------------------- // In-memory AgentFs backed by a Map @@ -2430,3 +2438,120 @@ describe('runInstall — SKILLS registry / DEFAULT_SKILLS contract', () => { expect(ONBOARD_CODEX_LINE).toContain('**First-time setup:**'); }); }); + +// --------------------------------------------------------------------------- +// runStatus — `agent status` (issue #123) +// --------------------------------------------------------------------------- + +describe('runStatus — agent status (issue #123)', () => { + const statusOpts = { + profile: 'default' as const, + output: 'json' as const, + debug: false, + dryRun: false, + }; + + /** Run status against the given fs and return the printed rows. */ + async function statusRows(agentFs: AgentFs): Promise<{ rows: StatusResult[]; thrown: unknown }> { + const { capture, deps } = makeCapture(); + let thrown: unknown; + try { + await runStatus(statusOpts, { cwd: CWD, fs: agentFs, ...deps }); + } catch (err) { + thrown = err; + } + return { rows: JSON.parse(capture.stdout.join('')) as StatusResult[], thrown }; + } + + it('nothing installed: every row is absent and the command exits 0', async () => { + const { fs: agentFs } = makeMemFs(); + const { rows, thrown } = await statusRows(agentFs); + expect(thrown).toBeUndefined(); + expect(rows).toHaveLength(Object.keys(TARGETS).length * DEFAULT_SKILLS.length); + expect(rows.every(row => row.state === 'absent')).toBe(true); + }); + + it('fresh installs read ok (own-file and codex managed section), exit 0', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'codex'], + skills: [...DEFAULT_SKILLS], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(thrown).toBeUndefined(); + for (const skill of DEFAULT_SKILLS) { + expect(rows.find(r => r.target === 'claude' && r.skill === skill)?.state).toBe('ok'); + expect(rows.find(r => r.target === 'codex' && r.skill === skill)?.state).toBe('ok'); + expect(rows.find(r => r.target === 'cursor' && r.skill === skill)?.state).toBe('absent'); + } + }); + + it('stale: a marker whose hash matches an OLDER body reads stale and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const oldBody = '# TestSprite Verification Loop\n\nold body from a previous CLI release\n'; + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + renderOwnFileWithMarker( + 'claude', + 'testsprite-verify', + buildSkillMarker('testsprite-verify', oldBody), + oldBody, + ), + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'stale', + ); + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(1); + expect((thrown as CLIError).message).toContain('need attention'); + }); + + it('modified: current hash but edited bytes reads modified and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const canonical = renderForTarget('claude', 'testsprite-verify').content; + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + `${canonical}\n\n`, + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'modified', + ); + expect((thrown as CLIError).exitCode).toBe(1); + }); + + it('unmarked: an artifact without a marker line reads unmarked and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + '# hand-rolled skill file with no marker\n', + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'unmarked', + ); + expect((thrown as CLIError).exitCode).toBe(1); + }); + + it('rejects an explicit empty --dir (exit 5), matching the resolve-to-cwd hazard', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + await expect( + runStatus({ ...statusOpts, dir: ' ' }, { cwd: CWD, fs: agentFs, ...deps }), + ).rejects.toMatchObject({ exitCode: 5 }); + }); +}); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 8213755..1b4e878 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -11,10 +11,15 @@ import { TARGETS, SKILLS, DEFAULT_SKILLS, + MARKER_SKILL_SEPARATOR, pathFor, loadSkillBodyFor, + bodyHash12, buildCodexAggregate, + buildSkillMarker, + parseSkillMarker, renderForTarget, + renderOwnFileWithMarker, MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, } from '../lib/agent-targets.js'; @@ -150,11 +155,15 @@ async function writeBackup(agentFs: AgentFs, abs: string, existing: string): Pro // --------------------------------------------------------------------------- /** - * Build the section block to inject (sentinels + body + trailing newline). + * Build the section block to inject (sentinels + marker + body + trailing + * newline). The provenance marker line sits just inside the BEGIN sentinel so + * `agent status` can fingerprint the section. The same skill set + CLI version + * + body always produce byte-identical output, so the classifySection + * 'unchanged' fast-path keeps working across re-installs. * Uses \n throughout; the caller handles CRLF normalisation. */ -function buildSection(body: string): string { - return `${MANAGED_SECTION_BEGIN}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; +function buildSection(body: string, markerLine: string): string { + return `${MANAGED_SECTION_BEGIN}\n${markerLine}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; } /** @@ -437,7 +446,14 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr let codexSectionCache: string | undefined; const getCodexSection = (): string => { if (codexSectionCache === undefined) { - codexSectionCache = buildSection(buildCodexAggregate(skills)); + const aggregate = buildCodexAggregate(skills); + // ONE marker for the whole managed section: it names every aggregated + // skill ('+'-joined) and hashes the canonical aggregate body, so + // `agent status` can attribute and fingerprint the section per skill. + codexSectionCache = buildSection( + aggregate, + buildSkillMarker(skills.join(MARKER_SKILL_SEPARATOR), aggregate), + ); } return codexSectionCache; }; @@ -789,6 +805,236 @@ export async function runList(opts: CommonOptions, deps: AgentDeps = {}): Promis }); } +// --------------------------------------------------------------------------- +// runStatus (issue #123: detect silently stale installed skill files) +// --------------------------------------------------------------------------- + +/** + * Health of one installed skill artifact, as reported by `agent status`. + * + * Decision order (first match wins): + * - 'absent' : nothing at the landing path (codex: no managed section, + * including an AGENTS.md that exists without our sentinels). + * - 'corrupt' : codex only. Dangling or duplicated sentinels, the same + * classification `agent install` refuses on; status REPORTS it + * instead of refusing. + * - 'unmarked' : artifact present but carries no testsprite-skill marker + * (installed before markers existed), or the landing path is + * occupied by a non-regular file (never followed). + * - 'stale' : marker present, but its hash differs from the current + * canonical body: a re-install would change the content. Edits + * on top of an OLD install also read stale (older renders + * cannot be reproduced); the remedy is the same re-install. + * - 'modified' : marker hash matches the current body, but the artifact bytes + * differ from the canonical render carrying that same marker + * line: the user edited the artifact after install. + * - 'ok' : marker hash matches and the bytes equal the canonical render + * with the file's own marker line (a version-string-only lag + * with an unchanged body still reads ok). + * + * For the codex managed section, ONE marker names every aggregated skill + * ('+'-joined); skills not named by the marker report 'absent'. + */ +export type SkillArtifactState = 'ok' | 'stale' | 'modified' | 'unmarked' | 'absent' | 'corrupt'; + +export interface StatusResult { + target: AgentTarget; + skill: string; + path: string; + state: SkillArtifactState; +} + +interface StatusOptions extends CommonOptions { + dir?: string; +} + +/** + * Classify one own-file artifact per the {@link SkillArtifactState} contract. + * Comparisons are byte-exact, matching the installer's own skipped/blocked + * comparison for own-file targets. + */ +async function classifyOwnFileState( + agentFs: AgentFs, + abs: string, + target: AgentTarget, + skill: string, + bodyForSkill: (skill: string) => string, +): Promise { + const stat = await agentFs.lstat(abs); + if (stat === null) return 'absent'; + // Occupied by a directory or symlink: not something our installer wrote, and + // never followed (mirrors the installer's fail-closed stance on symlinks). + if (!stat.isFile) return 'unmarked'; + + const existing = await agentFs.readFile(abs); + const marker = parseSkillMarker(existing); + if (marker === null) return 'unmarked'; + + const canonicalBody = bodyForSkill(skill); + if (marker.hash12 !== bodyHash12(canonicalBody)) return 'stale'; + + // Hash matches the current body: pristine iff the file equals the canonical + // render carrying its own marker line, so a marker whose version string lags + // behind an unchanged body still reads ok. + const reRender = renderOwnFileWithMarker(target, skill, marker.line, canonicalBody); + return existing === reRender ? 'ok' : 'modified'; +} + +/** + * Classify the codex managed section per skill. The section is ONE artifact + * carrying ONE marker that names every aggregated skill, so a single + * inspection answers all skill rows; the returned function maps a skill name + * to its state. Comparisons are CRLF-insensitive on the section bytes. + */ +async function classifyManagedSectionStates( + agentFs: AgentFs, + abs: string, +): Promise<(skill: string) => SkillArtifactState> { + const constantState = + (state: SkillArtifactState): ((skill: string) => SkillArtifactState) => + () => + state; + + const stat = await agentFs.lstat(abs); + if (stat === null) return constantState('absent'); + // Occupied by a directory or symlink: never followed (fail-closed). + if (!stat.isFile) return constantState('unmarked'); + + const existing = await agentFs.readFile(abs); + + // Current canonical section for the default skill set. classifySection's + // 'unchanged' answers the common all-defaults-fresh case; its + // corrupt/append classification is reused verbatim for status verdicts. + const defaultAggregate = buildCodexAggregate(DEFAULT_SKILLS); + const defaultSection = buildSection( + defaultAggregate, + buildSkillMarker(DEFAULT_SKILLS.join(MARKER_SKILL_SEPARATOR), defaultAggregate), + ); + const sectionState = classifySection(existing, defaultSection); + + if (sectionState.kind === 'corrupt') return constantState('corrupt'); + // No standalone sentinels anywhere: the managed section is not installed. + if (sectionState.kind === 'append') return constantState('absent'); + if (sectionState.kind === 'unchanged') { + // Byte-identical to today's default install. + return skill => ((DEFAULT_SKILLS as readonly string[]).includes(skill) ? 'ok' : 'absent'); + } + if (sectionState.kind !== 'replace') { + // 'create' is unreachable when the file exists; treat defensively as absent. + return constantState('absent'); + } + + // Sentinels are present but the section differs from today's default + // canonical: slice the live section bytes out of the file and inspect its + // own marker (before/after are exact byte prefix/suffix around the section). + const sectionContent = existing.slice( + sectionState.before.length, + existing.length - sectionState.after.length, + ); + const marker = parseSkillMarker(sectionContent); + if (marker === null) return constantState('unmarked'); + + const installedSkills = marker.skill.split(MARKER_SKILL_SEPARATOR); + const coversSkill = (skill: string): boolean => installedSkills.includes(skill); + + // A marker naming a skill this CLI does not ship cannot be re-rendered; + // report the named skills stale (a re-install refreshes the section). + if (installedSkills.some(name => SKILLS[name] === undefined)) { + return skill => (coversSkill(skill) ? 'stale' : 'absent'); + } + + const canonicalAggregate = buildCodexAggregate(installedSkills); + if (marker.hash12 !== bodyHash12(canonicalAggregate)) { + return skill => (coversSkill(skill) ? 'stale' : 'absent'); + } + + // Hash matches the current aggregate: the section is pristine iff its bytes + // equal a re-render carrying its own marker line (version-string-only lag + // with an unchanged body still reads ok). + const pristine = + sectionContent.replace(/\r\n/g, '\n') === buildSection(canonicalAggregate, marker.line); + return skill => (coversSkill(skill) ? (pristine ? 'ok' : 'modified') : 'absent'); +} + +/** + * `agent status`: one row per (target × default skill), each classified per + * the {@link SkillArtifactState} contract. Exit contract: returns normally + * (exit 0) when every row is 'ok' or 'absent'; throws CLIError exit 1 when any + * row is stale/modified/unmarked/corrupt, so the command can gate CI. + */ +export async function runStatus(opts: StatusOptions, deps: AgentDeps = {}): Promise { + const agentFs = deps.fs ?? defaultAgentFs; + const out = makeOutput(opts.output, deps); + + // An explicit but empty --dir must not silently resolve to cwd + // (path.resolve('') === cwd). + if (opts.dir !== undefined && opts.dir.trim() === '') { + throw localValidationError('dir', 'must not be empty'); + } + const dir = opts.dir !== undefined ? opts.dir.trim() : (deps.cwd ?? process.cwd()); + const root = path.resolve(dir); + + // Canonical own-file bodies, read once per skill (same lazy caching pattern + // as runInstall's bodyForSkill). + const skillBodyCache = new Map(); + const bodyForSkill = (skill: string): string => { + let cachedBody = skillBodyCache.get(skill); + if (cachedBody === undefined) { + cachedBody = loadSkillBodyFor(skill); + skillBodyCache.set(skill, cachedBody); + } + return cachedBody; + }; + + const results: StatusResult[] = []; + for (const [target, spec] of Object.entries(TARGETS) as [ + AgentTarget, + { mode: string; path: string }, + ][]) { + if (spec.mode === 'managed-section') { + const stateFor = await classifyManagedSectionStates(agentFs, path.resolve(root, spec.path)); + for (const skill of DEFAULT_SKILLS) { + results.push({ target, skill, path: spec.path, state: stateFor(skill) }); + } + continue; + } + for (const skill of DEFAULT_SKILLS) { + const relPath = pathFor(target, skill); + results.push({ + target, + skill, + path: relPath, + state: await classifyOwnFileState( + agentFs, + path.resolve(root, relPath), + target, + skill, + bodyForSkill, + ), + }); + } + } + + out.print(results, data => { + const items = data as StatusResult[]; + const header = `${'TARGET'.padEnd(14)} ${'SKILL'.padEnd(20)} ${'STATE'.padEnd(10)} PATH`; + const rows = items.map( + row => `${row.target.padEnd(14)} ${row.skill.padEnd(20)} ${row.state.padEnd(10)} ${row.path}`, + ); + return [header, ...rows].join('\n'); + }); + + const needingAttention = results.filter( + result => result.state !== 'ok' && result.state !== 'absent', + ); + if (needingAttention.length > 0) { + throw new CLIError( + `${needingAttention.length} skill artifact(s) need attention (stale/modified/unmarked/corrupt); re-run \`testsprite agent install\` (add --force for own-file targets) to refresh them.`, + 1, + ); + } +} + // --------------------------------------------------------------------------- // Command factory // --------------------------------------------------------------------------- @@ -852,6 +1098,17 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { await runList(resolveCommonOptions(command), deps); }); + agent + .command('status') + .description( + 'Check installed TestSprite skill files against this CLI version: ok, stale, modified, unmarked, absent, or corrupt (exits 1 when anything needs attention, so it can gate CI)', + ) + .option('--dir ', 'Project root to inspect (default: cwd)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: { dir?: string }, command: Command) => { + await runStatus({ ...resolveCommonOptions(command), dir: cmdOpts.dir }, deps); + }); + return agent; } diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 8ef0e36..38c74e2 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -1,5 +1,7 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; +import { VERSION } from '../version.js'; import { DEFAULT_SKILLS, MANAGED_SECTION_BEGIN, @@ -9,13 +11,17 @@ import { SKILL_NAME, SKILLS, TARGETS, + bodyHash12, buildCodexAggregate, + buildSkillMarker, codexContentFor, loadCodexSkillBody, loadSkillBody, loadSkillBodyFor, + parseSkillMarker, pathFor, renderForTarget, + renderOwnFileWithMarker, } from './agent-targets.js'; // --------------------------------------------------------------------------- @@ -699,3 +705,110 @@ describe('renderForTarget for testsprite-onboard', () => { expect(() => renderForTarget('claude', 'testsprite-unknown')).toThrow('unknown skill'); }); }); + +// --------------------------------------------------------------------------- +// Install marker (issue #123): format, parsing, and render placement +// --------------------------------------------------------------------------- + +describe('buildSkillMarker / parseSkillMarker / bodyHash12', () => { + it('marker line is an HTML comment carrying name, vVERSION, and a 12-hex hash', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + expect(marker).toBe( + ``, + ); + expect(marker).toMatch( + /^$/, + ); + }); + + it('bodyHash12 equals the first 12 hex chars of the body SHA-256', () => { + const fullHex = createHash('sha256').update(STUB_BODY, 'utf8').digest('hex'); + expect(bodyHash12(STUB_BODY)).toBe(fullHex.slice(0, 12)); + }); + + it('parseSkillMarker round-trips a built marker embedded in surrounding content', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + const parsed = parseSkillMarker(`# heading\n${marker}\nbody text\n`); + expect(parsed).not.toBeNull(); + expect(parsed?.skill).toBe('testsprite-verify'); + expect(parsed?.version).toBe(VERSION); + expect(parsed?.hash12).toBe(bodyHash12(STUB_BODY)); + expect(parsed?.line).toBe(marker); + }); + + it('parseSkillMarker strips a trailing CR so CRLF checkouts parse identically', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + const parsed = parseSkillMarker(`${marker}\r\nrest\r\n`); + expect(parsed?.line).toBe(marker); + }); + + it('parseSkillMarker returns null when no marker line is present', () => { + expect(parseSkillMarker('# Just a heading\n\nProse without any marker.\n')).toBeNull(); + }); + + it('parseSkillMarker ignores the managed-section sentinels (also HTML comments)', () => { + expect(parseSkillMarker(`${MANAGED_SECTION_BEGIN}\nbody\n${MANAGED_SECTION_END}\n`)).toBeNull(); + }); +}); + +describe('render marker placement (own-file targets)', () => { + it('claude render carries the marker on the line right after the closing frontmatter fence', () => { + const { content } = renderForTarget('claude', 'testsprite-verify', STUB_BODY); + const closingFence = '\n---\n'; + const fenceEnd = content.indexOf(closingFence) + closingFence.length; + expect(content.slice(fenceEnd).startsWith(''; + const withForeign = renderOwnFileWithMarker( + 'claude', + 'testsprite-verify', + foreignMarker, + STUB_BODY, + ); + expect(withForeign).toContain(foreignMarker); + // Marker line aside, the bytes match the canonical render exactly. + const canonical = renderForTarget('claude', 'testsprite-verify', STUB_BODY).content; + const currentMarker = buildSkillMarker('testsprite-verify', STUB_BODY); + expect(withForeign.replace(foreignMarker, currentMarker)).toBe(canonical); + }); + + it('renderOwnFileWithMarker rejects the managed-section target', () => { + expect(() => renderOwnFileWithMarker('codex', 'testsprite-verify', 'marker', 'body')).toThrow( + 'own-file', + ); + }); + + it('renderOwnFileWithMarker throws on an unknown skill', () => { + expect(() => renderOwnFileWithMarker('claude', 'testsprite-unknown', 'marker')).toThrow( + 'unknown skill', + ); + }); +}); diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 663a998..0273a04 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -1,4 +1,6 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; +import { VERSION } from '../version.js'; export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex' | 'kiro'; @@ -211,6 +213,85 @@ export const MANAGED_SECTION_BEGIN = ''; export const MANAGED_SECTION_END = ''; +// --------------------------------------------------------------------------- +// Install marker (stale-skill detection, issue #123) +// --------------------------------------------------------------------------- + +/** + * Hex characters of the canonical body's SHA-256 kept in the install marker. + * 12 hex chars (48 bits) is ample for drift DETECTION (equality against bodies + * this CLI ships); the marker is provenance metadata, not a security boundary. + */ +const MARKER_HASH_HEX_LENGTH = 12; + +/** + * When one marker covers several skills (the codex managed section aggregates + * every installed skill), their names are joined with this separator in the + * marker's skill field. Skill names never contain '+' (see {@link SKILLS} keys). + */ +export const MARKER_SKILL_SEPARATOR = '+'; + +/** + * Marker line shape: ``. + * An HTML comment is inert in every target format (SKILL.md, .mdc, .clinerules + * markdown, AGENTS.md). Built via `new RegExp` so the hash length stays bound + * to {@link MARKER_HASH_HEX_LENGTH}. + */ +const SKILL_MARKER_LINE_RE = new RegExp( + `^$`, +); + +/** + * First {@link MARKER_HASH_HEX_LENGTH} hex chars of the SHA-256 of a canonical + * skill body. The hash covers the CANONICAL BODY ONLY (pre-wrap, pre-marker), + * so writing the marker into the rendered artifact never changes the hash the + * marker itself carries. + */ +export function bodyHash12(canonicalBody: string): string { + return createHash('sha256') + .update(canonicalBody, 'utf8') + .digest('hex') + .slice(0, MARKER_HASH_HEX_LENGTH); +} + +/** + * Build the provenance marker line for a skill (or a + * {@link MARKER_SKILL_SEPARATOR}-joined skill set) and its canonical body. + * `agent status` compares this fingerprint against the bodies the running CLI + * ships to detect silently stale installs. + */ +export function buildSkillMarker(skillName: string, canonicalBody: string): string { + return ``; +} + +/** A marker line parsed back into its fields. */ +export interface ParsedSkillMarker { + /** Skill name, or several names joined with {@link MARKER_SKILL_SEPARATOR}. */ + skill: string; + /** CLI version that wrote the artifact. */ + version: string; + /** First 12 hex chars of the canonical body's SHA-256 at install time. */ + hash12: string; + /** The exact marker line (trailing CR/whitespace stripped) as found. */ + line: string; +} + +/** + * Find the first testsprite-skill marker line in `content`, or null when the + * content carries none (a pre-marker install). Lines are matched whole with + * trailing CR/whitespace stripped, so CRLF checkouts parse identically. + */ +export function parseSkillMarker(content: string): ParsedSkillMarker | null { + for (const rawLine of content.split('\n')) { + const line = rawLine.trimEnd(); + const matched = SKILL_MARKER_LINE_RE.exec(line); + if (matched) { + return { skill: matched[1]!, version: matched[2]!, hash12: matched[3]!, line }; + } + } + return null; +} + type ReadFn = (url: URL) => string; const defaultRead: ReadFn = (url: URL) => readFileSync(url, 'utf8'); @@ -285,15 +366,67 @@ export function loadCodexSkillBody(read: ReadFn = defaultRead): string { // renderForTarget // --------------------------------------------------------------------------- +/** + * Place the marker line inside a wrapped own-file render. + * + * - Wraps that emit YAML frontmatter (claude/antigravity/cursor): the marker + * lands on the line right after the closing `---` fence, before the body. + * - Wrapless targets (cline, body verbatim): the marker is appended as the + * LAST line instead. Cline surfaces the file's first heading as the rule + * title, so a leading comment would displace the body's H1. + */ +function injectMarkerLine(wrapped: string, markerLine: string): string { + if (wrapped.startsWith('---\n')) { + // The name/description frontmatter values are single-line, so the first + // `\n---\n` after the opening fence is always the closing fence. + const closingFence = '\n---\n'; + const fenceIdx = wrapped.indexOf(closingFence); + if (fenceIdx !== -1) { + const insertAt = fenceIdx + closingFence.length; + return `${wrapped.slice(0, insertAt)}${markerLine}\n${wrapped.slice(insertAt)}`; + } + } + const separator = wrapped.endsWith('\n') ? '' : '\n'; + return `${wrapped}${separator}${markerLine}\n`; +} + +/** + * Exact own-file bytes for a skill on a target, carrying the GIVEN marker line. + * `agent status` uses this to re-render the current canonical body with a + * file's own (possibly older-versioned) marker: when only the marker's version + * string lags but the body is unchanged, the artifact still compares pristine. + */ +export function renderOwnFileWithMarker( + target: AgentTarget, + skill: string, + markerLine: string, + body?: string, +): string { + const spec = TARGETS[target]; + if (spec.mode !== 'own-file') { + throw new Error(`renderOwnFileWithMarker: ${target} is not an own-file target`); + } + const skillSpec = SKILLS[skill]; + if (!skillSpec) throw new Error(`unknown skill: ${skill}`); + const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); + return injectMarkerLine( + spec.wrap(skillSpec.name, skillSpec.description, resolvedBody), + markerLine, + ); +} + /** * The exact bytes to write for one skill on one target. * * - own-file targets: `body` defaults to the skill's own-file asset, wrapped in - * the target's frontmatter/header. + * the target's frontmatter/header, and carrying a provenance marker line so + * `agent status` can tell fresh, stale, and hand-edited installs apart. * - codex (managed-section): returns the skill's codex contribution unwrapped - * (plain Markdown, no frontmatter). The real install does NOT call this for - * codex — it aggregates all skills via {@link buildCodexAggregate} — but it is - * kept single-skill here for tests and parity. Pass an explicit `body` to override. + * and marker-free (plain Markdown, no frontmatter). The real install does NOT + * call this for codex: it aggregates all skills via + * {@link buildCodexAggregate} and writes ONE marker just inside the BEGIN + * sentinel. It is kept single-skill here for tests and parity. Pass an + * explicit `body` to override. */ export function renderForTarget( t: AgentTarget, @@ -309,5 +442,8 @@ export function renderForTarget( return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; } const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); - return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; + return { + path, + content: renderOwnFileWithMarker(t, skill, buildSkillMarker(skill, resolvedBody), resolvedBody), + }; } diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index f8e9fa2..0c387fa 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -14,6 +14,9 @@ Commands: first-run onboarding) into a project for a coding agent list List supported agent targets and skills, their status, and landing paths + status [options] Check installed TestSprite skill files against this CLI + version: ok, stale, modified, unmarked, absent, or corrupt + (exits 1 when anything needs attention, so it can gate CI) help [command] display help for command " `; From f732fdb1663b4dddb3a75db6e471359fed6fa054 Mon Sep 17 00:00:00 2001 From: kshitij-heizen Date: Mon, 6 Jul 2026 01:04:26 +0530 Subject: [PATCH 041/102] fix(bundle): only sweep bundle-owned files in commit, preserving user files in --out dir (#162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit commitBundle's stale-file sweep removed EVERY directory entry not part of the fresh bundle, so 'test failure get --out

' / 'test artifact get --out ' pointed at a pre-existing, populated directory silently deleted the user's unrelated files (exit 0, no warning) — on the very first write, not just re-commits. Scope the sweep to entries the bundle format owns (result.json, failure.json, video.mp4, meta.json, steps, .tmp, .partial, code.). Stale bundle files are still cleaned — an old video.mp4 when the new bundle has no video, a code.py when the new bundle writes code.ts — but foreign files and directories are never touched. Fixes #159 Co-authored-by: Kshitij Bhardwaj Co-authored-by: Claude Fable 5 --- src/lib/bundle.test.ts | 77 +++++++++++++++++++++++++++++++++++++++++- src/lib/bundle.ts | 36 +++++++++++++++++--- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index 5b746da..0c0680c 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -8,7 +8,7 @@ * the full http+fetch path is wired against MSW). */ -import { existsSync, mkdtempSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -18,6 +18,7 @@ import { assertNoEscape, BUNDLE_SCHEMA_VERSION, buildMeta, + isBundleOwnedEntry, pickCodeExtension, resolveBundleDir, STREAM_URL_MAX_RETRIES, @@ -694,6 +695,37 @@ describe('streamUrlToFile retry', () => { }); }); +describe('isBundleOwnedEntry', () => { + it('owns the fixed bundle file set', () => { + for (const entry of [ + 'result.json', + 'failure.json', + 'video.mp4', + 'meta.json', + 'steps', + '.tmp', + '.partial', + ]) { + expect(isBundleOwnedEntry(entry)).toBe(true); + } + }); + + it('owns code. for any single-token extension', () => { + expect(isBundleOwnedEntry('code.ts')).toBe(true); + expect(isBundleOwnedEntry('code.js')).toBe(true); + expect(isBundleOwnedEntry('code.py')).toBe(true); + }); + + it('does not own foreign entries', () => { + expect(isBundleOwnedEntry('notes.txt')).toBe(false); + expect(isBundleOwnedEntry('src')).toBe(false); + expect(isBundleOwnedEntry('.git')).toBe(false); + expect(isBundleOwnedEntry('code.tar.gz')).toBe(false); + expect(isBundleOwnedEntry('mycode.ts')).toBe(false); + expect(isBundleOwnedEntry('code.')).toBe(false); + }); +}); + describe('step artifact path validation', () => { // A fetchImpl that fails the test if called — proves validation rejects // before any write happens. @@ -807,6 +839,49 @@ describe('step artifact path validation', () => { expect(existsSync(join(res.dir, 'meta.json'))).toBe(true); }); + describe('commit sweep ownership (data-loss guard)', () => { + it('preserves pre-existing foreign files and directories in the --out dir', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'notes.txt'), 'important notes\n', 'utf8'); + mkdirSync(join(dir, 'src')); + writeFileSync(join(dir, 'src', 'app.js'), "console.log('app')\n", 'utf8'); + + const res = await writeBundle(stepCtx(3), { + dir, + failedOnly: false, + fetchImpl: throwIfFetched, + }); + + // The bundle landed… + expect(existsSync(join(res.dir, 'meta.json'))).toBe(true); + expect(existsSync(join(res.dir, 'result.json'))).toBe(true); + // …and the user's unrelated files survived the commit sweep. + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('important notes\n'); + expect(readFileSync(join(dir, 'src', 'app.js'), 'utf8')).toBe("console.log('app')\n"); + }); + + it('still sweeps a stale bundle-owned video.mp4 the new bundle does not write', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'video.mp4'), 'stale-bytes', 'utf8'); + + // stepCtx has videoUrl: null → the fresh bundle ships no video. + await writeBundle(stepCtx(3), { dir, failedOnly: false, fetchImpl: throwIfFetched }); + + expect(existsSync(join(dir, 'video.mp4'))).toBe(false); + }); + + it('sweeps a stale code file with a different extension than the new bundle writes', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'code.py'), '# stale python code\n', 'utf8'); + + // baseCtx.code.language is 'typescript' → the fresh bundle writes code.ts. + await writeBundle(stepCtx(3), { dir, failedOnly: false, fetchImpl: throwIfFetched }); + + expect(existsSync(join(dir, 'code.ts'))).toBe(true); + expect(existsSync(join(dir, 'code.py'))).toBe(false); + }); + }); + describe('assertNoEscape', () => { it('returns the resolved path for an in-bounds segment', () => { const base = mkdtempSync(join(tmpdir(), 'bundle-test-')); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index 7795115..9c03b07 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -543,6 +543,30 @@ async function freshTmpDir(dir: string): Promise { * caught reading the dir during it sees no meta and refuses to consume * (per §7.3). That's what we want. */ +/** + * Whether a top-level directory entry belongs to the bundle format — + * i.e. something a prior `writeBundle` could have produced and this + * commit is therefore allowed to clean up. `code.` is matched by + * pattern (not the current run's extension) so a stale `code.py` is + * still swept when the new bundle writes `code.ts`. Everything else in + * the directory is the user's and must never be deleted (`--out` can + * point at a pre-existing, populated directory). + */ +export function isBundleOwnedEntry(entry: string): boolean { + if ( + entry === 'result.json' || + entry === 'failure.json' || + entry === 'video.mp4' || + entry === 'meta.json' || + entry === 'steps' || + entry === '.tmp' || + entry === '.partial' + ) { + return true; + } + return /^code\.[A-Za-z0-9]+$/.test(entry); +} + async function commitBundle( tmpDir: string, dir: string, @@ -553,20 +577,22 @@ async function commitBundle( // (2) Sweep stale top-level files that the new bundle won't write. // If the prior run wrote `video.mp4` and the new run has no video, - // an in-place rename leaves the old video lingering. Enumerate - // current top-level entries and remove anything that isn't being - // freshly renamed in. + // an in-place rename leaves the old video lingering. Only entries the + // bundle format OWNS are candidates: `--out` may point at a directory + // that also holds the user's unrelated files, and those must survive + // the commit (deleting them would be silent data loss). const topLevel = files.filter(f => !f.startsWith('steps/')); const newTopLevelSet = new Set(topLevel); newTopLevelSet.add('meta.json'); // about to land last, do not delete const existing = await readdir(dir).catch(() => [] as string[]); for (const entry of existing) { // Preserve the writer's own scratch dir + the .partial marker - // (we'll re-evaluate .partial at the end of commit). Anything else - // not-listed in the new bundle is stale. + // (we'll re-evaluate .partial at the end of commit). Any other + // bundle-owned entry not-listed in the new bundle is stale. if (entry === '.tmp' || entry === '.partial') continue; if (newTopLevelSet.has(entry)) continue; if (entry === 'steps') continue; // handled below + if (!isBundleOwnedEntry(entry)) continue; // foreign file — never touch await rm(join(dir, entry), { recursive: true, force: true }); } From 93c621c6ba021dc513c31506c35fd632c394b536 Mon Sep 17 00:00:00 2001 From: kshitij-heizen Date: Mon, 6 Jul 2026 01:04:42 +0530 Subject: [PATCH 042/102] fix(rerun): reject explicit IDs with --all and --status/--skip-terminal without --all (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two silent-footgun gaps in test rerun's flag validation: 1. Explicit test IDs combined with --all silently discarded the listed IDs — the --all branch resolves the full project test set and overwrites them — so 'rerun test_abc --all' dispatched a batch rerun of EVERY test in the project, burning rerun/auto-heal credits with no error. Both siblings already guard this exact ambiguity (test run's positional+--all guard, delete-batch's ids+--all guard). 2. --status and --skip-terminal without --all were silently ignored — including INVALID --status values, which were never validated — while the same misuse of rerun's own --filter (and delete-batch's --status) exits 5. All three narrowing filters now share the same guard. Both reject with VALIDATION_ERROR (exit 5) before any network dispatch. Fixes #160 Co-authored-by: Kshitij Bhardwaj Co-authored-by: Claude Fable 5 --- src/commands/test.rerun.spec.ts | 120 ++++++++++++++++++++++++++++++++ src/commands/test.ts | 30 ++++++++ 2 files changed, 150 insertions(+) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index d36a233..129d859 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -280,6 +280,126 @@ describe('runTestRerun — validation', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); }); + it('exit 5 (VALIDATION_ERROR) when explicit test IDs are combined with --all', async () => { + // The --all branch resolves the FULL project test set and overwrites the + // listed ids, so 'rerun test_abc --all' would silently rerun the ENTIRE + // project instead of test_abc — burning rerun/auto-heal credits. The + // guard throws BEFORE any network/dispatch. (Mirrors `test run`'s + // positional+--all guard and delete-batch's ids+--all guard.) + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_abc'], + all: true, + projectId: 'proj_1', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'test-ids' }), + }); + }); + + it('exit 5 (VALIDATION_ERROR) when --status is passed WITHOUT --all', async () => { + // --status is an --all-only narrowing filter. With explicit ids it was + // silently ignored (both tests dispatched, filter dropped) — same + // failure mode as the --filter guard above. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a', 'test_b'], + all: false, + statusFilter: 'failed', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'status' }), + }); + }); + + it('exit 5 (VALIDATION_ERROR) for an INVALID --status value without --all (was silently accepted)', async () => { + // Before the guard, an invalid --status token without --all was never + // even validated: 'rerun test_a --status notastatus' exited 0 while the + // same flag on delete-batch exits 5. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + statusFilter: 'notastatus', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('exit 5 (VALIDATION_ERROR) when --skip-terminal is passed WITHOUT --all', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + skipTerminal: true, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'skip-terminal' }), + }); + }); + it('exit 5 when --all without --project', async () => { const creds = makeCreds(); try { diff --git a/src/commands/test.ts b/src/commands/test.ts index 7fe985d..22d0170 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5980,6 +5980,18 @@ export async function runTestRerun( 'provide at least one , or use --all to rerun all tests in the project', ); } + // Explicit ids + --all is ambiguous: the --all branch resolves the FULL + // project test set and overwrites the listed ids, so the user's narrowing + // intent would be silently replaced by a whole-project batch rerun — + // burning rerun/auto-heal credits. Reject early. (Mirrors `test run`'s + // positional+--all guard and delete-batch's ids+--all data-loss guard.) + if (opts.all && opts.testIds.length > 0) { + throw localValidationError( + 'test-ids', + 'pass either explicit test IDs or --all, not both — --all reruns every test in the ' + + 'project and would ignore the listed IDs. Drop the IDs, or drop --all.', + ); + } if (opts.all && !opts.projectId) { throw localValidationError( 'project', @@ -5998,6 +6010,24 @@ export async function runTestRerun( 'Remove --filter, or add --all --project .', ); } + // --status and --skip-terminal are --all-only narrowing filters with the + // same silent-ignore failure mode as --filter above: without --all the + // explicit ids get reran unfiltered (and an invalid --status value is + // never even validated). Reject both, mirroring the --filter guard. + if (opts.statusFilter !== undefined && !opts.all) { + throw localValidationError( + 'status', + '--status only applies with --all (it narrows which project tests get reran). ' + + 'Remove --status, or add --all --project .', + ); + } + if (opts.skipTerminal && !opts.all) { + throw localValidationError( + 'skip-terminal', + '--skip-terminal only applies with --all (it narrows which project tests get reran). ' + + 'Remove --skip-terminal, or add --all --project .', + ); + } if ( !Number.isInteger(opts.maxConcurrency) || opts.maxConcurrency < 1 || From 759e85c2e046af365c7904376ab577d9e38e0cb3 Mon Sep 17 00:00:00 2001 From: mo01115285816-cyber Date: Sun, 5 Jul 2026 22:34:58 +0300 Subject: [PATCH 043/102] docs(usage): add --debug example + exit codes to help text (#171) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'testsprite usage' command help text listed three examples but omitted the --debug flag (a global option useful for diagnosing auth and network issues). It also didn't document the exit codes, which matters for CI/CD scripts that gate on the return value. This PR adds: - A --debug example showing what it traces (HTTP method/path, request id, latency) — useful when debugging 'auth error' or 'transport failure' messages. - An explicit exit-codes section (0 success, 3 auth error, 10 network failure) so users scripting against the CLI know what to expect. Pure documentation improvement — no behavioral change, no new deps. Tested: existing unit tests pass (npm test). Co-authored-by: MOAAMN SAYED --- src/commands/usage.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/commands/usage.ts b/src/commands/usage.ts index d479cab..a97dbba 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -200,7 +200,12 @@ export function createUsageCommand(deps: UsageDeps = {}): Command { '\nExamples:\n' + ' testsprite usage # show balance + plan\n' + ' testsprite usage --output json # machine-readable balance\n' + + ' testsprite usage --debug # trace HTTP method/path, request id, latency\n' + ' testsprite credits # alias for usage\n' + + '\nExit codes:\n' + + ' 0 success (or --dry-run)\n' + + ' 3 auth error — run `testsprite setup` to configure credentials\n' + + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + '\nNote: credit balance requires a backend update to /me. Until shipped,\n' + " check your portal's Billing page (/dashboard/settings/billing) for your balance.", ) From 5ff639978187dc521aecf344de132dfbc5db0ed1 Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:38:16 +0100 Subject: [PATCH 044/102] fix(cli): route interactive prompts and prelude to stderr (keep stdout pure) (#31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interactive prompts (`prompt.ts` — the API-key prompt during `setup`/`auth configure`, the target prompt during `agent install`) wrote the question and masking to STDOUT, and the "Configuring profile …" prelude defaulted to stdout too. On the interactive path that mixes UI text into stdout — and under `--output json` it breaks the contract that stdout is a single JSON document, so a consumer doing `JSON.parse(stdout)` fails. Default both to stderr: prompts and informational preludes are interactive UI, not result data. stdout now carries only the command's result (the §8.1 stdout-purity principle the repo already enforces elsewhere). stderr is still the user's TTY, so prompts remain visible; the secret is still never echoed. Callers that inject explicit streams are unaffected. Adds regression tests: promptText writes the question to stderr by default, and the configure prelude lands on stderr (not the result stdout). --- src/commands/auth.test.ts | 29 ++++++++++++++++++++ src/commands/auth.ts | 5 +++- src/lib/prompt.test.ts | 56 +++++++++++++++++++++++++++++++++++++++ src/lib/prompt.ts | 10 +++++-- 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 8a7ae8b..1605f58 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -278,6 +278,35 @@ describe('runConfigure', () => { expect(capture.prelude.join('')).toContain('Configuring profile "default"'); }); + it('routes the interactive prelude to stderr by default, keeping stdout for the result', async () => { + // Regression: the prelude used to default to process.stdout, polluting the + // result stream (and the JSON document under --output json). With no + // injected preludeWrite/stderr, the default must land on stderr, not stdout. + const stdout: string[] = []; + const errChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + try { + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false }, + { + stdout: line => stdout.push(line), + prompt: { secret: vi.fn(async () => 'sk-typed') }, + fetchImpl: meOkFetch, + credentialsPath, + env: {}, + }, + ); + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + } + expect(errChunks.join('')).toContain('Configuring profile "default"'); + expect(stdout.join('\n')).not.toContain('Configuring profile'); + }); + it('interactive path resolves the endpoint from TESTSPRITE_API_URL without prompting', async () => { const { capture, deps } = makeCapture(); const prompt = { secret: vi.fn(async () => 'sk-typed') }; diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 1ec3e40..bf2eb88 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -75,7 +75,10 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): const env = deps.env ?? process.env; const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const out = makeOutput(opts.output, deps); - const prelude = deps.preludeWrite ?? ((chunk: string) => process.stdout.write(chunk)); + // The "Configuring profile …" prelude is informational, not result data, so + // it defaults to stderr — stdout stays a pure result stream (the configured + // JSON/text), which matters under `--output json` (§8.1 stdout purity). + const prelude = deps.preludeWrite ?? ((chunk: string) => process.stderr.write(chunk)); const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); // Normalize the env endpoint: an empty / whitespace-only TESTSPRITE_API_URL is diff --git a/src/lib/prompt.test.ts b/src/lib/prompt.test.ts index 45275e3..bb8cfb5 100644 --- a/src/lib/prompt.test.ts +++ b/src/lib/prompt.test.ts @@ -69,6 +69,33 @@ describe('promptText', () => { const output = new CaptureStream(); expect(await promptText('? ', { input, output })).toBe('eof-no-newline'); }); + + it('writes the question to stderr by default — keeps stdout pure for --output json', async () => { + // Regression: prompts used to default to stdout, which polluted the JSON + // result on the interactive setup/configure path. Interactive UI belongs + // on stderr. Manually swap the global stream writers (prompt reads + // process.stderr/stdout at call time, so this is reliably intercepted). + const errChunks: string[] = []; + const outChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + (process.stdout as unknown as { write: (c: string) => boolean }).write = c => { + outChunks.push(String(c)); + return true; + }; + try { + await promptText('Q: ', { input: Readable.from(['x\n']) }); // no output → default + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + (process.stdout as unknown as { write: typeof origOut }).write = origOut; + } + expect(errChunks.join('')).toContain('Q: '); + expect(outChunks.join('')).not.toContain('Q: '); + }); }); describe('promptSecret (non-TTY behavior)', () => { @@ -88,6 +115,35 @@ describe('promptSecret (non-TTY behavior)', () => { expect(written).not.toContain('sk-hidden-12345'); }); + it('writes the prompt to stderr by default — keeps stdout pure for --output json', async () => { + // Same regression as promptText: the secret prompt is interactive UI and + // must default to stderr so stdout carries only the command result. Manual + // stream swap (vi.spyOn does not intercept process.stderr.write here). + const errChunks: string[] = []; + const outChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + (process.stdout as unknown as { write: (c: string) => boolean }).write = c => { + outChunks.push(String(c)); + return true; + }; + try { + await promptSecret('Key: ', { input: Readable.from(['sk-x\n']) }); // no output → default + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + (process.stdout as unknown as { write: typeof origOut }).write = origOut; + } + expect(errChunks.join('')).toContain('Key: '); + expect(outChunks.join('')).not.toContain('Key: '); + // The typed secret is never echoed to either real stream. + expect(errChunks.join('')).not.toContain('sk-x'); + expect(outChunks.join('')).not.toContain('sk-x'); + }); + it('honors DEL/backspace before submission', async () => { const DEL = String.fromCharCode(0x7f); const input = Readable.from([`abc${DEL}d\n`]); diff --git a/src/lib/prompt.ts b/src/lib/prompt.ts index 3561b0a..bccead9 100644 --- a/src/lib/prompt.ts +++ b/src/lib/prompt.ts @@ -15,14 +15,20 @@ const pendingPromptInput = new WeakMap(); export async function promptText(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; - const output = streams.output ?? process.stdout; + // Prompts are interactive UI, not data — write the question (and any echo) + // to stderr so stdout carries only the command's result. This keeps + // `--output json` stdout a single pure JSON document even on the interactive + // setup / configure path (§8.1 stdout purity). stderr is still the user's + // TTY, so the prompt remains visible. + const output = streams.output ?? process.stderr; output.write(question); return readLine(input, output, false); } export async function promptSecret(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; - const output = streams.output ?? process.stdout; + // See promptText: interactive prompt + masking go to stderr, not stdout. + const output = streams.output ?? process.stderr; output.write(question); const inputAsTTY = input as Readable & RawModeCapable; From 17650be02c06d9fef2742039a16bb678595bdf89 Mon Sep 17 00:00:00 2001 From: merlinsantiago982-cmd Date: Mon, 6 Jul 2026 03:38:34 +0800 Subject: [PATCH 045/102] fix: harden failure bundle artifact downloads (#60) * block artifact download redirects * redact artifact download urls --------- Co-authored-by: merlinsantiago982-cmd --- src/lib/bundle.test.ts | 56 ++++++++++++++++++++++++++++++++++-------- src/lib/bundle.ts | 20 +++++++++++---- 2 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index 0c0680c..aec451a 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -641,17 +641,24 @@ describe('streamUrlToFile retry', () => { calls++; throw new Error('ENETUNREACH dns lookup failed'); }; - await expect( - streamUrlToFile( - 'https://example.com/x', + let caught: unknown; + + try { + await streamUrlToFile( + 'https://example.com/x?X-Amz-Signature=secret-token', '/tmp/will-not-be-written', fetchImpl as typeof globalThis.fetch, { sleep: noSleep }, - ), - ).rejects.toMatchObject({ + ); + } catch (err) { + caught = err; + } + + expect(caught).toMatchObject({ name: 'TransportError', message: expect.stringContaining('ENETUNREACH'), }); + expect(caught).not.toMatchObject({ message: expect.stringContaining('secret-token') }); expect(calls).toBe(STREAM_URL_MAX_RETRIES); }); @@ -661,17 +668,46 @@ describe('streamUrlToFile retry', () => { calls++; return new Response('Forbidden', { status: 403 }); }; - await expect( - streamUrlToFile( - 'https://example.com/x', + let caught: unknown; + const presignedUrl = 'https://example.com/x?X-Amz-Signature=secret-token#download'; + + try { + await streamUrlToFile( + presignedUrl, '/tmp/will-not-be-written', fetchImpl as typeof globalThis.fetch, { sleep: noSleep }, - ), - ).rejects.toMatchObject({ code: 'UNAVAILABLE' }); + ); + } catch (err) { + caught = err; + } + + expect(caught).toMatchObject({ + code: 'UNAVAILABLE', + details: { status: 403, artifactUrl: 'https://example.com/x' }, + }); + const details = (caught as { details?: Record }).details; + expect(details).not.toHaveProperty('url'); + expect(JSON.stringify(details)).not.toContain('secret-token'); expect(calls).toBe(1); }); + it('disables automatic redirects so unsafe redirect targets cannot bypass URL validation', async () => { + const dir = mkdtempSync(join(tmpdir(), 'stream-test-')); + const dest = join(dir, 'out.bin'); + const redirects: Array = []; + const fetchImpl = async (_url: Parameters[0], init?: RequestInit) => { + redirects.push(init?.redirect); + return new Response('hello', { status: 200 }); + }; + + await streamUrlToFile('https://example.com/x', dest, fetchImpl as typeof globalThis.fetch, { + sleep: noSleep, + }); + + expect(redirects).toEqual(['error']); + }); + it('sleeps between retries', async () => { const sleepDelays: number[] = []; const fetchImpl = async () => { diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index 9c03b07..a3c0808 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -772,17 +772,18 @@ export async function streamUrlToFile( deps?: { sleep?: (ms: number) => Promise }, ): Promise { const sleepFn = deps?.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + const artifactUrl = redactArtifactUrlForDetails(url); for (let attempt = 1; attempt <= STREAM_URL_MAX_RETRIES; attempt++) { let response: Response; try { - response = await fetchImpl(url); + response = await fetchImpl(url, { redirect: 'error' }); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (attempt < STREAM_URL_MAX_RETRIES) { await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + throw new TransportError(`Failed to download presigned URL ${artifactUrl}: ${message}`); } if (!response.ok) { // Non-2xx: the URL itself is bad (expired, unauthorized, not found). @@ -794,7 +795,7 @@ export async function streamUrlToFile( nextAction: 'Re-run `testsprite test failure get`. Presigned URLs in the bundle expire after 15 minutes.', requestId: 'local', - details: { status: response.status, url }, + details: { status: response.status, artifactUrl }, }, }); } @@ -814,7 +815,7 @@ export async function streamUrlToFile( await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + throw new TransportError(`Failed to download presigned URL ${artifactUrl}: ${message}`); } } await mkdir(dirname(filePath), { recursive: true }); @@ -836,11 +837,20 @@ export async function streamUrlToFile( await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed mid-download of ${url}: ${message}`); + throw new TransportError(`Failed mid-download of ${artifactUrl}: ${message}`); } } } +function redactArtifactUrlForDetails(url: string): string { + try { + const parsed = new URL(url); + return `${parsed.origin}${parsed.pathname}`; + } catch { + return ''; + } +} + function isPresignedUrl(value: string): boolean { return value.startsWith('https://'); } From d72290d6d92d2806b33608d67af304782597f753 Mon Sep 17 00:00:00 2001 From: Lexiie Date: Mon, 6 Jul 2026 02:41:12 +0700 Subject: [PATCH 046/102] fix(test): guard default artifact run id path (#172) * fix(test): validate artifact run id default path * fix(test): reject windows dot artifact run ids * fix(test): reject windows dot-suffix artifact run ids * style(test): format artifact run id guard --------- Co-authored-by: Lexiie <28455136+Lexiie@users.noreply.github.com> --- src/commands/test.artifact.spec.ts | 48 ++++++++++++++++++++++++++++++ src/commands/test.ts | 26 +++++++++++++--- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/src/commands/test.artifact.spec.ts b/src/commands/test.artifact.spec.ts index c058079..c0b9283 100644 --- a/src/commands/test.artifact.spec.ts +++ b/src/commands/test.artifact.spec.ts @@ -21,6 +21,7 @@ import { assertOutDirParentExists, createTestArtifactCommand, createTestCommand, + resolveDefaultArtifactDir, runArtifactGet, runFailureGet, } from './test.js'; @@ -322,6 +323,53 @@ describe('runArtifactGet', () => { } }); + it('rejects path-like runId before auth or fetch when default --out is used', async () => { + const fetchImpl = vi.fn(); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: '../../outside', + failedOnly: false, + }, + { fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it.each([ + '.', + '..', + '. ', + '.. ', + '...', + '.. .', + '. .', + '../outside', + '..\\outside', + 'nested/run', + 'nested\\run', + 'bad\0id', + ])('rejects unsafe default artifact runId segment %j', runId => { + expect(() => resolveDefaultArtifactDir(runId, '/repo')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'run-id' }), + }), + ); + }); + + it('keeps the documented default directory for path-safe runIds', () => { + expect(resolveDefaultArtifactDir(SAMPLE_RUN_ID, '/repo')).toBe( + join('/repo', '.testsprite', 'runs', SAMPLE_RUN_ID), + ); + }); + // ---- --failed-only passed through to writeBundle ---- it('passes --failed-only through to writeBundle (steps filtered to failed ± 1)', async () => { diff --git a/src/commands/test.ts b/src/commands/test.ts index 22d0170..5582603 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -7169,6 +7169,25 @@ export interface ArtifactGetResult { bundle?: WriteBundleResult; } +export function resolveDefaultArtifactDir(runId: string, cwd: string = process.cwd()): string { + requireNonEmpty('run-id', runId); + const windowsNormalizedSegment = runId.replace(/[ .]+$/u, ''); + if ( + windowsNormalizedSegment === '' || + windowsNormalizedSegment === '.' || + windowsNormalizedSegment === '..' || + runId.includes('/') || + runId.includes('\\') || + runId.includes('\0') + ) { + throw localValidationError( + 'run-id', + 'must be a single path-safe segment for the default output directory; pass --out to choose a custom path', + ); + } + return join(cwd, '.testsprite', 'runs', runId); +} + /** * Validate that the parent directory of `resolvedDir` exists and is a * directory. Surfaces `VALIDATION_ERROR` (exit 5) — matches the convention @@ -7218,14 +7237,11 @@ export async function runArtifactGet( deps: TestDeps = {}, ): Promise { const out = makeOutput(opts.output, deps); - const client = makeClient(opts, deps); const { runId } = opts; // Resolve output dir: explicit --out or the default .testsprite/runs// const resolvedDir = - opts.out !== undefined - ? resolveBundleDir(opts.out) - : join(process.cwd(), '.testsprite', 'runs', runId); + opts.out !== undefined ? resolveBundleDir(opts.out) : resolveDefaultArtifactDir(runId); // --dry-run: no network, no disk write. // The client (makeClient) is already wired with createDryRunFetch() when @@ -7271,6 +7287,8 @@ export async function runArtifactGet( await assertOutDirParentExists(resolvedDir); } + const client = makeClient(opts, deps); + // Fetch the run-scoped failure bundle. const { body: context, requestId: fetchRequestId } = await client.getWithMeta( `/runs/${encodeURIComponent(runId)}/failure`, From 2ddb03d5674ed4bfd7d0e7655170ce8b8d2bdd54 Mon Sep 17 00:00:00 2001 From: Resque Date: Sun, 5 Jul 2026 23:41:57 +0400 Subject: [PATCH 047/102] feat(cli): add runtime Node.js version check with clear error message (#11) * feat(cli): add runtime Node.js version check with clear error message * refactor(version-guard): extract to a documented module tested against the real implementation --- src/index.ts | 11 +++++++++ src/version-guard.test.ts | 51 +++++++++++++++++++++++++++++++++++++++ src/version-guard.ts | 41 +++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 src/version-guard.test.ts create mode 100644 src/version-guard.ts diff --git a/src/index.ts b/src/index.ts index 7842ec9..bc15ec5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node + import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; @@ -15,6 +16,16 @@ import { Output, isOutputMode } from './lib/output.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; import { VERSION } from './version.js'; +import { shouldRejectNodeVersion } from './version-guard.js'; + +// Guard: exit early with a clear message on unsupported Node.js versions, +// rather than failing later with a cryptic ESM/runtime error. +if (shouldRejectNodeVersion(process.versions.node)) { + process.stderr.write( + `Error: testsprite requires Node.js >= 20 (found ${process.versions.node}).\nInstall the latest LTS from https://nodejs.org\n`, + ); + process.exit(1); +} const program = new Command(); diff --git a/src/version-guard.test.ts b/src/version-guard.test.ts new file mode 100644 index 0000000..19f0eab --- /dev/null +++ b/src/version-guard.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { + MIN_SUPPORTED_NODE_MAJOR, + parseMajorVersion, + shouldRejectNodeVersion, +} from './version-guard.js'; + +// These tests exercise the REAL guard functions used by src/index.ts, +// imported here rather than re-declared, so a regression in the source is +// actually caught. + +describe('parseMajorVersion', () => { + it('extracts the leading major from a semver string', () => { + expect(parseMajorVersion('20.11.1')).toBe(20); + expect(parseMajorVersion('18.0.0')).toBe(18); + expect(parseMajorVersion('22.3.0')).toBe(22); + }); + + it('returns NaN for a non-numeric version string', () => { + expect(Number.isNaN(parseMajorVersion('not-a-version'))).toBe(true); + }); +}); + +describe('shouldRejectNodeVersion', () => { + it('rejects majors below the supported floor', () => { + expect(shouldRejectNodeVersion('18.19.1')).toBe(true); + expect(shouldRejectNodeVersion('16.20.2')).toBe(true); + expect(shouldRejectNodeVersion('14.21.3')).toBe(true); + }); + + it('accepts the supported floor and above', () => { + expect(shouldRejectNodeVersion('20.0.0')).toBe(false); + expect(shouldRejectNodeVersion('20.11.0')).toBe(false); + expect(shouldRejectNodeVersion('21.0.0')).toBe(false); + expect(shouldRejectNodeVersion('22.1.0')).toBe(false); + }); + + it(`treats exactly ${MIN_SUPPORTED_NODE_MAJOR} as supported (boundary)`, () => { + expect(shouldRejectNodeVersion(`${MIN_SUPPORTED_NODE_MAJOR}.0.0`)).toBe(false); + expect(shouldRejectNodeVersion(`${MIN_SUPPORTED_NODE_MAJOR - 1}.9.9`)).toBe(true); + }); + + it('does not reject an unparseable version (guard never blocks on garbage)', () => { + expect(shouldRejectNodeVersion('not-a-version')).toBe(false); + }); + + it('the running Node satisfies the guard (meta-test)', () => { + // The test suite itself runs on a supported Node, so the guard must pass. + expect(shouldRejectNodeVersion(process.versions.node)).toBe(false); + }); +}); diff --git a/src/version-guard.ts b/src/version-guard.ts new file mode 100644 index 0000000..a7fe464 --- /dev/null +++ b/src/version-guard.ts @@ -0,0 +1,41 @@ +/** + * Node.js runtime version guard. + * + * The CLI targets modern Node (see `engines.node` in package.json). Running on + * an older runtime tends to fail later with a cryptic ESM/syntax error, so the + * entrypoint (`src/index.ts`) uses {@link shouldRejectNodeVersion} to exit early + * with a clear, actionable message instead. + * + * The logic lives here (rather than inline) so it can be unit-tested against the + * real implementation the entrypoint uses — not a copy. + */ + +/** Minimum Node.js major version supported by the CLI (matches package.json `engines.node`). */ +export const MIN_SUPPORTED_NODE_MAJOR = 20; + +/** + * Parse the leading major version number from a Node.js version string. + * + * @param nodeVersion - a dot-separated version string such as `process.versions.node` + * (e.g. `"20.11.1"`). A leading `v` is not expected (Node does not include one here). + * @returns the major version as a number, or `NaN` if the string has no numeric leading segment. + */ +export function parseMajorVersion(nodeVersion: string): number { + return Number(nodeVersion.split('.')[0]); +} + +/** + * Decide whether the given Node.js version is too old to run the CLI. + * + * A version is rejected only when its major number is a real value below + * {@link MIN_SUPPORTED_NODE_MAJOR}. An unparseable string yields `NaN`, which is + * treated as "do not reject" so the guard never blocks on a version string it + * cannot understand (the runtime would surface any real incompatibility itself). + * + * @param nodeVersion - a `process.versions.node` style string (e.g. `"18.19.1"`). + * @returns `true` when the runtime is below the supported floor and should be rejected. + */ +export function shouldRejectNodeVersion(nodeVersion: string): boolean { + const major = parseMajorVersion(nodeVersion); + return !Number.isNaN(major) && major < MIN_SUPPORTED_NODE_MAJOR; +} From 8eb4acff70e216ed6b63bfd19fb1fde7107b5c7f Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Sun, 5 Jul 2026 23:27:00 +0100 Subject: [PATCH 048/102] feat(agent): add windsurf as an install target (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windsurf (Cascade) reads workspace rules from `.windsurf/rules/*.md`. Add it as an own-file agent target so `testsprite agent install --target windsurf` (and `setup --agent windsurf`) installs the TestSprite skills into a Windsurf project. Reworked onto the v0.2.0 multi-skill agent-targets API (pathFor / SKILLS / DEFAULT_SKILLS). Rule files use Cascade frontmatter with `trigger: model_decision` — the equivalent of the Cursor `.mdc` `alwaysApply: false` mode (description shown up front; full body pulled in on relevance). Budget handling: a `.windsurf/rules/*.md` file caps at ~12 K characters and Cascade silently truncates beyond it, which would cut the full ~22 KB verify skill in half. The windsurf target therefore renders the COMPACT body per skill (new `compactBody` flag + `compactBodyFor`): a skill that ships a trimmed codex asset (`testsprite-verify` → ~5 KB) uses it, while a skill whose codex contribution is only a one-liner (`testsprite-onboard`, ~6.5 KB full) keeps its full body — both land well under the cap. `agent.ts` and `renderForTarget` select the same body so installed bytes match the render. Everything else derives from the TARGETS map automatically (agent list, the setup --agent choices, skill-nudge install detection). Updated the hardcoded help strings, the --help snapshot, the agent-targets/agent unit tests (incl. Cascade-frontmatter and per-skill budget tests), the e2e matrix guards / content-integrity (gated on compactBody), and the README/DOCUMENTATION target lists (incl. the --force own-file list). --- DOCUMENTATION.md | 7 ++- README.md | 46 +++++++------- src/commands/agent.test.ts | 23 +++---- src/commands/agent.ts | 22 ++++++- src/lib/agent-targets.test.ts | 54 +++++++++++++++- src/lib/agent-targets.ts | 63 ++++++++++++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 12 ++-- test/e2e/agent-install.e2e.test.ts | 20 +++++- test/e2e/setup.e2e.test.ts | 1 + 9 files changed, 193 insertions(+), 55 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 7a89216..8c0cf1a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -112,16 +112,17 @@ testsprite agent install claude # install the skill for Claude Code testsprite agent install codex # install into AGENTS.md for Codex (managed-section) testsprite agent install cursor # .cursor/rules/testsprite-verify.mdc testsprite agent install cline # .clinerules/testsprite-verify.md +testsprite agent install windsurf # .windsurf/rules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md -testsprite agent list # list all 6 targets with status + mode + path +testsprite agent list # list all 7 targets with status + mode + path ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental). +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental). The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf) backs up the existing file to `.bak` first. ## Command reference diff --git a/README.md b/README.md index 1e20a1d..79f8aea 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ npm install -g @testsprite/testsprite-cli testsprite setup ``` -`testsprite setup` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `antigravity`, `codex`, etc.) — one command, so your agent is wired to verify its own work. Non-interactive (CI / onboarding scripts): +`testsprite setup` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `windsurf`, `antigravity`, `codex`, etc.) — one command, so your agent is wired to verify its own work. Non-interactive (CI / onboarding scripts): ```bash TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude @@ -89,28 +89,28 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test wait` | Block on a `runId` until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index f8ed59f..7da57cf 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -769,12 +769,13 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 6 targets × 2 default skills = 12 rows - expect(json).toHaveLength(12); + // 7 targets × 2 default skills = 14 rows + expect(json).toHaveLength(14); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); + expect(targets).toContain('windsurf'); expect(targets).toContain('antigravity'); expect(targets).toContain('kiro'); expect(targets).toContain('codex'); @@ -916,11 +917,11 @@ describe('createAgentCommand wiring', () => { }); // --------------------------------------------------------------------------- -// All five own-file targets installed at once +// All own-file targets installed at once // --------------------------------------------------------------------------- -describe('runInstall — all five own-file targets', () => { - it('installs all five own-file targets in one invocation', async () => { +describe('runInstall — all own-file targets', () => { + it('installs every own-file target in one invocation', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -930,7 +931,7 @@ describe('runInstall — all five own-file targets', () => { output: 'text', debug: false, dryRun: false, - target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro'], + target: [...OWN_FILE_TARGETS], skills: ['testsprite-verify'], force: false, }, @@ -948,11 +949,11 @@ describe('runInstall — all five own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all five own-file targets +// Dry-run for all six own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the five own-file targets (default 2 skills = 10 would-write lines)', async () => { + it('writes nothing for any of the six own-file targets (default 2 skills = 12 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -962,7 +963,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -972,9 +973,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 5 targets × 2 default skills = 10 would-write lines + // 6 targets × 2 default skills = 12 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(10); + expect(wouldWriteLines.length).toBe(12); }); }); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 1b4e878..0e80a91 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -15,6 +15,7 @@ import { pathFor, loadSkillBodyFor, bodyHash12, + compactBodyFor, buildCodexAggregate, buildSkillMarker, parseSkillMarker, @@ -443,6 +444,21 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr } return b; }; + // Budget-capped own-file targets (e.g. windsurf) render the compact per-skill + // body so the rule file isn't truncated by the agent. Cached separately; must + // match renderForTarget's default selection so written bytes equal the asserted + // render. + const compactBodyCache = new Map(); + const compactBodyForSkill = (skill: string): string => { + let b = compactBodyCache.get(skill); + if (b === undefined) { + b = compactBodyFor(skill); + compactBodyCache.set(skill, b); + } + return b; + }; + const ownFileBodyFor = (t: AgentTarget, skill: string): string => + TARGETS[t].compactBody ? compactBodyForSkill(skill) : bodyForSkill(skill); let codexSectionCache: string | undefined; const getCodexSection = (): string => { if (codexSectionCache === undefined) { @@ -651,7 +667,7 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (abs !== root && !abs.startsWith(root + path.sep)) { throw new CLIError(`refusing to write outside --dir: ${relPath}`, 5); } - const content = renderForTarget(t, skill, bodyForSkill(skill)).content; + const content = renderForTarget(t, skill, ownFileBodyFor(t, skill)).content; if (opts.dryRun) { // Apply the SAME symlink fail-close guard as the real install path @@ -1045,7 +1061,7 @@ function collect(v: string, prev: string[]): string[] { export function createAgentCommand(deps: AgentDeps = {}): Command { const agent = new Command('agent').description( - 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Codex)', + 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Windsurf, Antigravity, Codex)', ); agent @@ -1055,7 +1071,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, kiro, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, codex (comma-separated or repeated)', collect, [], ) diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 38c74e2..0417166 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -80,18 +80,19 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all six required keys', () => { + it('has all seven required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro']); + expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro', 'windsurf']); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, antigravity, kiro, and codex are experimental', () => { + it('cursor, cline, windsurf, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); + expect(TARGETS.windsurf.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); @@ -110,6 +111,7 @@ describe('TARGETS', () => { expect(TARGETS.cursor.mode).toBe('own-file'); expect(TARGETS.cline.mode).toBe('own-file'); expect(TARGETS.kiro.mode).toBe('own-file'); + expect(TARGETS.windsurf.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -293,6 +295,52 @@ describe('renderForTarget("cline")', () => { }); }); +describe('renderForTarget("windsurf")', () => { + const result = renderForTarget('windsurf', 'testsprite-verify', STUB_BODY); + + it('returns the .windsurf/rules path', () => { + expect(result.path).toBe('.windsurf/rules/testsprite-verify.md'); + }); + + it('uses the Cascade frontmatter (trigger: model_decision + description)', () => { + expect(result.content.startsWith('---\n')).toBe(true); + expect(result.content).toContain('trigger: model_decision'); + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); + + it('does NOT carry the Claude/Cursor frontmatter keys', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); // claude key + expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key + }); +}); + +describe('windsurf renders within the rules-file budget', () => { + // Regression: a `.windsurf/rules/*.md` file caps at ~12 K characters and + // Cascade silently truncates beyond that. The full verify body (~22 KB) would + // be cut in half, so windsurf renders the COMPACT body for verify (its trimmed + // codex asset) and the full body for onboard (which already fits). Uses the + // REAL bodies (no stub) so the size reflects what a user receives. + for (const skill of DEFAULT_SKILLS) { + it(`${skill} fits under 12 000 characters`, () => { + const r = renderForTarget('windsurf', skill); + expect(r.content.length).toBeLessThan(12_000); + }); + } + + it('verify uses the compact body (smaller than the full claude render)', () => { + const windsurf = renderForTarget('windsurf', 'testsprite-verify'); + const claude = renderForTarget('claude', 'testsprite-verify'); + expect(windsurf.content.length).toBeLessThan(claude.content.length); + // The full-body-only intro line is absent from the compact body... + expect(claude.content).toContain('The verification loop that flies'); + expect(windsurf.content).not.toContain('The verification loop that flies'); + // ...but the load-bearing command survives. + expect(windsurf.content).toContain('testsprite test run'); + }); +}); + // --------------------------------------------------------------------------- // Content integrity — load-bearing command strings must survive any body trim // --------------------------------------------------------------------------- diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 0273a04..d9a85d5 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -2,7 +2,14 @@ import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { VERSION } from '../version.js'; -export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex' | 'kiro'; +export type AgentTarget = + | 'claude' + | 'cursor' + | 'cline' + | 'antigravity' + | 'codex' + | 'kiro' + | 'windsurf'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -14,11 +21,19 @@ export interface TargetSpec { */ path: string; /** - * 'own-file': the CLI owns the whole file (claude/cursor/cline/antigravity). + * 'own-file': the CLI owns the whole file (claude/cursor/cline/antigravity/windsurf). * 'managed-section': the CLI writes only a sentinel-delimited section inside * a potentially user-authored file (codex target, AGENTS.md). */ mode: 'own-file' | 'managed-section'; + /** + * When true, render the budget-friendly body (see {@link compactBodyFor}) + * instead of the full own-file skill body. Used for own-file targets whose + * rule files are size-capped — currently `windsurf` (`.windsurf/rules/*.md` + * files cap at ~12 K characters and Cascade silently truncates beyond that, + * which would cut the full ~22 KB verify skill in half). + */ + compactBody?: boolean; /** * Wrap a skill body in this target's frontmatter/header. Takes the skill's * `name`+`description` (own-file targets emit them as frontmatter) and the body. @@ -122,6 +137,18 @@ function wrapMdc(_name: string, description: string, body: string): string { return `---\ndescription: ${description}\nalwaysApply: false\n---\n\n${body}\n`; } +/** + * Windsurf (Cascade) reads workspace rules from `.windsurf/rules/*.md` with YAML + * frontmatter. `trigger: model_decision` is the Cascade equivalent of the Cursor + * `.mdc` `alwaysApply: false` mode: only the `description` is surfaced up front, + * and Cascade pulls in the full rule body when the description shows it is + * relevant — exactly the on-demand activation these skills want. (The other + * triggers are `always_on`, `manual`, and `glob`.) + */ +function wrapWindsurf(_name: string, description: string, body: string): string { + return `---\ntrigger: model_decision\ndescription: ${description}\n---\n\n${body}\n`; +} + // --------------------------------------------------------------------------- // Landing paths // --------------------------------------------------------------------------- @@ -144,6 +171,8 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.clinerules/${skill}.md`; case 'kiro': return `.kiro/skills/${skill}/SKILL.md`; + case 'windsurf': + return `.windsurf/rules/${skill}.md`; case 'codex': return 'AGENTS.md'; } @@ -182,6 +211,15 @@ export const TARGETS: Record = { // claude/antigravity, so it shares the wrapSkill wrapper. wrap: wrapSkill, }, + windsurf: { + status: 'experimental', + path: pathFor('windsurf', SKILL_NAME), + mode: 'own-file', + // Windsurf rules files are budget-capped (~12 K chars per `.windsurf/rules/*.md`), + // so render the compact body per skill (see compactBodyFor). + compactBody: true, + wrap: wrapWindsurf, + }, /** * codex target — managed-section mode. * @@ -318,6 +356,24 @@ export function loadSkillBodyFor(skill: string, read: ReadFn = defaultRead): str return readSkillAsset(spec.bodyFile, read); } +/** + * Budget-friendly body for an own-file target whose rule files are size-capped + * (e.g. windsurf). For a skill that ships a trimmed codex asset (`codex.kind === + * 'full'`, e.g. `testsprite-verify` — full body ~22 KB, codex ~5 KB) we render + * that compact asset so the wrapped file stays under the cap. For skills whose + * codex contribution is only a one-liner (`'line'`/`'none'`, e.g. + * `testsprite-onboard`), the one-liner is useless as a standalone rule and the + * full own-file body (~6.5 KB) already fits the budget — so the full body is + * used. + */ +export function compactBodyFor(skill: string, read: ReadFn = defaultRead): string { + const spec = SKILLS[skill]; + if (!spec) throw new Error(`unknown skill: ${skill}`); + return spec.codex.kind === 'full' + ? readSkillAsset(spec.codex.file, read) + : loadSkillBodyFor(skill, read); +} + /** * Resolve a skill's codex (AGENTS.md) contribution as a Markdown string. * 'full' → read the `*.codex.md` asset; 'line' → the inline one-liner; 'none' → ''. @@ -441,7 +497,8 @@ export function renderForTarget( const resolvedBody = body !== undefined ? body : codexContentFor(skill); return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; } - const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); + const resolvedBody = + body !== undefined ? body : spec.compactBody ? compactBodyFor(skill) : loadSkillBodyFor(skill); return { path, content: renderOwnFileWithMarker(t, skill, buildSkillMarker(skill, resolvedBody), resolvedBody), diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 0c387fa..d89e7dc 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -4,7 +4,7 @@ exports[`--help snapshots > agent 1`] = ` "Usage: testsprite agent [options] [command] Install TestSprite guidance into coding-agent config (Claude Code, Cursor, -Cline, Antigravity, Codex) +Cline, Windsurf, Antigravity, Codex) Options: -h, --help display help for command @@ -29,7 +29,7 @@ into a project for a coding agent Options: --target Agent target(s): claude, cursor, cline, antigravity, kiro, - codex (comma-separated or repeated) (default: []) + windsurf, codex (comma-separated or repeated) (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -115,8 +115,8 @@ Options: --from-env Read TESTSPRITE_API_KEY from the environment instead of prompting (default: false) --agent Coding-agent target to install: claude, antigravity, - cursor, cline, kiro, codex (default: claude) (default: - "claude") + cursor, cline, kiro, windsurf, codex (default: claude) + (default: "claude") --no-agent Skip the agent skill install (configure credentials only) --force Overwrite an existing skill file (a .bak backup is kept) --dir Project root for the skill install (default: current @@ -604,8 +604,8 @@ Commands: project Manage TestSprite projects test Inspect TestSprite tests agent Install TestSprite guidance into coding-agent - config (Claude Code, Cursor, Cline, Antigravity, - Codex) + config (Claude Code, Cursor, Cline, Windsurf, + Antigravity, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) help [command] display help for command diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 0513b8b..094aec2 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -169,11 +169,20 @@ describe('content integrity', () => { content.trimStart().startsWith('#'), `cline: should start with a markdown heading`, ).toBe(true); + } else if (target === 'windsurf') { + // Windsurf Cascade frontmatter: trigger + description (no name/alwaysApply) + expect(content.startsWith('---'), `windsurf: should start with ---`).toBe(true); + expect(content).toContain('trigger: model_decision'); + expect(content).toContain('description:'); } - // (b) branding — the renamed H1 must be present + // (b) branding — the renamed H1 must be present in every body variant expect(content).toContain('TestSprite Verification Loop'); - expect(content).toContain('The verification loop that flies'); + // The full-body intro line lives only in the FULL body; compact-body targets + // (e.g. windsurf, budget-capped) ship the trimmed verify body and omit it. + if (!TARGETS[target].compactBody) { + expect(content).toContain('The verification loop that flies'); + } // (c) Load-bearing command strings expect(content, `${target}: missing 'testsprite test run'`).toContain('testsprite test run'); @@ -198,6 +207,10 @@ describe('content integrity', () => { expect(content).toContain('alwaysApply: false'); } else if (target === 'cline') { expect(content.startsWith('---'), `cline/onboard: must NOT start with ---`).toBe(false); + } else if (target === 'windsurf') { + expect(content.startsWith('---'), `windsurf/onboard: should start with ---`).toBe(true); + expect(content).toContain('trigger: model_decision'); + expect(content).toContain('description:'); } // Load-bearing onboard string: the skill body must reference setup @@ -790,7 +803,7 @@ describe('agent list', () => { }>; expect(Array.isArray(parsed)).toBe(true); - // Expected: 5 targets × 2 skills = 10 rows + // Expected: 7 targets × 2 skills = 14 rows const expectedCount = Object.keys(TARGETS).length * DEFAULT_SKILLS.length; expect(parsed.length).toBe(expectedCount); @@ -825,6 +838,7 @@ describe('matrix coverage guard', () => { 'cursor', 'cline', 'kiro', + 'windsurf', 'codex', ]); }); diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index a30c3fb..3b35751 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -228,6 +228,7 @@ describe('matrix coverage guard', () => { 'cursor', 'cline', 'kiro', + 'windsurf', 'codex', ]); }); From 768da46510d4c4d15ce2a70ff50acc7ef0c4c245 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:27:15 -0400 Subject: [PATCH 049/102] feat(cli): honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY behind corporate and CI proxies (#169) * feat(cli): honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY behind corporate and CI proxies * fix(proxy): degrade to default dispatcher when proxy agent init fails instead of crashing startup * fix(proxy): pin undici to ^7.16.0 for Node 20 compatibility (8.x requires Node >=22.19) --- package-lock.json | 53 +++++++++----------------------------- package.json | 1 + src/index.ts | 5 ++++ src/lib/proxy.test.ts | 59 +++++++++++++++++++++++++++++++++++++++++++ src/lib/proxy.ts | 58 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 135 insertions(+), 41 deletions(-) create mode 100644 src/lib/proxy.test.ts create mode 100644 src/lib/proxy.ts diff --git a/package-lock.json b/package-lock.json index 456c498..b254e0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.1.2", + "version": "0.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.1.2", + "version": "0.2.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", + "undici": "^7.16.0", "valibot": "^1.4.1" }, "bin": { @@ -1024,9 +1025,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1041,9 +1039,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1058,9 +1053,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1075,9 +1067,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1092,9 +1081,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1109,9 +1095,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1126,9 +1109,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1143,9 +1123,6 @@ "ppc64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1160,9 +1137,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1177,9 +1151,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1194,9 +1165,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1211,9 +1179,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1228,9 +1193,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3855,6 +3817,15 @@ "typescript": ">=4.8.4 <6.1.0" } }, + "node_modules/undici": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index 63c5aac..341e713 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", + "undici": "^7.16.0", "valibot": "^1.4.1" }, "devDependencies": { diff --git a/src/index.ts b/src/index.ts index bc15ec5..f36e4de 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; import { Output, isOutputMode } from './lib/output.js'; +import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; import { VERSION } from './version.js'; @@ -149,6 +150,10 @@ program.hook('preAction', (_thisCommand, actionCommand) => { }); }); +// Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch +// ignores them by default). No-op when no proxy variable is set. +maybeInstallProxyAgent(); + try { await program.parseAsync(process.argv); } catch (err) { diff --git a/src/lib/proxy.test.ts b/src/lib/proxy.test.ts new file mode 100644 index 0000000..5c686af --- /dev/null +++ b/src/lib/proxy.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from 'vitest'; +import { maybeInstallProxyAgent } from './proxy.js'; + +describe('maybeInstallProxyAgent', () => { + it('installs an agent when HTTPS_PROXY is set', () => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }); + + it.each(['https_proxy', 'HTTP_PROXY', 'http_proxy'] as const)( + 'also honors the %s spelling', + name => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { [name]: 'http://proxy.corp.example.com:8080' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }, + ); + + it('does nothing when no proxy variable is set (default path unchanged)', () => { + const install = vi.fn(); + expect(maybeInstallProxyAgent({ env: {}, install })).toBe(false); + expect(maybeInstallProxyAgent({ env: { HTTPS_PROXY: '' }, install })).toBe(false); + expect(install).not.toHaveBeenCalled(); + }); + + it('falls back (returns false, warns, never throws) when installing the agent fails', () => { + // A malformed/unsupported proxy value makes the agent throw at startup; the + // CLI must degrade to a proxy-less dispatcher, not crash every command. + const errs: string[] = []; + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080' }, + install: () => { + throw new Error('unsupported proxy scheme'); + }, + stderr: line => errs.push(line), + }); + expect(installed).toBe(false); + expect(errs.join('\n')).toContain('ignoring proxy environment'); + }); + + it('still installs when NO_PROXY is set (exemptions are applied per request by undici)', () => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080', NO_PROXY: 'localhost,127.0.0.1' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/proxy.ts b/src/lib/proxy.ts new file mode 100644 index 0000000..b536761 --- /dev/null +++ b/src/lib/proxy.ts @@ -0,0 +1,58 @@ +/** + * Proxy support (issue #119): honor HTTPS_PROXY / HTTP_PROXY / NO_PROXY. + * + * Node's built-in fetch (undici) deliberately ignores the proxy environment + * variables, so behind a corporate or CI proxy every request dies with + * `fetch failed` after a full retry cycle. Installing undici's + * `EnvHttpProxyAgent` as the global dispatcher restores the conventional + * behavior (including NO_PROXY exemptions) for every fetch the CLI makes. + * + * Only active when a proxy env var is actually present, so the default path + * stays byte-identical and pays zero startup cost. Dependency note for + * reviewers: this adds `undici` as an explicit runtime dependency (the same + * engine Node already bundles); the alternative, a hand-rolled CONNECT + * tunnel, would re-implement what undici ships and maintains. + */ +import type { Dispatcher } from 'undici'; +import { EnvHttpProxyAgent, setGlobalDispatcher } from 'undici'; + +export interface ProxyDeps { + env?: NodeJS.ProcessEnv; + /** Dispatcher installer. Defaults to undici's setGlobalDispatcher. */ + install?: (agent: Dispatcher) => void; + /** Warning sink. Defaults to `process.stderr`. */ + stderr?: (line: string) => void; +} + +/** + * Install the env-driven proxy dispatcher when any proxy variable is set + * (both canonical upper-case and conventional lower-case spellings). + * Returns whether an agent was installed (observable for tests/debugging). + */ +export function maybeInstallProxyAgent(deps: ProxyDeps = {}): boolean { + const env = deps.env ?? process.env; + const hasProxy = [env.HTTPS_PROXY, env.https_proxy, env.HTTP_PROXY, env.http_proxy].some( + value => typeof value === 'string' && value.length > 0, + ); + if (!hasProxy) return false; + const install = deps.install ?? setGlobalDispatcher; + // EnvHttpProxyAgent reads HTTPS_PROXY/HTTP_PROXY/NO_PROXY itself, per + // request, so NO_PROXY exemptions apply without extra plumbing here. + // + // A malformed or unsupported proxy value (e.g. `socks5://...`) makes the + // agent throw. Because this runs at startup, an unguarded throw would abort + // every command before the CLI's own error handling — so fall back to the + // default (proxy-less) dispatcher and warn instead of crashing. + try { + install(new EnvHttpProxyAgent()); + return true; + } catch (error) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr( + `warning: ignoring proxy environment (could not initialize proxy agent): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return false; + } +} From c6f946e574dc402abf2be1f5d8f56a3a219393cb Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:27:31 -0400 Subject: [PATCH 050/102] feat(cli): non-blocking "new version available" notice (24h-cached npm check, opt-out, CI-safe) (#181) --- DOCUMENTATION.md | 12 ++ src/index.ts | 13 +- src/lib/update-check.test.ts | 231 ++++++++++++++++++++++++++++ src/lib/update-check.ts | 285 +++++++++++++++++++++++++++++++++++ 4 files changed, 540 insertions(+), 1 deletion(-) create mode 100644 src/lib/update-check.test.ts create mode 100644 src/lib/update-check.ts diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 8c0cf1a..0b0fc27 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -455,8 +455,20 @@ These apply to every command: | `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | | `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | | `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | | `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +### Update notice + +Interactive runs print a one-line "new version available" notice on stderr when +a newer release exists. To learn this, the CLI contacts the public npm registry +(`registry.npmjs.org`) at most once per 24 hours; the request carries the +package name only — never your API key, project data, or command line. The +check is skipped in CI, when stderr is not a TTY, under `--output json` / +`--dry-run`, and entirely when `TESTSPRITE_NO_UPDATE_NOTIFIER` is set. Any +failure is silent: the notice can never break or delay a command. This is the +only outbound call the CLI makes besides your configured API endpoint. + ### Scopes API-key scopes gate the write and run surfaces: diff --git a/src/index.ts b/src/index.ts index f36e4de..fb935c8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,7 @@ import { Output, isOutputMode } from './lib/output.js'; import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; +import { maybeNotifyUpdate } from './lib/update-check.js'; import { VERSION } from './version.js'; import { shouldRejectNodeVersion } from './version-guard.js'; @@ -140,14 +141,24 @@ program.hook('preAction', (_thisCommand, actionCommand) => { profile?: string; dryRun?: boolean; }; + const commandPath = commandPathOf(actionCommand); maybeEmitSkillNudge({ - commandPath: commandPathOf(actionCommand), + commandPath, output: isOutputMode(globals.output) ? globals.output : 'text', dryRun: globals.dryRun ?? false, profile: globals.profile ?? 'default', cwd: process.cwd(), env: process.env, }); + + // Best-effort update notice (see lib/update-check.ts): self-gates on the + // opt-out env, CI, TTY, and a 24h cache; the wiring adds the flag-level + // gates the lib cannot see. Skipped for `completion` (its stdout is eval'd + // by shells), under --output json, and under --dry-run. Deliberately not + // awaited: an advisory must never delay the real command. + if (globals.output !== 'json' && globals.dryRun !== true && commandPath !== 'completion') { + void maybeNotifyUpdate(); + } }); // Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch diff --git a/src/lib/update-check.test.ts b/src/lib/update-check.test.ts new file mode 100644 index 0000000..dbce172 --- /dev/null +++ b/src/lib/update-check.test.ts @@ -0,0 +1,231 @@ +/** + * Unit tests for the update notice (issue #122). Every effect is injected: + * no real network, filesystem, clock, or TTY is touched. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { UpdateCheckDeps } from './update-check.js'; +import { + UPDATE_CHECK_OPT_OUT_ENV, + UPDATE_CHECK_TTL_MS, + compareSemver, + fetchLatestVersion, + maybeNotifyUpdate, + shouldCheckForUpdate, +} from './update-check.js'; + +/** In-memory fs + deterministic clock harness for the cache round-trip. */ +function makeHarness(overrides: UpdateCheckDeps = {}) { + const files = new Map(); + const stderrLines: string[] = []; + const deps: UpdateCheckDeps = { + env: {}, + now: () => 1_000_000, + cachePath: '/fake/.testsprite/update-check.json', + readFile: path => { + const content = files.get(path); + if (content === undefined) throw new Error('ENOENT'); + return content; + }, + writeFile: (path, content) => { + files.set(path, content); + }, + mkdir: () => undefined, + isTTY: true, + stderr: line => stderrLines.push(line), + currentVersion: '0.2.0', + fetchImpl: async () => + new Response(JSON.stringify({ version: '0.2.0' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ...overrides, + }; + return { deps, files, stderrLines }; +} + +describe('shouldCheckForUpdate gates', () => { + it('opt-out env set to any non-empty value disables (even "0")', () => { + const { deps } = makeHarness({ env: { [UPDATE_CHECK_OPT_OUT_ENV]: '0' } }); + expect(shouldCheckForUpdate(deps)).toBe(false); + }); + + it('CI set disables; CI="false" re-enables', () => { + expect(shouldCheckForUpdate(makeHarness({ env: { CI: 'true' } }).deps)).toBe(false); + expect(shouldCheckForUpdate(makeHarness({ env: { CI: '' } }).deps)).toBe(false); + expect(shouldCheckForUpdate(makeHarness({ env: { CI: 'false' } }).deps)).toBe(true); + }); + + it('non-TTY stderr disables', () => { + expect(shouldCheckForUpdate(makeHarness({ isTTY: false }).deps)).toBe(false); + }); + + it('a fresh cache suppresses; a stale cache does not', () => { + const fresh = makeHarness(); + fresh.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 1_000_000 - UPDATE_CHECK_TTL_MS + 5_000 }), + ); + expect(shouldCheckForUpdate(fresh.deps)).toBe(false); + + const stale = makeHarness(); + stale.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 1_000_000 - UPDATE_CHECK_TTL_MS - 5_000 }), + ); + expect(shouldCheckForUpdate(stale.deps)).toBe(true); + }); + + it('missing, corrupt, wrong-shape, or future-stamped caches count as stale', () => { + expect(shouldCheckForUpdate(makeHarness().deps)).toBe(true); // missing + const corrupt = makeHarness(); + corrupt.files.set('/fake/.testsprite/update-check.json', '{not json'); + expect(shouldCheckForUpdate(corrupt.deps)).toBe(true); + const wrongShape = makeHarness(); + wrongShape.files.set('/fake/.testsprite/update-check.json', JSON.stringify({ nope: true })); + expect(shouldCheckForUpdate(wrongShape.deps)).toBe(true); + const future = makeHarness(); + future.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 9_999_999_999 }), + ); + expect(shouldCheckForUpdate(future.deps)).toBe(true); + }); +}); + +describe('fetchLatestVersion', () => { + it('returns the version from a valid registry body', async () => { + const { deps } = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '1.2.3' }), { status: 200 }), + }); + await expect(fetchLatestVersion(deps)).resolves.toBe('1.2.3'); + }); + + it('returns undefined on non-2xx, thrown fetch, and wrong-shape body', async () => { + const notOk = makeHarness({ fetchImpl: async () => new Response('nope', { status: 500 }) }); + await expect(fetchLatestVersion(notOk.deps)).resolves.toBeUndefined(); + + const throwing = makeHarness({ + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + await expect(fetchLatestVersion(throwing.deps)).resolves.toBeUndefined(); + + const wrongShape = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ notVersion: 1 }), { status: 200 }), + }); + await expect(fetchLatestVersion(wrongShape.deps)).resolves.toBeUndefined(); + }); +}); + +describe('compareSemver', () => { + it('orders numerically and treats prerelease as older than its release', () => { + expect(compareSemver('0.2.0', '0.3.0')).toBe(-1); + expect(compareSemver('1.0.0', '0.9.9')).toBe(1); + expect(compareSemver('0.2.0', '0.2.0')).toBe(0); + expect(compareSemver('0.10.0', '0.9.0')).toBe(1); // numeric, not lexicographic + expect(compareSemver('1.0.0-rc.1', '1.0.0')).toBe(-1); + expect(compareSemver('1.0.0', '1.0.0-rc.1')).toBe(1); + }); + + it('unparseable input on either side compares as 0 (never a false notice)', () => { + expect(compareSemver('garbage', '1.0.0')).toBe(0); + expect(compareSemver('1.0.0', '')).toBe(0); + }); +}); + +describe('maybeNotifyUpdate', () => { + it('prints exactly one stderr line naming both versions when newer, and stamps the cache', async () => { + const harness = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '0.3.1' }), { status: 200 }), + }); + await maybeNotifyUpdate(harness.deps); + expect(harness.stderrLines).toHaveLength(1); + expect(harness.stderrLines[0]).toContain('0.2.0 -> 0.3.1'); + expect(harness.stderrLines[0]).toContain(UPDATE_CHECK_OPT_OUT_ENV); + const cache = JSON.parse(harness.files.get('/fake/.testsprite/update-check.json')!) as { + lastCheckMs: number; + latestKnown?: string; + }; + expect(cache.lastCheckMs).toBe(1_000_000); + expect(cache.latestKnown).toBe('0.3.1'); + }); + + it('stays silent on an equal or older registry version', async () => { + const equal = makeHarness(); + await maybeNotifyUpdate(equal.deps); + expect(equal.stderrLines).toHaveLength(0); + + const older = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '0.1.9' }), { status: 200 }), + }); + await maybeNotifyUpdate(older.deps); + expect(older.stderrLines).toHaveLength(0); + }); + + it('a failed probe stays silent but still stamps the cache (retry once per TTL)', async () => { + const harness = makeHarness({ + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + await maybeNotifyUpdate(harness.deps); + expect(harness.stderrLines).toHaveLength(0); + const cache = JSON.parse(harness.files.get('/fake/.testsprite/update-check.json')!) as { + lastCheckMs: number; + latestKnown?: string; + }; + expect(cache.lastCheckMs).toBe(1_000_000); + expect(cache.latestKnown).toBeUndefined(); + }); + + it('does nothing when a gate blocks (no fetch fired)', async () => { + const fetchImpl = vi.fn(async () => new Response('{}', { status: 200 })); + const harness = makeHarness({ env: { CI: '1' }, fetchImpl }); + await maybeNotifyUpdate(harness.deps); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(harness.stderrLines).toHaveLength(0); + }); + + it('never rejects, even when every injected dependency throws', async () => { + const harness = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '9.9.9' }), { status: 200 }), + writeFile: () => { + throw new Error('EROFS'); + }, + stderr: () => { + throw new Error('broken stderr sink'); + }, + }); + await expect(maybeNotifyUpdate(harness.deps)).resolves.toBeUndefined(); + }); + + it('uses the default fs readers/writers when none are injected (real cache round-trip)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'update-check-')); + // Point cachePath at a not-yet-existing subdir so the default mkdir + // (recursive) and writeFile arrows both run, and the first read (file + // absent) exercises the default readFile arrow's ENOENT path. + const cachePath = join(dir, 'nested', 'update-check.json'); + const stderrLines: string[] = []; + try { + await maybeNotifyUpdate({ + env: {}, + now: () => 1_000_000, + isTTY: true, + currentVersion: '0.0.1', + cachePath, + stderr: line => stderrLines.push(line), + fetchImpl: async () => new Response(JSON.stringify({ version: '9.9.9' }), { status: 200 }), + }); + // Cache was persisted by the default writeFile through the default mkdir. + expect(JSON.parse(readFileSync(cachePath, 'utf8'))).toMatchObject({ latestKnown: '9.9.9' }); + expect(stderrLines.join('\n')).toContain('0.0.1 -> 9.9.9'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/update-check.ts b/src/lib/update-check.ts new file mode 100644 index 0000000..e928913 --- /dev/null +++ b/src/lib/update-check.ts @@ -0,0 +1,285 @@ +/** + * Non-blocking "new version available" notice (issue #122), following the + * pattern of the gh and npm CLIs: at most one npm-registry probe per 24 hours, + * result cached on disk, advisory printed to stderr so stdout stays parseable. + * + * Behavior (`maybeNotifyUpdate`): + * 1. Gate through `shouldCheckForUpdate`; every gate below must pass. + * 2. Probe the npm registry for the `latest` dist-tag (1.5s hard timeout). + * 3. Stamp the cache with `lastCheckMs` (plus `latestKnown` when the probe + * succeeded) so the next 24h of invocations skip the network entirely. + * The stamp happens even after a failed probe: a dead registry must not + * trigger a retry on every command. + * 4. When `latest` is strictly newer than the running version, write exactly + * one advisory line to stderr. The function never throws or rejects and + * never alters the exit status of the command it rides along with. + * + * Gates, in order (`shouldCheckForUpdate`): + * - `TESTSPRITE_NO_UPDATE_NOTIFIER` set to any non-empty value: opted out. + * Presence-style, mirroring gh's GH_NO_UPDATE_NOTIFIER: even "0" disables. + * - `CI` set to anything except the literal "false": CI logs are not the + * place for update nags. `CI=false` explicitly re-enables the notice. + * - stderr is not a TTY: piped or redirected output stays clean. + * - the on-disk cache is fresh (last probe within the TTL). A missing, + * unreadable, corrupt, or wrong-shape cache counts as stale, and so does a + * `lastCheckMs` in the future (clock rollback or corrupt data). + * + * Why not the npm `update-notifier` package: this CLI's runtime dependency + * budget is commander + valibot only (package.json). `update-notifier` would + * add a transitive dependency tree for what Node 20 already ships as + * primitives (fetch, AbortSignal.timeout, sync fs). Owning the ~100 lines + * keeps the install size flat and every effect injectable for tests. + * + * All effects (env, network, clock, fs, tty, stderr sink) are injectable via + * `UpdateCheckDeps`, the same dependency-injection style as `skill-nudge.ts`. + */ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import * as v from 'valibot'; +import { VERSION } from '../version.js'; +import type { FetchImpl } from './http.js'; + +/** Re-check interval: 24 hours, expressed in milliseconds. */ +export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000; + +/** + * Env var that disables the update notice entirely. Presence-style: any + * non-empty value (including "0") opts out. Deliberately stricter than the + * truthy-style `TESTSPRITE_NO_SKILL_WARNING` because it matches the + * convention users already know from gh's GH_NO_UPDATE_NOTIFIER. + */ +export const UPDATE_CHECK_OPT_OUT_ENV = 'TESTSPRITE_NO_UPDATE_NOTIFIER'; + +/** + * Hard cap for the registry probe, in milliseconds. The probe rides along a + * real command, so unlike the 120s API budget in `http.ts` it gets a tiny + * window: a slow registry means "no update info", never a visible delay. + */ +const REGISTRY_TIMEOUT_MS = 1_500; + +/** + * npm registry `latest` dist-tag endpoint for this package. Package name from + * package.json (`@testsprite/testsprite-cli`); the scope separator must be + * URL-encoded as %2F per the npm registry API. + */ +const REGISTRY_LATEST_URL = 'https://registry.npmjs.org/@testsprite%2Ftestsprite-cli/latest'; + +/** On-disk cache shape at `cachePath`; unknown keys are stripped on read. */ +const UPDATE_CHECK_CACHE_SCHEMA = v.object({ + lastCheckMs: v.number(), + latestKnown: v.optional(v.string()), +}); + +export type UpdateCheckCache = v.InferOutput; + +/** Minimal slice of the registry response the notice needs. */ +const REGISTRY_LATEST_BODY_SCHEMA = v.object({ version: v.string() }); + +export interface UpdateCheckDeps { + env?: NodeJS.ProcessEnv; + fetchImpl?: FetchImpl; + /** Clock, epoch milliseconds. */ + now?: () => number; + /** Cache file; lives next to credentials/config under ~/.testsprite. */ + cachePath?: string; + readFile?: (path: string) => string; + writeFile?: (path: string, content: string) => void; + /** Must create missing parent directories (recursive). */ + mkdir?: (dir: string) => void; + /** Whether stderr is an interactive terminal. */ + isTTY?: boolean; + /** Sink for the single advisory line. */ + stderr?: (line: string) => void; + /** Version the running binary reports. */ + currentVersion?: string; +} + +type ResolvedUpdateCheckDeps = Required; + +function resolveUpdateCheckDeps(deps: UpdateCheckDeps): ResolvedUpdateCheckDeps { + return { + env: deps.env ?? process.env, + fetchImpl: deps.fetchImpl ?? globalThis.fetch, + now: deps.now ?? Date.now, + cachePath: deps.cachePath ?? join(homedir(), '.testsprite', 'update-check.json'), + readFile: deps.readFile ?? ((path: string) => readFileSync(path, 'utf8')), + writeFile: + deps.writeFile ?? ((path: string, content: string) => writeFileSync(path, content, 'utf8')), + mkdir: + deps.mkdir ?? + ((dir: string) => { + mkdirSync(dir, { recursive: true }); + }), + isTTY: deps.isTTY ?? process.stderr.isTTY === true, + stderr: deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)), + currentVersion: deps.currentVersion ?? VERSION, + }; +} + +/** + * Read and validate the cache file. Every failure mode (missing file, + * unreadable file, invalid JSON, wrong shape) returns undefined, which the + * caller treats as "stale, probe again". + */ +function readUpdateCheckCache(resolved: ResolvedUpdateCheckDeps): UpdateCheckCache | undefined { + try { + const raw = resolved.readFile(resolved.cachePath); + const body: unknown = JSON.parse(raw); + const parsed = v.safeParse(UPDATE_CHECK_CACHE_SCHEMA, body); + return parsed.success ? parsed.output : undefined; + } catch { + // Missing or unreadable cache: treat as stale. + return undefined; + } +} + +/** + * Persist the cache, creating the parent directory when missing. Best-effort: + * every error (read-only home, quota, fs races) is swallowed. A failed write + * only means the next invocation probes the registry again. + */ +function writeUpdateCheckCache(resolved: ResolvedUpdateCheckDeps, cache: UpdateCheckCache): void { + try { + resolved.mkdir(dirname(resolved.cachePath)); + resolved.writeFile(resolved.cachePath, `${JSON.stringify(cache)}\n`); + } catch { + // Cache persistence is optional; never surface fs errors to the command. + } +} + +/** + * True when every gate documented in the module header passes: no opt-out + * env, not CI (unless CI=false), stderr is a TTY, and the cached check is + * stale or absent. Order matters: the cheap env gates run before any fs read. + */ +export function shouldCheckForUpdate(deps: UpdateCheckDeps = {}): boolean { + const resolved = resolveUpdateCheckDeps(deps); + + const optOutValue = resolved.env[UPDATE_CHECK_OPT_OUT_ENV]; + if (optOutValue !== undefined && optOutValue !== '') return false; + + // Any set CI value except the literal "false" counts as CI. The empty + // string still signals a CI-managed environment; silence wins in doubt. + const ciValue = resolved.env.CI; + if (ciValue !== undefined && ciValue !== 'false') return false; + + if (!resolved.isTTY) return false; + + const cache = readUpdateCheckCache(resolved); + if (cache !== undefined) { + const elapsedMs = resolved.now() - cache.lastCheckMs; + // A negative elapsed (lastCheckMs in the future) means clock rollback or + // corrupt data: treat as stale instead of suppressing the check forever. + // A NaN lastCheckMs fails both comparisons and lands on stale too. + if (elapsedMs >= 0 && elapsedMs < UPDATE_CHECK_TTL_MS) return false; + } + + return true; +} + +/** + * Probe the npm registry for the `latest` dist-tag version of this package. + * Hard 1.5s timeout via AbortSignal.timeout. ANY failure (network error, + * timeout, non-2xx status, invalid JSON, wrong shape) resolves to undefined; + * this function never rejects. + */ +export async function fetchLatestVersion(deps: UpdateCheckDeps = {}): Promise { + const resolved = resolveUpdateCheckDeps(deps); + try { + const response = await resolved.fetchImpl(REGISTRY_LATEST_URL, { + signal: AbortSignal.timeout(REGISTRY_TIMEOUT_MS), + }); + if (!response.ok) return undefined; + const body: unknown = await response.json(); + const parsed = v.safeParse(REGISTRY_LATEST_BODY_SCHEMA, body); + return parsed.success ? parsed.output.version : undefined; + } catch { + // Offline, DNS failure, abort, or a non-JSON body: no update info. + return undefined; + } +} + +interface ParsedSemver { + major: number; + minor: number; + patch: number; + hasPrerelease: boolean; +} + +/** + * x.y.z with optional prerelease (after "-") and optional build metadata + * (after "+"), per the semver 2.0.0 grammar. A leading "v" is tolerated + * because humans type it; the npm registry never returns one. + */ +const SEMVER_RE = /^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/; + +function parseSemver(version: string): ParsedSemver | undefined { + const match = SEMVER_RE.exec(version.trim()); + if (!match) return undefined; + const [, majorRaw, minorRaw, patchRaw, prerelease] = match; + if (majorRaw === undefined || minorRaw === undefined || patchRaw === undefined) return undefined; + return { + major: Number(majorRaw), + minor: Number(minorRaw), + patch: Number(patchRaw), + hasPrerelease: prerelease !== undefined, + }; +} + +/** + * Compare two semver strings numerically. Returns -1 when versionA is older + * than versionB, 1 when newer, 0 when equal. A version carrying a prerelease + * tag sorts OLDER than the plain release with the same x.y.z core. Prerelease + * identifiers themselves are not ranked (two prereleases on the same core + * compare as 0): the registry `latest` tag points at releases, so identifier + * ordering never decides whether the notice fires. Unparseable input on + * either side compares as 0, so garbage can never produce a false notice. + */ +export function compareSemver(versionA: string, versionB: string): number { + const left = parseSemver(versionA); + const right = parseSemver(versionB); + if (left === undefined || right === undefined) return 0; + if (left.major !== right.major) return left.major < right.major ? -1 : 1; + if (left.minor !== right.minor) return left.minor < right.minor ? -1 : 1; + if (left.patch !== right.patch) return left.patch < right.patch ? -1 : 1; + if (left.hasPrerelease !== right.hasPrerelease) return left.hasPrerelease ? -1 : 1; + return 0; +} + +/** + * Fire-and-forget update notice. Gates, probes, stamps the cache, and prints + * at most one stderr line when the registry version is strictly newer than + * the running one. Never throws and never rejects: any failure in any + * injected dependency (clock, fs, network, the stderr sink itself) is + * swallowed, because an advisory must never break or delay a real command. + */ +export async function maybeNotifyUpdate(deps: UpdateCheckDeps = {}): Promise { + try { + const resolved = resolveUpdateCheckDeps(deps); + if (!shouldCheckForUpdate(resolved)) return; + + const latest = await fetchLatestVersion(resolved); + + // Stamp even on a failed probe so a dead registry is retried at most + // once per TTL window, not on every invocation. + writeUpdateCheckCache(resolved, { + lastCheckMs: resolved.now(), + ...(latest === undefined ? {} : { latestKnown: latest }), + }); + + if (latest === undefined) return; + if (compareSemver(latest, resolved.currentVersion) !== 1) return; + + // User-facing advisory copy (exact format specified by issue #122), not a + // diagnostic log line; stderr keeps stdout parseable for scripts. + resolved.stderr( + `A new version of testsprite-cli is available: ${resolved.currentVersion} -> ${latest}. ` + + `Run npm install -g @testsprite/testsprite-cli to update. ` + + `(Disable with ${UPDATE_CHECK_OPT_OUT_ENV}=1)`, + ); + } catch { + // An update notice must never break, delay, or alter the exit status of + // the command it accompanies. Swallow everything. + } +} From 4b724619e27a360a00e6c578db3e5068bb566d91 Mon Sep 17 00:00:00 2001 From: Resque Date: Mon, 6 Jul 2026 02:27:46 +0400 Subject: [PATCH 051/102] feat(cli): add 'test flaky' repeat-run flaky-test detector (#132) * feat(cli): add 'test flaky' repeat-run flaky-test detector * fix(flaky): cap --runs at 10 per maintainer scope (#115) Rescope the flaky detector's --runs bound from 1-100 to 1-10 as requested in the #115 triage: uncapped FE replays amplify free executions. Updates the MAX_FLAKY_RUNS constant (which drives the validation, error message, and --runs help text), docs, changelog, and the runs-bound tests. Regenerates the help snapshot, which also adds the previously-missing 'test flaky' entry. * test(snapshot): refresh flaky help snapshot after rebase onto main Rebasing onto current main (which added the global --request-timeout option, #17) changes the 'Global options' line rendered in the test flaky --help output. Regenerate the snapshot so the help snapshot test stays green on CI. * docs(changelog): resolve leftover merge-conflict markers (keep JUnit + flaky 1-10) --- CHANGELOG.md | 1 + DOCUMENTATION.md | 23 ++ src/commands/test.flaky.spec.ts | 389 ++++++++++++++++++ src/commands/test.test.ts | 1 + src/commands/test.ts | 234 +++++++++++ src/lib/flaky.test.ts | 105 +++++ src/lib/flaky.ts | 133 ++++++ test/__snapshots__/help.snapshot.test.ts.snap | 41 ++ test/help.snapshot.test.ts | 1 + 9 files changed, 928 insertions(+) create mode 100644 src/commands/test.flaky.spec.ts create mode 100644 src/lib/flaky.test.ts create mode 100644 src/lib/flaky.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a4ab9b7..4f1a0a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ### Added - **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. +- **`testsprite test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs `, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky --runs 5 || exit 1`). Flags: `--runs ` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout ` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits. ## [0.2.0] - 2026-06-29 diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 0b0fc27..fb62cae 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -403,6 +403,29 @@ Flags: A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. +#### `testsprite test flaky ` + +Detect a **flaky** test by replaying it several times and reporting how often it passes. Each attempt is a rerun with auto-heal **off** (a strict verbatim replay), so healed drift can't disguise a nondeterministic pass/fail — this measures the replay stability of the saved script against the configured URL. Frontend replays are free verbatim script replays; backend tests re-run their dependency closure and may cost credits (a one-line stderr advisory is printed before the run). + +```bash +# Replay 10 times and print a stability score +testsprite test flaky test_xxxxxxxx --runs 10 + +# Fast "is it flaky at all?" — stop at the first non-passing attempt +testsprite test flaky test_xxxxxxxx --runs 10 --until-fail + +# Machine-readable stability report for CI +testsprite test flaky test_xxxxxxxx --runs 10 --output json +``` + +Flags: + +- `--runs ` — number of replays (1–10, default 5). +- `--until-fail` — stop at the first attempt that does not pass. +- `--timeout ` — per-attempt polling deadline (same semantics as `test wait`). + +`--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. + #### `testsprite test wait ` Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. diff --git a/src/commands/test.flaky.spec.ts b/src/commands/test.flaky.spec.ts new file mode 100644 index 0000000..0c4450e --- /dev/null +++ b/src/commands/test.flaky.spec.ts @@ -0,0 +1,389 @@ +/** + * Unit tests for `test flaky` — the repeat-run flaky-test detector. + * + * All HTTP is mocked via `makeFlakyFetch`. The polling loop's sleep is injected + * through `TestDeps.sleep` to avoid real delays. Each rerun POST returns a + * unique runId; each run GET returns a terminal status scripted per attempt, + * so a test can assert stable / flaky / failing verdicts deterministically. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { CLIError, ApiError } from '../lib/errors.js'; +import type { FlakyReport } from '../lib/flaky.js'; +import type { FetchImpl } from '../lib/http.js'; +import { runFlaky } from './test.js'; + +type FetchInput = Parameters[0]; +type RunStatus = 'passed' | 'failed' | 'blocked' | 'cancelled'; + +function urlOf(input: FetchInput): string { + return typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; +} + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +/** + * Build a fetch that: + * - GET /tests/{id} → the test record ('frontend' | 'backend') + * - POST /tests/{id}/runs/rerun → a queued rerun with runId run_ (n increments) + * - GET /runs/run_ → a terminal run with statuses[k-1] + * + * `notFoundOnTrigger` makes the rerun POST return 404 (no replayable run). + */ +function makeFlakyFetch(opts: { + statuses: RunStatus[]; + testType?: 'frontend' | 'backend'; + notFoundOnTrigger?: boolean; +}): { fetchImpl: FetchImpl; triggerCount: () => number } { + let triggers = 0; + const testType = opts.testType ?? 'frontend'; + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = urlOf(input); + const method = (init.method ?? 'GET').toUpperCase(); + + if (method === 'GET' && /\/tests\/[^/]+$/.test(url.split('?')[0]!)) { + return jsonResponse(200, { + id: 'test_x', + projectId: 'project_abc', + name: 'sample', + type: testType, + createdFrom: 'portal', + status: 'passed', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }); + } + + if (method === 'POST' && url.includes('/runs/rerun')) { + if (opts.notFoundOnTrigger) { + return jsonResponse(404, { + error: { + code: 'NOT_FOUND', + message: 'no replayable run', + nextAction: 'run it', + requestId: 'req_1', + details: {}, + }, + }); + } + triggers += 1; + return jsonResponse(200, { + runId: `run_${triggers}`, + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + }); + } + + const runMatch = /\/runs\/(run_\d+)/.exec(url); + if (method === 'GET' && runMatch) { + const runId = runMatch[1]!; + const idx = Number(runId.replace('run_', '')) - 1; + const status = opts.statuses[idx] ?? 'passed'; + return jsonResponse(200, { + runId, + testId: 'test_x', + projectId: 'project_abc', + userId: 'user_1', + status, + source: 'cli', + createdAt: '2026-06-03T10:00:00.000Z', + startedAt: '2026-06-03T10:00:01.000Z', + finishedAt: '2026-06-03T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'rerun:prior', + failedStepIndex: status === 'passed' ? null : 2, + failureKind: status === 'passed' ? null : 'assertion', + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 5, passedCount: 5, failedCount: 0 }, + }); + } + + return jsonResponse(404, { + error: { code: 'NOT_FOUND', message: 'unmatched', requestId: 'x', details: {} }, + }); + }) as FetchImpl; + + return { fetchImpl, triggerCount: () => triggers }; +} + +function makeCreds(): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-flaky-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync( + credentialsPath, + `[default]\napi_url = http://localhost:13509\napi_key = sk-user-test\n`, + { + mode: 0o600, + }, + ); + return { credentialsPath }; +} + +const instantSleep = (): Promise => Promise.resolve(); + +function makeDeps(fetchImpl: FetchImpl): { + deps: { + credentialsPath: string; + fetchImpl: FetchImpl; + sleep: () => Promise; + stdout: (l: string) => void; + stderr: (l: string) => void; + }; + stdout: string[]; + stderr: string[]; +} { + const stdout: string[] = []; + const stderr: string[] = []; + const { credentialsPath } = makeCreds(); + return { + deps: { + credentialsPath, + fetchImpl, + sleep: instantSleep, + stdout: (l: string) => stdout.push(l), + stderr: (l: string) => stderr.push(l), + }, + stdout, + stderr, + }; +} + +// --------------------------------------------------------------------------- +// Surface +// --------------------------------------------------------------------------- + +describe('createTestCommand — flaky subcommand exposed', () => { + it('exposes flaky with its flags', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const flaky = test.commands.find(c => c.name() === 'flaky'); + expect(flaky).toBeDefined(); + const flagNames = flaky!.options.map(o => o.long); + expect(flagNames).toContain('--runs'); + expect(flagNames).toContain('--until-fail'); + expect(flagNames).toContain('--timeout'); + }); +}); + +// --------------------------------------------------------------------------- +// Behavior +// --------------------------------------------------------------------------- + +describe('runFlaky', () => { + it('reports STABLE and exits 0 when every attempt passes', async () => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: ['passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + const report = (await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + )) as FlakyReport; + expect(report.verdict).toBe('stable'); + expect(report.runs).toBe(3); + expect(triggerCount()).toBe(3); + }); + + it('reports FLAKY and throws exit 1 on a mix of pass/fail', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'failed', 'passed'] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).exitCode).toBe(1); + expect((err as CLIError).message).toContain('flaky'); + }); + + it('reports FAILING and throws exit 1 when no attempt passes', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['failed', 'failed'] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).message).toContain('failing'); + }); + + it('--until-fail stops at the first non-passing attempt', async () => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: ['passed', 'failed', 'passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 5, + untilFail: true, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + // Stopped after attempt 2 (the failure) — only 2 triggers fired. + expect(triggerCount()).toBe(2); + expect(err).toBeInstanceOf(CLIError); + }); + + it('prints a backend credit advisory to stderr', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'passed'], testType: 'backend' }); + const { deps, stderr } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ); + expect(stderr.some(l => l.includes('backend test') && l.includes('credits'))).toBe(true); + }); + + it('emits a machine-readable JSON stability report', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'failed', 'passed'] }); + const { deps, stdout } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch(() => undefined); // swallow the exit-1 throw; we only assert stdout + const parsed = JSON.parse(stdout.join('\n')) as FlakyReport; + expect(parsed.testId).toBe('test_x'); + expect(parsed.runs).toBe(3); + expect(parsed.passed).toBe(2); + expect(parsed.verdict).toBe('flaky'); + expect(parsed.failures).toHaveLength(1); + expect(parsed.failures[0]!.failureKind).toBe('assertion'); + }); + + it('throws exit 4 when the test has no replayable run', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [], notFoundOnTrigger: true }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('NOT_FOUND'); + }); + + it('rejects --runs below the range (0) with a validation error (exit 5)', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 0, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(5); + }); + + it('rejects --runs above the cap (11) with a validation error (exit 5)', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 11, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(5); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 38c16e5..6d8d734 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -124,6 +124,7 @@ describe('createTestCommand — surface', () => { 'delete-batch', 'diff', 'failure', + 'flaky', 'get', 'list', 'plan', diff --git a/src/commands/test.ts b/src/commands/test.ts index 5582603..508d8be 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -75,6 +75,14 @@ import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; import { loadConfig } from '../lib/config.js'; +import { + flakyExitCode, + renderFlakyText, + summarizeFlaky, + type FlakyAttempt, + type FlakyOutcome, + type FlakyReport, +} from '../lib/flaky.js'; /** * `details` debug block per the CLI OpenAPI `Test` schema @@ -8117,6 +8125,64 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + // ------------------------------------------------------------------------- + // `test flaky` — repeat-run flaky-test detector + // ------------------------------------------------------------------------- + + test + .command('flaky ') + .description( + 'Repeatedly replay a test to measure stability and surface flakiness.\n' + + 'Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail.\n' + + '\nExit codes:\n' + + ' 0 stable (every attempt passed)\n' + + ' 1 flaky or failing (at least one attempt did not pass)\n' + + ' 3 auth error\n' + + ' 4 test not found (no replayable run — trigger `testsprite test run ` first)\n' + + ' 5 validation error', + ) + .option( + '--runs ', + `number of replays to run (1-${MAX_FLAKY_RUNS}, default ${DEFAULT_FLAKY_RUNS})`, + ) + .option( + '--until-fail', + 'stop at the first non-passing attempt (fast "is it flaky at all?" check)', + false, + ) + .option( + '--timeout ', + `per-attempt max seconds to wait (1-${MAX_RUN_TIMEOUT_SECONDS}, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`, + ) + .addHelpText( + 'after', + '\nNotes:\n' + + ' • Frontend replays are free verbatim script replays (no credit); backend replays\n' + + ' re-run the dependency closure and may cost credits — a one-line advisory is printed.\n' + + ' • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass;\n' + + ' this measures replay stability of the saved script against the configured URL.\n' + + ' • `--output json` emits a machine-readable stability report for CI gating.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + testIdArg: string, + cmdOpts: { runs?: string; untilFail?: boolean; timeout?: string }, + command: Command, + ) => { + await runFlaky( + { + ...resolveCommonOptions(command), + testId: testIdArg, + runs: parseNumericFlag(cmdOpts.runs, 'runs') ?? DEFAULT_FLAKY_RUNS, + untilFail: cmdOpts.untilFail === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + }, + ); + test.addCommand(createTestCodeCommand(deps)); test.addCommand(createTestPlanCommand(deps)); test.addCommand(createTestFailureCommand(deps)); @@ -8125,6 +8191,174 @@ export function createTestCommand(deps: TestDeps = {}): Command { return test; } +// --------------------------------------------------------------------------- +// `test flaky` — repeat-run flaky-test detector +// --------------------------------------------------------------------------- + +/** Upper bound on `--runs` so a repeat-runner can't amplify free FE replays. */ +const MAX_FLAKY_RUNS = 10; +/** Default replay count when `--runs` is omitted. */ +const DEFAULT_FLAKY_RUNS = 5; + +interface RunTestFlakyOptions extends CommonOptions { + testId: string; + /** Number of replays to run (1..MAX_FLAKY_RUNS). */ + runs: number; + /** Stop at the first non-passing attempt. */ + untilFail: boolean; + /** Per-attempt polling deadline in seconds. */ + timeoutSeconds: number; +} + +/** + * `test flaky ` — replay a test N times and report a stability score. + * + * Each attempt is a `POST /tests/{id}/runs/rerun` with auto-heal OFF (a strict + * verbatim replay) followed by `pollRunUntilTerminal`. Frontend replays are + * free verbatim script replays; backend replays re-run the dependency closure + * (a one-line credit advisory is printed). The pure scoring lives in + * `lib/flaky.ts`; this function is the I/O orchestrator. + * + * Exit code: 0 when every observed attempt passed (stable), else 1 — so CI can + * gate a merge on flakiness. + */ +export async function runFlaky( + opts: RunTestFlakyOptions, + deps: TestDeps = {}, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + if (typeof opts.testId !== 'string' || opts.testId.length === 0) { + throw localValidationError('test-id', 'is required'); + } + if (!Number.isInteger(opts.runs) || opts.runs < 1 || opts.runs > MAX_FLAKY_RUNS) { + throw localValidationError('runs', `must be an integer between 1 and ${MAX_FLAKY_RUNS}`); + } + + if (opts.dryRun) { + out.print({ + dryRun: true, + command: 'test flaky', + testId: opts.testId, + runs: opts.runs, + untilFail: opts.untilFail, + method: 'POST', + path: `/api/cli/v1/tests/${opts.testId}/runs/rerun`, + note: `Would replay the test up to ${opts.runs}x with auto-heal OFF and report a stability score.`, + }); + return undefined; + } + + // Under the implicit wait, raise the per-request timeout to cover --timeout + // so a slow trigger / long-poll under load isn't cut at the 120s default. + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + + // Best-effort test-type detection for the credit advisory. A probe failure + // never blocks the run — we just skip the advisory. + let isBackend = false; + try { + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + isBackend = test.type === 'backend'; + } catch { + // best-effort — proceed without the advisory. + } + if (isBackend) { + stderrFn( + `[advisory] ${opts.testId} is a backend test — each replay re-runs its dependency closure ` + + `and may cost credits. Frontend replays are free verbatim script replays; backend replays are not.`, + ); + } + + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + const attempts: FlakyAttempt[] = []; + + for (let i = 1; i <= opts.runs; i++) { + const idempotencyKey = `cli-flaky-${randomUUID()}`; + + let rerunResp: RerunResponse; + try { + // auto-heal is intentionally OFF: flaky detection needs a strict verbatim + // replay so healed drift cannot mask a nondeterministic pass/fail. + rerunResp = await client.triggerRerun(opts.testId, { source: 'cli' }, { idempotencyKey }); + } catch (err) { + // A missing replayable run is fatal for the whole command (mirror rerun): + // there is nothing to repeat, so point the user at a fresh `test run`. + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + throw ApiError.fromEnvelope({ + error: { + code: 'NOT_FOUND', + message: `Test ${opts.testId} has no replayable run (unknown/cross-tenant id, or it has never completed a clean run).`, + nextAction: `Trigger a fresh run first: testsprite test run ${opts.testId}`, + requestId: err.requestId ?? 'local', + details: { testId: opts.testId, reason: 'no_replayable_run' }, + }, + }); + } + // Any other trigger error is recorded as an errored attempt so a single + // transient blip doesn't abort a long stability probe. + const code = err instanceof ApiError ? err.code : 'ERROR'; + attempts.push({ attempt: i, runId: null, outcome: 'error', failureKind: code }); + ticker.update(`Attempt ${i}/${opts.runs} — error (${code})`); + if (opts.untilFail) break; + continue; + } + + const runId = rerunResp.runId; + // Backend run rows never finalize server-side; resolve the verdict from the + // testId-scoped result on non-terminal ticks (same fallback as `test rerun`). + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => opts.testId, + resolveNotBefore: () => rerunResp.enqueuedAt, + }); + + let outcome: FlakyOutcome; + let failureKind: string | null = null; + try { + const finalRun = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: opts.timeoutSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + resolveAlternate, + }); + outcome = finalRun.status as FlakyOutcome; + failureKind = finalRun.failureKind; + } catch (err) { + // A per-attempt deadline (poll TimeoutError) or a client-side request + // timeout both count as a non-passing "timeout" outcome for this attempt. + if (err instanceof TimeoutError || err instanceof RequestTimeoutError) { + outcome = 'timeout'; + } else { + throw err; + } + } + + attempts.push({ attempt: i, runId, outcome, failureKind }); + const passedSoFar = attempts.filter(a => a.outcome === 'passed').length; + ticker.update(`Attempt ${i}/${opts.runs} — ${outcome} (${passedSoFar} passed so far)`); + + if (opts.untilFail && outcome !== 'passed') break; + } + + ticker.finalize(); + + const report = summarizeFlaky(opts.testId, attempts); + out.print(report, data => renderFlakyText(data as FlakyReport)); + + const exitCode = flakyExitCode(report); + if (exitCode !== 0) { + throw new CLIError( + `Test ${opts.testId} is ${report.verdict} — ${report.passed}/${report.runs} attempts passed`, + exitCode, + ); + } + return report; +} + interface RunFlagOpts { targetUrl?: string; wait?: boolean; diff --git a/src/lib/flaky.test.ts b/src/lib/flaky.test.ts new file mode 100644 index 0000000..a6acd9c --- /dev/null +++ b/src/lib/flaky.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest'; +import { flakyExitCode, renderFlakyText, summarizeFlaky, type FlakyAttempt } from './flaky.js'; + +function pass(attempt: number): FlakyAttempt { + return { attempt, runId: `run_${attempt}`, outcome: 'passed' }; +} +function fail(attempt: number, failureKind = 'assertion'): FlakyAttempt { + return { attempt, runId: `run_${attempt}`, outcome: 'failed', failureKind }; +} + +describe('summarizeFlaky', () => { + it('reports STABLE when every attempt passed', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2), pass(3)]); + expect(report.verdict).toBe('stable'); + expect(report.runs).toBe(3); + expect(report.passed).toBe(3); + expect(report.failed).toBe(0); + expect(report.stableRatio).toBe(1); + expect(report.failures).toEqual([]); + expect(flakyExitCode(report)).toBe(0); + }); + + it('reports FLAKY on a mix of pass and fail', () => { + const report = summarizeFlaky('test_x', [pass(1), fail(2), pass(3)]); + expect(report.verdict).toBe('flaky'); + expect(report.passed).toBe(2); + expect(report.failed).toBe(1); + expect(report.stableRatio).toBe(0.6667); + expect(report.failures).toEqual([ + { attempt: 2, runId: 'run_2', outcome: 'failed', failureKind: 'assertion' }, + ]); + expect(flakyExitCode(report)).toBe(1); + }); + + it('reports FAILING when no attempt passed', () => { + const report = summarizeFlaky('test_x', [fail(1), fail(2)]); + expect(report.verdict).toBe('failing'); + expect(report.passed).toBe(0); + expect(report.stableRatio).toBe(0); + expect(flakyExitCode(report)).toBe(1); + }); + + it('treats an empty attempt list as FAILING with a 0 ratio', () => { + const report = summarizeFlaky('test_x', []); + expect(report.verdict).toBe('failing'); + expect(report.runs).toBe(0); + expect(report.stableRatio).toBe(0); + expect(flakyExitCode(report)).toBe(1); + }); + + it('counts timeout and error outcomes as non-passing', () => { + const attempts: FlakyAttempt[] = [ + pass(1), + { attempt: 2, runId: 'run_2', outcome: 'timeout' }, + { attempt: 3, runId: null, outcome: 'error', failureKind: 'UNAVAILABLE' }, + ]; + const report = summarizeFlaky('test_x', attempts); + expect(report.verdict).toBe('flaky'); + expect(report.passed).toBe(1); + expect(report.failed).toBe(2); + expect(report.failures.map(f => f.outcome)).toEqual(['timeout', 'error']); + // error attempt with no runId is preserved as null in the report + expect(report.failures[1]).toEqual({ + attempt: 3, + runId: null, + outcome: 'error', + failureKind: 'UNAVAILABLE', + }); + }); + + it('rounds stableRatio to 4 decimal places', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2), fail(3)]); + expect(report.stableRatio).toBe(0.6667); + }); + + it('reflects a short-circuited (--until-fail) run — fewer runs than requested', () => { + // Only two attempts were observed before the probe stopped at the failure. + const report = summarizeFlaky('test_x', [pass(1), fail(2)]); + expect(report.runs).toBe(2); + expect(report.verdict).toBe('flaky'); + }); +}); + +describe('renderFlakyText', () => { + it('summarizes a stable run on one line with no failure list', () => { + const text = renderFlakyText(summarizeFlaky('test_login', [pass(1), pass(2)])); + expect(text).toBe('Ran test_login 2x — 2 passed, 0 failed → STABLE (100% stable)'); + }); + + it('lists failing attempts with runId and failureKind', () => { + const text = renderFlakyText( + summarizeFlaky('test_login', [pass(1), fail(2, 'network_timeout')]), + ); + expect(text).toContain('→ FLAKY (50% stable)'); + expect(text).toContain('failed attempts:'); + expect(text).toContain('#2 run_2 failed failureKind=network_timeout'); + }); + + it('shows (no runId) when an errored attempt never got a runId', () => { + const text = renderFlakyText( + summarizeFlaky('test_login', [{ attempt: 1, runId: null, outcome: 'error' }]), + ); + expect(text).toContain('#1 (no runId) error'); + }); +}); diff --git a/src/lib/flaky.ts b/src/lib/flaky.ts new file mode 100644 index 0000000..402eaf3 --- /dev/null +++ b/src/lib/flaky.ts @@ -0,0 +1,133 @@ +/** + * Pure aggregation + rendering for `test flaky` — the repeat-run flaky-test + * detector. This module performs no I/O: the orchestrator in + * `commands/test.ts` feeds it the per-attempt outcomes, and it returns a + * machine-readable stability report plus the text rendering and exit code. + * + * Keeping the scoring logic pure makes it trivially unit-testable in isolation + * (deterministic, no network / credentials), matching the repo's mock-based + * test convention. + */ + +/** + * Outcome of a single flaky-detector attempt. The first four mirror the + * terminal `RunStatus` values; `timeout` and `error` are orchestration + * outcomes (per-attempt deadline exceeded, or a non-fatal trigger/transport + * error that was recorded rather than aborting the whole probe). + */ +export type FlakyOutcome = 'passed' | 'failed' | 'blocked' | 'cancelled' | 'timeout' | 'error'; + +/** Overall stability verdict across all observed attempts. */ +export type FlakyVerdict = 'stable' | 'flaky' | 'failing'; + +/** One recorded attempt in a flaky run. */ +export interface FlakyAttempt { + /** 1-based attempt index. */ + attempt: number; + /** The runId of this attempt, or `null` when the trigger never returned one. */ + runId: string | null; + outcome: FlakyOutcome; + /** + * Server `failureKind` for non-passing runs (or the error code for `error` + * outcomes). `null` / omitted when the attempt passed or the kind is unknown. + */ + failureKind?: string | null; +} + +/** A single non-passing attempt as surfaced in the report. */ +export interface FlakyFailure { + attempt: number; + runId: string | null; + outcome: FlakyOutcome; + failureKind: string | null; +} + +/** + * Machine-readable stability report. This is also the exact `--output json` + * shape, so dashboards / agents / CI can consume it directly. + */ +export interface FlakyReport { + testId: string; + /** + * Attempts actually observed. May be fewer than requested when + * `--until-fail` short-circuits on the first non-passing attempt. + */ + runs: number; + passed: number; + failed: number; + /** `passed / runs`, rounded to 4 decimal places. `0` when `runs === 0`. */ + stableRatio: number; + verdict: FlakyVerdict; + /** Non-passing attempts, in attempt order. */ + failures: FlakyFailure[]; +} + +/** + * Aggregate per-attempt outcomes into a stability report. + * + * Verdict rules: + * - every attempt passed → `stable` + * - no attempt passed → `failing` + * - a mix of pass and non-pass → `flaky` + * An empty attempt list (no runs observed) is reported as `failing` with a + * `0` ratio — there is no evidence the test is stable. + */ +export function summarizeFlaky(testId: string, attempts: FlakyAttempt[]): FlakyReport { + const runs = attempts.length; + const passed = attempts.filter(a => a.outcome === 'passed').length; + const failed = runs - passed; + const stableRatio = runs === 0 ? 0 : Math.round((passed / runs) * 10000) / 10000; + const verdict: FlakyVerdict = + runs > 0 && passed === runs ? 'stable' : passed === 0 ? 'failing' : 'flaky'; + const failures: FlakyFailure[] = attempts + .filter(a => a.outcome !== 'passed') + .map(a => ({ + attempt: a.attempt, + runId: a.runId, + outcome: a.outcome, + failureKind: a.failureKind ?? null, + })); + return { testId, runs, passed, failed, stableRatio, verdict, failures }; +} + +/** + * Exit code for the command: `0` only when the verdict is `stable` (every + * observed attempt passed). Anything else is non-zero so CI can gate a merge + * on flakiness (`testsprite test flaky --runs 5 || exit 1`). + */ +export function flakyExitCode(report: FlakyReport): number { + return report.verdict === 'stable' ? 0 : 1; +} + +/** Human-readable label for a verdict. */ +function verdictLabel(verdict: FlakyVerdict): string { + switch (verdict) { + case 'stable': + return 'STABLE'; + case 'flaky': + return 'FLAKY'; + case 'failing': + return 'FAILING'; + } +} + +/** + * Render a report to human-readable text. JSON-mode callers ship the report + * verbatim via `out.print`; this is the text-mode rendering. + */ +export function renderFlakyText(report: FlakyReport): string { + const pct = Math.round(report.stableRatio * 100); + const lines: string[] = [ + `Ran ${report.testId} ${report.runs}x — ${report.passed} passed, ${report.failed} failed → ` + + `${verdictLabel(report.verdict)} (${pct}% stable)`, + ]; + if (report.failures.length > 0) { + lines.push(' failed attempts:'); + for (const f of report.failures) { + const kind = f.failureKind ? ` failureKind=${f.failureKind}` : ''; + const rid = f.runId ?? '(no runId)'; + lines.push(` #${f.attempt} ${rid} ${f.outcome}${kind}`); + } + } + return lines.join('\n'); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index d89e7dc..39e57dd 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -270,6 +270,15 @@ Commands: 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get + flaky [options] Repeatedly replay a test to measure stability and surface flakiness. + Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. + + Exit codes: + 0 stable (every attempt passed) + 1 flaky or failing (at least one attempt did not pass) + 3 auth error + 4 test not found (no replayable run — trigger \`testsprite test run \` first) + 5 validation error code Inspect and edit generated test code plan Manage FE test plan-steps (FE-only) failure Export the latest-failure agent bundle @@ -345,6 +354,38 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeou " `; +exports[`--help snapshots > test flaky 1`] = ` +"Usage: testsprite test flaky [options] + +Repeatedly replay a test to measure stability and surface flakiness. +Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. + +Exit codes: + 0 stable (every attempt passed) + 1 flaky or failing (at least one attempt did not pass) + 3 auth error + 4 test not found (no replayable run — trigger \`testsprite test run \` first) + 5 validation error + +Options: + --runs number of replays to run (1-10, default 5) + --until-fail stop at the first non-passing attempt (fast "is it flaky at + all?" check) (default: false) + --timeout per-attempt max seconds to wait (1-3600, default 600) + -h, --help display help for command + +Notes: + • Frontend replays are free verbatim script replays (no credit); backend replays + re-run the dependency closure and may cost credits — a one-line advisory is printed. + • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass; + this measures replay stability of the saved script against the configured URL. + • \`--output json\` emits a machine-readable stability report for CI gating. + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test get 1`] = ` "Usage: testsprite test get [options] diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index 2448c91..ee89b3b 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -39,6 +39,7 @@ const cases: Array<[string, string[]]> = [ ['test result', ['test', 'result', '--help']], ['test failure get', ['test', 'failure', 'get', '--help']], ['test rerun', ['test', 'rerun', '--help']], + ['test flaky', ['test', 'flaky', '--help']], // R5: regression guard for commands that gained new flag wording ['test create-batch', ['test', 'create-batch', '--help']], ['test run', ['test', 'run', '--help']], From b9e9601c4926e21c89066d30d56312524036ecb0 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Sun, 5 Jul 2026 18:28:01 -0400 Subject: [PATCH 052/102] feat(test): add "test lint" offline plan/steps validator (#176) * feat(test): add "test lint" offline plan/steps validator * fix(lint): report physical JSONL line numbers (blank lines no longer shift them) --- src/commands/test.test.ts | 81 +++++++++ src/commands/test.ts | 169 ++++++++++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 5 + 3 files changed, 255 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 6d8d734..c78c7a8 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -31,6 +31,7 @@ import { runFailureGet, runFailureSummary, runGet, + runLint, runList, runPlanPut, runResult, @@ -126,6 +127,7 @@ describe('createTestCommand — surface', () => { 'failure', 'flaky', 'get', + 'lint', 'list', 'plan', 'rerun', @@ -2961,6 +2963,85 @@ describe('runDiff', () => { }); }); +describe('runLint', () => { + const VALID_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Checkout works', + planSteps: [ + { type: 'action', description: 'Open the cart' }, + { type: 'assertion', description: 'Assert the total is visible' }, + ], + }); + const INVALID_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Broken', + planSteps: [{ type: 'hover', description: 'Bad step type' }], + }); + + it('a directory with valid and invalid plans reports EVERY problem and exits 5', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-')); + writeFileSync(join(dir, 'a-valid.json'), VALID_PLAN, 'utf8'); + writeFileSync(join(dir, 'b-invalid.json'), INVALID_PLAN, 'utf8'); + writeFileSync(join(dir, 'c-notjson.json'), '{oops', 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + checked: number; + valid: number; + issues: Array<{ file: string; field: string }>; + }; + // All three files were checked (no first-error-fatal bailout). + expect(report.checked).toBe(3); + expect(report.valid).toBe(1); + expect(report.issues.map(issue => issue.file).sort()).toEqual([ + 'b-invalid.json', + 'c-notjson.json', + ]); + }); + + it('an all-valid directory resolves with exit 0 and no network/credentials needed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-ok-')); + writeFileSync(join(dir, 'a.json'), VALID_PLAN, 'utf8'); + const report = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: () => undefined }, + ); + expect(report).toMatchObject({ checked: 1, valid: 1, issues: [] }); + }); + + it('a JSONL file reports each bad line with its PHYSICAL line number (blank lines skipped, not renumbered)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-jsonl-')); + const file = join(dir, 'plans.jsonl'); + // A blank separator line sits between entries: line 1 valid, line 2 blank, + // line 3 bad JSON, line 4 invalid plan. Reported numbers must be 3 and 4. + writeFileSync(file, `${VALID_PLAN}\n\nnot json at all\n${INVALID_PLAN}\n`, 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, plans: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { checked: number; issues: Array<{ file: string }> }; + expect(report.checked).toBe(3); + expect(report.issues.some(issue => issue.file.endsWith(':3'))).toBe(true); + expect(report.issues.some(issue => issue.file.endsWith(':4'))).toBe(true); + // The blank line itself is not an entry and never reports. + expect(report.issues.some(issue => issue.file.endsWith(':2'))).toBe(false); + }); + + it('requires exactly one input source', async () => { + await expect( + runLint({ profile: 'default', output: 'json', debug: false }, { stdout: () => undefined }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + describe('runResult', () => { it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index 508d8be..f11e688 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3862,6 +3862,144 @@ function renderRunDiffText(diff: CliRunDiff): string { return lines.join('\n'); } +export interface LintOptions extends CommonOptions { + planFrom?: string; + planFromDir?: string; + plans?: string; + steps?: string; +} + +export interface CliLintIssue { + file: string; + field: string; + reason: string; +} + +export interface CliLintReport { + checked: number; + valid: number; + issues: CliLintIssue[]; +} + +/** + * `test lint` (issue #98): validate plan/steps files fully OFFLINE with the + * SAME validators the create paths run, but collecting EVERY problem instead + * of dying on the first one, and without any network write. The create-batch + * reader is first-error-fatal and only reachable through a command that POSTs, + * so authoring a 12-plan directory meant one error per paid round-trip. Zero + * network, zero credentials: exit 0 when everything is valid, 5 otherwise, so + * it drops into a pre-commit hook or CI step before `create-batch`. + */ +export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const sources = [opts.planFrom, opts.planFromDir, opts.plans, opts.steps].filter( + source => source !== undefined, + ); + if (sources.length !== 1) { + throw localValidationError( + 'plan-from', + 'exactly one of --plan-from, --plan-from-dir, --plans, or --steps is required', + ); + } + + const issues: CliLintIssue[] = []; + let checked = 0; + // Run one existing validator, converting its typed throw into a report row + // (same envelopes, so `details.field` pointers like planSteps[2].type + // survive verbatim). + const collect = (file: string, validate: () => void): void => { + checked += 1; + try { + validate(); + } catch (err) { + if (err instanceof ApiError) { + issues.push({ + file, + field: String(err.getDetail('field') ?? '(file)'), + reason: String(err.getDetail('reason') ?? err.nextAction ?? err.message), + }); + } else { + issues.push({ + file, + field: '(file)', + reason: err instanceof Error ? err.message : String(err), + }); + } + } + }; + + if (opts.planFrom !== undefined) { + const planFrom = opts.planFrom; + collect(planFrom, () => void readPlanFromGuarded(planFrom)); + } else if (opts.steps !== undefined) { + const steps = opts.steps; + collect(steps, () => void readPlanStepsFileGuarded(steps)); + } else if (opts.planFromDir !== undefined) { + const dir = resolveAbsolute(opts.planFromDir); + let entries: string[]; + try { + entries = readdirSync(dir) + .filter(name => name.endsWith('.json')) + .sort(); + } catch { + throw localValidationError('plan-from-dir', `cannot read directory: ${dir}`); + } + if (entries.length === 0) { + throw localValidationError('plan-from-dir', 'contains no *.json plan files'); + } + for (const entry of entries) { + collect(entry, () => void readPlanFromGuarded(join(dir, entry))); + } + } else if (opts.plans !== undefined) { + // JSONL: validate PER LINE so every bad line reports (the create path's + // reader stays throw-on-first; this is the collecting counterpart). + const absolute = resolveAbsolute(opts.plans); + let content: string; + try { + content = readFileSync(absolute, 'utf8'); + } catch { + throw localValidationError('plans', `cannot read file: ${absolute}`); + } + // Index lines BEFORE dropping blanks so every reported `file:N` points at + // the PHYSICAL line in the file (a blank separator line must not shift all + // subsequent line numbers). + const numberedLines = content + .split('\n') + .map((rawLine, physicalIndex) => ({ line: rawLine.trim(), lineNo: physicalIndex + 1 })) + .filter(entry => entry.line.length > 0); + if (numberedLines.length === 0) throw localValidationError('plans', 'contains no plan lines'); + for (const { line, lineNo } of numberedLines) { + collect(`${opts.plans}:${lineNo}`, () => { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + throw localValidationError( + 'plans', + `line ${lineNo} is not valid JSON`, + undefined, + 'field', + ); + } + assertPlanShape(parsed, { specIndex: lineNo - 1 }); + }); + } + } + + const filesWithIssues = new Set(issues.map(issue => issue.file)).size; + const report: CliLintReport = { checked, valid: checked - filesWithIssues, issues }; + out.print(report, () => + [ + ...issues.map(issue => `${issue.file}: ${issue.field}: ${issue.reason}`), + `${report.valid}/${report.checked} valid, ${issues.length} problem(s)`, + ].join('\n'), + ); + if (issues.length > 0) { + throw new CLIError(`lint: ${issues.length} problem(s) across ${report.checked} file(s)`, 5); + } + return report; +} + export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -7669,6 +7807,37 @@ export function createTestCommand(deps: TestDeps = {}): Command { await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps); }); + test + .command('lint') + .description( + 'Validate plan/steps files offline with the same validators `create` runs, collecting EVERY problem. No network, no credentials. Exit 0 when all valid, 5 otherwise.', + ) + .option('--plan-from ', 'single plan JSON file') + .option( + '--plan-from-dir ', + 'directory of *.json plan files (each checked, all errors reported)', + ) + .option('--plans ', 'JSONL file with one plan spec per line (each line checked)') + .option('--steps ', 'plan-steps JSON file (the shape `test plan put` ingests)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + cmdOpts: { planFrom?: string; planFromDir?: string; plans?: string; steps?: string }, + command: Command, + ) => { + await runLint( + { + ...resolveCommonOptions(command), + planFrom: cmdOpts.planFrom, + planFromDir: cmdOpts.planFromDir, + plans: cmdOpts.plans, + steps: cmdOpts.steps, + }, + deps, + ); + }, + ); + test .command('result ') .description( diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 39e57dd..9fb6b2d 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -208,6 +208,11 @@ Commands: failedStepIndex, per-step status flips, codeVersion drift. Exit 0 when verdicts match, 1 when they differ. + lint [options] Validate plan/steps files offline with + the same validators \`create\` runs, + collecting EVERY problem. No network, + no credentials. Exit 0 when all valid, + 5 otherwise. result [options] Get the latest result for a test (default) or list prior runs (--history). --output json shape differs by mode: From f7b10b724442b81b46275a51b4f0a37051ba19af Mon Sep 17 00:00:00 2001 From: Sahil Rakhaiya <144577420+SahilRakhaiya05@users.noreply.github.com> Date: Tue, 7 Jul 2026 01:39:51 +0530 Subject: [PATCH 053/102] fix(config): treat empty env vars as unset in loadConfig (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(config): treat empty env vars as unset in loadConfig * test: address review — dry-run whoami uses offline client; fix whitespace env key expectation --- src/commands/auth.test.ts | 14 ++++++++++++++ src/commands/auth.ts | 7 ++++--- src/commands/init.ts | 3 ++- src/lib/client-factory.test.ts | 21 ++++++++++++++++++++- src/lib/config.test.ts | 22 ++++++++++++++++++++++ src/lib/config.ts | 15 +++++++++++++-- 6 files changed, 75 insertions(+), 7 deletions(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 1605f58..c0b4e00 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -850,6 +850,20 @@ describe('runWhoami', () => { expect(printed).toEqual(sampleMe); }); + it('dry-run: whitespace-only TESTSPRITE_API_URL falls through to prod default endpoint', async () => { + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + { + ...deps, + env: { TESTSPRITE_API_URL: ' ' }, + credentialsPath, + }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('endpoint: https://api.testsprite.com'); + }); + it('L1788: text output includes the resolved endpoint URL', async () => { writeProfile( 'default', diff --git a/src/commands/auth.ts b/src/commands/auth.ts index bf2eb88..59cd43f 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -17,7 +17,7 @@ import { readProfile, writeProfile, } from '../lib/credentials.js'; -import { loadConfig } from '../lib/config.js'; +import { loadConfig, normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; @@ -85,7 +85,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // treated as unset. Without this, `''` (e.g. `export TESTSPRITE_API_URL=` in a // shell profile) is non-nullish and would short-circuit the `??` chains below to // an empty endpoint instead of falling through to the profile / prod default. - const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); // Dry-run: do not prompt, do not read env, do not write credentials. // Print the canned success shape so an agent sees exactly the JSON it @@ -208,7 +208,8 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi // displayed URL always matches where requests actually go (dogfood L1788). let resolvedEndpoint: string; if (opts.dryRun) { - resolvedEndpoint = opts.endpointUrl ?? env.TESTSPRITE_API_URL ?? 'https://api.testsprite.com'; + resolvedEndpoint = + opts.endpointUrl ?? normalizeEnvVar(env.TESTSPRITE_API_URL) ?? 'https://api.testsprite.com'; } else { const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const config = loadConfig({ diff --git a/src/commands/init.ts b/src/commands/init.ts index e01215a..aff0644 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -15,6 +15,7 @@ import { parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; +import { normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; @@ -39,7 +40,7 @@ const DEFAULT_API_URL = 'https://api.testsprite.com'; */ function resolveReportedEndpoint(opts: InitOptions, deps: InitDeps): string { const env = deps.env ?? process.env; - const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); let existing: string | undefined; try { existing = readProfile(opts.profile, { path: deps.credentialsPath })?.apiUrl; diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index ab5fd99..d5135cc 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -339,7 +339,6 @@ describe('makeHttpClient - API key validation', () => { ['smart dash', 'sk-user-abc\u2013def'], ['smart quote', 'sk-user-\u201cabc\u201d'], ['emoji', 'sk-user-abc\u{1f600}'], - ['whitespace-only', ' '], ])('rejects a malformed configured API key with %s before fetch/retry', (_label, apiKey) => { const fetchImpl = vi.fn(); let caught: unknown; @@ -362,6 +361,26 @@ describe('makeHttpClient - API key validation', () => { expect(apiErr.exitCode).toBe(5); expect(apiErr.nextAction).toContain('api-key'); }); + + it('treats a whitespace-only TESTSPRITE_API_KEY env var as unset (AUTH_REQUIRED)', () => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: ' ' } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + expect((caught as ApiError).code).toBe('AUTH_REQUIRED'); + }); }); describe('assertValidApiKeyHeaderValue', () => { diff --git a/src/lib/config.test.ts b/src/lib/config.test.ts index 463ddcc..cf56eaf 100644 --- a/src/lib/config.test.ts +++ b/src/lib/config.test.ts @@ -81,6 +81,28 @@ describe('loadConfig', () => { const config = loadConfig({ profile: 'dev', env: {}, credentialsPath }); expect(config.apiKey).toBe('sk-dev'); }); + + it('treats empty / whitespace TESTSPRITE_API_URL as unset (falls through to profile)', () => { + writeProfile( + 'default', + { apiKey: 'sk-file', apiUrl: 'https://api.example.com:8443' }, + { path: credentialsPath }, + ); + const config = loadConfig({ + env: { TESTSPRITE_API_URL: ' ' }, + credentialsPath, + }); + expect(config.apiUrl).toBe('https://api.example.com:8443'); + }); + + it('treats empty / whitespace TESTSPRITE_API_KEY as unset (falls through to profile)', () => { + writeProfile('default', { apiKey: 'sk-file' }, { path: credentialsPath }); + const config = loadConfig({ + env: { TESTSPRITE_API_KEY: '' }, + credentialsPath, + }); + expect(config.apiKey).toBe('sk-file'); + }); }); describe('defaultConfigPath', () => { diff --git a/src/lib/config.ts b/src/lib/config.ts index 363c394..7f8065f 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -17,6 +17,11 @@ export interface LoadConfigOptions { const DEFAULT_API_URL = 'https://api.testsprite.com'; +/** Treat empty / whitespace-only env values as unset for `??` resolution chains. */ +export function normalizeEnvVar(value: string | undefined): string | undefined { + return value?.trim() || undefined; +} + export function defaultConfigPath(): string { return join(homedir(), '.testsprite', 'config'); } @@ -38,9 +43,15 @@ export function loadConfig(options: LoadConfigOptions = {}): Config { const credentialsPath = options.credentialsPath ?? defaultCredentialsPath(); const fileEntry = readProfile(profile, { path: credentialsPath }); + // Empty / whitespace-only env vars are treated as unset so they do not + // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` in a shell + // profile). Matches the normalization in auth configure and init/setup. + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); + const envApiKey = normalizeEnvVar(env.TESTSPRITE_API_KEY); + return { - apiUrl: options.endpointUrl ?? env.TESTSPRITE_API_URL ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, - apiKey: env.TESTSPRITE_API_KEY ?? fileEntry?.apiKey, + apiUrl: options.endpointUrl ?? envApiUrl ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, + apiKey: envApiKey ?? fileEntry?.apiKey, profile, }; } From c6d37b7e5cfc85cd56b3ac2f5d39c7177e019a46 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:10:25 -0500 Subject: [PATCH 054/102] fix(http): clear per-attempt timeout timers (#137) * fix(http): clear per-attempt timeout timers * fix(http): preserve timeout race classification --- src/lib/http.test.ts | 73 +++++++++ src/lib/http.ts | 359 ++++++++++++++++++++++++------------------- 2 files changed, 277 insertions(+), 155 deletions(-) diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 0320367..a9c1193 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -449,6 +449,52 @@ describe('HttpClient per-request timeout', () => { expect(callCount).toBe(1); }); + it('clears the per-attempt timeout after a successful response body is read', async () => { + let capturedSignal: AbortSignal | undefined; + const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { + capturedSignal = init?.signal ?? undefined; + return jsonResponse({ ok: true }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => Promise.resolve(), + random: () => 0, + requestTimeoutMs: 25, + }); + + await expect(client.get('/me')).resolves.toEqual({ ok: true }); + expect(capturedSignal?.aborted).toBe(false); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(capturedSignal?.aborted).toBe(false); + }); + + it('clears a failed attempt timeout before retry backoff sleeps', async () => { + const attemptSignals: AbortSignal[] = []; + let calls = 0; + const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { + if (init?.signal) attemptSignals.push(init.signal); + calls += 1; + if (calls === 1) return errorEnvelopeResponse(500, 'INTERNAL'); + return jsonResponse({ ok: true }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => new Promise(resolve => setTimeout(resolve, 50)), + random: () => 0, + requestTimeoutMs: 25, + }); + + await expect(client.get('/me')).resolves.toEqual({ ok: true }); + expect(attemptSignals).toHaveLength(2); + expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); + }); + it('caller-supplied AbortSignal still propagates as AbortError (not RequestTimeoutError)', async () => { const controller = new AbortController(); // Abort immediately @@ -475,6 +521,33 @@ describe('HttpClient per-request timeout', () => { expect((err as Error).name).toBe('AbortError'); }); + it('preserves RequestTimeoutError when the caller aborts after the request timeout wins', async () => { + const controller = new AbortController(); + const fetchImpl = vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + const reason = init.signal?.reason; + controller.abort(new Error('caller aborted after timeout')); + const err = new Error(reason?.message ?? 'timed out'); + err.name = reason?.name ?? 'TimeoutError'; + reject(err); + }); + }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => Promise.resolve(), + random: () => 0, + requestTimeoutMs: 1, + }); + const err = await client.get('/me', { signal: controller.signal }).catch(e => e); + expect(err).toBeInstanceOf(RequestTimeoutError); + expect((err as RequestTimeoutError).exitCode).toBe(7); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + it('defaults to REQUEST_TIMEOUT_DEFAULT_MS when no requestTimeoutMs is supplied', () => { // Verify the default is wired without actually waiting 120s. // We test via the exported constant rather than exercising the stall. diff --git a/src/lib/http.ts b/src/lib/http.ts index b5dabb1..af1c25a 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -395,9 +395,15 @@ export class HttpClient { timeoutSignal: AbortSignal, callerSignal: AbortSignal | undefined, requestId: string, + effectiveSignal: AbortSignal = timeoutSignal, ): void { if (isAbortError(err) || isTimeoutError(err)) { - if (timeoutSignal.aborted && (callerSignal == null || !callerSignal.aborted)) { + const timeoutWon = + timeoutSignal.aborted && + (callerSignal == null || + !callerSignal.aborted || + effectiveSignal.reason === timeoutSignal.reason); + if (timeoutWon) { throw new RequestTimeoutError(this.requestTimeoutMs, requestId); } throw err; @@ -428,190 +434,200 @@ export class HttpClient { // signal. The polling path supplies its own deadline-aware signal per // iteration — this timeout (120s default) is safely larger than any single // long-poll window (<=25s via ?waitSeconds), so it never bites polling. - const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs); + const requestTimeout = createRequestTimeout(this.requestTimeoutMs); + const timeoutSignal = requestTimeout.signal; const effectiveSignal = options.signal != null ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal; try { - response = await this.fetchImpl(url, { - method, - headers: this.buildHeaders(requestId, options), - body: options.body !== undefined ? JSON.stringify(options.body) : undefined, - signal: effectiveSignal, - }); - } catch (err) { - // Distinguish a client-side request timeout from a caller-supplied abort. - // - // Node 22 `AbortSignal.timeout()` throws a `DOMException` with - // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. - // A caller-supplied abort sets `name === 'AbortError'`. - // We treat both abort variants together: if the timeout signal fired and - // the caller hadn't already aborted, surface a clear RequestTimeoutError. - // A timeout/abort during the fetch itself: classify it (RequestTimeoutError - // when our deadline fired; otherwise rethrow the caller's abort unmodified). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - // If a RequestTimeoutError already propagated from somewhere (e.g. from a - // nested call or from a test-injected fetchImpl), pass it through unchanged - // rather than re-wrapping it as a TransportError. - if (err instanceof RequestTimeoutError) throw err; - const message = err instanceof Error ? err.message : String(err); - this.debug({ - kind: 'error', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - durationMs: Date.now() - startedAt, - }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) throw new TransportError(message, requestId); - this.transition( - `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, - ); - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - delayMs: decision.delayMs, - }); - await this.sleep(decision.delayMs); - continue; - } + try { + response = await this.fetchImpl(url, { + method, + headers: this.buildHeaders(requestId, options), + body: options.body !== undefined ? JSON.stringify(options.body) : undefined, + signal: effectiveSignal, + }); + } catch (err) { + // Distinguish a client-side request timeout from a caller-supplied abort. + // + // The request-timeout controller aborts with an Error/DOMException whose + // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. + // A caller-supplied abort sets `name === 'AbortError'`. + // We treat both abort variants together: if the timeout signal fired and + // the caller hadn't already aborted, surface a clear RequestTimeoutError. + // A timeout/abort during the fetch itself: classify it (RequestTimeoutError + // when our deadline fired; otherwise rethrow the caller's abort unmodified). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + // If a RequestTimeoutError already propagated from somewhere (e.g. from a + // nested call or from a test-injected fetchImpl), pass it through unchanged + // rather than re-wrapping it as a TransportError. + if (err instanceof RequestTimeoutError) throw err; + const message = err instanceof Error ? err.message : String(err); + this.debug({ + kind: 'error', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + durationMs: Date.now() - startedAt, + }); + const decision = transportRetryDecision(attempt, this.random); + if (!decision.retry) throw new TransportError(message, requestId); + this.transition( + `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + requestTimeout.clear(); + await this.sleep(decision.delayMs); + continue; + } - const durationMs = Date.now() - startedAt; - if (response.ok) { - this.debug({ - kind: 'response', - method, - url, - attempt, - status: response.status, - requestId, - durationMs, - }); + const durationMs = Date.now() - startedAt; + if (response.ok) { + this.debug({ + kind: 'response', + method, + url, + attempt, + status: response.status, + requestId, + durationMs, + }); + try { + return { body: (await response.json()) as T, requestId, status: response.status }; + } catch (err) { + // A timeout/abort can fire mid-body-read (headers received, stream stalls). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + // Otherwise the successful response body was not valid JSON — a + // misconfigured endpoint, a proxy / captive-portal / login page that + // returns HTML with a 200 status, or an empty body. Surface a typed + // error carrying the requestId instead of letting the raw SyntaxError + // escape to index.ts, where it would print a bare `{"error":"..."}` + // and break the --output json envelope contract. + throw malformedResponseError(response, requestId, err); + } + } + + let rawBody: unknown; try { - return { body: (await response.json()) as T, requestId, status: response.status }; + rawBody = await safeReadJson(response); } catch (err) { - // A timeout/abort can fire mid-body-read (headers received, stream stalls). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - // Otherwise the successful response body was not valid JSON — a - // misconfigured endpoint, a proxy / captive-portal / login page that - // returns HTML with a 200 status, or an empty body. Surface a typed - // error carrying the requestId instead of letting the raw SyntaxError - // escape to index.ts, where it would print a bare `{"error":"..."}` - // and break the --output json envelope contract. - throw malformedResponseError(response, requestId, err); + // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse + // errors), so a timeout fired mid-body-read on a non-OK response lands here. + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + throw err; } - } - let rawBody: unknown; - try { - rawBody = await safeReadJson(response); - } catch (err) { - // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse - // errors), so a timeout fired mid-body-read on a non-OK response lands here. - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - throw err; - } + // Edge proxies / load balancers return 408/502/504 without our error + // envelope on transient outages. Per the CLI error spec §7 these are + // transport-level retries, not facade errors — fold them in here so + // we get the bounded backoff budget instead of a single INTERNAL bail. + if (rawBody === null && isTransportEdgeStatus(response.status)) { + this.debug({ + kind: 'error', + method, + url, + attempt, + status: response.status, + requestId, + errorCode: 'TRANSPORT', + durationMs, + }); + const decision = transportRetryDecision(attempt, this.random); + if (!decision.retry) { + throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); + } + this.transition( + `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + requestTimeout.clear(); + await this.sleep(decision.delayMs); + continue; + } - // Edge proxies / load balancers return 408/502/504 without our error - // envelope on transient outages. Per the CLI error spec §7 these are - // transport-level retries, not facade errors — fold them in here so - // we get the bounded backoff budget instead of a single INTERNAL bail. - if (rawBody === null && isTransportEdgeStatus(response.status)) { + const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); + // Clamp server-directed Retry-After to [1s, 300s] and surface on the + // thrown error so outer callers (e.g. runBatchRun outer retry loop) + // can honor it without re-reading the now-consumed HTTP response. + const retryAfterMsForError = + retryAfterSec !== undefined + ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 + : undefined; + const apiError = ApiError.fromEnvelope( + rawBody, + response.status, + retryAfterMsForError, + // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing + // links) resolve the environment-correct portal domain. + this.baseUrl, + ); this.debug({ kind: 'error', method, url, attempt, status: response.status, + errorCode: apiError.code, requestId, - errorCode: 'TRANSPORT', durationMs, }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) { - throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); - } - this.transition( - `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + const retryOnConflict = options.retryOnConflict !== false; + const retryOnRateLimit = options.retryOnRateLimit !== false; + const decision = apiRetryDecision( + apiError.code, + attempt, + retryAfterSec, + this.random, + retryOnConflict, + retryOnRateLimit, ); + if (!decision.retry) throw apiError; + const delaySec = Math.round(decision.delayMs / 1000); + if (apiError.code === 'RATE_LIMITED') { + this.transition( + `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, + ); + } else if (apiError.code === 'INTERNAL') { + this.transition( + `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } else if (apiError.code === 'UNAVAILABLE') { + this.transition( + `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } this.debug({ kind: 'retry', method, url, attempt, requestId, - errorCode: 'TRANSPORT', + errorCode: apiError.code, delayMs: decision.delayMs, }); + requestTimeout.clear(); await this.sleep(decision.delayMs); - continue; + } finally { + requestTimeout.clear(); } - - const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); - // Clamp server-directed Retry-After to [1s, 300s] and surface on the - // thrown error so outer callers (e.g. runBatchRun outer retry loop) - // can honor it without re-reading the now-consumed HTTP response. - const retryAfterMsForError = - retryAfterSec !== undefined ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 : undefined; - const apiError = ApiError.fromEnvelope( - rawBody, - response.status, - retryAfterMsForError, - // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing - // links) resolve the environment-correct portal domain. - this.baseUrl, - ); - this.debug({ - kind: 'error', - method, - url, - attempt, - status: response.status, - errorCode: apiError.code, - requestId, - durationMs, - }); - const retryOnConflict = options.retryOnConflict !== false; - const retryOnRateLimit = options.retryOnRateLimit !== false; - const decision = apiRetryDecision( - apiError.code, - attempt, - retryAfterSec, - this.random, - retryOnConflict, - retryOnRateLimit, - ); - if (!decision.retry) throw apiError; - const delaySec = Math.round(decision.delayMs / 1000); - if (apiError.code === 'RATE_LIMITED') { - this.transition( - `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, - ); - } else if (apiError.code === 'INTERNAL') { - this.transition( - `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, - ); - } else if (apiError.code === 'UNAVAILABLE') { - this.transition( - `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, - ); - } - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: apiError.code, - delayMs: decision.delayMs, - }); - await this.sleep(decision.delayMs); } } @@ -687,6 +703,39 @@ function newRequestId(): string { return `cli_${randomUUID()}`; } +interface RequestTimeoutHandle { + signal: AbortSignal; + clear: () => void; +} + +function createRequestTimeout(timeoutMs: number): RequestTimeoutHandle { + const controller = new AbortController(); + const timer = setTimeout(() => { + controller.abort(makeTimeoutReason()); + }, timeoutMs); + unrefTimer(timer); + + return { + signal: controller.signal, + clear: () => clearTimeout(timer), + }; +} + +function makeTimeoutReason(): Error { + if (typeof DOMException !== 'undefined') { + return new DOMException('The operation timed out.', 'TimeoutError'); + } + const err = new Error('The operation timed out.'); + err.name = 'TimeoutError'; + return err; +} + +function unrefTimer(timer: ReturnType): void { + if (typeof timer !== 'object' || timer === null || !('unref' in timer)) return; + const unref = (timer as { unref?: () => void }).unref; + if (typeof unref === 'function') unref.call(timer); +} + async function safeReadJson(response: Response): Promise { try { return await response.json(); From 78ad224ba55231d383f7bc44f949b8ad016235d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:10:58 -0700 Subject: [PATCH 055/102] chore(ci): bump marocchino/sticky-pull-request-comment (#187) Bumps [marocchino/sticky-pull-request-comment](https://github.com/marocchino/sticky-pull-request-comment) from 2.9.4 to 3.0.5. - [Release notes](https://github.com/marocchino/sticky-pull-request-comment/releases) - [Commits](https://github.com/marocchino/sticky-pull-request-comment/compare/773744901bac0e8cbb5a0dc842800d45e9b2b405...5770ad5eb8f42dd2c4f34da00c94c5381e49af88) --- updated-dependencies: - dependency-name: marocchino/sticky-pull-request-comment dependency-version: 3.0.5 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test-coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 4eb15a9..ebb3331 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -73,7 +73,7 @@ jobs: fi - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 if: github.event_name == 'pull_request' && steps.coverage-summary.outputs.coverage_generated == 'true' with: recreate: true From 108a91315762ccb8c1c157cbc5e15fb1914a7401 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Mon, 6 Jul 2026 23:11:31 +0300 Subject: [PATCH 056/102] fix(batch): classify RequestTimeoutError as timeout in --all --wait fan-out (#154) * fix(batch): classify RequestTimeoutError as timeout in --all --wait fan-out When test run --all --wait or test rerun --all --wait hit a per-request timeout during batch fan-out polling, RequestTimeoutError rejected the fan-out before out.print(). Classify it as status:'timeout' in pollFreshAccepted and pollAccepted so stdout always lists every dispatched runId. Co-authored-by: Cursor * Remove unused readFileSync import from test file --------- Co-authored-by: Contributor Co-authored-by: Cursor --- src/commands/test.rerun.spec.ts | 471 ++------------------------------ src/commands/test.run.spec.ts | 226 +++------------ src/commands/test.ts | 32 +++ 3 files changed, 90 insertions(+), 639 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 129d859..9f16c4b 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4820,326 +4820,38 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { }); // --------------------------------------------------------------------------- -// [fix-exitcode] pollAccepted preserves ApiError exit codes (not hardcoded 1) +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- -describe('[fix-exitcode] polling error exit codes preserved in batch rerun results', () => { - it('AUTH_REQUIRED during polling → batch escalates to exitCode 3', async () => { +describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { const creds = makeCreds(); - const passRun = makeTerminalRun('run_pass_a1', 'passed'); const batchResp: BatchRerunResponse = { accepted: [ - { testId: 'test_1', runId: 'run_auth_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_pass_a1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_auth_fail')) return errorBody('AUTH_REQUIRED'); - if (url.includes('/runs/run_pass_a1')) return { body: passRun }; - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e as { exitCode?: number; message?: string }); - - expect((err as { exitCode?: number }).exitCode).toBe(3); - }); - - it('RATE_LIMITED during polling → non-auth, batch exits 1', async () => { - const creds = makeCreds(); - const passRun = makeTerminalRun('run_pass_a2', 'passed'); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_rl', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_pass_a2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_rl')) return errorBody('RATE_LIMITED'); - if (url.includes('/runs/run_pass_a2')) return { body: passRun }; - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(1); - }); - - it('NOT_FOUND during run polling → non-auth, batch exits 1', async () => { - const creds = makeCreds(); - const passRun = makeTerminalRun('run_pass_a3', 'passed'); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_nf', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_pass_a3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_nf')) return errorBody('NOT_FOUND'); - if (url.includes('/runs/run_pass_a3')) return { body: passRun }; - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(1); - }); -}); - -// --------------------------------------------------------------------------- -// [fix-auth-escalation] batch auth failure escalates to exit 3 -// --------------------------------------------------------------------------- - -describe('[fix-auth-escalation] auth error in batch rerun polling escalates to exit 3', () => { - it('auth failure in batch poll → batch exits 3, not 1', async () => { - const creds = makeCreds(); - const passRun = makeTerminalRun('run_other', 'passed'); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_auth', runId: 'run_auth', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_other', runId: 'run_other', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_auth')) return errorBody('AUTH_REQUIRED'); - if (url.includes('/runs/run_other')) return { body: passRun }; - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_auth', 'test_other'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(3); - }); - - it('mixed batch: one pass, one auth failure → exits 3 (auth wins)', async () => { - const creds = makeCreds(); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_pass', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_auth2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - const passRun = makeTerminalRun('run_pass', 'passed'); - passRun.testId = 'test_1'; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_pass')) return { body: passRun }; - if (url.includes('/runs/run_auth2')) return errorBody('AUTH_REQUIRED'); - return errorBody('NOT_FOUND'); - }); - - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(3); - expect((err as { message?: string }).message).toMatch(/auth error/i); - }); - - it('non-auth failure → exits 1 (no escalation)', async () => { - const creds = makeCreds(); - const failRun = makeTerminalRun('run_fail', 'failed'); - failRun.testId = 'test_1'; - const passRun = makeTerminalRun('run_pass_c3', 'passed'); - passRun.testId = 'test_2'; - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_fail', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_pass_c3', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, ], deferred: [], conflicts: [], closure: { byProject: [] }, }; - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/run_fail')) return { body: failRun }; - if (url.includes('/runs/run_pass_c3')) return { body: passRun }; + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } return errorBody('NOT_FOUND'); }); + const stdoutLines: string[] = []; const err = await runTestRerun( { testIds: ['test_1', 'test_2'], all: false, wait: true, - timeoutSeconds: 10, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 5, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ).catch(e => e); - - expect((err as { exitCode?: number }).exitCode).toBe(1); - }); -}); - -// --------------------------------------------------------------------------- -// [fix-D4] initial chunk idempotency key bounded to ≤256 chars -// --------------------------------------------------------------------------- - -describe('[fix-D4] initial chunk dispatch idempotency key bounded to 256 chars', () => { - it('short key with multiple chunks passes through unchanged', async () => { - const creds = makeCreds(); - const receivedKeys: string[] = []; - - // 51 test IDs forces 2 chunks (MAX_BATCH_RERUN_IDS = 50) - const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); - const batchResp: BatchRerunResponse = { - accepted: testIds.slice(0, 50).map(id => ({ - testId: id, - runId: `run_${id}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - })), - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - const batchResp2: BatchRerunResponse = { - accepted: [ - { - testId: testIds[50]!, - runId: `run_${testIds[50]}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - let callCount = 0; - - const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/batch/rerun')) { - const h = new Headers(init.headers ?? {}); - const key = h.get('idempotency-key') ?? ''; - receivedKeys.push(key); - callCount++; - return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; - } - return errorBody('NOT_FOUND'); - }); - - await runTestRerun( - { - testIds, - all: false, - wait: false, - timeoutSeconds: 600, + timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false, @@ -5149,154 +4861,23 @@ describe('[fix-D4] initial chunk dispatch idempotency key bounded to 256 chars', dryRun: false, debug: false, verbose: false, - idempotencyKey: 'short-key', }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - - expect(receivedKeys).toHaveLength(2); - expect(receivedKeys[0]).toBe('short-key:chunk0'); - expect(receivedKeys[1]).toBe('short-key:chunk1'); - expect(receivedKeys[0]!.length).toBeLessThanOrEqual(256); - expect(receivedKeys[1]!.length).toBeLessThanOrEqual(256); - }); - - it('249-char key + :chunk0 suffix would exceed 256 → key truncated to keep total ≤256', async () => { - const creds = makeCreds(); - const receivedKeys: string[] = []; - - // key is 249 chars; `:chunk0` is 7 chars → 256 total (edge case, fits exactly) - const longKey = 'k'.repeat(249); - const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); - const batchResp: BatchRerunResponse = { - accepted: testIds.slice(0, 50).map(id => ({ - testId: id, - runId: `run_${id}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - })), - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - const batchResp2: BatchRerunResponse = { - accepted: [ - { - testId: testIds[50]!, - runId: `run_${testIds[50]}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - let callCount = 0; - - const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/batch/rerun')) { - const h = new Headers(init.headers ?? {}); - receivedKeys.push(h.get('idempotency-key') ?? ''); - callCount++; - return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; - } - return errorBody('NOT_FOUND'); - }); - - await runTestRerun( { - testIds, - all: false, - wait: false, - timeoutSeconds: 600, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 10, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - idempotencyKey: longKey, + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - - expect(receivedKeys).toHaveLength(2); - for (const key of receivedKeys) { - expect(key.length).toBeLessThanOrEqual(256); - } - // suffix must be preserved - expect(receivedKeys[0]).toMatch(/:chunk0$/); - expect(receivedKeys[1]).toMatch(/:chunk1$/); - }); - - it('256-char key + :chunk0 suffix → base truncated so total is exactly 256', async () => { - const creds = makeCreds(); - const receivedKeys: string[] = []; + ).catch(e => e); - // Max-length user key: 256 chars. `:chunk0` = 7 chars → need to truncate base to 249. - const maxKey = 'x'.repeat(256); - const testIds = Array.from({ length: 51 }, (_, i) => `test_${i}`); - const batchResp: BatchRerunResponse = { - accepted: testIds.slice(0, 50).map(id => ({ - testId: id, - runId: `run_${id}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - })), - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - const batchResp2: BatchRerunResponse = { - accepted: [ - { - testId: testIds[50]!, - runId: `run_${testIds[50]}`, - enqueuedAt: '2026-06-03T10:00:00.000Z', - }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; }; - let callCount = 0; - - const fetchImpl = makeFetch((url, init) => { - if (url.includes('/tests/batch/rerun')) { - const h = new Headers(init.headers ?? {}); - receivedKeys.push(h.get('idempotency-key') ?? ''); - callCount++; - return { status: 202, body: callCount === 1 ? batchResp : batchResp2 }; - } - return errorBody('NOT_FOUND'); - }); - - await runTestRerun( - { - testIds, - all: false, - wait: false, - timeoutSeconds: 600, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 10, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - idempotencyKey: maxKey, - }, - { ...creds, sleep: instantSleep, fetchImpl, stdout: () => {}, stderr: () => {} }, - ); - - expect(receivedKeys).toHaveLength(2); - for (const key of receivedKeys) { - expect(key.length).toBeLessThanOrEqual(256); - } - expect(receivedKeys[0]).toMatch(/:chunk0$/); - expect(receivedKeys[1]).toMatch(/:chunk1$/); + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 2f2ea9d..c0ff083 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -5,7 +5,7 @@ * sleep injection is wired through `TestDeps.sleep` to avoid real delays. */ -import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; @@ -3693,60 +3693,33 @@ describe('dashboardUrl on run completion', () => { }); }); -describe('runTestRunAll — JUnit report export', () => { - const BATCH_FRESH_RESP: BatchRunFreshResponse = { - accepted: [ - { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, - { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, - ], - conflicts: [], - deferred: [], - skippedFrontend: [], - skippedIntegration: [], - }; - - function makeTerminalRun(runId: string, testId: string, status: string): RunResponse { - return { - runId, - testId, - projectId: 'project_be', - userId: 'user_1', - status: status as RunResponse['status'], - source: 'cli', - createdAt: '2026-06-09T10:00:00.000Z', - startedAt: '2026-06-09T10:00:01.000Z', - finishedAt: '2026-06-09T10:00:30.000Z', - codeVersion: 'v1', - targetUrl: 'https://api.example.com', - createdFrom: 'cli', - failedStepIndex: null, - failureKind: null, - error: null, - videoUrl: null, - stepSummary: { - total: 3, - completed: 3, - passedCount: status === 'passed' ? 3 : 0, - failedCount: 0, - }, - }; - } +// --------------------------------------------------------------------------- +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty +// --------------------------------------------------------------------------- - it('--wait --report junit writes XML after polling', async () => { +describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { const { credentialsPath } = makeCreds(); - const dir = mkdtempSync(join(tmpdir(), 'junit-run-all-')); - const reportPath = join(dir, 'results.xml'); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; const fetchImpl = makeFetch((url, init) => { - if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; - const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; - if (runId === 'run_fresh_01') - return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; - if (runId === 'run_fresh_02') - return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'passed') }; + if ((init.method ?? 'GET') === 'POST') return { body: batchResp }; + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_all'); + } return errorBody('NOT_FOUND'); }); + const stdoutLines: string[] = []; - await runTestRunAll( + const err = await runTestRunAll( { profile: 'default', output: 'json', @@ -3755,158 +3728,23 @@ describe('runTestRunAll — JUnit report export', () => { wait: true, timeoutSeconds: 60, maxConcurrency: 5, - report: 'junit', - reportFile: reportPath, }, { credentialsPath, fetchImpl, - stdout: () => undefined, + stdout: line => stdoutLines.push(line), stderr: () => undefined, sleep: instantSleep, }, - ); - - const xml = readFileSync(reportPath, 'utf8'); - expect(xml).toContain(' { - const { credentialsPath } = makeCreds(); - const dir = mkdtempSync(join(tmpdir(), 'junit-run-fail-')); - const reportPath = join(dir, 'results.xml'); - const fetchImpl = makeFetch((url, init) => { - if ((init.method ?? 'GET') === 'POST') return { body: BATCH_FRESH_RESP }; - const runId = url.split('/runs/')[1]?.split('?')[0] ?? ''; - if (runId === 'run_fresh_01') - return { body: makeTerminalRun('run_fresh_01', 'test_be_01', 'passed') }; - if (runId === 'run_fresh_02') - return { body: makeTerminalRun('run_fresh_02', 'test_be_02', 'failed') }; - return errorBody('NOT_FOUND'); - }); - - await expect( - runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_be', - wait: true, - timeoutSeconds: 60, - maxConcurrency: 5, - report: 'junit', - reportFile: reportPath, - }, - { - credentialsPath, - fetchImpl, - stdout: () => undefined, - stderr: () => undefined, - sleep: instantSleep, - }, - ), - ).rejects.toMatchObject({ exitCode: 1 }); - - const xml = readFileSync(reportPath, 'utf8'); - expect(xml).toContain('failures="1"'); - expect(xml).toContain('name="test_be_02"'); - }); - - it('rejects --report without --wait', async () => { - await expect( - runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_be', - wait: false, - timeoutSeconds: 60, - maxConcurrency: 5, - report: 'junit', - reportFile: './results.xml', - }, - {}, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); - - it('rejects --report-suite-name without --report', async () => { - await expect( - runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - projectId: 'project_be', - wait: true, - timeoutSeconds: 60, - maxConcurrency: 5, - reportSuiteName: 'orphan-suite', - }, - {}, - ), - ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); - }); - - it('--dry-run --report junit writes canned sample XML', async () => { - const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-')); - const reportPath = join(dir, 'results.xml'); - - await runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - dryRun: true, - projectId: 'project_be', - wait: true, - timeoutSeconds: 60, - maxConcurrency: 5, - report: 'junit', - reportFile: reportPath, - }, - { - stdout: () => undefined, - stderr: () => undefined, - }, - ); - - const xml = readFileSync(reportPath, 'utf8'); - expect(xml).toContain('name="test_fresh_wave_01"'); - expect(xml).toContain('failures="1"'); - }); - - it('--dry-run --report junit --report-suite-name overrides canned suite name', async () => { - const dir = mkdtempSync(join(tmpdir(), 'junit-run-dry-suite-')); - const reportPath = join(dir, 'results.xml'); - - await runTestRunAll( - { - profile: 'default', - output: 'json', - debug: false, - dryRun: true, - projectId: 'project_be', - wait: true, - timeoutSeconds: 60, - maxConcurrency: 5, - report: 'junit', - reportFile: reportPath, - reportSuiteName: 'ci-checkout-suite', - }, - { - stdout: () => undefined, - stderr: () => undefined, - }, - ); + ).catch(e => e); - const xml = readFileSync(reportPath, 'utf8'); - expect(xml).toContain('; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_fresh_01', 'run_fresh_02']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); diff --git a/src/commands/test.ts b/src/commands/test.ts index f11e688..9982651 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5951,6 +5951,22 @@ export async function runTestRunAll( }, }; } + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7) so the fan-out completes and stdout carries every runId. + // Without this, RequestTimeoutError rejects the fan-out before out.print(), + // leaving JSON consumers with empty stdout (mirrors create-batch --run). + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: err.message, + exitCode: err.exitCode, + }, + }; + } if (err instanceof ApiError) { // Preserve the real exit code + envelope (AUTH_INVALID=3, NOT_FOUND=4, // RATE_LIMITED=11, …) instead of flattening every member failure to 1 @@ -7160,6 +7176,22 @@ export async function runTestRerun( }, }; } + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7) so the fan-out completes and stdout carries every runId. + // Without this, RequestTimeoutError rejects the fan-out before out.print(), + // leaving JSON consumers with empty stdout (mirrors create-batch --run). + return { + testId: entry.testId, + runId: entry.runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: err.message, + exitCode: err.exitCode, + }, + }; + } if (err instanceof ApiError) { // Preserve the real exit code (AUTH_INVALID=3, RATE_LIMITED=11, …) so the // batch exit-code aggregator can escalate auth failures correctly. Mirroring From 8ac2ff0132e01c2c63e4a411ba4c0fbfc48a7a1d Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:12:12 -0400 Subject: [PATCH 057/102] =?UTF-8?q?feat(test):=20make=20"test=20wait"=20va?= =?UTF-8?q?riadic=20=E2=80=94=20attach=20to=20several=20runs=20in=20one=20?= =?UTF-8?q?invocation=20(#178)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(test): make "test wait" variadic — attach to several runs in one invocation * fix(wait): honor the shared deadline for queued members and hint only resumable runs --- src/commands/test.test.ts | 155 +++++++++++ src/commands/test.ts | 247 +++++++++++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 12 +- 3 files changed, 404 insertions(+), 10 deletions(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index c78c7a8..0368e06 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -36,6 +36,7 @@ import { runPlanPut, runResult, runSteps, + runTestWaitMany, runUpdate, } from './test.js'; @@ -3042,6 +3043,160 @@ describe('runLint', () => { }); }); +describe('runTestWaitMany', () => { + const terminalRun = (runId: string, status: string) => ({ + runId, + testId: `test_of_${runId}`, + projectId: 'project_alice', + userId: 'u1', + status, + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }); + + it('polls every run, keeps input order, and exits 1 when one finished non-passed', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => ({ + body: url.includes('run_bad') + ? terminalRun('run_bad', 'failed') + : terminalRun('run_ok', 'passed'), + })); + const out: string[] = []; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_ok', 'run_bad'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + summary: { passed: number; failed: number }; + }; + expect(payload.results.map(row => row.runId)).toEqual(['run_ok', 'run_bad']); + expect(payload.summary).toMatchObject({ passed: 1, failed: 1 }); + }); + + it('a member whose poll errors is captured as error: and the others survive (exit 7)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + if (url.includes('run_gone')) { + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'no such run', + nextAction: 'check the id', + requestId: 'req_x', + details: {}, + }, + }, + }; + } + return { body: terminalRun('run_ok', 'passed') }; + }); + const out: string[] = []; + const errs: string[] = []; + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_ok', 'run_gone'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => errs.push(line), + }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + }; + // The failing member did not abort the pool: the passed verdict survived. + expect(payload.results[0]).toMatchObject({ runId: 'run_ok', status: 'passed' }); + expect(payload.results[1]!.status).toBe('error:NOT_FOUND'); + // The re-attach hint names the errored member (resumable) but NOT the + // already-terminal passed one. + const hint = errs.find(line => line.includes('Re-attach with:')); + expect(hint).toContain('run_gone'); + expect(hint).not.toContain('run_ok'); + }); + + it('members dequeued after the shared deadline are not granted extra poll time', async () => { + const { credentialsPath } = makeCreds(); + let fetches = 0; + const fetchImpl = makeFetch(() => { + fetches += 1; + return { body: terminalRun('run_any', 'passed') }; + }); + // timeoutSeconds 0: the shared deadline is already in the past when the + // pool starts, so every member must resolve to timeout WITHOUT polling + // (previously each dequeued member was granted a fresh 1s minimum). + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_a', 'run_b', 'run_c'], + timeoutSeconds: 0, + maxConcurrency: 1, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + expect(fetches).toBe(0); + }); + + it('an auth error escalates the exit code to 3', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 401, + body: { + error: { + code: 'AUTH_INVALID', + message: 'bad key', + nextAction: 'run setup', + requestId: 'req_y', + details: {}, + }, + }, + })); + const rejection = await runTestWaitMany( + { + profile: 'default', + output: 'json', + debug: false, + runIds: ['run_a', 'run_b'], + timeoutSeconds: 30, + maxConcurrency: 2, + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 3 }); + }); +}); + describe('runResult', () => { it('JSON mode prints the §6.5 LatestResult shape verbatim', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index 9982651..57d90f5 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -5331,6 +5331,211 @@ export async function runTestRun( return finalRun; } +/** One row of the `test wait ` multi-run payload. */ +export interface CliMultiWaitResult { + runId: string; + /** Terminal run status, or 'timeout', or 'error:' when the poll failed. */ + status: string; + /** Test the run belongs to, when the poll observed it. */ + testId?: string; +} + +export interface RunTestWaitManyOptions extends CommonOptions { + runIds: string[]; + timeoutSeconds: number; + maxConcurrency: number; +} + +/** + * `test wait ` with two or more ids: attach to N already-dispatched + * runs in ONE invocation. This closes the loop the CLI itself opens: every + * batch/closure timeout prints one `testsprite test wait ` hint PER + * member, which previously meant N sequential blocking invocations. The runs + * are polled concurrently under a bounded pool with ONE shared deadline + * (`--timeout` bounds the whole invocation, not each member), each member's + * poll is total (a transient error on one run never discards the others), and + * the exit code is the worst status across members: auth errors escalate to + * exit 3, any timeout or poll error exits 7, any non-passed terminal exits 1. + * Distinct from a run journal (issue #80): no persistence, just N known ids. + */ +export async function runTestWaitMany( + opts: RunTestWaitManyOptions, + deps: TestDeps = {}, +): Promise<{ results: CliMultiWaitResult[]; summary: Record }> { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const results: CliMultiWaitResult[] = opts.runIds.map(runId => ({ + runId, + status: 'passed', + })); + const payload = { + results, + summary: { total: results.length, passed: results.length, failed: 0, timedOut: 0, errors: 0 }, + }; + out.print(payload, () => results.map(r => `${r.runId} ${r.status}`).join('\n')); + return payload; + } + + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + + // One shared deadline across every member (the whole point of the shared + // pool: `--timeout 600` means the invocation ends within ~600s, not + // 600s x ceil(N/concurrency)). + const deadlineMs = Date.now() + opts.timeoutSeconds * 1000; + + type WaitOutcome = + | { kind: 'result'; run: RunResponse } + | { kind: 'timeout' } + | { kind: 'error'; code: string; exitCode: number }; + + const pollOne = async (runId: string): Promise => { + // A member dequeued AFTER the shared deadline has passed must not be + // granted a fresh minimum poll window (with --max-concurrency 1 that + // would extend the invocation by ~1s per queued run past --timeout). + const remainingSeconds = Math.ceil((deadlineMs - Date.now()) / 1000); + if (remainingSeconds <= 0) return { kind: 'timeout' }; + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: run => run.testId, + resolveNotBefore: run => run.createdAt, + onResolved: () => undefined, + }); + try { + const run = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: remainingSeconds, + sleep: deps.sleep, + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + ticker.update(`Run ${run.runId} — ${run.status} (elapsed=${elapsed}s)`); + }, + resolveAlternate, + }); + return { kind: 'result', run }; + } catch (err) { + if (err instanceof TimeoutError) return { kind: 'timeout' }; + if (err instanceof RequestTimeoutError) throw err; + if (err instanceof ApiError) return { kind: 'error', code: err.code, exitCode: err.exitCode }; + return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; + } + }; + + const outcomes = new Map(); + let inFlight = 0; + let nextIdx = 0; + try { + await new Promise((resolve, reject) => { + const startNext = (): void => { + while (inFlight < opts.maxConcurrency && nextIdx < opts.runIds.length) { + const runId = opts.runIds[nextIdx++]!; + inFlight++; + pollOne(runId) + .then(outcome => { + outcomes.set(runId, outcome); + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= opts.runIds.length) resolve(); + }) + // pollOne is total except for RequestTimeoutError (handled below). + .catch(reject); + } + }; + startNext(); + if (opts.runIds.length === 0) resolve(); + }); + } catch (fanOutErr) { + if (fanOutErr instanceof RequestTimeoutError) { + // Same contract as the batch pollers: leave stdout parseable before + // exiting 7. Members that already settled keep their real status; only + // the still-unfinished ids are marked running and named in the hint + // (re-attaching to an already-terminal run would be a wasted command). + ticker.finalize('Multi-run wait — request timed out'); + const partial = { + results: opts.runIds.map((runId): CliMultiWaitResult => { + const outcome = outcomes.get(runId); + if (outcome === undefined) return { runId, status: 'running' }; + if (outcome.kind === 'timeout') return { runId, status: 'timeout' }; + if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; + return { runId, status: outcome.run.status, testId: outcome.run.testId }; + }), + summary: { total: opts.runIds.length }, + }; + out.print(partial, () => partial.results.map(r => `${r.runId} ${r.status}`).join('\n')); + const unfinished = partial.results + .filter(r => r.status === 'running' || r.status === 'timeout') + .map(r => r.runId); + if (unfinished.length > 0) { + stderrFn(`Re-attach with: testsprite test wait ${unfinished.join(' ')}`); + } + } + throw fanOutErr; + } + ticker.finalize(); + + const results: CliMultiWaitResult[] = opts.runIds.map(runId => { + const outcome = outcomes.get(runId); + if (outcome === undefined || outcome.kind === 'timeout') return { runId, status: 'timeout' }; + if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; + return { runId, status: outcome.run.status, testId: outcome.run.testId }; + }); + const passed = results.filter(r => r.status === 'passed').length; + const timedOut = results.filter(r => r.status === 'timeout').length; + const errors = results.filter(r => r.status.startsWith('error:')).length; + const failed = results.length - passed - timedOut - errors; + const payload = { + results, + summary: { total: results.length, passed, failed, timedOut, errors }, + }; + out.print(payload, () => + [ + ...results.map(r => `${r.runId} ${r.status}`), + '', + `${passed}/${results.length} passed, ${failed} failed/blocked, ${timedOut} timed out, ${errors} poll errors`, + ].join('\n'), + ); + + // Every member that did not reach a terminal verdict is re-attachable: + // timeouts (still running server-side) and poll errors (e.g. a transient + // transport failure) both belong in the hint; terminal runs do not. + const unfinishedIds = results + .filter(r => r.status === 'timeout' || r.status.startsWith('error:')) + .map(r => r.runId); + if (unfinishedIds.length > 0) { + stderrFn(`Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}`); + } + + // Worst-status exit: auth escalates (a rejected key fails every member the + // same way), then timeout/poll-error (7, resumable), then plain failure (1). + const authError = [...outcomes.values()].find( + o => + o.kind === 'error' && + (o.code === 'AUTH_REQUIRED' || o.code === 'AUTH_INVALID' || o.code === 'AUTH_FORBIDDEN'), + ); + if (authError !== undefined && authError.kind === 'error') { + throw new CLIError( + `Multi-run wait: authentication failed (${authError.code})`, + authError.exitCode, + ); + } + if (timedOut > 0 || errors > 0) { + throw new CLIError( + `Multi-run wait: ${timedOut} timed out, ${errors} poll error(s) out of ${results.length} runs`, + 7, + ); + } + if (failed > 0) { + throw new CLIError(`Multi-run wait: ${failed} run(s) finished non-passed`, 1); + } + return payload; +} + /** * `test wait ` — M3.3 piece-3. * @@ -8172,26 +8377,53 @@ export function createTestCommand(deps: TestDeps = {}): Command { }); test - .command('wait ') + .command('wait ') .description( - 'Wait for a run to reach a terminal status.\n' + + 'Wait for one or more runs to reach a terminal status.\n' + + '\nWith several run-ids the runs are polled concurrently under one shared\n' + + '--timeout and a {results, summary} envelope is printed (worst status wins\n' + + 'the exit code), so every re-attach hint the CLI prints can be pasted as\n' + + 'ONE command.\n' + '\nExit codes:\n' + ' 0 passed\n' + ' 1 failed / blocked / cancelled\n' + ' 3 auth error\n' + - ' 4 run not found\n' + - ' 7 timeout — resume with: testsprite test wait \n' + + ' 4 run not found (single run-id; with several ids a per-member poll error\n' + + ' is recorded as error: in its row and folded into exit 7)\n' + + ' 7 timeout or per-member poll error — resume with: testsprite test wait \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', ) .option('--timeout ', `max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`) + .option( + '--max-concurrency ', + 'with several run-ids, max concurrent polls (1-100, default: 10)', + ) .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (runId: string, cmdOpts: WaitFlagOpts, command: Command) => { - await runTestWait( + .action(async (runIds: string[], cmdOpts: WaitFlagOpts, command: Command) => { + // One id keeps the historical single-run path byte-identical (same + // output shape, same exit codes); two or more fan out. + if (runIds.length === 1) { + await runTestWait( + { + ...resolveCommonOptions(command), + runId: runIds[0]!, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + return; + } + const maxConcurrency = parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? 10; + if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 100) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + await runTestWaitMany( { ...resolveCommonOptions(command), - runId, + runIds, timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + maxConcurrency, }, deps, ); @@ -8577,6 +8809,7 @@ interface RunFlagOpts { interface WaitFlagOpts { timeout?: string; + maxConcurrency?: string; } interface RerunFlagOpts { diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 9fb6b2d..a16d0c4 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -251,14 +251,20 @@ Commands: 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get - wait [options] Wait for a run to reach a terminal status. + wait [options] Wait for one or more runs to reach a terminal status. + + With several run-ids the runs are polled concurrently under one shared + --timeout and a {results, summary} envelope is printed (worst status wins + the exit code), so every re-attach hint the CLI prints can be pasted as + ONE command. Exit codes: 0 passed 1 failed / blocked / cancelled 3 auth error - 4 run not found - 7 timeout — resume with: testsprite test wait + 4 run not found (single run-id; with several ids a per-member poll error + is recorded as error: in its row and folded into exit 7) + 7 timeout or per-member poll error — resume with: testsprite test wait 10 transport/network failure (UNAVAILABLE) — retry the command On failure/blocked/cancelled, run: testsprite test artifact get From e819e4543e5993f7548965b15cd9a03c1ee87b95 Mon Sep 17 00:00:00 2001 From: Awokoya Olawale Davidson <99369614+Davidson3556@users.noreply.github.com> Date: Mon, 6 Jul 2026 21:12:45 +0100 Subject: [PATCH 058/102] feat(agent): add GitHub Copilot as an install target (#194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(agent): add GitHub Copilot as an install target Adds `copilot` to the agent-install targets. GitHub Copilot reads path-specific custom instructions from `.github/instructions/*.instructions.md` (VS Code / Visual Studio / Copilot Chat), with YAML frontmatter carrying an `applyTo` glob. The skill installs to `.github/instructions/testsprite-verify.instructions.md` (and the onboard skill alongside) with `applyTo: '**'` so the guidance attaches to every request in the repo. Because `applyTo: '**'` is always-on (Copilot has no on-demand 'model decides' mode like Cursor/Windsurf), the target renders the COMPACT body — the same reasoning behind windsurf's compact render — keeping the always-injected context small (~6 KB vs the ~23 KB full body). Slots into the existing TARGETS machinery, so `agent list`, `setup --agent`, and skill-nudge install-detection pick it up automatically. Updates the AgentTarget union, pathFor, TARGETS, help/docs, unit + e2e matrix guards, and the help snapshot. Fixes #193 * fix(agent): include Kiro in the agent command description The top-level `agent` command description listed the other targets but omitted Kiro (a pre-existing gap), leaving it inconsistent with the `--target` help text and the docs. Align the description with the `--target` list so every supported target is named. --- DOCUMENTATION.md | 7 +-- README.md | 44 ++++++++-------- src/commands/agent.test.ts | 15 +++--- src/commands/agent.ts | 4 +- src/lib/agent-targets.test.ts | 51 +++++++++++++++++-- src/lib/agent-targets.ts | 30 ++++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 13 ++--- test/e2e/agent-install.e2e.test.ts | 11 +++- test/e2e/setup.e2e.test.ts | 1 + 9 files changed, 131 insertions(+), 45 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index fb62cae..6b798ef 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -115,14 +115,15 @@ testsprite agent install cline # .clinerules/testsprite-verify.md testsprite agent install windsurf # .windsurf/rules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md -testsprite agent list # list all 7 targets with status + mode + path +testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md +testsprite agent list # list all 8 targets with status + mode + path ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental). +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. ## Command reference diff --git a/README.md b/README.md index 79f8aea..404f237 100644 --- a/README.md +++ b/README.md @@ -89,28 +89,28 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test wait` | Block on a `runId` until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 7da57cf..4f5fdd3 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -769,13 +769,14 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 7 targets × 2 default skills = 14 rows - expect(json).toHaveLength(14); + // 8 targets × 2 default skills = 16 rows + expect(json).toHaveLength(16); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); expect(targets).toContain('windsurf'); + expect(targets).toContain('copilot'); expect(targets).toContain('antigravity'); expect(targets).toContain('kiro'); expect(targets).toContain('codex'); @@ -949,11 +950,11 @@ describe('runInstall — all own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all six own-file targets +// Dry-run for all seven own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the six own-file targets (default 2 skills = 12 would-write lines)', async () => { + it('writes nothing for any of the seven own-file targets (default 2 skills = 14 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -963,7 +964,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf', 'copilot'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -973,9 +974,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 6 targets × 2 default skills = 12 would-write lines + // 7 targets × 2 default skills = 14 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(12); + expect(wouldWriteLines.length).toBe(14); }); }); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 0e80a91..0a40c4b 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -1061,7 +1061,7 @@ function collect(v: string, prev: string[]): string[] { export function createAgentCommand(deps: AgentDeps = {}): Command { const agent = new Command('agent').description( - 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Windsurf, Antigravity, Codex)', + 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Kiro, Windsurf, Copilot, Codex)', ); agent @@ -1071,7 +1071,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, copilot, codex (comma-separated or repeated)', collect, [], ) diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 0417166..51bf9b5 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -80,19 +80,29 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all seven required keys', () => { + it('has all eight required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor', 'kiro', 'windsurf']); + expect(keys).toEqual([ + 'antigravity', + 'claude', + 'cline', + 'codex', + 'copilot', + 'cursor', + 'kiro', + 'windsurf', + ]); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, windsurf, antigravity, kiro, and codex are experimental', () => { + it('cursor, cline, windsurf, copilot, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); expect(TARGETS.windsurf.status).toBe('experimental'); + expect(TARGETS.copilot.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); @@ -112,6 +122,7 @@ describe('TARGETS', () => { expect(TARGETS.cline.mode).toBe('own-file'); expect(TARGETS.kiro.mode).toBe('own-file'); expect(TARGETS.windsurf.mode).toBe('own-file'); + expect(TARGETS.copilot.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -341,11 +352,45 @@ describe('windsurf renders within the rules-file budget', () => { }); }); +describe('renderForTarget("copilot")', () => { + const result = renderForTarget('copilot', 'testsprite-verify', STUB_BODY); + + it('returns the .github/instructions path', () => { + expect(result.path).toBe('.github/instructions/testsprite-verify.instructions.md'); + }); + + it('uses the Copilot frontmatter (applyTo + description)', () => { + expect(result.content.startsWith('---\n')).toBe(true); + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + expect(result.content).toContain("applyTo: '**'"); + }); + + it('does NOT carry the Claude/Cursor/Windsurf frontmatter keys', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); // claude key + expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key + expect(fm).not.toContain('trigger:'); // windsurf Cascade key + }); + + it('renders the compact verify body (applyTo:** is always-on, so keep it small)', () => { + // Uses the REAL bodies (no stub): copilot always-injects, so like windsurf it + // ships the trimmed verify body while keeping the load-bearing command. + const copilot = renderForTarget('copilot', 'testsprite-verify'); + const claude = renderForTarget('claude', 'testsprite-verify'); + expect(copilot.content.length).toBeLessThan(claude.content.length); + expect(copilot.content).not.toContain('The verification loop that flies'); + expect(copilot.content).toContain('testsprite test run'); + }); +}); + // --------------------------------------------------------------------------- // Content integrity — load-bearing command strings must survive any body trim // --------------------------------------------------------------------------- describe('content integrity — own-file targets', () => { + // Full-body own-file targets. Compact-body targets (windsurf, copilot) are + // excluded — they render the trimmed verify body; see their dedicated tests. const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity' | 'kiro'> = [ 'claude', 'cursor', diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index d9a85d5..7e6f94d 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -9,7 +9,8 @@ export type AgentTarget = | 'antigravity' | 'codex' | 'kiro' - | 'windsurf'; + | 'windsurf' + | 'copilot'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -149,6 +150,19 @@ function wrapWindsurf(_name: string, description: string, body: string): string return `---\ntrigger: model_decision\ndescription: ${description}\n---\n\n${body}\n`; } +/** + * GitHub Copilot reads path-specific custom instructions from + * `.github/instructions/*.instructions.md` (VS Code / Visual Studio / GitHub + * Copilot Chat). Each file carries YAML frontmatter with `applyTo` — a glob that + * scopes when the instructions attach. `applyTo: '**'` attaches the guidance to + * every request in the repo, which is what a persistent verification skill wants + * (there is no on-demand "model decides" mode for Copilot instruction files, so + * always-apply is the correct idiom). `description` is surfaced in Copilot's UI. + */ +function wrapCopilot(_name: string, description: string, body: string): string { + return `---\ndescription: ${description}\napplyTo: '**'\n---\n\n${body}\n`; +} + // --------------------------------------------------------------------------- // Landing paths // --------------------------------------------------------------------------- @@ -173,6 +187,8 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.kiro/skills/${skill}/SKILL.md`; case 'windsurf': return `.windsurf/rules/${skill}.md`; + case 'copilot': + return `.github/instructions/${skill}.instructions.md`; case 'codex': return 'AGENTS.md'; } @@ -220,6 +236,18 @@ export const TARGETS: Record = { compactBody: true, wrap: wrapWindsurf, }, + copilot: { + status: 'experimental', + path: pathFor('copilot', SKILL_NAME), + mode: 'own-file', + // GitHub Copilot path-specific instructions: frontmatter carries `applyTo`. + // `applyTo: '**'` means the file is ALWAYS injected into Copilot requests + // (there is no on-demand "model decides" mode like Cursor/Windsurf), so + // render the compact body to keep the always-on context cost small — the + // same reasoning that drives windsurf's compact render. + compactBody: true, + wrap: wrapCopilot, + }, /** * codex target — managed-section mode. * diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index a16d0c4..231d531 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -4,7 +4,7 @@ exports[`--help snapshots > agent 1`] = ` "Usage: testsprite agent [options] [command] Install TestSprite guidance into coding-agent config (Claude Code, Cursor, -Cline, Windsurf, Antigravity, Codex) +Cline, Antigravity, Kiro, Windsurf, Copilot, Codex) Options: -h, --help display help for command @@ -29,7 +29,8 @@ into a project for a coding agent Options: --target Agent target(s): claude, cursor, cline, antigravity, kiro, - windsurf, codex (comma-separated or repeated) (default: []) + windsurf, copilot, codex (comma-separated or repeated) + (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -115,8 +116,8 @@ Options: --from-env Read TESTSPRITE_API_KEY from the environment instead of prompting (default: false) --agent Coding-agent target to install: claude, antigravity, - cursor, cline, kiro, windsurf, codex (default: claude) - (default: "claude") + cursor, cline, kiro, windsurf, copilot, codex (default: + claude) (default: "claude") --no-agent Skip the agent skill install (configure credentials only) --force Overwrite an existing skill file (a .bak backup is kept) --dir Project root for the skill install (default: current @@ -656,8 +657,8 @@ Commands: project Manage TestSprite projects test Inspect TestSprite tests agent Install TestSprite guidance into coding-agent - config (Claude Code, Cursor, Cline, Windsurf, - Antigravity, Codex) + config (Claude Code, Cursor, Cline, Antigravity, + Kiro, Windsurf, Copilot, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) help [command] display help for command diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 094aec2..4fb4584 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -174,6 +174,11 @@ describe('content integrity', () => { expect(content.startsWith('---'), `windsurf: should start with ---`).toBe(true); expect(content).toContain('trigger: model_decision'); expect(content).toContain('description:'); + } else if (target === 'copilot') { + // GitHub Copilot instructions frontmatter: applyTo glob + description + expect(content.startsWith('---'), `copilot: should start with ---`).toBe(true); + expect(content).toContain("applyTo: '**'"); + expect(content).toContain('description:'); } // (b) branding — the renamed H1 must be present in every body variant @@ -211,6 +216,9 @@ describe('content integrity', () => { expect(content.startsWith('---'), `windsurf/onboard: should start with ---`).toBe(true); expect(content).toContain('trigger: model_decision'); expect(content).toContain('description:'); + } else if (target === 'copilot') { + expect(content.startsWith('---'), `copilot/onboard: should start with ---`).toBe(true); + expect(content).toContain("applyTo: '**'"); } // Load-bearing onboard string: the skill body must reference setup @@ -803,7 +811,7 @@ describe('agent list', () => { }>; expect(Array.isArray(parsed)).toBe(true); - // Expected: 7 targets × 2 skills = 14 rows + // Expected: 8 targets × 2 skills = 16 rows const expectedCount = Object.keys(TARGETS).length * DEFAULT_SKILLS.length; expect(parsed.length).toBe(expectedCount); @@ -839,6 +847,7 @@ describe('matrix coverage guard', () => { 'cline', 'kiro', 'windsurf', + 'copilot', 'codex', ]); }); diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index 3b35751..5a07434 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -229,6 +229,7 @@ describe('matrix coverage guard', () => { 'cline', 'kiro', 'windsurf', + 'copilot', 'codex', ]); }); From edc31c8e0e82599a9b95d15ba761eede61683517 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:13:18 -0400 Subject: [PATCH 059/102] feat(cli): add "testsprite doctor" environment diagnostic (#183) * feat(cli): add "testsprite doctor" environment diagnostic One-shot preflight: checks CLI version, Node runtime, profile, API endpoint, credentials, live connectivity (GET /me), and verify-skill install. Prints an OK/WARN/FAIL report and exits non-zero when any check fails, so it gates a CI step or agent preflight. Reuses the real resolution helpers; the API key is never printed. Fixes #73 * fix(doctor): address review findings - Node check reuses the CLI runtime guard (shouldRejectNodeVersion) instead of a hardcoded major floor, so the verdict matches what the entrypoint enforces at startup; the precise 20.19+/22.13+/24+ engines are enforced by npm engine-strict. - --request-timeout raises a validation error on malformed input instead of silently defaulting, matching the other commands. - The --output json test asserts the API key never appears in the JSON path (distinct from the text renderer already covered). --- src/commands/doctor.test.ts | 229 ++++++++++++++ src/commands/doctor.ts | 285 ++++++++++++++++++ src/index.ts | 2 + test/__snapshots__/help.snapshot.test.ts.snap | 2 + 4 files changed, 518 insertions(+) create mode 100644 src/commands/doctor.test.ts create mode 100644 src/commands/doctor.ts diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts new file mode 100644 index 0000000..230398a --- /dev/null +++ b/src/commands/doctor.test.ts @@ -0,0 +1,229 @@ +/** + * Unit tests for `testsprite doctor`. + * + * The command reuses the real resolution helpers (loadConfig, makeHttpClient, + * isVerifySkillInstalled), so these tests inject env/credentials/fetch/fs and + * assert on the rendered report + the exit-on-failure contract. + */ + +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { CLIError } from '../lib/errors.js'; +import { writeProfile } from '../lib/credentials.js'; +import type { DoctorDeps, DoctorReport } from './doctor.js'; +import { createDoctorCommand, runDoctor } from './doctor.js'; + +interface CapturedOutput { + stdout: string[]; + stderr: string[]; +} + +function makeCapture(): { capture: CapturedOutput; deps: Pick } { + const capture: CapturedOutput = { stdout: [], stderr: [] }; + return { + capture, + deps: { + stdout: line => capture.stdout.push(line), + stderr: line => capture.stderr.push(line), + }, + }; +} + +function makeFetch(body: unknown, status = 200): DoctorDeps['fetchImpl'] { + return vi.fn( + async () => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as DoctorDeps['fetchImpl']; +} + +const OK_ME = { userId: 'u-doc', keyId: 'k-doc' }; + +/** Base deps shared by the healthy-path tests: node OK, skill installed, empty env. */ +function healthyDeps(credentialsPath: string, extra: Partial = {}): DoctorDeps { + return { + env: {}, + credentialsPath, + cwd: '/project', + nodeVersion: '22.9.0', + existsSync: () => true, // skill landing file present + fetchImpl: makeFetch(OK_ME), + ...extra, + }; +} + +let credentialsPath: string; + +beforeEach(() => { + credentialsPath = join(mkdtempSync(join(tmpdir(), 'testsprite-doctor-')), 'credentials'); +}); + +describe('runDoctor — healthy environment', () => { + it('returns an all-passing report and does not throw', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + expect(report.failures).toBe(0); + expect(report.warnings).toBe(0); + const out = capture.stdout.join('\n'); + expect(out).toContain('[OK]'); + expect(out).toContain('All checks passed.'); + expect(out).toContain('reached GET /me'); + }); + + it('never prints the API key anywhere in the report', async () => { + writeProfile('default', { apiKey: 'sk-super-secret-value' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + const all = capture.stdout.join('\n') + capture.stderr.join('\n'); + expect(all).not.toContain('sk-super-secret-value'); + }); + + it('emits a machine-readable report under --output json without leaking the API key', async () => { + writeProfile('default', { apiKey: 'sk-json-secret-value' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runDoctor( + { profile: 'default', output: 'json', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + const raw = capture.stdout.join(''); + // Security: the JSON serialization path is distinct from the text renderer, + // so assert the key never leaks here either. + expect(raw).not.toContain('sk-json-secret-value'); + const parsed = JSON.parse(raw) as DoctorReport; + expect(parsed.failures).toBe(0); + expect(Array.isArray(parsed.checks)).toBe(true); + expect( + parsed.checks.some(check => check.name === 'Connectivity' && check.status === 'ok'), + ).toBe(true); + }); +}); + +describe('runDoctor — failing checks exit non-zero', () => { + it('missing API key fails Credentials and throws CLIError (exit 1)', async () => { + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, // no profile written => no key + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + expect(rejection).toMatchObject({ exitCode: 1 }); + const out = capture.stdout.join('\n'); + expect(out).toContain('[FAIL]'); + expect(out).toContain('Credentials'); + }); + + it('invalid endpoint URL fails the API endpoint check', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false, endpointUrl: 'not-a-url' }, + { ...healthyDeps(credentialsPath), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('API endpoint'); + expect(out).toContain('not a valid'); + }); + + it('rejected API key surfaces as a Connectivity failure', async () => { + writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const authError = { + error: { code: 'AUTH_INVALID', message: 'Bad key.', requestId: 'req_x', details: {} }, + }; + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(authError, 401) }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('Connectivity'); + expect(out).toContain('API key rejected (AUTH_INVALID)'); + }); + + it('a non-auth /me error is reported as a Connectivity failure with its code', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const notFound = { + error: { code: 'NOT_FOUND', message: 'nope', requestId: 'req_y', details: {} }, + }; + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(notFound, 404) }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + expect(capture.stdout.join('\n')).toContain('GET /me failed (NOT_FOUND)'); + }); + + it('an outdated Node runtime fails the Node.js check', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { nodeVersion: '18.0.0' }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('Node.js'); + expect(out).toContain('below the required Node 20'); + }); +}); + +describe('runDoctor — warnings do not fail', () => { + it('missing verify skill is a warning, not a failure', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { existsSync: () => false }), ...deps }, + ); + expect(report.failures).toBe(0); + expect(report.warnings).toBeGreaterThanOrEqual(1); + const out = capture.stdout.join('\n'); + expect(out).toContain('[WARN]'); + expect(out).toContain('Verify skill'); + }); + + it('--dry-run skips connectivity and never calls fetch, missing key is a warning', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('fetch must not be called under --dry-run'); + }) as unknown as DoctorDeps['fetchImpl']; + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + { + env: {}, + credentialsPath, + cwd: '/project', + nodeVersion: '22.9.0', + existsSync: () => true, + fetchImpl, + ...deps, + }, + ); + expect(report.failures).toBe(0); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(capture.stdout.join('\n')).toContain('skipped under --dry-run'); + }); +}); + +describe('createDoctorCommand wiring', () => { + it('exposes the doctor command name', () => { + expect(createDoctorCommand().name()).toBe('doctor'); + }); + + it('--help describes the diagnostic', () => { + expect(createDoctorCommand().helpInformation()).toContain('Diagnose'); + }); +}); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts new file mode 100644 index 0000000..2fa0c57 --- /dev/null +++ b/src/commands/doctor.ts @@ -0,0 +1,285 @@ +/** + * `testsprite doctor` — one-shot environment diagnostic. + * + * Runs a fixed checklist (CLI version, Node.js runtime, active profile, API + * endpoint, credentials, live connectivity + key validity, and whether the + * verify skill is installed in the current project) and prints an OK/WARN/FAIL + * report. Exits non-zero when any check FAILS so it can gate a CI step or an + * agent preflight (`testsprite doctor && testsprite test run ...`). Warnings + * (e.g. skill not installed) do not fail the process. + * + * Every check is reused from the same helpers the real commands use, so the + * report reflects exactly what a subsequent command would resolve: `loadConfig` + * for profile/endpoint/key, `assertValidEndpointUrl` for the endpoint gate, + * `makeHttpClient` + `GET /me` for connectivity, and `isVerifySkillInstalled` + * for the skill check. + */ + +import { Command } from 'commander'; +import { + assertValidEndpointUrl, + makeHttpClient, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { loadConfig } from '../lib/config.js'; +import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; +import type { FetchImpl } from '../lib/http.js'; +import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; +import { VERSION } from '../version.js'; +import { MIN_SUPPORTED_NODE_MAJOR, shouldRejectNodeVersion } from '../version-guard.js'; + +export type DoctorStatus = 'ok' | 'warn' | 'fail'; + +export interface DoctorCheck { + /** Short, stable label (also the JSON key-ish name). */ + name: string; + status: DoctorStatus; + /** Human-readable one-line result. Never contains the API key. */ + detail: string; +} + +export interface DoctorReport { + checks: DoctorCheck[]; + failures: number; + warnings: number; +} + +/** Minimal projection of `GET /me` we read for the connectivity detail. */ +interface MeIdentity { + userId?: string; + keyId?: string; +} + +export interface DoctorDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + /** Project dir for the skill check. Defaults to `process.cwd()`. */ + cwd?: string; + /** Runtime version string (e.g. "22.9.0"). Defaults to `process.versions.node`. */ + nodeVersion?: string; + existsSync?: (p: string) => boolean; + readFileSync?: (p: string) => string; +} + +type CommonOptions = FactoryCommonOptions; + +export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const env = deps.env ?? process.env; + const cwd = deps.cwd ?? process.cwd(); + const nodeVersion = deps.nodeVersion ?? process.versions.node; + + const config = loadConfig({ + profile: opts.profile, + endpointUrl: opts.endpointUrl, + env, + credentialsPath: deps.credentialsPath, + }); + const endpointCheck = checkEndpoint(config.apiUrl); + const hasKey = Boolean(config.apiKey); + + const checks: DoctorCheck[] = [ + { name: 'CLI version', status: 'ok', detail: VERSION }, + checkNodeVersion(nodeVersion), + { name: 'Profile', status: 'ok', detail: config.profile }, + endpointCheck, + checkCredentials(hasKey, config.profile, opts.dryRun ?? false), + await checkConnectivity(opts, deps, { + hasKey, + endpointOk: endpointCheck.status === 'ok', + }), + checkSkill(cwd, deps), + ]; + + const failures = checks.filter(check => check.status === 'fail').length; + const warnings = checks.filter(check => check.status === 'warn').length; + const report: DoctorReport = { checks, failures, warnings }; + + out.print(report, () => renderDoctor(report)); + + if (failures > 0) { + // Non-zero exit so `testsprite doctor && ...` gates a CI step or an agent + // preflight. The full report already printed above; this line is the stderr + // summary index.ts renders before exiting 1. + throw new CLIError(`doctor: ${failures} check(s) failed, ${warnings} warning(s)`, 1); + } + return report; +} + +function checkNodeVersion(nodeVersion: string): DoctorCheck { + // Reuse the CLI's own runtime guard so the verdict matches exactly what the + // entrypoint enforces at startup, rather than a divergent hardcoded check. + // The precise engines floor (20.19+/22.13+/24+) is enforced by npm at install + // time via .npmrc engine-strict. sourceRef: src/version-guard.ts. + const rejected = shouldRejectNodeVersion(nodeVersion); + return { + name: 'Node.js', + status: rejected ? 'fail' : 'ok', + detail: rejected + ? `v${nodeVersion} is below the required Node ${MIN_SUPPORTED_NODE_MAJOR}; upgrade Node.js` + : `v${nodeVersion} (>=${MIN_SUPPORTED_NODE_MAJOR} required)`, + }; +} + +function checkEndpoint(apiUrl: string): DoctorCheck { + try { + assertValidEndpointUrl(apiUrl); + return { name: 'API endpoint', status: 'ok', detail: apiUrl }; + } catch { + return { + name: 'API endpoint', + status: 'fail', + detail: `"${apiUrl}" is not a valid http(s) URL`, + }; + } +} + +function checkCredentials(hasKey: boolean, profile: string, dryRun: boolean): DoctorCheck { + if (hasKey) { + // Never print any part of the key (security). Confirm presence only. + return { + name: 'Credentials', + status: 'ok', + detail: `API key configured (profile "${profile}")`, + }; + } + // Under --dry-run no key is expected, so a missing key is not a failure. + return { + name: 'Credentials', + status: dryRun ? 'warn' : 'fail', + detail: dryRun + ? 'no API key (not needed under --dry-run)' + : 'no API key found; run `testsprite setup` (or set TESTSPRITE_API_KEY)', + }; +} + +function checkSkill(cwd: string, deps: DoctorDeps): DoctorCheck { + const installed = isVerifySkillInstalled(cwd, { + existsSync: deps.existsSync, + readFileSync: deps.readFileSync, + }); + return { + name: 'Verify skill', + status: installed ? 'ok' : 'warn', + detail: installed + ? 'installed in this project' + : 'not installed here; run `testsprite setup` so your agent verifies its changes', + }; +} + +async function checkConnectivity( + opts: CommonOptions, + deps: DoctorDeps, + ctx: { hasKey: boolean; endpointOk: boolean }, +): Promise { + const name = 'Connectivity'; + if (opts.dryRun) return { name, status: 'warn', detail: 'skipped under --dry-run' }; + if (!ctx.hasKey) return { name, status: 'warn', detail: 'skipped; no API key to test with' }; + if (!ctx.endpointOk) return { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' }; + + try { + const client = makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); + const me = await client.get('/me'); + const who = me.userId ? ` (userId ${me.userId})` : ''; + return { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }; + } catch (error) { + if (error instanceof ApiError) { + if ( + error.code === 'AUTH_REQUIRED' || + error.code === 'AUTH_INVALID' || + error.code === 'AUTH_FORBIDDEN' + ) { + return { name, status: 'fail', detail: `API key rejected (${error.code})` }; + } + return { name, status: 'fail', detail: `GET /me failed (${error.code})` }; + } + return { + name, + status: 'fail', + detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, + }; + } +} + +const STATUS_LABEL: Record = { + ok: '[OK] ', + warn: '[WARN]', + fail: '[FAIL]', +}; + +function renderDoctor(report: DoctorReport): string { + const nameWidth = Math.max(...report.checks.map(check => check.name.length)); + const lines: string[] = ['TestSprite doctor', '']; + for (const check of report.checks) { + lines.push(` ${STATUS_LABEL[check.status]} ${check.name.padEnd(nameWidth)} ${check.detail}`); + } + lines.push(''); + lines.push( + report.failures === 0 && report.warnings === 0 + ? 'All checks passed.' + : `${report.failures} failure(s), ${report.warnings} warning(s).`, + ); + return lines.join('\n'); +} + +export function createDoctorCommand(deps: DoctorDeps = {}): Command { + const cmd = new Command('doctor') + .description( + 'Diagnose CLI setup: version, Node, profile, endpoint, credentials, connectivity, skill', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite doctor # run all checks (exit 1 if any fails)\n' + + ' testsprite doctor --output json # machine-readable report\n' + + ' testsprite doctor && testsprite test run # gate a command on a healthy setup', + ) + .action(async (_cmdOpts, command: Command) => { + await runDoctor(resolveCommonOptions(command), deps); + }); + + return cmd; +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + return { + profile: globals.profile ?? 'default', + output: globals.output ?? 'text', + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const seconds = Number(raw); + if (!Number.isFinite(seconds) || seconds <= 0) { + // Match the other commands: a malformed --request-timeout is a validation + // error, not a silently-ignored default. + throw localValidationError( + 'request-timeout', + `must be a positive number of seconds (got "${raw}")`, + ); + } + return Math.round(seconds * 1000); +} + +function makeOutput(mode: OutputMode, deps: DoctorDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} diff --git a/src/index.ts b/src/index.ts index fb935c8..78010c9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; +import { createDoctorCommand } from './commands/doctor.js'; import { createDeprecatedInitCommand, createSetupCommand, @@ -89,6 +90,7 @@ program.addCommand(createProjectCommand({})); program.addCommand(createTestCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); +program.addCommand(createDoctorCommand()); // Buffer Commander error messages instead of writing immediately. The catch // block re-emits in the correct format (JSON or text) once the requested diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 231d531..323dd5c 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -661,6 +661,8 @@ Commands: Kiro, Windsurf, Copilot, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) + doctor Diagnose CLI setup: version, Node, profile, + endpoint, credentials, connectivity, skill help [command] display help for command " `; From 946f5b326453b55aaeac97dffb5bac2aa844901f Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:13:57 -0400 Subject: [PATCH 060/102] feat(cli): graceful termination signals + broken-pipe guard (#185) * feat(cli): graceful termination signals + broken-pipe guard Install handlers for SIGINT/SIGTERM/SIGHUP that print a one-line explanation (any started run keeps executing server-side; resume with `testsprite test list` or `testsprite test wait `) and exit with the conventional 128+signum code (SIGINT -> 130). Also guard EPIPE on stdout/stderr so piping to a reader that closes early (`... | head`) exits cleanly instead of dumping a raw `write EPIPE` stack. process and streams are injectable, so both are unit-tested without spawning a subprocess or sending a real signal. Fixes #75 * fix(interrupt): flush the signal message synchronously before exit A signal handler calls process.exit() immediately after writing the interrupt hint. When stderr is a pipe, an async process.stderr.write() may not flush before the process terminates, so the hint could be lost. The default stderr writer now uses fs.writeSync (best-effort, guarded against EPIPE) so the hint is reliably emitted. Added a test mocking fs.writeSync to assert the synchronous write on the default path. --- src/index.ts | 9 +++ src/lib/interrupt.test.ts | 119 +++++++++++++++++++++++++++++++++++ src/lib/interrupt.ts | 126 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100644 src/lib/interrupt.test.ts create mode 100644 src/lib/interrupt.ts diff --git a/src/index.ts b/src/index.ts index 78010c9..806f6e4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ import { createProjectCommand } from './commands/project.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; +import { installBrokenPipeGuard, installSignalHandlers } from './lib/interrupt.js'; import { Output, isOutputMode } from './lib/output.js'; import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; @@ -163,6 +164,14 @@ program.hook('preAction', (_thisCommand, actionCommand) => { } }); +// Clean process lifecycle: a clear message + conventional exit code on SIGINT / +// SIGTERM / SIGHUP (instead of Node's silent abrupt kill) so an interrupted +// `test run --wait` explains the run continues server-side; plus an EPIPE guard +// so piping to a reader that closes early (`| head`) exits cleanly instead of +// dumping a raw `write EPIPE` stack. +installSignalHandlers(); +installBrokenPipeGuard(); + // Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch // ignores them by default). No-op when no proxy variable is set. maybeInstallProxyAgent(); diff --git a/src/lib/interrupt.test.ts b/src/lib/interrupt.test.ts new file mode 100644 index 0000000..1fcf8eb --- /dev/null +++ b/src/lib/interrupt.test.ts @@ -0,0 +1,119 @@ +import { EventEmitter } from 'node:events'; +import { writeSync } from 'node:fs'; +import { describe, expect, it, vi } from 'vitest'; +import { + SIGINT_EXIT_CODE, + TERMINATION_EXIT_CODES, + formatInterruptMessage, + installBrokenPipeGuard, + installSignalHandlers, +} from './interrupt.js'; + +// installSignalHandlers' default stderr writes via fs.writeSync (synchronous, so +// the hint survives a piped stderr before exit); mock it to assert on that path. +vi.mock('node:fs', async importOriginal => { + const actual = (await importOriginal()) as Record; + return { ...actual, writeSync: vi.fn() }; +}); + +describe('formatInterruptMessage', () => { + it('defaults to SIGINT and explains the run continues server-side', () => { + const message = formatInterruptMessage(); + expect(message).toContain('Interrupted (SIGINT)'); + expect(message).toContain('test wait'); + expect(message).toContain('test list'); + }); + + it('names the specific signal when given one', () => { + expect(formatInterruptMessage('SIGTERM')).toContain('Interrupted (SIGTERM)'); + expect(formatInterruptMessage('SIGHUP')).toContain('Interrupted (SIGHUP)'); + }); +}); + +describe('installSignalHandlers', () => { + it('registers SIGINT, SIGTERM and SIGHUP with the conventional 128+signum exit codes', () => { + const handlers = new Map void>(); + const stderr: string[] = []; + const exit = vi.fn(); + + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + stderr: line => stderr.push(line), + exit, + }); + + expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); + + handlers.get('SIGINT')!(); + expect(exit).toHaveBeenLastCalledWith(130); + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenLastCalledWith(143); + handlers.get('SIGHUP')!(); + expect(exit).toHaveBeenLastCalledWith(129); + + // Each handler emits a leading blank line then the explanation. + expect(stderr[0]).toBe(''); + expect(stderr.join('\n')).toContain('Interrupted (SIGINT)'); + expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); + expect(stderr.join('\n')).toContain('Interrupted (SIGHUP)'); + expect(SIGINT_EXIT_CODE).toBe(130); + expect(TERMINATION_EXIT_CODES.SIGTERM).toBe(143); + expect(TERMINATION_EXIT_CODES.SIGHUP).toBe(129); + }); + + it('writes the hint synchronously via writeSync before exit (survives a piped stderr)', () => { + vi.mocked(writeSync).mockClear(); + const handlers = new Map void>(); + const exit = vi.fn(); + // No stderr dep: exercise the synchronous default path. + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + exit, + }); + handlers.get('SIGINT')!(); + expect(exit).toHaveBeenCalledWith(130); + const written = vi + .mocked(writeSync) + .mock.calls.map(call => String(call[1])) + .join(''); + expect(written).toContain('Interrupted (SIGINT)'); + }); +}); + +describe('installBrokenPipeGuard', () => { + function makeEpipe(): NodeJS.ErrnoException { + return Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }); + } + + it('exits 0 on stdout EPIPE (clean SIGPIPE-equivalent for `| head`)', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + stdout.emit('error', makeEpipe()); + expect(exit).toHaveBeenCalledWith(0); + }); + + it('re-throws a non-EPIPE stdout error instead of silently swallowing it', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + expect(() => + stdout.emit('error', Object.assign(new Error('boom'), { code: 'ENOSPC' })), + ).toThrow('boom'); + expect(exit).not.toHaveBeenCalled(); + }); + + it('swallows stderr EPIPE without exiting or throwing', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + expect(() => stderr.emit('error', makeEpipe())).not.toThrow(); + expect(exit).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/interrupt.ts b/src/lib/interrupt.ts new file mode 100644 index 0000000..cc5b4d5 --- /dev/null +++ b/src/lib/interrupt.ts @@ -0,0 +1,126 @@ +/** + * Process lifecycle hardening: graceful termination signals and broken-pipe. + * + * Termination signals: without a handler, Node terminates the process abruptly + * with no output, so a user (Ctrl+C), a CI runner or `docker stop` (SIGTERM), or + * a closed terminal/SSH session (SIGHUP) that interrupts a long + * `test run --wait` is left unsure whether the run was cancelled or is still + * executing server-side (it is: the CLI only polls; the run lives on the + * backend). The handler prints a one-line explanation plus how to resume, then + * exits with the conventional `128 + signal` code. + * + * Broken pipe: when output is piped to a reader that closes early + * (`testsprite ... | head`), the kernel raises `EPIPE` on the next stdout write. + * Node turns an `'error'` with no listener into an uncaughtException and dumps a + * raw `write EPIPE` stack (exit 1). The guard swallows it and exits 0, the + * conventional SIGPIPE-equivalent result for "the reader went away". + * + * `process` and the streams are injectable so the wiring is unit-testable + * without spawning a subprocess or sending a real signal. + */ + +import { writeSync } from 'node:fs'; + +/** + * Termination signals handled, mapped to their conventional `128 + signum` + * exit code. sourceRef: POSIX signal numbers (SIGHUP=1, SIGINT=2, SIGTERM=15). + */ +export const TERMINATION_EXIT_CODES = { + SIGINT: 130, // 128 + 2 + SIGTERM: 143, // 128 + 15 + SIGHUP: 129, // 128 + 1 +} as const; + +export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; + +/** Back-compat alias: SIGINT's conventional exit code. */ +export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; + +export function formatInterruptMessage(signal: TerminationSignal = 'SIGINT'): string { + return ( + `Interrupted (${signal}). Any run already started keeps executing on the server; ` + + 'check it with `testsprite test list` or `testsprite test wait `.' + ); +} + +export interface InterruptDeps { + /** Signal registrar. Defaults to `process.on`. */ + on?: (signal: TerminationSignal, handler: () => void) => void; + /** Line-oriented stderr writer (appends a newline). */ + stderr?: (line: string) => void; + /** Process exit. Defaults to `process.exit`. */ + exit?: (code: number) => void; +} + +/** + * Register handlers for SIGINT, SIGTERM and SIGHUP. Idempotent enough for a + * single top-level call in `index.ts`; not designed to be installed twice. + */ +export function installSignalHandlers(deps: InterruptDeps = {}): void { + const on = + deps.on ?? + ((signal: TerminationSignal, handler: () => void) => { + process.on(signal, handler); + }); + const stderr = + deps.stderr ?? + ((line: string) => { + // A signal handler calls process.exit() right after writing, which can + // truncate an async process.stderr.write() when stderr is a pipe. Write + // synchronously so the interrupt hint is flushed before the process exits. + try { + writeSync(process.stderr.fd, `${line}\n`); + } catch { + // Best-effort: if stderr is already gone (EPIPE), still exit cleanly. + } + }); + const exit = deps.exit ?? ((code: number) => process.exit(code)); + + for (const signal of Object.keys(TERMINATION_EXIT_CODES) as TerminationSignal[]) { + on(signal, () => { + // Blank line first so the message starts on its own row rather than + // trailing the progress ticker's in-place line. + stderr(''); + stderr(formatInterruptMessage(signal)); + exit(TERMINATION_EXIT_CODES[signal]); + }); + } +} + +export interface BrokenPipeDeps { + /** stdout stream. Defaults to `process.stdout`. */ + stdout?: NodeJS.EventEmitter; + /** stderr stream. Defaults to `process.stderr`. */ + stderr?: NodeJS.EventEmitter; + /** Process exit. Defaults to `process.exit`. */ + exit?: (code: number) => void; +} + +/** + * Guard against `EPIPE` on stdout/stderr so piping to a reader that closes + * early (`testsprite ... | head`) exits cleanly instead of crashing with an + * unhandled `write EPIPE` stack. Only `EPIPE` is swallowed; any other stream + * error is left to surface normally. + */ +export function installBrokenPipeGuard(deps: BrokenPipeDeps = {}): void { + const stdout = deps.stdout ?? process.stdout; + const stderr = deps.stderr ?? process.stderr; + const exit = deps.exit ?? ((code: number) => process.exit(code)); + + stdout.on('error', (error: NodeJS.ErrnoException) => { + // Reader went away (`| head`, `| less` then q): exit cleanly like SIGPIPE + // rather than dumping an unhandled `write EPIPE` stack. Any other stdout + // error is a genuine, actionable failure, so re-throw it (Node's default). + if (error.code === 'EPIPE') { + exit(0); + return; + } + throw error; + }); + stderr.on('error', (error: NodeJS.ErrnoException) => { + // stderr closed: nothing can be reported over it, so swallow EPIPE. Any + // other error re-throws so a genuine failure is not silently hidden. + if (error.code === 'EPIPE') return; + throw error; + }); +} From 3305dfa57ab258b11311d15197a80dd9eb654911 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:14:29 -0400 Subject: [PATCH 061/102] feat(test): add "test scaffold" to emit a schema-correct starter plan or backend skeleton (#180) --- src/commands/test.test.ts | 90 +++++++++++ src/commands/test.ts | 145 +++++++++++++++++- test/__snapshots__/help.snapshot.test.ts.snap | 4 + 3 files changed, 238 insertions(+), 1 deletion(-) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 0368e06..6ae831c 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -35,6 +35,7 @@ import { runList, runPlanPut, runResult, + runScaffold, runSteps, runTestWaitMany, runUpdate, @@ -134,6 +135,7 @@ describe('createTestCommand — surface', () => { 'rerun', 'result', 'run', + 'scaffold', 'steps', 'update', 'wait', @@ -2325,6 +2327,94 @@ describe('runCodePut', () => { }); }); +describe('runScaffold', () => { + it('frontend scaffold is a valid CliPlanInput and prints as JSON on stdout', async () => { + const out: string[] = []; + const result = await runScaffold( + { profile: 'default', output: 'text', debug: false, scaffoldType: 'frontend', force: false }, + { stdout: line => out.push(line), stderr: () => undefined, env: {} }, + ); + const plan = result as { projectId: string; type: string; planSteps: Array<{ type: string }> }; + expect(plan.type).toBe('frontend'); + // Placeholder project id when TESTSPRITE_PROJECT_ID is unset. + expect(plan.projectId).toContain('testsprite project list'); + expect(plan.planSteps.length).toBeGreaterThanOrEqual(2); + // Every emitted step type must come from the real enum (no drift). + for (const step of plan.planSteps) expect(['action', 'assertion']).toContain(step.type); + // At least one assertion step so the scaffold is a meaningful test. + expect(plan.planSteps.some(step => step.type === 'assertion')).toBe(true); + // stdout body parses back to the same plan (`> plan.json` works). + expect(JSON.parse(out.join('\n'))).toEqual(plan); + }); + + it('pre-fills projectId from TESTSPRITE_PROJECT_ID when set', async () => { + const result = await runScaffold( + { profile: 'default', output: 'json', debug: false, scaffoldType: 'frontend', force: false }, + { + stdout: () => undefined, + stderr: () => undefined, + env: { TESTSPRITE_PROJECT_ID: 'project_env' }, + }, + ); + expect((result as { projectId: string }).projectId).toBe('project_env'); + }); + + it('backend scaffold defines a requests test with a status assertion AND calls it', async () => { + const out: string[] = []; + const result = await runScaffold( + { profile: 'default', output: 'text', debug: false, scaffoldType: 'backend', force: false }, + { stdout: line => out.push(line), stderr: () => undefined, env: {} }, + ); + const code = (result as { code: string }).code; + expect(code).toContain('import requests'); + expect(code).toContain('assert response.status_code == 200'); + // The onboarding rule: the function must be CALLED, not just defined. + expect(code).toContain('\ntest_health_endpoint()'); + expect(out.join('\n')).toContain('import requests'); + }); + + it('--out writes the file and refuses to overwrite without --force', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-')); + const target = join(dir, 'plan.json'); + const opts = { + profile: 'default', + output: 'text', + debug: false, + scaffoldType: 'frontend', + out: target, + force: false, + } as const; + const deps = { stdout: () => undefined, stderr: () => undefined, env: {} }; + await runScaffold({ ...opts }, deps); + const written = JSON.parse(readFileSync(target, 'utf8')) as { type: string }; + expect(written.type).toBe('frontend'); + // Second run without --force must not clobber the (possibly edited) file. + await expect(runScaffold({ ...opts }, deps)).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + // --force overwrites. + await expect(runScaffold({ ...opts, force: true }, deps)).resolves.toBeDefined(); + }); + + it('--out pointing at an existing path (here a directory) rejects without --force', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-dir-')); + await expect( + runScaffold( + { + profile: 'default', + output: 'text', + debug: false, + scaffoldType: 'frontend', + out: dir, + force: false, + }, + { stdout: () => undefined, stderr: () => undefined, env: {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + describe('runSteps', () => { it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index 57d90f5..1c6edd8 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1,4 +1,11 @@ -import { createWriteStream, readFileSync, readdirSync, statSync, type WriteStream } from 'node:fs'; +import { + createWriteStream, + existsSync, + readFileSync, + readdirSync, + statSync, + type WriteStream, +} from 'node:fs'; import { rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path'; import { randomUUID } from 'node:crypto'; @@ -4000,6 +4007,114 @@ export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const env = deps.env ?? process.env; + // Pre-fill the project id from TESTSPRITE_PROJECT_ID when the caller's + // environment carries one; otherwise a clearly-marked placeholder the user + // swaps after running `testsprite project list`. + const projectId = + typeof env.TESTSPRITE_PROJECT_ID === 'string' && env.TESTSPRITE_PROJECT_ID.length > 0 + ? env.TESTSPRITE_PROJECT_ID + : ''; + + let payload: CliPlanInput | CliBackendScaffold; + let body: string; + if (opts.scaffoldType === 'frontend') { + const plan: CliPlanInput = { + projectId, + type: 'frontend', + name: 'My first frontend test', + description: 'Replace with one sentence describing what this test verifies.', + priority: 'p2', + planSteps: [ + { + type: 'action', + description: 'Navigate to /login and sign in with a seeded test account', + }, + { type: 'action', description: 'Open the first product page and click "Add to cart"' }, + { type: 'assertion', description: 'Assert that the cart badge shows 1 item' }, + ], + }; + payload = plan; + body = `${JSON.stringify(plan, null, 2)}\n`; + } else { + const code = [ + 'import requests', + '', + '# Replace with your API base URL (must be reachable from the internet).', + 'BASE_URL = "https://staging.example.com"', + '', + '', + 'def test_health_endpoint() -> None:', + ' response = requests.get(f"{BASE_URL}/health", timeout=30)', + ' assert response.status_code == 200, f"expected 200, got {response.status_code}"', + '', + '', + '# The test function MUST be called: TestSprite executes this file top to', + '# bottom, so a defined-but-never-called function would pass vacuously.', + 'test_health_endpoint()', + '', + ].join('\n'); + payload = { type: 'backend', language: 'python', code }; + body = code; + } + + if (opts.out !== undefined) { + const resolved = isAbsolute(opts.out) ? opts.out : resolve(process.cwd(), opts.out); + // Never clobber silently: scaffolds are starting points the user edits, so + // an accidental re-run must not erase their work. --force opts in. + if (!opts.force && existsSync(resolved)) { + throw localValidationError('out', `already exists: ${resolved}. Pass --force to overwrite`); + } + const sink = openOutputFile(opts.out); // reuses the directory/parent guards + const fileOut = makeFileOutput(opts.output, sink); + await fileOut.writeChunk(body); + await closeOutputFile(sink, true); + stderrFn(`Scaffold written to ${resolved}`); + return payload; + } + + // No --out: the scaffold body IS the stdout payload (`> plan.json` works). + out.print(payload, () => body.trimEnd()); + return payload; +} + export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -8007,6 +8122,34 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('scaffold') + .description( + 'Emit a schema-correct starter test definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials.', + ) + .option('--type ', 'frontend|backend (default: frontend)') + .option('--out ', 'write the scaffold to a file instead of stdout') + .option('--force', 'overwrite an existing --out file', false) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite test scaffold > first-test.plan.json\n' + + ' testsprite test scaffold --type backend --out tests/health.py\n' + + ' testsprite test scaffold --out plan.json # then edit, and create with --plan-from plan.json', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: ScaffoldFlagOpts, command: Command) => { + await runScaffold( + { + ...resolveCommonOptions(command), + scaffoldType: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES) ?? 'frontend', + out: cmdOpts.out, + force: cmdOpts.force === true, + }, + deps, + ); + }); + test .command('steps ') .description( diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 323dd5c..fcc3d85 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -201,6 +201,10 @@ Commands: (--plan-from, FE-only, M3.2 piece-5) create-batch [options] Create multiple FE tests from a JSONL of plan specs (FE-only) + scaffold [options] Emit a schema-correct starter test + definition (frontend plan JSON by + default, or a backend Python skeleton). + Pure-local: no network, no credentials. steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) From 3d65ff5a0e25399a7b87e8c75b6f8eb90f0ef2f8 Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Thu, 9 Jul 2026 15:29:24 -0700 Subject: [PATCH 062/102] release: v0.3.0 Private-Snapshot-RevId: d22d9f18f5d9712921097d5829c28d056e3ffd31 --- .gitattributes | 6 + .github/PULL_REQUEST_TEMPLATE.md | 4 +- .github/workflows/ci-nudge.yml | 129 ++++++ .github/workflows/issue-triage.yml | 23 +- .github/workflows/pr-triage.yml | 189 +++++++++ .github/workflows/stale.yml | 64 +++ CHANGELOG.md | 47 ++- CONTRIBUTING.md | 48 ++- DOCUMENTATION.md | 190 +++++++-- README.md | 52 ++- package.json | 2 +- skills/testsprite-onboard.skill.md | 21 + skills/testsprite-verify.skill.md | 33 ++ src/commands/agent.test.ts | 50 ++- src/commands/agent.ts | 21 +- src/commands/init.test.ts | 17 +- src/commands/init.ts | 27 +- src/commands/project.test.ts | 268 ++++++++++++- src/commands/project.ts | 376 +++++++++++++++++- src/commands/test.test.ts | 72 ++++ src/commands/test.ts | 87 +++- src/commands/usage.ts | 6 +- src/index.ts | 2 +- src/lib/agent-targets.test.ts | 4 +- src/lib/bundle.test.ts | 6 +- src/lib/credentials.test.ts | 14 +- src/lib/http.ts | 6 +- src/lib/skill-nudge.test.ts | 18 +- src/version.ts | 2 +- test/__snapshots__/help.snapshot.test.ts.snap | 52 ++- test/cli.subprocess.test.ts | 12 +- test/helpers/hermetic-env.ts | 41 ++ vitest.config.ts | 3 + 33 files changed, 1696 insertions(+), 196 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/ci-nudge.yml create mode 100644 .github/workflows/pr-triage.yml create mode 100644 .github/workflows/stale.yml create mode 100644 test/helpers/hermetic-env.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..468c8ed --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Check out all text files with LF on every platform. Tests compare bytes +# from checked-out files (skill templates, snapshots); a CRLF working tree +# (core.autocrlf on Windows) broke those comparisons. +* text=auto eol=lf + +*.png binary diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c490f9b..1c165f6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,7 +12,9 @@ agree on the approach before you invest time — see CONTRIBUTING.md. ## Related issue - + ## Type of change diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml new file mode 100644 index 0000000..d01ede3 --- /dev/null +++ b/.github/workflows/ci-nudge.yml @@ -0,0 +1,129 @@ +# "Fix this and we merge" nudge (InsForge `agent-zhang-beihai` pattern, part 3). +# When a PR's CI finishes red, posts ONE sticky comment listing exactly which +# jobs failed and the local one-liner that reproduces/fixes each (the automated +# version of the hand-written "run `npm run format` and you're green" review +# comments). The comment flips to a green confirmation once all checks pass. +# +# Runs in base-repo context via workflow_run — no PR code is checked out, so +# fork PRs are safe. State is recomputed from check-runs each time, so the two +# CI workflows (CI + Test Coverage) can complete in any order. +# +# Bot identity: same App-token-first / GITHUB_TOKEN-fallback pattern as +# pr-triage.yml (the App needs the "Pull requests: Read & write" permission to +# post as testsprite-hob[bot]; until then comments come from github-actions[bot]). +name: CI failure nudge + +on: + workflow_run: + workflows: ['CI', 'Test Coverage'] + types: [completed] + +permissions: + checks: read # read the head SHA's check-run state + pull-requests: write + issues: write # PR comments ride the issues API + +env: + MARKER: '' + +jobs: + nudge: + # Public repo only; PR-triggered runs only (pushes to main have no PR to nudge). + if: >- + github.repository == 'TestSprite/testsprite-cli' && + github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + with: + script: | + const { owner, repo } = context.repo; + const run = context.payload.workflow_run; + const marker = process.env.MARKER; + + const appClient = process.env.APP_TOKEN + ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + : null; + async function write(fn) { + if (appClient) { + try { return await fn(appClient.rest); } + catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } + } + return await fn(github.rest); + } + + // Resolve the PR for this run. `workflow_run.pull_requests` is empty + // for fork PRs, so fall back to the commit→PRs lookup. + let pr = (run.pull_requests || [])[0]; + if (!pr) { + const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit( + { owner, repo, commit_sha: run.head_sha }); + pr = data.find(p => p.state === 'open'); + } else { + pr = (await github.rest.pulls.get({ owner, repo, pull_number: pr.number })).data; + } + if (!pr || pr.state !== 'open' || pr.user.type === 'Bot') return; + + // Job name → the local command that reproduces/fixes it. Also the + // allowlist of CI jobs this nudge watches — keep in sync with ci.yml. + const FIX = { + 'Lint & Format': 'run `npm run lint:fix && npm run format`, then commit', + 'Typecheck': 'run `npm run typecheck` and fix the reported type errors', + 'Unit Tests': 'run `npm test` and fix the failing tests', + 'Build': 'run `npm run build` and fix the compile errors', + 'Local E2E Tests': 'run `npm run test:e2e` (it builds first)', + 'Coverage (>= 80%)': 'run `npm run test:coverage` — new code needs tests until every metric is back at 80%', + }; + + // Recompute full CI state from this SHA's check runs, narrowed to the + // CI jobs above — other workflows (e.g. pr-triage) also attach + // github-actions check runs to the PR head and must not count here. + const checks = await github.paginate(github.rest.checks.listForRef, + { owner, repo, ref: run.head_sha, per_page: 100 }); + const ours = checks.filter(c => c.app && c.app.slug === 'github-actions' && FIX[c.name]); + const failing = ours.filter(c => ['failure', 'timed_out'].includes(c.conclusion)); + const pending = ours.filter(c => c.status !== 'completed'); + + const comments = await github.paginate(github.rest.issues.listComments, + { owner, repo, issue_number: pr.number, per_page: 100 }); + const sticky = comments.find(c => (c.body || '').includes(marker)); + + if (failing.length === 0) { + // Only speak up on success if we previously flagged a failure, and + // only once everything has actually finished. + if (sticky && pending.length === 0 && !sticky.body.includes('all green')) { + await write(rest => rest.issues.updateComment({ owner, repo, comment_id: sticky.id, + body: `${marker}\n✅ CI is **all green** now — thanks, @${pr.user.login}!` })); + } + return; + } + + const lines = failing + .sort((a, b) => a.name.localeCompare(b.name)) + .map(c => { + const fix = FIX[c.name] || 'see the logs for details'; + return `- **${c.name}** — ${fix} ([logs](${c.html_url}))`; + }); + const body = `${marker}\nThanks, @${pr.user.login}! CI is red on this PR — ` + + `here's what failed and how to reproduce it locally:\n\n${lines.join('\n')}\n\n` + + `Everything runs on Node 22 after \`npm ci\`. Push a fix and this comment ` + + `flips green automatically once all checks pass.`; + + if (sticky) { + if (sticky.body !== body) { + await write(rest => rest.issues.updateComment( + { owner, repo, comment_id: sticky.id, body })); + } + } else { + await write(rest => rest.issues.createComment( + { owner, repo, issue_number: pr.number, body })); + } diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 63323c0..6b6ce62 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -1,13 +1,15 @@ # P1-2 issue auto-assign + 3-slot-cap bot (InsForge `agent-zhang-beihai` pattern). -# Posts as the `alfheim-agent` GitHub App ⇒ `alfheim-agent[bot]`. +# Posts as the `testsprite-hob` GitHub App ⇒ `testsprite-hob[bot]`. # # Setup (one-time, by an org admin): -# 1. Create a GitHub App named `alfheim-agent` (org Settings → Developer settings → +# 1. Create a GitHub App named `testsprite-hob` (org Settings → Developer settings → # GitHub Apps → New). Permissions: Issues = Read & write, Metadata = Read. No webhook. # Generate a private key; install the App on the public testsprite-cli repo. # 2. Add two repo (or org) secrets: -# ALFHEIM_AGENT_APP_ID = the App's numeric App ID -# ALFHEIM_AGENT_PRIVATE_KEY = the App's .pem private key (full contents) +# TESTSPRITE_HOB_APP_ID = the App's numeric App ID +# TESTSPRITE_HOB_PRIVATE_KEY = the App's .pem private key (full contents) +# (The ALFHEIM_AGENT_* fallbacks are this App's original secret names from +# before it was renamed — same App ID + key; either naming works.) # Until those exist, the bot gracefully falls back to github-actions[bot] (still works). # # Fires on each new issue comment; assigns the commenter when they claim an issue @@ -30,23 +32,26 @@ env: jobs: triage: # issues only (issue_comment also fires on PRs), and never react to a bot's own - # comment — incl. our own alfheim-agent[bot], which (unlike GITHUB_TOKEN) would + # comment — incl. our own testsprite-hob[bot], which (unlike GITHUB_TOKEN) would # otherwise re-trigger this workflow. `type == 'Bot'` covers both bot identities. if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' }} runs-on: ubuntu-latest steps: - # Mint a token for the alfheim-agent App so comments post as alfheim-agent[bot]. + # Mint a token for the testsprite-hob App so comments post as testsprite-hob[bot]. # continue-on-error: until the App + secrets exist this no-ops and we fall back below. - id: app-token continue-on-error: true uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - app-id: ${{ secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # Scope the minted token to what this job uses (issues API only) instead + # of inheriting the App's full installation permissions. + permission-issues: write - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 with: - # App token when available (→ alfheim-agent[bot]); else the default + # App token when available (→ testsprite-hob[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. github-token: ${{ steps.app-token.outputs.token || github.token }} script: | diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml new file mode 100644 index 0000000..a9cd23d --- /dev/null +++ b/.github/workflows/pr-triage.yml @@ -0,0 +1,189 @@ +# PR-side twin of issue-triage.yml (InsForge `agent-zhang-beihai` pattern, part 2). +# Enforces the issue-first workflow on community PRs: every non-docs PR must +# carry a closing link ("Closes #123") to an issue that is ASSIGNED to the PR +# author (claimed via `/assign`, which issue-triage.yml handles). Violations +# get a `needs-issue` label + a sticky comment, and — for PRs opened after +# GATE_SINCE — a failing check, so unclaimed work is visibly not review-ready. +# PRs opened before GATE_SINCE are grandfathered: nudge only, never a red check. +# +# Assignment happens on the ISSUE, so it cannot re-trigger this PR workflow; +# the comment tells the contributor to edit the PR description or push a +# commit (`edited` / `synchronize`) to re-run the gate. +# +# Runs with NO checkout — metadata-only, so it is safe under pull_request_target +# (which is required for fork PRs to get a write-capable token). +# +# Bot identity: posts as `testsprite-hob[bot]` when the App token works. The +# App currently has Issues R/W + Metadata only — commenting/labeling a PULL +# REQUEST needs the "Pull requests: Read & write" App permission (issues-API +# endpoints are permission-checked by target type). Until an org admin adds +# that permission and re-approves the installation, every call gracefully falls +# back to the default GITHUB_TOKEN and posts as github-actions[bot]. +# Secrets: TESTSPRITE_HOB_* preferred; the ALFHEIM_AGENT_* fallbacks are the +# original names from before the App was renamed (same App ID + key). +name: PR triage (issue-link gate) + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +permissions: + pull-requests: write # add/remove the needs-issue label + issues: write # create/update the nudge comment (PR comments ride the issues API) + +env: + LABEL: 'needs-issue' + MARKER: '' + # PRs created before this instant are grandfathered (nudge, no failing check). + GATE_SINCE: '2026-07-04T00:00:00Z' + +jobs: + gate: + # Public repo only — this file also lives in the private mirror, where PRs + # are internal work that never links public issues. Skip bot authors + # (dependabot etc.), maintainers, and docs-only changes (CONTRIBUTING + # exempts docs/small fixes from the issue-first ask). + if: >- + github.repository == 'TestSprite/testsprite-cli' && + github.event.pull_request.state == 'open' && + github.event.pull_request.user.type != 'Bot' && + !contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) && + !startsWith(github.event.pull_request.title, 'docs') + runs-on: ubuntu-latest + steps: + # Mint an App token so actions post as testsprite-hob[bot]. Until the App + + # secrets + Pull-requests permission exist this no-ops / gets 403 and the + # script below falls back to the default token per-call. + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + with: + # `github` client = default GITHUB_TOKEN: used for all reads (the App + # can't read PRs without the Pull-requests permission) and as the + # write fallback. App client (when mintable) is preferred for writes. + script: | + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + const label = process.env.LABEL; + const marker = process.env.MARKER; + const author = pr.user.login; + + const appClient = process.env.APP_TOKEN + ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + : null; + // Prefer the App identity; fall back to github-actions[bot] when the + // App is absent or lacks the Pull-requests permission (403/404). + async function write(fn) { + if (appClient) { + try { return await fn(appClient.rest); } + catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } + } + return await fn(github.rest); + } + + // Linked issues: GitHub-computed closing references carry number + + // assignees in one query. + const gql = await github.graphql( + `query($owner:String!,$repo:String!,$num:Int!){ + repository(owner:$owner,name:$repo){ + pullRequest(number:$num){ + closingIssuesReferences(first:10){ + nodes{ number assignees(first:10){ nodes{ login } } } + } + } + } + }`, { owner, repo, num: pr.number }); + const linkedIssues = gql.repository.pullRequest.closingIssuesReferences.nodes + .map(n => ({ number: n.number, assignees: n.assignees.nodes.map(a => a.login) })); + + // Body-text fallback for the brief window before GitHub computes the + // link (same-repo bare "#N" references only). + const bodyRefs = [...(pr.body || '') + .matchAll(/(close[sd]?|fix(e[sd])?|resolve[sd]?)\s*:?\s+#(\d+)/gi)] + .map(m => Number(m[3])) + .filter(n => !linkedIssues.some(i => i.number === n)) + .slice(0, 5); + for (const num of bodyRefs) { + try { + const { data } = await github.rest.issues.get({ owner, repo, issue_number: num }); + if (data.pull_request) continue; // "#N" pointed at a PR, not an issue + linkedIssues.push({ number: num, assignees: (data.assignees || []).map(a => a.login) }); + } catch (e) { /* unknown number — ignore */ } + } + + const linked = linkedIssues.length > 0; + const assignedToAuthor = linkedIssues.some(i => i.assignees.includes(author)); + const state = assignedToAuthor ? 'ok' : (linked ? 'unassigned' : 'unlinked'); + + const hasLabel = (pr.labels || []).some(l => l.name === label); + const comments = await github.paginate(github.rest.issues.listComments, + { owner, repo, issue_number: pr.number, per_page: 100 }); + const nudge = comments.find(c => (c.body || '').includes(marker)); + const stateLine = ``; + + const contributingUrl = + `https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#contribution-model`; + const rerunHint = 'After fixing it, edit the PR description or push a commit to re-run this check.'; + + let body; + if (state === 'ok') { + body = `${marker}\n${stateLine}\n` + + `✅ This PR is linked to an issue assigned to @${author} — thanks! ` + + `The \`${label}\` label has been removed.`; + } else if (state === 'unassigned') { + const list = linkedIssues.map(i => { + const holders = i.assignees.filter(a => a !== author); + return `#${i.number}` + (holders.length ? ` (currently assigned to @${holders.join(', @')})` : ' (unassigned)'); + }).join(', '); + body = `${marker}\n${stateLine}\n` + + `Thanks for the PR, @${author}! It links an issue, but that issue isn't assigned to you yet: ${list}. ` + + `Per our workflow, **claim the issue first by commenting \`/assign\` on it** (the triage bot assigns you automatically). ` + + `If it's already assigned to someone else, please coordinate with them or pick another issue — ` + + `unclaimed-issue PRs are not reviewed. ${rerunHint} ` + + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; + } else { + body = `${marker}\n${stateLine}\n` + + `Thanks for the PR, @${author}! A quick note on our workflow: for **features and behavior changes** ` + + `we require contributors to **open an issue first, claim it by commenting \`/assign\` on the issue, ` + + `then submit a PR that links it** (e.g. \`Closes #123\`). This PR isn't linked to any issue yet, ` + + `so it is not review-ready. ${rerunHint} ` + + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; + } + + // Sticky comment: create once, update when the state changes. + if (!nudge) { + if (state !== 'ok') { + await write(rest => rest.issues.createComment( + { owner, repo, issue_number: pr.number, body })); + } + } else if (!nudge.body.includes(stateLine)) { + await write(rest => rest.issues.updateComment( + { owner, repo, comment_id: nudge.id, body })); + } + + // Label tracks the gate state. + if (state === 'ok' && hasLabel) { + await write(rest => rest.issues.removeLabel( + { owner, repo, issue_number: pr.number, name: label })).catch(() => {}); + } + if (state !== 'ok' && !hasLabel) { + await write(rest => rest.issues.addLabels( + { owner, repo, issue_number: pr.number, labels: [label] })); + } + + // Hard gate for PRs opened after the cutoff; older PRs are nudged only. + const gated = new Date(pr.created_at) >= new Date(process.env.GATE_SINCE); + if (state !== 'ok' && gated) { + core.setFailed(state === 'unlinked' + ? 'No closing-linked issue. Open/claim an issue, add "Closes #" to the PR description, then re-run.' + : 'Linked issue is not assigned to the PR author. Comment /assign on the issue, then re-run.'); + } else if (state !== 'ok') { + core.notice(`Grandfathered PR (opened before ${process.env.GATE_SINCE}): gate not enforced, nudge only.`); + } diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..fbaa607 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,64 @@ +# P1-3 — stale-bot: nudge then close inactive issues / PRs (actions/stale). +# +# Generous windows for an open-source repo (first-response SLA is 5 business days, +# see CONTRIBUTING). Activity removes the `stale` label automatically, so a single +# reply resets the clock. Newcomer- and security-relevant work is exempt from +# closing. `good first issue` / `help wanted` stay open for whoever picks them up. +# +# This is a SYNCED asset (ships to the public mirror). It is scheduled, so unlike +# the event-gated triage bot it is fenced to the PUBLIC repo with a repository +# guard — on private atlas it is a no-op (no nagging internal issues/PRs). +name: Stale + +on: + schedule: + - cron: '30 1 * * *' # daily 01:30 UTC + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + stale: + if: ${{ github.repository == 'TestSprite/testsprite-cli' }} + runs-on: ubuntu-latest + permissions: + issues: write # comment + (un)label + close stale issues + pull-requests: write # comment + (un)label + close stale PRs + steps: + - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 + with: + # ── issues ────────────────────────────────────────────────────── + days-before-issue-stale: 60 + days-before-issue-close: 14 + stale-issue-label: stale + stale-issue-message: > + This issue has had no activity for 60 days, so it's been marked + `stale`. If it's still relevant, just leave a comment (or remove the + `stale` label) and we'll keep it open — otherwise it will be closed + in 14 days. Thanks for helping us keep the tracker tidy! + close-issue-message: > + Closing as `stale` after no activity. This isn't a judgement on the + idea — please reopen or open a fresh issue if it's still relevant. + + # ── pull requests (more grace — external contributors may be slow) ─ + days-before-pr-stale: 45 + days-before-pr-close: 21 + stale-pr-label: stale + stale-pr-message: > + This PR has had no activity for 45 days, so it's been marked `stale`. + Push a commit or leave a comment to keep it open — otherwise it will + be closed in 21 days. We'd still love to merge it; ping a maintainer + if you're blocked on a review. + close-pr-message: > + Closing as `stale` after no activity. Reopen any time you can pick it + back up — your work isn't lost. + + # ── shared behaviour ──────────────────────────────────────────── + exempt-issue-labels: 'pinned,security,in-progress,good first issue,help wanted,hackathon' + exempt-pr-labels: 'pinned,security,in-progress,hackathon' + exempt-draft-pr: true + remove-stale-when-updated: true + ascending: true # oldest first — fairest under the per-run op cap + operations-per-run: 60 + enable-statistics: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f1a0a8..e78347e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,48 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +## [0.3.0] - 2026-07-08 + ### Added -- **JUnit XML report export for batch `--wait` runs.** `test run --all` and batch `test rerun` (`--all` or multiple test ids) accept `--report junit --report-file ` to write a CI-friendly XML sidecar after polling completes. `--output json` is unchanged; the report is written even when the batch exits non-zero. `--dry-run` writes a canned sample without network calls. -- **`testsprite test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs `, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal OFF (strict verbatim) so a healed drift can't mask a nondeterministic pass/fail. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness (`testsprite test flaky --runs 5 || exit 1`). Flags: `--runs ` (1–10), `--until-fail` (stop at the first non-passing attempt), `--timeout ` (per-attempt), and `--output json` for a machine-readable stability report. Frontend replays are free verbatim script replays; a one-line advisory is printed for backend tests, whose closure reruns may cost credits. +- **`testsprite doctor`** — one-command environment diagnostic that checks your Node version, credentials, endpoint reachability, and installed agent skills, and reports what's misconfigured. +- **`test scaffold`** — emit a schema-correct starter plan (frontend) or a backend test skeleton to bootstrap a new test without hand-writing the JSON. +- **`test lint`** — offline validator for plan / steps files; catches malformed test definitions before they are sent to the server. +- **`test diff `** — compare two runs of the same test to isolate what changed between a passing and a failing run. +- **`test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs`, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal off (strict verbatim) so a nondeterministic pass/fail can't be masked. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness. Flags: `--runs` (1–10), `--until-fail`, `--timeout`, `--output json`. +- **JUnit XML report export for batch runs.** `test run --all` and batch `test rerun` accept `--report junit --report-file ` to write a CI-friendly XML sidecar after `--wait` polling completes. The report is written even when the batch exits non-zero; `--output json` is unchanged; `--dry-run` writes a canned sample without network calls. +- **`test wait` is now variadic** — pass several run ids to attach to and poll multiple runs in a single invocation. +- **`agent status`** — report which TestSprite skills are installed for each agent target and whether they are current; installed skills are now stamped with a version/hash marker. +- **New `agent install` targets:** GitHub Copilot, Windsurf, and Kiro (experimental), alongside the existing Claude / Cursor / Cline / Codex / Antigravity targets. +- **`project credential` / `project auto-auth`** — configure a project's backend credentials (static credential, free) or a recurring auto-auth token (Pro) from the CLI, with surfaced auth warnings and managed-credential guidance. +- **Proxy support** — the CLI now honors `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` for use behind corporate and CI proxies. +- **`NO_COLOR` support** — colored output is suppressed when `NO_COLOR` is set, per no-color.org. +- **"New version available" notice** — a non-blocking, 24h-cached npm version check prints an upgrade hint on stderr. Opt out with the documented env var; automatically silenced in CI and under `--output json`. + +### Changed + +- **Node.js 20.19+, 22.13+, or 24+ is now the minimum supported runtime.** The CLI checks the running Node version at startup and exits with a clear message on an unsupported version; builds and CI run against Node 20 and 22. +- **Graceful shutdown** — the CLI handles termination signals cleanly and guards against broken-pipe (`EPIPE`) errors when its output is piped to a closing consumer (e.g. `| head`). +- **Interactive prompts and preamble now go to stderr**, keeping stdout pure for machine consumers even in interactive mode. +- **Empty environment variables are treated as unset** when resolving config, so `TESTSPRITE_API_URL=` no longer overrides the built-in default with an empty string. +- `agent install` defaults `--target` to `claude` in non-interactive / CI contexts (matching `setup`). +- The `usage` command no longer implies backend test runs are free. +- `setup`'s "Next steps" guidance no longer suggests `test list` before any project exists. + +### Fixed + +- **Timeouts & polling:** `RequestTimeoutError` is now classified as a timeout in the `--all --wait` fan-out; per-attempt timeout timers are cleared so they can't fire late; `run --all --wait` no longer polls still-queued runs past the shared deadline; a partial result is emitted on stdout when `run --wait` / `test wait` times out (so a redirected file is never zero-byte). +- **Batch rerun:** the exit code is preserved and auth errors escalate correctly; explicit ids combined with `--all` — or `--status` / `--skip-terminal` without `--all` — are rejected with a clear validation error; auto-minted idempotency keys are surfaced under `--output json`. +- **HTTP:** non-JSON `200` responses map to a typed error envelope instead of crashing the parser. +- **Failure bundles / artifacts:** artifact downloads retry on transient errors and guard the default run-id path; the `--out` directory no longer sweeps unrelated pre-existing files (data-loss fix); run-scoped per-step error text and step type are surfaced. +- **Input validation (fail fast, before any network call):** malformed API keys, invalid `--request-timeout`, directory `--code-file` / `--out` paths, blank or whitespace-only `--name` (test and project create/update), blank inline project passwords, fractional pagination flags / page sizes, and `--since` overflow are all rejected up front with `VALIDATION_ERROR` rather than crashing or failing late server-side. `--output` is validated uniformly across all command groups. +- **Setup / auth:** the endpoint is validated before the key check; the typed API-error envelope is preserved when key verification fails; the per-request timeout is honored during `configure`. +- **Misc:** cursor pagination no longer drops empty pages; trailing-dot hostnames are treated as loopback by the local-target guard; buffered input is preserved between interactive prompts; the Codex managed-section skill check requires a complete section; `code get` strips a leading BOM and rejects an empty `--out`. + +### Security + +- **INI injection:** CR/LF characters are stripped from credential values before they are written to `~/.testsprite/credentials`. +- **Symlink fail-close:** the own-file `agent install` path applies its symlink containment guard under `--dry-run` as well, so a planted symlink cannot place or clobber files outside `--dir`. ## [0.2.0] - 2026-06-29 @@ -149,6 +187,9 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for - Commander `help [command]` exits 0 (previously exited 5 on `test help` / `project help`). -[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...HEAD +[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.2...v0.2.0 +[0.1.2]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/TestSprite/testsprite-cli/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2110f0b..bed91fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,14 +20,54 @@ Discussions**, so the issue tracker stays a clean, actionable list. ## Contribution model -- **Small fixes and improvements:** just open a pull request. No issue required. -- **Large or breaking changes** (new commands, changed flags/output, new - dependencies, refactors): **open an issue first** to discuss the design. This - avoids wasted work on something we'd ask you to rework or that's out of scope. +- **Docs and small fixes** (typos, doc corrections, comment-only changes): + just open a pull request. No issue required (linking one is still + appreciated). +- **Features and behavior changes** (new commands or flags, changed output, + new dependencies, refactors) follow **issue-first**: + 1. Find an existing issue, or open a new one describing the change. + 2. Claim it by commenting `/assign` on the issue — the literal slash + command on its own line. The triage bot assigns you automatically; + free-text requests ("can I take this?") are **not** detected. + 3. If the issue is new, wait for triage — we check proposals against + [VISION.md](./VISION.md) and the [standing policies](#standing-scope-policies) + below before any code is written, so you don't invest in something + we'd ask you to rework or decline. + 4. Open your PR with a closing link (e.g. `Closes #123`) in the + description. +- **PR gate:** a bot checks every non-docs community PR for a closing-linked + issue that is **assigned to the PR author**. PRs that don't meet this get + the `needs-issue` label and a failing `PR triage` check, and **are not + reviewed** until it's fixed — file or claim the issue, add the closing + link, then edit the PR description (or push a commit) to re-run the check. +- Suspected **security vulnerabilities** are the exception to "file an + issue": report them privately per [SECURITY.md](./SECURITY.md) instead. - We **do** accept community code contributions — this is an actively maintained open-source CLI, not a read-only distribution mirror. - See [VISION.md](./VISION.md) for what is in and out of scope. +### Standing scope policies + +Pre-decided policies, so proposals don't have to relitigate them: + +- **Runtime dependencies are budgeted.** The CLI ships with a deliberately + tiny runtime dependency set (`commander`, `valibot`, plus `undici` — + approved for `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` support). Any new + runtime dependency needs explicit maintainer sign-off **in the issue, + before the PR**. Utility modules land only together with the consumer + that uses them — standalone libraries are declined. +- **`agent install` targets.** Shipped: `claude`, `antigravity`, `cursor`, + `cline`, `codex`, `kiro`, `windsurf`, `copilot`. Accepted and in progress: + `gemini`. + A proposal for a new target needs (1) the editor's official rules/skill + file mechanism, documented, and (2) the proposer prepared to maintain the + target going forward. +- **Outbound network calls.** The CLI talks only to the configured + TestSprite API endpoint. The one approved exception is an opt-out-able + npm registry version check (at most once per 24h, carrying nothing but + the package name, fully silenced by its env opt-out and in CI/JSON/dry-run + modes or when stderr is not a TTY). Any other outbound call is out of scope. + We aim to give every issue and PR a **first response within 5 business days** (best-effort). If something has gone quiet longer than that, a polite nudge on the thread or in Discord is welcome. diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 6b798ef..2f6cd6c 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -15,6 +15,7 @@ The full reference for the TestSprite CLI: install verification, manual setup, e - [Read commands](#read-commands) - [Write commands](#write-commands) - [Run commands](#run-commands) + - [Account & diagnostics](#account--diagnostics) - [Configuration](#configuration) - [Output & scripting](#output--scripting) - [Exit codes](#exit-codes) @@ -117,10 +118,15 @@ testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.m testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md testsprite agent list # list all 8 targets with status + mode + path +testsprite agent status # check installed skills against this CLI version ``` Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). +Omitting `--target` in a non-interactive shell (CI, agent subprocess) defaults to `claude` with an `[info]` note on stderr; in a terminal the CLI prompts (empty answer = `claude`). + +`agent status` checks every installed skill file against the current CLI version and reports one of `ok`, `stale`, `modified`, `unmarked`, `absent`, or `corrupt` per target. It exits `1` when anything needs attention, so `testsprite agent status && …` can gate a CI step; `--dir ` inspects a different project root. + The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. @@ -220,6 +226,15 @@ testsprite test result test_xxxxxxxx --history --source cli --since 7d --output testsprite test result test_xxxxxxxx --history --dry-run --output json ``` +#### `testsprite test diff ` + +Compare two runs of a test and print what regressed: verdict, `failureKind`, `failedStepIndex`, per-step status flips, and `codeVersion` drift. Exit `0` when the verdicts match, `1` when they differ — so a script can assert "this rerun behaves like the last known-good run" in one call. + +```bash +testsprite test diff run_aaaa run_bbbb --output json +testsprite test diff run_aaaa run_bbbb --dry-run --output json +``` + #### `testsprite test failure get ` The latest-failure agent entry point. Returns one consistent snapshot of the latest failing run as a self-contained bundle: the result, the failed step plus its immediate neighbors with screenshots and DOM snapshots, the test source, a video pointer, a root-cause hypothesis, a recommended fix target, and correlation metadata. For the bundle of a _specific_ run an agent just triggered, prefer `test artifact get ` — it is keyed by `runId` and cannot be raced by another run that lands afterward. @@ -253,7 +268,28 @@ testsprite test failure summary test_xxxxxxxx --dry-run --output json ### Write commands -Require the `write:tests` scope. +Require the `write:tests` scope (project commands require `write:projects`), except `test scaffold` and `test lint`, which are pure-local authoring helpers — no network, no credentials, no scope. + +#### `testsprite test scaffold` + +Emit a schema-correct starter test definition — a frontend plan JSON by default, or a backend Python skeleton with `--type backend`. Pure-local: no network, no credentials. Edit the scaffold, then create the test with `--plan-from` / `--code-file`. + +```bash +testsprite test scaffold > first-test.plan.json +testsprite test scaffold --type backend --out tests/health.py +testsprite test scaffold --out plan.json --force # overwrite an existing file +``` + +#### `testsprite test lint` + +Validate plan/steps files offline with the same validators `test create` runs, collecting **every** problem instead of stopping at the first. No network, no credentials. Exit `0` when all inputs are valid, `5` otherwise. + +```bash +testsprite test lint --plan-from ./checkout.plan.json +testsprite test lint --plan-from-dir ./plans/ # every *.json checked, all errors reported +testsprite test lint --plans ./plans.jsonl # one plan spec per line +testsprite test lint --steps ./refined.plan.json # the shape `test plan put` ingests +``` #### `testsprite test create` @@ -292,7 +328,7 @@ testsprite test update test_xxxxxxxx --dry-run --output json #### `testsprite test delete ` / `test delete-batch` -Soft-delete one test (or many). `--confirm` is required; absent it, the CLI exits 5 with a local validation error. +Permanently delete one test (or many) — there is **no restore window**. `--confirm` is required; absent it, the CLI exits 5 with a local validation error. ```bash testsprite test delete test_xxxxxxxx --confirm @@ -322,20 +358,66 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Note the asymmetry: `--description` is **create-only** — `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`, but not `--description`. ```bash testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` +#### `testsprite project credential ` + +Set the **static backend credential** injected into every backend test in the project (free tier). Supported types: `public` (no credential), `"Bearer token"`, `"API key"`, `"basic token"`. + +```bash +testsprite project credential proj_xxxxxxxx --type "Bearer token" --credential-file ./token.txt +testsprite project credential proj_xxxxxxxx --type public +testsprite project credential proj_xxxxxxxx --type "API key" --credential sk-live-... --dry-run --output json +``` + +`--credential ` or `--credential-file ` supplies the value (required unless `--type public`). Prefer `--credential-file` in scripts so the secret never lands in shell history. + +#### `testsprite project auto-auth ` + +Configure the **recurring-token (auto-refresh) login** for backend tests (Pro): a fresh token is fetched on each run and injected into every backend test, so long-lived suites survive token expiry. + +```bash +# Password login: POST the login endpoint, extract the token, inject as a Bearer header +testsprite project auto-auth proj_xxxxxxxx \ + --method password --inject bearer \ + --login-url https://api.example.com/login --login-method POST \ + --login-content-type application/json \ + --login-body-template '{"user":"{{username}}","pass":"{{password}}"}' \ + --username ci@example.com --password-file ./pw.txt \ + --token-path '$.data.accessToken' + +# OAuth refresh-token flow +testsprite project auto-auth proj_xxxxxxxx \ + --method refresh_token --inject header --inject-key X-Auth-Token \ + --token-endpoint https://auth.example.com/oauth/token \ + --client-id my-client --client-secret-file ./secret.txt \ + --refresh-token-file ./refresh.txt --scope api.read + +# AWS Cognito refresh +testsprite project auto-auth proj_xxxxxxxx \ + --method aws_cognito_refresh --inject bearer \ + --client-id my-app-client --refresh-token-file ./refresh.txt --region us-east-1 + +# Turn it off (stored config is kept) +testsprite project auto-auth proj_xxxxxxxx --disable +``` + +Required flags: `--method ` and `--inject ` (`--inject-key ` names the header/cookie when not `bearer`). Method-specific flags: password login uses `--login-url/--login-method/--login-content-type/--login-body-template/--username/--password[-file]/--token-path`; OAuth uses `--token-endpoint/--client-id/--client-secret[-file]/--refresh-token[-file]/--scope`; Cognito adds `--region`. File variants (`--password-file`, `--client-secret-file`, `--refresh-token-file`) keep secrets out of shell history. + ### Run commands Require the `run:tests` scope. #### `testsprite test run ` -Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout` (with a `nextAction` pointing at `test wait ` so an agent can resume). +Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout`. On a timeout the CLI still prints the partial run object (with `runId`) to stdout **before** exiting 7, plus a `nextAction` pointing at `test wait ` — so a script always has the id to resume with, and stdout is never empty. + +`--all --project ` runs every test in the project in wave order. On the current unified engine that means **all tests, frontend and backend**; on the legacy backend-only engine, frontend tests can't run — they are skipped and enumerated in `skippedFrontend` with a stderr advisory. ```bash # Trigger and return immediately @@ -348,7 +430,7 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json -# Batch BE run with JUnit XML for CI (sidecar; --output json unchanged) +# Batch run with JUnit XML for CI (sidecar; --output json unchanged) testsprite test run --all --project proj_xxxxxxxx --wait \ --report junit --report-file ./results.xml --output json @@ -427,16 +509,17 @@ Flags: `--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. -#### `testsprite test wait ` +#### `testsprite test wait ` -Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. +Block until one **or more** runs reach a terminal status. With a single `run-id` the behavior is unchanged: same exit-code matrix as `test run --wait`. With several ids, the runs are polled concurrently under one shared `--timeout` and the CLI prints a `{ results, summary }` envelope — the worst status wins the exit code — so every re-attach hint the CLI prints can be pasted back as one command. `--max-concurrency ` (1–100, default 10) caps concurrent polls. Used to resume polling after a timed-out `--wait`, or when an agent already holds `runId`s from previous invocations. ```bash testsprite test wait run_01hx3z9p8q4k2y7a --timeout 600 --output json +testsprite test wait run_aaaa run_bbbb run_cccc --timeout 900 --output json testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json ``` -Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. #### `testsprite test artifact get ` @@ -451,6 +534,30 @@ testsprite test artifact get run_01hx3z9p8q4k2y7a --dry-run --output json Returns 404 (CLI exit 4) when the run passed (`details.reason: "no_failing_run"`), is still in flight (`run_not_ready`), was cancelled (`cancelled_no_artifacts`), or its test was deleted (`no_code`). +### Account & diagnostics + +#### `testsprite usage` (alias: `testsprite credits`) + +Account pre-flight before a large batch: resolves the active key to its identity (`userId`, `keyId`, `env`) and surfaces the credit balance / plan fields when the backend supplies them. Useful right before a `test run --all` fan-out. + +```bash +testsprite usage --output json +testsprite credits +testsprite usage --dry-run --output json +``` + +#### `testsprite doctor` + +One-shot environment diagnostic. Runs a fixed checklist — CLI version, Node.js runtime, active profile, API endpoint, credentials, live connectivity + key validity (`GET /me`), and whether the verify skill is installed in the current project — and prints an OK/WARN/FAIL report. Exits non-zero only when a check **fails** (warnings, e.g. skill not installed, don't fail the process), so it can gate a CI step or an agent preflight: + +```bash +testsprite doctor +testsprite doctor --output json +testsprite doctor && testsprite test run test_xxxxxxxx --wait +``` + +Every check reuses the same helpers the real commands use, so the report reflects exactly what a subsequent command would resolve. + ## Configuration ### Profiles & credentials @@ -473,14 +580,17 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ------------------------------- | --------------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | -| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | -| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| Variable | Purpose | +| ----------------------------------------- | ---------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support — API traffic is routed through the configured proxy | +| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | +| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | ### Update notice @@ -497,13 +607,14 @@ only outbound call the CLI makes besides your configured API endpoint. API-key scopes gate the write and run surfaces: -| Scope | Required by | -| --------------- | -------------------------------------------------------------------- | -| `read:me` | `auth whoami` | -| `read:projects` | `project list / get` | -| `read:tests` | every `test *` read command | -| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | -| `run:tests` | `test run / rerun / wait / artifact get` | +| Scope | Required by | +| ---------------- | -------------------------------------------------------------------- | +| `read:me` | `auth status`, `usage`, `doctor` (connectivity check) | +| `read:projects` | `project list / get` | +| `read:tests` | every `test *` read command | +| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | +| `write:projects` | `project create / update / credential / auto-auth` | +| `run:tests` | `test run / rerun / flaky / wait / artifact get` | New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. @@ -521,20 +632,25 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not ## Exit codes -| Code | Meaning | -| ---- | --------------------------------------- | -| `0` | Success | -| `1` | Generic failure / non-passed run status | -| `2` | Not yet implemented | -| `3` | Auth error | -| `4` | Not found | -| `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | -| `7` | Timeout / unsupported | -| `10` | Service unavailable | -| `11` | Rate limited (retriable) | -| `12` | Insufficient credits (non-retriable) | -| `13` | Feature gated (paid plan required) | +| Code | Meaning | +| --------------------- | --------------------------------------------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | +| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | + +### Signals & pipes + +On SIGINT (Ctrl-C), SIGTERM, or SIGHUP the CLI prints `Interrupted (). Any run already started keeps executing on the server; check it with 'testsprite test list' or 'testsprite test wait '.` and exits `128 + signal`. **Ctrl-C does not cancel the server-side run** — execution (and any credit spend) continues; there is no cancel command today, so re-attach with `test wait ` instead of re-triggering. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. ## Design principles diff --git a/README.md b/README.md index 404f237..78d293a 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude > **Pointing a coding agent (Claude Code, Cursor, Codex, Cline, …) at TestSprite?** Have it run `testsprite setup` first — that installs the verification skill, so the agent knows how to create, run, and triage tests on its own (instead of guessing from this README). New here? Start with the **[getting-started overview](https://docs.testsprite.com/cli/getting-started/overview)**. +> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). + From there, the loop runs on its own — an example session, typed by the coding agent: ```bash @@ -89,28 +91,34 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | +| Group | Command | What it does | +| --------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | +| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | +| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata / permanently delete (no restore window; `--confirm` required) | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` | Manage projects | +| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | +| | `test wait` | Block on one or more `runId`s until terminal | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/package.json b/package.json index 341e713..e78ce17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.2.0", + "version": "0.3.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", diff --git a/skills/testsprite-onboard.skill.md b/skills/testsprite-onboard.skill.md index cc9342c..df04c18 100644 --- a/skills/testsprite-onboard.skill.md +++ b/skills/testsprite-onboard.skill.md @@ -91,6 +91,27 @@ Each file is a COMPLETE plan and must include `projectId` (from step 2), `type: **Backend** — one `.py` file per endpoint, using `requests` with concrete assertions on status code and response body. +**Backend auth — read the injected `__AUTH_HEADERS__`, NEVER hardcode any credential.** This +covers **every** secret the API needs — Bearer/JWT tokens **and** API keys (`sk-…`, +`x-api-key`), basic-auth blobs, cookies. TestSprite prepends a managed credential block +(`__AUTH_CREDENTIAL__` / `__AUTH_TYPE__` / `__AUTH_HEADERS__`) to every backend test from the +project's Authentication settings, and `__AUTH_HEADERS__` already holds the right header(s) for +the configured type (Bearer → `{"Authorization": "Bearer …"}`; API key → `{"X-API-Key": "…"}`; +basic → `{"Authorization": "Basic …"}`). Spread it into your request headers — never paste a +literal `Bearer …` / `sk-…` / key value into the script: + +```python +r = requests.get(f"{TARGET_URL}/orders", headers={**__AUTH_HEADERS__}) +``` + +Configure the credential once on the project (ask the user for the value — never invent it or +reuse a key you happened to see): a static credential with +`testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential `, +or an auto-refreshing login with `testsprite project auto-auth …` so scheduled/repeat +runs keep working after the token expires. A hardcoded token expires within hours and a hardcoded +key can't be rotated centrally — `test create` emits a `[warn]` on an inlined credential; treat it +as a must-fix. + **Assertion rule (this is the whole game for FE):** every `assertion` step must name a **concrete, observable** outcome — an element, text, URL, count, or status. Never write `"verify it works"`, `"check the page loads"`, or other narrative that an AI judge can diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index 650a88b..fe7c661 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -165,6 +165,39 @@ only the Python **standard library + `requests` + `pytest` + `numpy` + `scipy`** - Get values from the API's responses (and captured variables), not by importing and calling the app's internals. +**Authentication — read the injected credential, NEVER hardcode any credential.** This +applies to **every** secret the API needs — Bearer/JWT tokens **and** API keys, basic-auth +blobs, session cookies. Do not paste a literal `Bearer …`, `sk-…`, `x-api-key` value, or any +other credential into the test. Before your script runs, TestSprite prepends a managed +credential block built from the project's Authentication settings, and `__AUTH_HEADERS__` +already contains the right header(s) for the configured auth type: + +```python +# Auto-injected credentials — do not modify +__AUTH_CREDENTIAL__ = "..." +__AUTH_TYPE__ = "Bearer token" # or "API key" / "basic token" / "public" +__AUTH_HEADERS__ = {"Authorization": "Bearer ..."} # API key → {"X-API-Key": "..."}; basic → {"Authorization": "Basic ..."} +``` + +Spread `__AUTH_HEADERS__` into every authenticated request — it adapts to whatever auth type +the project is configured for, so the same line works for Bearer, API-key, or basic auth: + +```python +r = requests.get(f"{TARGET_URL}/profile", headers={**__AUTH_HEADERS__}) +``` + +Configure the credential **once on the project** (ask the user for the value — never invent +or reuse a key you happened to see), and the block stays correct + refreshable: + +- **Bearer / API key / basic (static):** + `testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential ` +- **Auto-refreshing login (recurring token):** `testsprite project auto-auth …` + +A hardcoded token expires within hours (and a hardcoded key can't be rotated centrally), so +the test breaks on later runs; the managed block is rewritten with a fresh value each run. +`test create` emits a `[warn]` when it detects an inlined credential literal — treat that as +a must-fix, not a nuisance. + **Backend tests that share state declare dependencies at create time.** For a one-off verification, prefer a single self-contained script (log in inside the same file). But when the coverage set splits naturally into producer → consumer diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 4f5fdd3..24d8ec6 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -661,29 +661,39 @@ describe('runInstall — multi-target', () => { // --------------------------------------------------------------------------- describe('runInstall — empty target', () => { - it('non-TTY with no target throws exit 5', async () => { - const { fs: agentFs } = makeMemFs(); + it('non-TTY with no target defaults to claude and installs the skill file', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: [], + force: false, + }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); + + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + expect(store.has(claudeAbs)).toBe(true); + expect(capture.stderr.join('\n')).toContain('defaulting to claude'); + }); + + it('non-TTY default writes the canonical claude content', async () => { + const { store, fs: agentFs } = makeMemFs(); const { deps } = makeCapture(); - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: [], - force: false, - }, - { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, - ); - } catch (err) { - thrown = err; - } + await runInstall( + { profile: 'default', output: 'text', debug: false, dryRun: false, target: [], force: false }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); - expect(thrown).toBeInstanceOf(ApiError); - expect((thrown as ApiError).exitCode).toBe(5); + const { path: relPath, content } = renderForTarget('claude', 'testsprite-verify'); + const abs = path.resolve(CWD, relPath); + expect(store.get(abs)).toBe(content); }); it('TTY with injected prompt returning "claude" installs claude', async () => { diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 0a40c4b..7d7e7c8 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -371,18 +371,19 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (rawTargets.length === 0) { const isTTY = deps.isTTY ?? Boolean(process.stdin.isTTY); if (!isTTY) { - throw localValidationError( - 'target', - `required; pass --target=claude (comma-separated or repeated for several). Supported: ${Object.keys(TARGETS).join(', ')}`, + stderrFn( + '[info] --target not specified; defaulting to claude. Pass --target= to select a different agent.', ); + resolvedTargetStrings = ['claude']; + } else { + const promptFn = deps.prompt ?? ((q: string) => promptText(q)); + const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); + const defaulted = answer || 'claude'; + resolvedTargetStrings = defaulted + .split(',') + .map(s => s.trim()) + .filter(Boolean); } - const promptFn = deps.prompt ?? ((q: string) => promptText(q)); - const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); - const defaulted = answer || 'claude'; - resolvedTargetStrings = defaulted - .split(',') - .map(s => s.trim()) - .filter(Boolean); } else { resolvedTargetStrings = rawTargets; } diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 75fdff0..617fd6f 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -199,8 +199,14 @@ describe('runInit — happy path (interactive)', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('TestSprite initialized.'); expect(stdout).toContain('profile:'); + // Next steps leads with creating a project; no command that fails without --project. expect(stdout).toContain('Next steps:'); - expect(stdout).toContain('testsprite test list'); + expect(stdout).toContain('testsprite project create --type frontend'); + expect(stdout).toContain('testsprite test run --all --project '); + expect(stdout).toContain('the testsprite-onboard skill is installed'); + // No "current project" wording, no bare test list. + expect(stdout).not.toContain('current project'); + expect(stdout).not.toContain('testsprite test list'); }); it('json mode: emits structured InitSummary object', async () => { @@ -306,6 +312,15 @@ describe('runInit — --no-agent', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('skipped (--no-agent)'); + // --no-agent points at manual test creation; must not claim the skill is installed. + expect(stdout).toContain('Next steps:'); + expect(stdout).toContain('testsprite project create --type frontend'); + expect(stdout).toContain('testsprite test create --project '); + expect(stdout).toContain('testsprite test run --all --project '); + expect(stdout).not.toContain('skill is installed'); + // No "current project" wording, no bare test list. + expect(stdout).not.toContain('current project'); + expect(stdout).not.toContain('testsprite test list'); }); it('text mode with agent: summary contains skills line with both default skills', async () => { diff --git a/src/commands/init.ts b/src/commands/init.ts index aff0644..c2d5678 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -407,12 +407,33 @@ function renderInitText(data: unknown): string { } lines.push(''); lines.push('Next steps:'); - lines.push(' testsprite test list # list tests in the current project'); - lines.push(' testsprite agent list # check installed agent targets'); + lines.push(' # 1. Create your first project (frontend example) — prints a projectId'); + lines.push( + ' testsprite project create --type frontend --name "My App" --url https://your-app.com', + ); + lines.push(''); if (s.agent) { lines.push( - ' testsprite agent install --target= # re-install or install additional targets', + ' # 2. Generate tests: ask your coding agent (the testsprite-onboard skill is installed),', + ); + lines.push(' # or create one yourself, then run them (use the projectId from step 1):'); + lines.push(' testsprite test run --all --project '); + lines.push(''); + lines.push(' # Manage installed agent skills'); + lines.push(' testsprite agent list'); + lines.push( + ' testsprite agent install --target= # re-install or install additional targets', + ); + } else { + lines.push(' # 2. Create a test, then run it (use the projectId from step 1):'); + lines.push(' testsprite test create --project ...'); + lines.push(' testsprite test run --all --project '); + lines.push( + ' # Tip: `testsprite agent install` sets up the onboarding skill for your coding agent', ); + lines.push(''); + lines.push(' # Manage installed agent skills'); + lines.push(' testsprite agent list'); } return lines.join('\n'); diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index f63928c..daea851 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -8,7 +8,9 @@ import { type CliProject, type CliUpdateProjectResponse, createProjectCommand, + runAutoAuth, runCreate, + runCredential, runGet, runList, runUpdate, @@ -72,10 +74,10 @@ describe('createProjectCommand', () => { errorSpy.mockRestore(); }); - it('exposes list, get, create and update subcommands', () => { + it('exposes list, get, create, update, credential and auto-auth subcommands', () => { const project = createProjectCommand(); const names = project.commands.map(c => c.name()).sort(); - expect(names).toEqual(['create', 'get', 'list', 'update']); + expect(names).toEqual(['auto-auth', 'create', 'credential', 'get', 'list', 'update']); }); it('list exposes the pagination flags from the design contract', () => { @@ -327,6 +329,21 @@ describe('runList', () => { }); }); +describe('DEV-244 — project update no longer accepts the dead --description flag', () => { + it('rejects --description on `project update` as an unknown option', async () => { + const project = createProjectCommand(); + const update = project.commands.find(c => c.name() === 'update')!; + project.exitOverride(); + update.exitOverride(); + + await expect( + project.parseAsync(['update', 'proj_x', '--description', 'should not exist'], { + from: 'user', + }), + ).rejects.toThrow(/unknown option.*--description/i); + }); +}); + describe('createProjectCommand --page-size option parser', () => { it('rejects non-numeric --page-size values via commander', async () => { const project = createProjectCommand(); @@ -864,7 +881,6 @@ describe('runUpdate', () => { debug: false, projectId: 'proj_text', name: 'New Name', - description: 'New desc', }, { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, ); @@ -929,3 +945,249 @@ describe('runUpdate', () => { expect(result.updatedFields).toBeUndefined(); }); }); + +describe('runCredential', () => { + interface Captured { + url: string; + method: string; + body: unknown; + headers: Headers; + } + function captureFetch(captured: Captured[], body: unknown) { + return makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { status: 200, body }; + }); + } + + it('PUTs /projects/:id/credential with authType + credential + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured, { + projectId: 'p1', + authType: 'Bearer token', + rewroteCount: 2, + }); + const res = await runCredential( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + authType: 'Bearer token', + credential: 'tok-123', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(res.rewroteCount).toBe(2); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.url).toContain('/projects/p1/credential'); + expect(put.body).toEqual({ authType: 'Bearer token', credential: 'tok-123' }); + expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-cred-[0-9a-f-]{36}$/); + }); + + it('public clears the credential (no credential in body, none required)', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured, { + projectId: 'p1', + authType: 'public', + rewroteCount: 0, + }); + await runCredential( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'public' }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.body).toEqual({ authType: 'public' }); + }); + + it('non-public without --credential → VALIDATION_ERROR (exit 5), no fetch', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runCredential( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'API key' }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); + + it('rejects an unknown --type locally (no fetch)', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runCredential( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + authType: 'jwt', + credential: 'x', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('runAutoAuth', () => { + interface Captured { + url: string; + method: string; + body: Record; + headers: Headers; + } + function captureFetch(captured: Captured[]) { + return makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : {}, + headers: new Headers(init.headers as Record), + }); + return { + status: 200, + body: { projectId: 'p1', enabled: true, method: 'aws_cognito_refresh', inject: 'bearer' }, + }; + }); + } + + it('PUTs /projects/:id/auto-auth with the config body + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'aws_cognito_refresh', + inject: 'bearer', + region: 'us-east-1', + clientId: 'abc', + refreshToken: 'rt-xyz', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.url).toContain('/projects/p1/auto-auth'); + expect(put.body).toEqual({ + enabled: true, + method: 'aws_cognito_refresh', + inject: 'bearer', + region: 'us-east-1', + clientId: 'abc', + refreshToken: 'rt-xyz', + }); + expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-autoauth-[0-9a-f-]{36}$/); + }); + + it('--disable sends enabled:false', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + disable: true, + method: 'password', + inject: 'bearer', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(captured.find(c => c.method === 'PUT')!.body.enabled).toBe(false); + }); + + it('reads a secret from --refresh-token-file', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-rt-')); + const rtFile = join(dir, 'rt.txt'); + writeFileSync(rtFile, ' rt-from-file\n'); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'refresh_token', + inject: 'bearer', + tokenEndpoint: 'https://idp.example.com/token', + refreshTokenFile: rtFile, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(captured.find(c => c.method === 'PUT')!.body.refreshToken).toBe('rt-from-file'); + }); + + it('rejects an unknown --method / --inject locally (no fetch)', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'magic', + inject: 'bearer', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('dogfood 2026-06-30 — whitespace-only --name is rejected (parity with `test create`)', () => { + const noNetwork = () => { + throw new Error('network should not be hit'); + }; + + it('runCreate rejects a whitespace-only --name (exit 5, no network)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runCreate( + { profile: 'default', output: 'json', debug: false, type: 'backend', name: ' ' }, + { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('runUpdate rejects a whitespace-only --name (exit 5, no network)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runUpdate( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', name: '\t \n' }, + { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); diff --git a/src/commands/project.ts b/src/commands/project.ts index ef4d5e8..96f2fc8 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -142,20 +142,16 @@ export async function runCreate( // (exit 10 UNAVAILABLE) — fail fast with a clear exit 5 instead. assertIdempotencyKey(opts.idempotencyKey); - // Reject empty / whitespace-only names so a junk record never reaches the - // backend — matches the `requireString` whitespace guard `test create` uses - // (dogfood P1 fix #1). Without this, `--name " "` passes the action - // handler's `if (!name)` check (a non-empty string is truthy) and is sent - // verbatim, creating a blank-named project. - if (opts.name !== undefined && opts.name.trim().length === 0) { - throw localValidationError('--name must not be empty or whitespace-only'); + // P1-3: client-side length checks matching server limits. + // Whitespace-only / empty rejection (parity with `test create`'s requireString; + // a truthy `--name " "` otherwise creates a blank-named project on the backend). + if (opts.name === undefined || opts.name.trim().length === 0) { + throw localValidationError('--name is required and must not be empty or whitespace-only'); } if (opts.password !== undefined && opts.password.trim().length === 0) { throw localValidationError('--password must not be empty or whitespace-only'); } - - // P1-3: client-side length checks matching server limits. - if (opts.name !== undefined && opts.name.length > 200) { + if (opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } if (opts.description !== undefined && opts.description.length > 2000) { @@ -248,7 +244,6 @@ interface UpdateOptions extends CommonOptions { username?: string; password?: string; passwordFile?: string; - description?: string; instruction?: string; idempotencyKey?: string; } @@ -264,6 +259,8 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. + // Reject a whitespace-only `--name` on update too (parity with create); name + // stays optional here, so only validate when the flag is supplied. if (opts.name !== undefined && opts.name.trim().length === 0) { throw localValidationError('--name must not be empty or whitespace-only'); } @@ -273,10 +270,6 @@ export async function runUpdate( if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } - if (opts.description !== undefined && opts.description.length > 2000) { - throw localValidationError('--description must be at most 2000 characters'); - } - // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); @@ -288,7 +281,6 @@ export async function runUpdate( targetUrl: opts.targetUrl !== undefined, username: opts.username !== undefined, password: passwordSupplied, - description: opts.description !== undefined, instruction: opts.instruction !== undefined, }; const presentFieldNames = Object.entries(mutableFields) @@ -296,7 +288,7 @@ export async function runUpdate( .map(([field]) => field); if (presentFieldNames.length === 0) { throw localValidationError( - 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', + 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, or --instruction.', ); } @@ -336,7 +328,6 @@ export async function runUpdate( targetUrl: opts.targetUrl, username: opts.username, password, - description: opts.description, instruction: opts.instruction, }; const body = Object.fromEntries( @@ -355,6 +346,228 @@ export async function runUpdate( return updated; } +// --------------------------------------------------------------------------- +// project credential — set the static backend credential +// --------------------------------------------------------------------------- + +const CLI_AUTH_TYPES = ['public', 'Bearer token', 'API key', 'basic token'] as const; + +export interface CliProjectCredentialResponse { + projectId: string; + authType: string; + rewroteCount: number; +} + +interface CredentialOptions extends CommonOptions { + projectId: string; + authType: string; + credential?: string; + credentialFile?: string; + idempotencyKey?: string; +} + +export async function runCredential( + opts: CredentialOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + assertIdempotencyKey(opts.idempotencyKey); + + if (!(CLI_AUTH_TYPES as readonly string[]).includes(opts.authType)) { + throw localValidationError(`--type must be one of: ${CLI_AUTH_TYPES.join(', ')}`); + } + + // Resolve the credential value (flag or file). Required for every type + // except `public` (which clears it). + let credential = opts.credential; + if (credential === undefined && opts.credentialFile !== undefined) { + credential = readFileSync(opts.credentialFile, 'utf8').trim(); + } + if (opts.authType !== 'public' && (credential === undefined || credential === '')) { + throw localValidationError( + '--credential (or --credential-file) is required unless --type is "public"', + ); + } + + const body: Record = { authType: opts.authType }; + if (opts.authType !== 'public' && credential !== undefined) body.credential = credential; + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-cred-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + if (opts.dryRun) { + const sample: CliProjectCredentialResponse = { + projectId: opts.projectId, + authType: opts.authType, + rewroteCount: 0, + }; + out.print(sample, data => renderCredentialText(data as CliProjectCredentialResponse)); + return sample; + } + + const client = makeClient(opts, deps); + const res = await client.put( + `/projects/${encodeURIComponent(opts.projectId)}/credential`, + { body, headers: { 'idempotency-key': idempotencyKey } }, + ); + out.print(res, data => renderCredentialText(data as CliProjectCredentialResponse)); + return res; +} + +function renderCredentialText(r: CliProjectCredentialResponse): string { + return [ + `projectId ${r.projectId}`, + `authType ${r.authType}`, + `rewroteCount ${r.rewroteCount}`, + ].join('\n'); +} + +// --------------------------------------------------------------------------- +// project auto-auth — configure the recurring-token (auto-refresh) login +// --------------------------------------------------------------------------- + +const AUTO_AUTH_METHODS = ['password', 'refresh_token', 'aws_cognito_refresh'] as const; +const AUTO_AUTH_INJECTS = ['bearer', 'header', 'cookie'] as const; + +export interface CliProjectAutoAuthResponse { + projectId: string; + enabled: boolean; + method: string; + inject: string; + /** + * Present when the server's trial refresh failed: `enabled` is then `false` + * and this carries the reason (e.g. a bad refresh token). The config is still + * stored, but auto-auth won't run until the login succeeds. + */ + lastRefreshError?: string; +} + +interface AutoAuthOptions extends CommonOptions { + projectId: string; + disable?: boolean; + method: string; + inject: string; + injectKey?: string; + // password method + loginUrl?: string; + loginMethod?: string; + loginContentType?: string; + loginBodyTemplate?: string; + username?: string; + password?: string; + passwordFile?: string; + tokenPath?: string; + // refresh_token method + tokenEndpoint?: string; + clientId?: string; + clientSecret?: string; + clientSecretFile?: string; + refreshToken?: string; + refreshTokenFile?: string; + scope?: string; + // aws_cognito_refresh method + region?: string; + idempotencyKey?: string; +} + +export async function runAutoAuth( + opts: AutoAuthOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + assertIdempotencyKey(opts.idempotencyKey); + + if (!(AUTO_AUTH_METHODS as readonly string[]).includes(opts.method)) { + throw localValidationError(`--method must be one of: ${AUTO_AUTH_METHODS.join(', ')}`); + } + if (!(AUTO_AUTH_INJECTS as readonly string[]).includes(opts.inject)) { + throw localValidationError(`--inject must be one of: ${AUTO_AUTH_INJECTS.join(', ')}`); + } + + // Resolve secrets from --*-file variants so they stay out of shell history. + const password = + opts.password ?? + (opts.passwordFile !== undefined ? readFileSync(opts.passwordFile, 'utf8').trim() : undefined); + const clientSecret = + opts.clientSecret ?? + (opts.clientSecretFile !== undefined + ? readFileSync(opts.clientSecretFile, 'utf8').trim() + : undefined); + const refreshToken = + opts.refreshToken ?? + (opts.refreshTokenFile !== undefined + ? readFileSync(opts.refreshTokenFile, 'utf8').trim() + : undefined); + + const enabled = opts.disable !== true; + const body: Record = { enabled, method: opts.method, inject: opts.inject }; + const maybe = (k: string, v: string | undefined): void => { + if (v !== undefined) body[k] = v; + }; + maybe('injectKey', opts.injectKey); + maybe('loginUrl', opts.loginUrl); + maybe('loginMethod', opts.loginMethod); + maybe('loginContentType', opts.loginContentType); + maybe('loginBodyTemplate', opts.loginBodyTemplate); + maybe('username', opts.username); + maybe('password', password); + maybe('tokenPath', opts.tokenPath); + maybe('tokenEndpoint', opts.tokenEndpoint); + maybe('clientId', opts.clientId); + maybe('clientSecret', clientSecret); + maybe('refreshToken', refreshToken); + maybe('scope', opts.scope); + maybe('region', opts.region); + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-autoauth-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + if (opts.dryRun) { + const sample: CliProjectAutoAuthResponse = { + projectId: opts.projectId, + enabled, + method: opts.method, + inject: opts.inject, + }; + out.print(sample, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); + return sample; + } + + const client = makeClient(opts, deps); + const res = await client.put( + `/projects/${encodeURIComponent(opts.projectId)}/auto-auth`, + { body, headers: { 'idempotency-key': idempotencyKey } }, + ); + out.print(res, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); + return res; +} + +function renderAutoAuthText(r: CliProjectAutoAuthResponse): string { + const lines = [ + `projectId ${r.projectId}`, + `enabled ${r.enabled}`, + `method ${r.method}`, + `inject ${r.inject}`, + ]; + if (r.lastRefreshError) { + lines.push(`lastRefreshError ${r.lastRefreshError}`); + } + // A disabled result after a write means the trial login failed — call it out + // so the user doesn't assume auto-auth is live. + if (!r.enabled) { + lines.push( + 'note auto-auth was stored but is DISABLED — the trial login failed. Fix the credentials (e.g. a valid refresh token) and re-run.', + ); + } + return lines.join('\n'); +} + export function createProjectCommand(deps: ProjectDeps = {}): Command { const project = new Command('project').description('Manage TestSprite projects'); @@ -448,7 +661,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--username ', 'new auth username') .option('--password ', 'new auth password') .option('--password-file ', 'read new password from file') - .option('--description ', 'new description') .option('--instruction ', 'new FE plan-gen instruction hint') .option( '--idempotency-key ', @@ -465,7 +677,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { username: cmdOpts.username, password: cmdOpts.password, passwordFile: cmdOpts.passwordFile, - description: cmdOpts.description, instruction: cmdOpts.instruction, idempotencyKey: cmdOpts.idempotencyKey, }, @@ -473,6 +684,99 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { ); }); + project + .command('credential ') + .description( + 'Set the static backend credential injected into every backend test\n' + + '(Bearer token / API key / Basic token / public). Free tier.', + ) + .requiredOption('--type ', 'public | "Bearer token" | "API key" | "basic token"') + .option('--credential ', 'credential value (required unless --type public)') + .option('--credential-file ', 'read the credential value from a file') + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: CredentialFlagOpts, command: Command) => { + await runCredential( + { + ...resolveCommonOptions(command), + projectId, + authType: cmdOpts.type, + credential: cmdOpts.credential, + credentialFile: cmdOpts.credentialFile, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + project + .command('auto-auth ') + .description( + 'Configure the recurring-token (auto-refresh login) for backend tests (Pro).\n' + + 'A fresh token is fetched on each run and injected into every backend test.', + ) + .requiredOption('--method ', 'password | refresh_token | aws_cognito_refresh') + .requiredOption('--inject ', 'bearer | header | cookie') + .option('--disable', 'turn auto-auth off (keeps stored config)') + .option('--inject-key ', 'header/cookie name when --inject is header/cookie') + // password method + .option('--login-url ', 'login endpoint (method=password)') + .option('--login-method ', 'POST | PUT (method=password)') + .option('--login-content-type ', 'application/json | application/x-www-form-urlencoded') + .option('--login-body-template ', 'login body template with {{username}}/{{password}}') + .option('--username ', 'login username (method=password)') + .option('--password ', 'login password (method=password)') + .option('--password-file ', 'read login password from a file') + .option('--token-path ', 'JSONPath to the token in the login response') + // refresh_token method + .option('--token-endpoint ', 'OAuth token endpoint (method=refresh_token)') + .option('--client-id ', 'OAuth client id') + .option('--client-secret ', 'OAuth client secret') + .option('--client-secret-file ', 'read OAuth client secret from a file') + .option('--refresh-token ', 'OAuth/Cognito refresh token') + .option('--refresh-token-file ', 'read the refresh token from a file') + .option('--scope ', 'OAuth scope') + // aws_cognito_refresh method + .option('--region ', "AWS region (method=aws_cognito_refresh, e.g. 'us-east-1')") + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: AutoAuthFlagOpts, command: Command) => { + await runAutoAuth( + { + ...resolveCommonOptions(command), + projectId, + disable: cmdOpts.disable, + method: cmdOpts.method, + inject: cmdOpts.inject, + injectKey: cmdOpts.injectKey, + loginUrl: cmdOpts.loginUrl, + loginMethod: cmdOpts.loginMethod, + loginContentType: cmdOpts.loginContentType, + loginBodyTemplate: cmdOpts.loginBodyTemplate, + username: cmdOpts.username, + password: cmdOpts.password, + passwordFile: cmdOpts.passwordFile, + tokenPath: cmdOpts.tokenPath, + tokenEndpoint: cmdOpts.tokenEndpoint, + clientId: cmdOpts.clientId, + clientSecret: cmdOpts.clientSecret, + clientSecretFile: cmdOpts.clientSecretFile, + refreshToken: cmdOpts.refreshToken, + refreshTokenFile: cmdOpts.refreshTokenFile, + scope: cmdOpts.scope, + region: cmdOpts.region, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + return project; } @@ -500,11 +804,41 @@ interface UpdateFlagOpts { username?: string; password?: string; passwordFile?: string; - description?: string; instruction?: string; idempotencyKey?: string; } +interface CredentialFlagOpts { + type: string; + credential?: string; + credentialFile?: string; + idempotencyKey?: string; +} + +interface AutoAuthFlagOpts { + disable?: boolean; + method: string; + inject: string; + injectKey?: string; + loginUrl?: string; + loginMethod?: string; + loginContentType?: string; + loginBodyTemplate?: string; + username?: string; + password?: string; + passwordFile?: string; + tokenPath?: string; + tokenEndpoint?: string; + clientId?: string; + clientSecret?: string; + clientSecretFile?: string; + refreshToken?: string; + refreshTokenFile?: string; + scope?: string; + region?: string; + idempotencyKey?: string; +} + function parseFlag(raw: string | undefined, flagName: string): number | undefined { if (raw === undefined) return undefined; const n = Number(raw); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 6ae831c..f7c7602 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -4631,6 +4631,48 @@ describe('runCreate', () => { expect(sent.headers.get('x-api-key')).toBe('sk-user-test'); }); + it('emits backend warnings[] to stderr without polluting stdout JSON', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('BEARER = "eyJhbGciOi.eyJzdWIiOiJ4In0.sig"\n'); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + ...SAMPLE_RESPONSE, + type: 'backend', + warnings: [ + 'This test appears to hardcode an auth credential — read auth from __AUTH_HEADERS__.', + ], + }, + }; + }); + const out: string[] = []; + const err: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'backend', + name: 'hardcoded be', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => err.push(line), + }, + ); + // Warning lands on stderr, prefixed `[warn]`. + expect(err.some(l => l.includes('[warn]') && l.includes('__AUTH_HEADERS__'))).toBe(true); + // stdout stays the parseable wire object — no warning noise. + expect(out.join('\n')).not.toContain('[warn]'); + }); + it('respects a caller-supplied --idempotency-key (for safe retries)', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('code body'); @@ -5674,6 +5716,36 @@ describe('runCreate — M4 BE dependency authoring flags', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('[FE guard] --produces rejection is well-formed: bare field, no ----produces, singular verb (dogfood 2026-06-30)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// fe code'); + const err = (await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_fe', + type: 'frontend', + name: 'fe test', + codeFile, + produces: ['some_var'], + }, + { + credentialsPath, + fetchImpl: () => Promise.resolve(new Response('{}')), + stdout: () => undefined, + stderr: () => undefined, + }, + ).catch((e: unknown) => e)) as ApiError; + expect(err).toBeInstanceOf(ApiError); + // details.field is the BARE flag name (was '--produces' → double-dashed subject). + expect(err.details).toMatchObject({ field: 'produces' }); + expect(err.nextAction).toContain('--produces'); + expect(err.nextAction).not.toContain('----'); // regression: '----produces' + expect(err.nextAction).toContain('is a backend-only flag'); // singular for one flag + expect(err.nextAction).not.toContain('backend..'); // regression: double period + }); + it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --needs', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('// fe code'); diff --git a/src/commands/test.ts b/src/commands/test.ts index 1c6edd8..0602afc 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -543,6 +543,12 @@ export interface CliCreateTestResponse { type: 'frontend' | 'backend'; codeVersion: string; createdAt: string; + /** + * Non-fatal advisories from the backend (e.g. the BE auth guardrail + * flagging a hardcoded credential). Rendered on stderr; the create + * still succeeded. + */ + warnings?: string[]; } export const CLI_CREATE_PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; @@ -752,14 +758,19 @@ export async function runCreate( // save a round-trip. if (opts.type === 'frontend') { const depFlags: string[] = []; - if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('--produces'); - if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('--needs'); - if (opts.category !== undefined) depFlags.push('--category'); + if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('produces'); + if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('needs'); + if (opts.category !== undefined) depFlags.push('category'); if (depFlags.length > 0) { + // Pass the BARE flag name to localValidationError — its kind:'flag' branch + // adds the `--` prefix, so '--produces' would render as '----produces'. + const flagList = depFlags.map(f => `--${f}`); + const verb = depFlags.length === 1 ? 'is a backend-only flag' : 'are backend-only flags'; + // No trailing period: localValidationError appends one after the reason. throw localValidationError( depFlags[0]!, - `${depFlags.join(', ')} are backend-only flags; frontend plans have no wave model. ` + - `Remove ${depFlags.join('/')} or use --type backend.`, + `${flagList.join(', ')} ${verb}; frontend plans have no wave model. ` + + `Remove ${flagList.join('/')} or use --type backend`, ); } } @@ -836,6 +847,11 @@ export async function runCreate( headers: { 'idempotency-key': idempotencyKey }, }); + // Surface backend advisories (e.g. a hardcoded-credential warning for BE + // tests) on stderr so they reach the agent without polluting stdout JSON. + // Emitted before the --run early return so they always show. + emitResponseWarnings(response.warnings, deps); + // --run chain (M3.3 piece-3). Per codex round-1 P1: suppress the // create's own print when chaining; `runTestRun` emits a single // merged envelope `{ ...createResponse, run: }` so @@ -998,6 +1014,16 @@ function renderCreateText(response: CliCreateTestResponse): string { ].join('\n'); } +/** + * Emit backend `warnings[]` advisories to stderr (one `[warn]` line each), + * keeping stdout — JSON or text — uncluttered. No-op when absent/empty. + */ +function emitResponseWarnings(warnings: string[] | undefined, deps: TestDeps): void { + if (!warnings || warnings.length === 0) return; + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + for (const w of warnings) stderrFn(`[warn] ${w}`); +} + /** * §6.X / M3.2 piece-6 — response from `PUT /tests/{id}/plan-steps`. * `planStepsHash` is a sha256 over the canonicalized new array so @@ -3400,6 +3426,12 @@ export interface CliPutTestCodeResponse { testId: string; codeVersion: string; updatedAt: string; + /** + * Non-fatal advisories (e.g. the BE auth guardrail flagging a hardcoded + * credential in the replaced code). Rendered on stderr; the update still + * succeeded. + */ + warnings?: string[]; } type CodePutLanguage = CliTestCode['language']; @@ -3577,6 +3609,7 @@ export async function runCodePut( }, }, ); + emitResponseWarnings(response.warnings, deps); out.print(response, data => renderCodePutText(data as CliPutTestCodeResponse)); return response; } catch (err) { @@ -5803,7 +5836,7 @@ export async function runTestWait( // --------------------------------------------------------------------------- interface RunTestRunAllOptions extends CommonOptions { - /** projectId to run all BE tests in. */ + /** projectId to run all tests in. */ projectId: string; /** --filter : only run tests whose name contains this substring (case-insensitive). */ nameFilter?: string; @@ -5940,7 +5973,7 @@ export async function runTestRunAll( stderrFn(`idempotency-key: ${idempotencyKey}`); } - // Resolve testIds: fetch all BE tests in the project, apply --filter. + // Resolve testIds: fetch all tests in the project, apply --filter. let testIds: string[] | undefined; if (opts.nameFilter !== undefined && opts.nameFilter !== '') { // We need to resolve the full test set to apply the name filter. @@ -5978,7 +6011,8 @@ export async function runTestRunAll( `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch run.`, ); } - // When no --filter, omit testIds → server runs ALL BE tests in the project. + // When no --filter, omit testIds → server runs ALL tests in the project + // (BE tests on the legacy V2 wave engine; FE + BE on the V3 unified engine). const batchResp = await client.triggerBatchRunFresh( { @@ -8369,7 +8403,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { .command('run [test-id]') .description( 'Trigger a test run. With --wait, polls until terminal status.\n' + - 'Use --all --project for a wave-ordered batch run of all BE tests (M4).\n' + + 'Use --all --project for a wave-ordered batch run of all tests in a project (M4).\n' + '\nExit codes:\n' + ' 0 passed (or queued without --wait)\n' + ' 1 failed / blocked / cancelled\n' + @@ -8397,7 +8431,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--all', - 'run all BE tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', + 'run all tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', false, ) .option( @@ -8424,11 +8458,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + - ' testsprite test run --all --project run all BE tests in wave order\n' + + ' testsprite test run --all --project run all project tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + - '(see `testsprite test create --help` for details).', + '(see `testsprite test create --help` for details).\n' + + '\nFrontend tests: the current unified engine runs FE tests too (they are billed\n' + + 'like any run). On the legacy backend-only engine FE tests cannot run — they are\n' + + "reported under skippedFrontend with an advisory; run those with 'test run '.", ) .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testIdArg: string | undefined, cmdOpts: RunFlagOpts, command: Command) => { @@ -8444,7 +8481,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (testIdArg === undefined && !isAll) { throw localValidationError( 'test-id', - 'provide a , or use --all --project to run all BE tests in a project', + 'provide a , or use --all --project to run all tests in a project', ); } // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). @@ -8473,14 +8510,16 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--all requires a project id — pass --project ', ); } - // --target-url has no effect on the --all batch path: it is BE-only - // (FE tests are skipped server-side) and a BE test's base URL is baked - // into its code. Silently dropping it could run the suite against an - // unintended environment in the caller's mind — reject loudly instead. + // --target-url has no effect on the --all batch path: a BE test's base + // URL is baked into its code, and the unified engine resolves each + // project's configured environment server-side (per-run URL overrides + // are not applied to batch FE runs either). Silently dropping it could + // run the suite against an unintended environment in the caller's mind + // — reject loudly instead. if (cmdOpts.targetUrl !== undefined && cmdOpts.targetUrl !== '') { throw localValidationError( 'target-url', - '--target-url has no effect with --all (the batch path is the BE-only wave engine; a BE test’s URL is baked into its code). Remove --target-url.', + '--target-url has no effect with --all (the batch path does not apply a per-run URL override — BE test URLs are baked into their code and the unified engine resolves the project environment server-side). Remove --target-url.', ); } await runTestRunAll( @@ -9973,7 +10012,11 @@ export function createTestArtifactCommand(deps: TestDeps): Command { 'Parent must exist. The bundle dir itself is created if absent.', ].join(' '), ) - .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .option( + '--failed-only', + 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + + 'so this is usually a no-op; use `test steps ` for the full run trail.', + ) .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (runId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { @@ -10003,7 +10046,11 @@ function createTestFailureCommand(deps: TestDeps): Command { '--out ', 'Directory to write the §7 disk layout into (default: print wire envelope to stdout)', ) - .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .option( + '--failed-only', + 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + + 'so this is usually a no-op; use `test steps ` for the full run trail.', + ) .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (testId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { diff --git a/src/commands/usage.ts b/src/commands/usage.ts index a97dbba..1980d26 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -154,8 +154,12 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { } if (u.creditsPerRun !== undefined) { lines.push(`cost per frontend run: ${u.creditsPerRun} credit(s)`); + // Backend runs DO consume credits (confirmed by design 2026-06-30 / DEV-289). + // The API exposes no backend-specific per-run cost field, and it differs from + // the frontend rate, so state that it bills without asserting a possibly-wrong + // number — check your balance before/after, or see the billing page. lines.push( - `cost per backend run: 0 credit(s) (backend tests bill at code-generation, not at run time)`, + `cost per backend run: also consumes credits (exact amount not reported by the API)`, ); } diff --git a/src/index.ts b/src/index.ts index 806f6e4..bf9a9aa 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,7 +52,7 @@ program .option('--debug', 'Print HTTP method/path, request id, latency, retry decisions to stderr') .option( '--dry-run', - 'Skip the network, credentials, and filesystem; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key.', + 'Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. Note: file inputs you pass (--plan-from/--plans/--steps) are still read and validated locally; only --code-file uses a placeholder.', ) .option( '--request-timeout ', diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 51bf9b5..2ac9158 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -34,7 +34,9 @@ import { * The description value is a single line (no folded/literal block scalars). */ function parseFrontmatterDescription(content: string): string | undefined { - const lines = content.split('\n'); + // Tolerate CRLF so a Windows checkout (autocrlf) doesn't leave a trailing + // \r on the description and break the byte-identical comparisons. + const lines = content.split(/\r?\n/); let inFrontmatter = false; for (const line of lines) { if (line.trim() === '---') { diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index aec451a..ff20e8c 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -10,7 +10,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { isAbsolute, join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; import { applyFailedOnly, @@ -593,8 +593,8 @@ describe('resolveBundleDir', () => { it('resolves a relative path against cwd', () => { const out = resolveBundleDir('./tmp/x'); - expect(out.endsWith('/tmp/x')).toBe(true); - expect(out.startsWith('/')).toBe(true); + expect(out).toBe(resolve(process.cwd(), 'tmp', 'x')); + expect(isAbsolute(out)).toBe(true); }); it('strips a trailing slash', () => { diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index 896d057..d50ad52 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -1,5 +1,5 @@ import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { @@ -139,8 +139,11 @@ describe('writeProfile', () => { it('creates the file with mode 0600 and writes the profile', () => { writeProfile(DEFAULT_PROFILE, { apiKey: 'sk-new' }, { path: credentialsPath }); expect(existsSync(credentialsPath)).toBe(true); - const mode = statSync(credentialsPath).mode & 0o777; - expect(mode).toBe(0o600); + // POSIX file modes don't exist on Windows (stat reports 0666). + if (process.platform !== 'win32') { + const mode = statSync(credentialsPath).mode & 0o777; + expect(mode).toBe(0o600); + } expect(readProfile(DEFAULT_PROFILE, { path: credentialsPath })).toEqual({ apiKey: 'sk-new' }); }); @@ -188,7 +191,8 @@ describe('ensureRestrictiveMode', () => { expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); }); - it('downgrades over-permissive modes', () => { + // POSIX-only premise: Windows has no 0644/0600 distinction to downgrade. + it.skipIf(process.platform === 'win32')('downgrades over-permissive modes', () => { mkdirSync(tmpRoot, { recursive: true }); writeFileSync(credentialsPath, 'data', { mode: 0o644 }); ensureRestrictiveMode(credentialsPath); @@ -199,7 +203,7 @@ describe('ensureRestrictiveMode', () => { describe('defaultCredentialsPath', () => { it('points at ~/.testsprite/credentials', () => { - expect(defaultCredentialsPath().endsWith('/.testsprite/credentials')).toBe(true); + expect(defaultCredentialsPath()).toBe(join(homedir(), '.testsprite', 'credentials')); }); }); diff --git a/src/lib/http.ts b/src/lib/http.ts index af1c25a..7a9051a 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -528,9 +528,9 @@ export class HttpClient { } // Edge proxies / load balancers return 408/502/504 without our error - // envelope on transient outages. Per the CLI error spec §7 these are - // transport-level retries, not facade errors — fold them in here so - // we get the bounded backoff budget instead of a single INTERNAL bail. + // envelope on transient outages. These are transport-level retries, + // not facade errors — fold them in here so we get the bounded backoff + // budget instead of a single INTERNAL bail. if (rawBody === null && isTransportEdgeStatus(response.status)) { this.debug({ kind: 'error', diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index c15b09d..2b26c0e 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -13,24 +13,30 @@ import { // isVerifySkillInstalled // --------------------------------------------------------------------------- +// The implementation joins paths with the native separator; normalize so the +// fakes below match on Windows (backslashes) as well as POSIX. +const toPosix = (p: string) => p.replaceAll('\\', '/'); + describe('isVerifySkillInstalled', () => { it('true when the claude own-file SKILL.md exists', () => { - const existsSync = (p: string) => p.endsWith('.claude/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => + toPosix(p).endsWith('.claude/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cursor .mdc landing file', () => { - const existsSync = (p: string) => p.endsWith('.cursor/rules/testsprite-verify.mdc'); + const existsSync = (p: string) => toPosix(p).endsWith('.cursor/rules/testsprite-verify.mdc'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cline landing file', () => { - const existsSync = (p: string) => p.endsWith('.clinerules/testsprite-verify.md'); + const existsSync = (p: string) => toPosix(p).endsWith('.clinerules/testsprite-verify.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the antigravity landing file', () => { - const existsSync = (p: string) => p.endsWith('.agents/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => + toPosix(p).endsWith('.agents/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); @@ -73,7 +79,7 @@ describe('isVerifySkillInstalled', () => { return false; }, }); - expect(seen.every(p => p.startsWith('/some/proj'))).toBe(true); + expect(seen.every(p => toPosix(p).startsWith('/some/proj'))).toBe(true); // One probe per target landing path. expect(seen).toHaveLength(Object.keys(TARGETS).length); }); @@ -198,6 +204,6 @@ describe('maybeEmitSkillNudge', () => { }); maybeEmitSkillNudge(ctx); expect(probed.length).toBeGreaterThan(0); - expect(probed.every(p => p.startsWith('/work/here'))).toBe(true); + expect(probed.every(p => toPosix(p).startsWith('/work/here'))).toBe(true); }); }); diff --git a/src/version.ts b/src/version.ts index efa916d..f60fdfd 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.2.0'; +export const VERSION = '0.3.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index fcc3d85..62efe5d 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -133,20 +133,29 @@ exports[`--help snapshots > project 1`] = ` Manage TestSprite projects Options: - -h, --help display help for command + -h, --help display help for command Commands: - list [options] List projects visible to the API key + list [options] List projects visible to the API key Exit codes: 0 success 3 auth error 5 validation error (e.g., bad --page-size) 10 transport/network failure (UNAVAILABLE) — retry the command - get Get a project by id - create [options] Create a new project - update [options] Update project metadata - help [command] display help for command + get Get a project by id + create [options] Create a new project + update [options] Update project metadata + credential [options] Set the static backend credential injected + into every backend test + (Bearer token / API key / Basic token / + public). Free tier. + auto-auth [options] Configure the recurring-token + (auto-refresh login) for backend tests + (Pro). + A fresh token is fetched on each run and + injected into every backend test. + help [command] display help for command " `; @@ -242,7 +251,7 @@ Commands: Note: a 404 "not found" response is counted as skipped in the summary, not an error. run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. - Use --all --project for a wave-ordered batch run of all BE tests (M4). + Use --all --project for a wave-ordered batch run of all tests in a project (M4). Exit codes: 0 passed (or queued without --wait) @@ -362,7 +371,9 @@ Write a self-contained failure-context bundle for a test's latest failing run Options: --out Directory to write the §7 disk layout into (default: print wire envelope to stdout) - --failed-only Keep only the failed step plus its immediate neighbors (±1) + --failed-only Trim to the failed step ±1. The bundle is already + failure-focused server-side, so this is usually a no-op; use + \`test steps \` for the full run trail. -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -548,7 +559,7 @@ exports[`--help snapshots > test run 1`] = ` "Usage: testsprite test run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. -Use --all --project for a wave-ordered batch run of all BE tests (M4). +Use --all --project for a wave-ordered batch run of all tests in a project (M4). Exit codes: 0 passed (or queued without --wait) @@ -572,9 +583,9 @@ Options: 600) --idempotency-key opaque key for safe retries (1–256 chars). Printed to stderr at --debug if auto-generated. - --all run all BE tests in the project (wave-ordered - fresh run; requires --project). Mutually - exclusive with . (default: false) + --all run all tests in the project (wave-ordered fresh + run; requires --project). Mutually exclusive with + . (default: false) --project project id (required with --all; returned by \`testsprite project list\`) --filter with --all: only run tests whose name contains @@ -589,13 +600,17 @@ Options: -h, --help display help for command Dependency-aware fresh run (M4): - testsprite test run --all --project run all BE tests in wave order + testsprite test run --all --project run all project tests in wave order testsprite test run --all --project --filter name-glob subset testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). +Frontend tests: the current unified engine runs FE tests too (they are billed +like any run). On the legacy backend-only engine FE tests cannot run — they are +reported under skippedFrontend with an advisory; run those with 'test run '. + Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " @@ -640,10 +655,13 @@ Options: without the full trace. --debug Print HTTP method/path, request id, latency, retry decisions to stderr - --dry-run Skip the network, credentials, and filesystem; - emit a canned sample matching the OpenAPI - contract. Useful for learning the CLI surface - without an API key. + --dry-run Skip the network and credentials; emit a canned + sample matching the OpenAPI contract. Useful for + learning the CLI surface without an API key. + Note: file inputs you pass + (--plan-from/--plans/--steps) are still read and + validated locally; only --code-file uses a + placeholder. --request-timeout Client-side per-request timeout in seconds (default: 120). Aborts any single fetch that does not complete within this deadline. Override diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index b0537b1..959f212 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -8,7 +8,7 @@ */ import { execFileSync, spawn } from 'node:child_process'; -import { existsSync, mkdtempSync, statSync } from 'node:fs'; +import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; @@ -364,7 +364,10 @@ function runCli(args: string[], envOverrides: Record = {}): Prom cwd: REPO_ROOT, env: { ...process.env, + // os.homedir() reads HOME on POSIX but USERPROFILE on Windows — + // set both so the child never sees the real ~/.testsprite. HOME: tmpHome, + USERPROFILE: tmpHome, TESTSPRITE_API_KEY: undefined, TESTSPRITE_API_URL: undefined, ...envOverrides, @@ -897,7 +900,10 @@ describe('setup --from-env subprocess', () => { expect(result.exitCode).toBe(0); const credentialsPath = join(tmpHome, '.testsprite', 'credentials'); expect(existsSync(credentialsPath)).toBe(true); - expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); + // POSIX file modes don't exist on Windows (stat reports 0666). + if (process.platform !== 'win32') { + expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); + } }, 30_000); it('exits 5 with VALIDATION_ERROR when --from-env is set without TESTSPRITE_API_KEY', async () => { @@ -1044,7 +1050,7 @@ describe('--dry-run subprocess smoke', () => { // skipped the prompt. const credPath = join(tmpHome, '.testsprite', 'credentials'); // Make sure any previous test didn't leave one behind. - if (existsSync(credPath)) execFileSync('rm', [credPath]); + rmSync(credPath, { force: true }); const result = await runCli(['setup', '--dry-run', '--no-agent', '--output', 'json']); expect(result.exitCode).toBe(0); expect(existsSync(credPath)).toBe(false); diff --git a/test/helpers/hermetic-env.ts b/test/helpers/hermetic-env.ts new file mode 100644 index 0000000..c798bb3 --- /dev/null +++ b/test/helpers/hermetic-env.ts @@ -0,0 +1,41 @@ +/** + * Unit-test env hermeticity (vitest `setupFiles`, runs before each test file). + * + * Two leaks this closes, both of which made results depend on the + * developer's machine: + * + * 1. Real `TESTSPRITE_*` env vars. `loadConfig` gives `TESTSPRITE_API_KEY` + * precedence over the credentials file, so a key exported in the + * developer's shell silently overrode test fixtures. + * 2. The real home directory. `os.homedir()` reads `HOME` on POSIX but + * `USERPROFILE` on Windows, so the documented `HOME=$(mktemp -d)` + * recipe never isolated Windows runs. Both vars are redirected to a + * throwaway dir so no test can read or write `~/.testsprite`. + * + * Tests that need these vars set them explicitly (on `process.env` or via + * injected `env` deps) after this runs. + */ +import { existsSync, mkdirSync, mkdtempSync } from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const realHome = homedir(); +const hermeticHome = mkdtempSync(join(tmpdir(), 'testsprite-unit-home-')); +if (process.platform === 'win32') { + // Node version shims (Volta) resolve LocalAppData under USERPROFILE and + // abort if it's missing, which would break the `npm run build` beforeAll + // in the subprocess/snapshot suites. + mkdirSync(join(hermeticHome, 'AppData', 'Local'), { recursive: true }); +} +// Same shim concern on macOS/Linux: Volta derives ~/.volta from HOME unless +// VOLTA_HOME is set. Pin it to the real install before redirecting HOME. +const realVoltaHome = join(realHome, '.volta'); +if (!process.env.VOLTA_HOME && existsSync(realVoltaHome)) { + process.env.VOLTA_HOME = realVoltaHome; +} +process.env.HOME = hermeticHome; +process.env.USERPROFILE = hermeticHome; + +for (const key of Object.keys(process.env)) { + if (key.startsWith('TESTSPRITE_')) delete process.env[key]; +} diff --git a/vitest.config.ts b/vitest.config.ts index add8f0e..bd9b2ad 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,9 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], + // Strip real TESTSPRITE_* env vars and redirect the home dir so results + // never depend on the developer's shell or ~/.testsprite (see the file). + setupFiles: ['./test/helpers/hermetic-env.ts'], // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). fileParallelism: false, From e91c5a107867f16ca183b11684c84d953d972cbf Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Thu, 16 Jul 2026 14:56:49 -0700 Subject: [PATCH 063/102] release: v0.4.0 Private-Snapshot-RevId: afa3bf501545df536cfa638a86ae096fb7c34880 --- .github/workflows/backport-dispatch.yml | 83 ++ .github/workflows/ci-nudge.yml | 30 +- .github/workflows/ci.yml | 64 ++ .github/workflows/issue-triage.yml | 12 +- .github/workflows/pr-triage.yml | 24 +- .github/workflows/release.yaml | 28 +- .github/workflows/test-coverage.yml | 2 + .gitleaks.toml | 19 + CHANGELOG.md | 21 + CONTRIBUTING.md | 27 +- DOCUMENTATION.md | 74 +- README.md | 59 +- package-lock.json | 4 +- package.json | 2 +- scripts/README.md | 29 + skills/testsprite-verify.codex.md | 6 + skills/testsprite-verify.skill.md | 19 +- src/commands/agent.test.ts | 156 ++-- src/commands/auth.test.ts | 57 ++ src/commands/auth.ts | 14 + src/commands/doctor.test.ts | 45 + src/commands/doctor.ts | 60 +- src/commands/project.test.ts | 168 +++- src/commands/project.ts | 130 ++- src/commands/test.cancel.spec.ts | 309 +++++++ src/commands/test.quickwins.spec.ts | 110 +++ src/commands/test.rerun.spec.ts | 98 ++- src/commands/test.run.spec.ts | 162 +++- src/commands/test.test.ts | 103 +++ src/commands/test.ts | 815 ++++++++++++++++-- src/commands/test.wait.spec.ts | 122 ++- src/index.ts | 45 +- src/lib/bundle.test.ts | 19 + src/lib/bundle.ts | 19 +- src/lib/client-factory.ts | 23 + src/lib/dry-run/samples.test.ts | 33 + src/lib/dry-run/samples.ts | 43 +- src/lib/errors.test.ts | 22 + src/lib/errors.ts | 45 + src/lib/failing-fe-resolver.spec.ts | 193 +++++ src/lib/failing-fe-resolver.ts | 67 +- src/lib/http.test.ts | 246 +++++- src/lib/http.ts | 133 ++- src/lib/interrupt.test.ts | 101 ++- src/lib/interrupt.ts | 124 ++- src/lib/poll.spec.ts | 104 ++- src/lib/poll.ts | 95 +- src/lib/runs.types.ts | 14 + src/lib/v3-advisory.test.ts | 24 + src/lib/v3-advisory.ts | 25 + src/lib/version-notice.test.ts | 110 +++ src/lib/version-notice.ts | 111 +++ src/version.ts | 2 +- test/__snapshots__/help.snapshot.test.ts.snap | 72 +- test/cli.subprocess.test.ts | 5 +- test/e2e/signal.e2e.test.ts | 188 ++++ test/help.snapshot.test.ts | 5 +- test/helpers/execNpm.ts | 17 + 58 files changed, 4410 insertions(+), 327 deletions(-) create mode 100644 .github/workflows/backport-dispatch.yml create mode 100644 scripts/README.md create mode 100644 src/commands/test.cancel.spec.ts create mode 100644 src/lib/v3-advisory.test.ts create mode 100644 src/lib/v3-advisory.ts create mode 100644 src/lib/version-notice.test.ts create mode 100644 src/lib/version-notice.ts create mode 100644 test/e2e/signal.e2e.test.ts create mode 100644 test/helpers/execNpm.ts diff --git a/.github/workflows/backport-dispatch.yml b/.github/workflows/backport-dispatch.yml new file mode 100644 index 0000000..8b6a99a --- /dev/null +++ b/.github/workflows/backport-dispatch.yml @@ -0,0 +1,83 @@ +# backport-dispatch.yml — OPTIONAL latency optimization for DEV-352's +# auto-backport bot. Authored in atlas (release/public path), ships to the +# PUBLIC repo via the snapshot per invariant I0 — this workflow only ever +# runs on the public repo, never on atlas (see the repo guard below). +# +# NOT YET ARMED (2026-07-15): the atlas-side auto-backport.yml has a +# scheduled SWEEP (every 30 min) that scans merged public PRs anonymously +# and backports anything unabsorbed — the whole pipeline is fully +# functional with ZERO public-repo credentials via that path alone. This +# workflow instantly notifies atlas the moment a PR merges instead of +# waiting for the next sweep (seconds vs. up to 30 minutes) — a nice-to-have, +# not a requirement. +# +# Arming it means placing a GitHub App credential with contents:write on +# the PRIVATE atlas repo into THIS PUBLIC repo's secrets — a real trust +# decision (a compromised public-repo secret store would let an attacker +# push to atlas) that is deliberately left to the operator, not decided by +# this file. Until TESTSPRITE_HOB_APP_ID/TESTSPRITE_HOB_PRIVATE_KEY (or the +# ALFHEIM_AGENT_* fallback) exist on the PUBLIC repo with contents:write on +# atlas, this workflow no-ops cleanly (see the "not armed" step) — it does +# NOT fail loud, so it stays quiet for every contributor watching the +# Actions tab until the operator decides to enable it. +# +# SECURITY: this workflow reads ONLY event metadata (PR number, merge SHA, +# base ref) — it never checks out the merged PR's code. That is what makes +# `pull_request_target` safe here: the base-repo context (secrets, a +# write-capable token) is required because a fork-originated merged PR gets +# ZERO secrets under a plain `pull_request` trigger, even for the `closed` +# activity type — but nothing in this job ever runs untrusted fork code, so +# the classic pull_request_target RCE/secret-exfiltration risk (checking +# out and executing the fork's own head ref under a privileged context) +# does not apply. Do not add actions/checkout to this workflow. +name: Notify atlas of a merged public PR + +on: + pull_request_target: + types: [closed] + +permissions: + contents: read + +jobs: + notify: + if: github.repository == 'TestSprite/testsprite-cli' && github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + owner: TestSprite + repositories: testsprite-cli-atlas + # Minimum required for POST /repos/{owner}/{repo}/dispatches (verified + # against docs.github.com/en/rest/using-the-rest-api/permissions-required-for-github-apps — + # the endpoint is gated on Contents:write, not Contents:read). + permission-contents: write + + - name: Dispatch to atlas + if: steps.app-token.outputs.token != '' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + # These two fields are fork-influenced (merge_commit_sha/base.ref come + # from a merged PR's event payload) — routed through env instead of + # spliced into the run: string to avoid the GHA script-injection class + # (a maliciously-crafted value could otherwise break out of the -F + # argument and inject arbitrary shell). PR_NUMBER is left inline: it is + # a GitHub-assigned integer, not attacker-authorable content. + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + gh api repos/TestSprite/testsprite-cli-atlas/dispatches \ + -f event_type=public-pr-merged \ + -F 'client_payload[pr_number]=${{ github.event.pull_request.number }}' \ + -F "client_payload[merge_commit_sha]=$MERGE_SHA" \ + -F "client_payload[base_ref]=$BASE_REF" + + - name: Not armed yet — the atlas sweep will pick this up + if: steps.app-token.outputs.token == '' + run: | + echo "::notice::No cross-repo credential configured on this repo yet — atlas's scheduled sweep (auto-backport.yml, every 30 min) will pick up this merge instead of an instant dispatch. See DEV-352 / DEV-348 / DEV-350 for the operator decision to arm this." diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml index d01ede3..f14461f 100644 --- a/.github/workflows/ci-nudge.yml +++ b/.github/workflows/ci-nudge.yml @@ -9,8 +9,10 @@ # CI workflows (CI + Test Coverage) can complete in any order. # # Bot identity: same App-token-first / GITHUB_TOKEN-fallback pattern as -# pr-triage.yml (the App needs the "Pull requests: Read & write" permission to -# post as testsprite-hob[bot]; until then comments come from github-actions[bot]). +# pr-triage.yml. The App has had the "Pull requests: Read & write" permission +# since 2026-07-02 (corrected 2026-07-15; this comment previously said the +# permission was still pending) — the fallback path stays as defense-in-depth +# for whenever the App/secrets are absent. name: CI failure nudge on: @@ -40,8 +42,20 @@ jobs: with: app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # The script below only ever calls the App client (`appClient`) for + # rest.issues.updateComment/createComment — PR comments ride the + # issues API (see the header comment above). All reads (checks, + # pulls, commit->PR lookup) go through the ambient `github` client, + # governed by this workflow's top-level `permissions:` block, not + # this minted token. Scoping to checks:read/pull-requests:write here + # (matching the job's top-level block literally) would risk the mint + # itself failing if the App's installation doesn't hold Checks + # permission — which would silently drop the App-token path + # entirely (continue-on-error swallows the failure) and always fall + # back to github-actions[bot], defeating this bot's own purpose. + permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: APP_TOKEN: ${{ steps.app-token.outputs.token }} with: @@ -82,14 +96,20 @@ jobs: 'Build': 'run `npm run build` and fix the compile errors', 'Local E2E Tests': 'run `npm run test:e2e` (it builds first)', 'Coverage (>= 80%)': 'run `npm run test:coverage` — new code needs tests until every metric is back at 80%', + 'Secret scan (gitleaks)': 'run `gitleaks detect --no-git --redact --source .` locally (config: .gitleaks.toml) and remove the secret — or, for a provable false positive, add a narrowly-anchored allowlist regex', }; + // Matrix jobs publish check runs as "Unit Tests (Node 20)" etc. — + // exact lookup alone would silently skip them (and the nudge would + // say "all green" while they are red). Try exact first, then the + // name with a trailing parenthetical stripped (review round 2). + const fixFor = (name) => FIX[name] ?? FIX[name.replace(/\s*\([^)]*\)$/, '')]; // Recompute full CI state from this SHA's check runs, narrowed to the // CI jobs above — other workflows (e.g. pr-triage) also attach // github-actions check runs to the PR head and must not count here. const checks = await github.paginate(github.rest.checks.listForRef, { owner, repo, ref: run.head_sha, per_page: 100 }); - const ours = checks.filter(c => c.app && c.app.slug === 'github-actions' && FIX[c.name]); + const ours = checks.filter(c => c.app && c.app.slug === 'github-actions' && fixFor(c.name)); const failing = ours.filter(c => ['failure', 'timed_out'].includes(c.conclusion)); const pending = ours.filter(c => c.status !== 'completed'); @@ -110,7 +130,7 @@ jobs: const lines = failing .sort((a, b) => a.name.localeCompare(b.name)) .map(c => { - const fix = FIX[c.name] || 'see the logs for details'; + const fix = fixFor(c.name) || 'see the logs for details'; return `- **${c.name}** — ${fix} ([logs](${c.html_url}))`; }); const body = `${marker}\nThanks, @${pr.user.login}! CI is red on this PR — ` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index edde076..3766b0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -37,6 +39,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -55,6 +59,8 @@ jobs: node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -68,6 +74,27 @@ jobs: env: CI: true + test-windows: + name: Unit Tests (Windows) + runs-on: windows-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: 22 + cache: 'npm' + + - run: npm ci + - run: npm run build + - run: npm run typecheck + - run: npm test + env: + CI: true + build: name: Build (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest @@ -76,6 +103,8 @@ jobs: node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -96,6 +125,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -107,3 +138,36 @@ jobs: - run: npm run test:e2e env: CI: true + + gitleaks: + name: Secret scan (gitleaks) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + # Same pinned version + checksum-verified install as + # divergence-sentinel.yml (the more recent/secure of the two gitleaks + # invocations already in this repo — release-build.yml pins an older + # 8.21.2 with no checksum check). Continuous, per-PR/per-push gate: a + # working-tree scan (--no-git), not a full-history scan — history + # scanning is too slow to run on every PR, and this mirrors the mode + # scripts/make-public-snapshot.sh already uses for the release-time scan + # (`gitleaks detect --no-git --no-banner --redact --source `). + - name: Install gitleaks + env: + GITLEAKS_VERSION: '8.28.0' + run: | + set -euo pipefail + BASE="https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}" + TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -sSL -o "$TARBALL" "${BASE}/${TARBALL}" + curl -sSL -o checksums.txt "${BASE}/gitleaks_${GITLEAKS_VERSION}_checksums.txt" + grep " ${TARBALL}\$" checksums.txt | sha256sum -c - + tar -xzf "$TARBALL" gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Scan working tree for secrets + run: gitleaks detect --no-git --no-banner --redact --source . diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 6b6ce62..0da78c8 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -31,10 +31,18 @@ env: jobs: triage: + # Public repo only — this file also lives in the private mirror (atlas), + # where issue-first claiming doesn't apply; the three sibling bots + # (pr-triage.yml, ci-nudge.yml, stale.yml) all carry this same fence and + # this one was missing it (found in the 2026-07-15 OSS-P2 audit — the + # /assign bot was live on atlas too until this fix). # issues only (issue_comment also fires on PRs), and never react to a bot's own # comment — incl. our own testsprite-hob[bot], which (unlike GITHUB_TOKEN) would # otherwise re-trigger this workflow. `type == 'Bot'` covers both bot identities. - if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' }} + if: >- + github.repository == 'TestSprite/testsprite-cli' && + !github.event.issue.pull_request && + github.event.comment.user.type != 'Bot' runs-on: ubuntu-latest steps: # Mint a token for the testsprite-hob App so comments post as testsprite-hob[bot]. @@ -49,7 +57,7 @@ jobs: # of inheriting the App's full installation permissions. permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: # App token when available (→ testsprite-hob[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml index a9cd23d..cfd114d 100644 --- a/.github/workflows/pr-triage.yml +++ b/.github/workflows/pr-triage.yml @@ -13,12 +13,12 @@ # Runs with NO checkout — metadata-only, so it is safe under pull_request_target # (which is required for fork PRs to get a write-capable token). # -# Bot identity: posts as `testsprite-hob[bot]` when the App token works. The -# App currently has Issues R/W + Metadata only — commenting/labeling a PULL -# REQUEST needs the "Pull requests: Read & write" App permission (issues-API -# endpoints are permission-checked by target type). Until an org admin adds -# that permission and re-approves the installation, every call gracefully falls -# back to the default GITHUB_TOKEN and posts as github-actions[bot]. +# Bot identity: posts as `testsprite-hob[bot]` when the App token works (the +# App has had the "Pull requests: Read & write" permission since 2026-07-02 — +# corrected 2026-07-15; this comment previously said the permission was still +# pending). The App-token-first / GITHUB_TOKEN-fallback code path below is +# kept regardless, as defense-in-depth for whenever the App/secrets are +# absent or a future installation loses the permission. # Secrets: TESTSPRITE_HOB_* preferred; the ALFHEIM_AGENT_* fallbacks are the # original names from before the App was renamed (same App ID + key). name: PR triage (issue-link gate) @@ -60,8 +60,18 @@ jobs: with: app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # Every write() call below (createComment/updateComment/addLabels/ + # removeLabel) is an `issues.*` Octokit method — GitHub gates all + # four (comments AND labels, even on a PR number) on the "Issues" + # repository permission, not "Pull requests" (verified against + # docs.github.com/en/rest/using-the-rest-api/permissions-required-for-github-apps). + # Scoping to issues:write alone is therefore both sufficient and + # minimal — narrower than this job's top-level `permissions:` block, + # which also carries pull-requests:write for the ambient-token + # fallback path (unused by this specific App-token mint). + permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 env: APP_TOKEN: ${{ steps.app-token.outputs.token }} with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a0f1346..8ef6dc9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,13 +14,33 @@ jobs: with: node-version: 22 registry-url: 'https://registry.npmjs.org' - # npm trusted publishing (OIDC) requires npm >= 11.5.1; Node 22 bundles npm 10.x - - run: npm install -g npm@latest + # npm trusted publishing (OIDC) requires npm >= 11.5.1; Node 22 never bundles + # that (it ships 10.9.x). Pin an EXACT version rather than floating on + # @latest: npm 12.0.0 shipped with a broken workspace symlink that pulled the + # wrong `sigstore` dependency into its own release tarball, so every + # `npm publish --provenance` failed with `Cannot find module 'sigstore'` + # (npm/cli#9722) — this is exactly what our v0.2.0 and v0.3.0 release.yaml + # runs hit, both AFTER `npm install -g npm@latest` reported success. Fixed + # in 12.0.1 (2026-07-10); bump this pin deliberately, never float it again. + - run: npm install -g npm@11.6.2 - run: npm ci - run: npm run lint - run: npm run typecheck - run: npm run format:check - run: npm run test:coverage - run: npm run build - # Auth via npm trusted publishing (OIDC) — no token needed; provenance is implied - - run: npm publish --provenance --access public + # Auth via npm trusted publishing (OIDC) — no token needed; provenance is + # implied. A prerelease tag (e.g. v0.3.0-rc.1, contains "-") must publish + # under an explicit dist-tag: current npm already refuses an implicit + # `latest` for a prerelease version, but failing mid-release is worse than + # not needing the guard, so this still passes --tag itself rather than + # relying on that enforcement alone. + - name: Publish + env: + REF_NAME: ${{ github.ref_name }} + run: | + if [[ "$REF_NAME" == *-* ]]; then + npm publish --provenance --access public --tag next + else + npm publish --provenance --access public + fi diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index ebb3331..0e20d9e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -16,6 +16,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 diff --git a/.gitleaks.toml b/.gitleaks.toml index b3f68df..b31d49c 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -19,4 +19,23 @@ regexes = [ '''sk-secret-[0-9]+''', # Documented placeholder keys used by dry-run/sample responses '''sk-user-(test|DRY-RUN)''', + # Example values inside internal (never-shipped, dropped-before-publish) + # design docs. These are only ever scanned by ci.yml's continuous gitleaks + # job (new, 2026-07) — the release-time scan + # (scripts/make-public-snapshot.sh) already drops those internal docs + # wholesale before it ever runs gitleaks, so these never previously + # tripped a scan. Deliberately described here without naming the internal + # doc paths themselves (this file ships to the public repo, and the + # snapshot's own internal-doc-reference scan would flag such a path). + # A "Bearer " curl example whose placeholder token literal is the + # word "dev-token" — not a real key. (Allowlist regexes match against the + # extracted secret value, not the surrounding line — verified empirically + # with `gitleaks detect`.) + '''^dev-token$''', + # A literal base64 pagination-cursor EXAMPLE value in an internal OpenAPI + # spec (decodes to {"ek":"project_a47b2c11"} — spec-example data, not a + # credential). Anchored ^…$ so it only ever matches this exact literal — + # an unanchored version would also suppress any REAL secret that happens + # to contain this substring (review round 2, 2026-07-15; reproduced). + '''^eyJlayI6InByb2plY3RfYTQ3YjJjMTEifQ==$''', ] diff --git a/CHANGELOG.md b/CHANGELOG.md index e78347e..45803e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +## [0.4.0] - 2026-07-16 + +### Added + +- **`test cancel `** — user-initiated cancel of in-flight runs (the real stop button; Ctrl-C only detaches). A single id renders the run card with status `cancelled` (plus an advisory when it was already cancelled); multiple ids print a `{cancelled, alreadyCancelled, conflicts, notFound}` summary. Exit codes: 4 when any id is not found, else 6 on conflicts, else 0. `--dry-run` supported. +- **Graceful Ctrl-C during `--wait`** — SIGINT/SIGTERM now detaches cleanly instead of killing the process mid-poll: the in-flight request aborts immediately, stdout gets the same partial `{runId, status: "running"}` envelope as the request-timeout path, and stderr states the truth — the server-side run keeps executing (and billing) — with a re-attach hint and a `test cancel` pointer. Exit 130/143/129 per the documented signal contract; a second signal forces a hard exit. Interrupting never cancels the server-side run — that's what `test cancel` is for. +- **`project delete --confirm`** — permanently delete a project and everything under it (its frontend/backend sub-projects, all their tests, and backend fixtures), mirroring the Portal's cascade delete. Requires `--confirm` (the CLI never prompts); `--dry-run` previews the response shape without a network call. Standard exit codes: 0 success, 3 auth, 4 not-found (or already-deleted), 5 validation (e.g. missing `--confirm`). +- **Backend stdout and traceback in results** — `test result` and the failure bundle now surface the backend test's captured stdout and Python traceback: full content in `--output json` (and in `result.json` / `failure.json` bundle files), and a bounded 20-line tail with a byte count in text mode. No change for frontend tests, passing runs, or older backends. +- **Backend dependency declarations are now readable and editable** — `test get` surfaces `produces` / `consumes` / `category`, and `test update` accepts `--produces` / `--needs` / `--category` (previously create-only). +- **Version-compatibility handshake** — the CLI reads the backend's advertised minimum-supported-version on every response and prints a one-line upgrade advisory on stderr when the running version is below the floor (honors the same opt-outs as the update notice; never alters exit status). A `CLIENT_TOO_OLD` rejection (HTTP 426) is now a first-class error: exit 14, non-retriable, rendered with upgrade guidance and the version gap. +- **V3 routing visibility** — `auth status` and `doctor` render a `routing: v2|v3` line when the backend reports the account's routing, and V3-routed accounts get one consolidated advisory listing the known V3-path behavior gaps. Text mode only — JSON consumers read `v3Enabled` from the `/me` payload; absent-safe against older backends. + +### Changed + +- **The `testsprite-verify` agent skill routes local-only changes to the TestSprite MCP** — the skill now states the reachability gate explicitly: the CLI verifies reachable deployed URLs only; when the change is only running locally, the skill hands off to the TestSprite MCP when available (an explicitly named tool always wins), instead of failing against localhost. + +### Fixed + +- **`project create --description` now fails fast with a clear validation error** — projects have no description field, so the flag's value was previously dropped silently; the error points at test-level descriptions (`test create --description`) instead. +- **Standalone backend run cards no longer show a misleading step summary** — `test run` / `test wait` / `test rerun` cards for backend tests render `steps: n/a (backend)` instead of `0/0 (passed=0, failed=0)` (backend tests have no per-step storage). + ## [0.3.0] - 2026-07-08 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bed91fc..1479099 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -109,11 +109,36 @@ npm run format:check # Prettier (check only) npm run typecheck # tsc --noEmit ``` +## Developing on Windows + +Native Windows (no WSL, no Git Bash required) is a fully supported dev +environment — you only need **git** and **Node ≥ 20**. `npm ci`, `npm run +build`, `npm test`, `npm run lint`, and `npm run typecheck` all run the same +way as on macOS/Linux, and CI runs the unit suite on `windows-latest` as the +reference environment (see [CI gates](#ci-gates-required-for-merge) below) — +if it's green there, it's green on your machine. + +A couple of things worth knowing: + +- Line endings are normalized to LF on checkout via `.gitattributes` + (`core.autocrlf` doesn't need any special local configuration). +- `--out`/bundle-path flags accept native Windows paths (`C:\Users\...`) + including a trailing backslash and a bare drive root (`C:\`). +- A small number of tests that create real filesystem symlinks are skipped on + Windows (creating a symlink there needs Administrator rights or Developer + Mode, which isn't guaranteed on hosted CI) — the safety behavior they cover + is still exercised on the Linux/macOS CI jobs. + +Hit something that doesn't work on Windows? Please file it — see +[Questions & support](#questions--support) above, and feel free to tag it +`good first issue` if it looks like an isolated fix; we'd rather know than +have you route around it silently. + ## CI gates (required for merge) - ESLint + Prettier clean - TypeScript type-check clean -- All unit tests passing +- All unit tests passing on Linux (Node 20 + 22) **and** on Windows (`windows-latest`, Node 22) - Coverage ≥ 80% on lines / statements / functions / branches - Build + smoke test of the CLI binary diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 2f6cd6c..bf6b1b5 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -180,7 +180,7 @@ Common flags: #### `testsprite test get ` -Get a single test by id. Test ids look like `test_xxxxxxxx` and come from `test list`. +Get a single test by id. Test ids look like `test_xxxxxxxx` and come from `test list`. Backend tests echo their dependency declarations — `produces` / `consumes` / `category` — when present. ```bash testsprite test get test_xxxxxxxx --output json @@ -210,7 +210,7 @@ Common flags: `--page-size`, `--starting-token`, `--max-items` — same shape as #### `testsprite test result ` -Get the latest result for a test — status, started / finished timestamps, video and failure-analysis URLs, summary counts (`passed / failed / skipped`), and correlation fields (`snapshotId`, `runId`, `codeVersion`). With `--include-analysis`, the response also carries an inline `analysis` block (root-cause hypothesis, recommended fix target, failure kind). +Get the latest result for a test — status, started / finished timestamps, video and failure-analysis URLs, summary counts (`passed / failed / skipped`), and correlation fields (`snapshotId`, `runId`, `codeVersion`). With `--include-analysis`, the response also carries an inline `analysis` block (root-cause hypothesis, recommended fix target, failure kind). Backend tests additionally surface the run's captured stdout (`apiOutput`) and Python traceback (`trace`): full content under `--output json` (and in `result.json` / `failure.json` inside failure bundles); text mode prints a bounded 20-line tail of each with a byte count. ```bash testsprite test result test_xxxxxxxx --output json @@ -293,7 +293,7 @@ testsprite test lint --steps ./refined.plan.json # the shape `test plan pu #### `testsprite test create` -Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. +Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. Backend tests can declare wave-ordering dependencies at create time — `--produces ` / `--needs ` (repeatable) and `--category ` — and amend them later via `test update`. ```bash # Backend test from a code file @@ -319,10 +319,11 @@ testsprite test create-batch --plan-from-dir ./plans/ --dry-run --output json #### `testsprite test update ` -Update test metadata (name, description). +Update test metadata (name, description, priority) — and, for **backend tests**, the dependency declarations: `--produces ` / `--needs ` (repeatable) and `--category `. Updated declarations are echoed back by `test get`. ```bash testsprite test update test_xxxxxxxx --name "Renamed test" --description "Updated" +testsprite test update test_be_xxxx --produces session_token --category setup testsprite test update test_xxxxxxxx --dry-run --output json ``` @@ -358,13 +359,22 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Note the asymmetry: `--description` is **create-only** — `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`, but not `--description`. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Projects have **no description field** — `--description` is rejected client-side with a validation error (descriptions live on tests: `test create --description`). `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`. ```bash testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` +#### `testsprite project delete ` + +Permanently delete a project and **everything under it** — its frontend/backend sub-projects, all their tests, and backend fixtures (mirrors the Portal's cascade delete). There is **no restore window**. `--confirm` is required (the CLI never prompts); absent it, the CLI exits 5 with a local validation error. `--dry-run` previews the response shape without a network call. Exit codes: 0 success, 3 auth, 4 not found (or already deleted), 5 validation. + +```bash +testsprite project delete proj_xxxxxxxx --confirm +testsprite project delete proj_xxxxxxxx --dry-run --output json +``` + #### `testsprite project credential ` Set the **static backend credential** injected into every backend test in the project (free tier). Supported types: `public` (no credential), `"Bearer token"`, `"API key"`, `"basic token"`. @@ -521,6 +531,16 @@ testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +#### `testsprite test cancel ` + +Cancel one or more in-flight runs — the counterpart to Ctrl-C, which only **detaches** (the server-side run keeps executing and billing). Cancelling is idempotent: an already-cancelled run reports `alreadyCancelled` as an advisory, not an error; a run that already reached a terminal verdict is a conflict — the verdict is never overwritten, and no credits are refunded. With one id, prints the run card; with several, prints a `{ cancelled, alreadyCancelled, conflicts, notFound }` summary. Exit codes: any unknown id → 4; else any conflict → 6; else 0. + +```bash +testsprite test cancel run_01hx3z9p8q4k2y7a +testsprite test cancel run_aaaa run_bbbb --output json +testsprite test cancel run_01hx3z9p8q4k2y7a --dry-run --output json +``` + #### `testsprite test artifact get ` Download the failure bundle for a specific `runId`. Same on-disk layout as `test failure get`, but addressed by `runId` instead of `testId`, so an agent can fetch the bundle for the exact run it just triggered — never a newer failure on the same test. Default `` is `./.testsprite/runs//`. The CLI enforces `meta.runId === ` as an integrity check; a mismatch exits 5 rather than silently writing the wrong bundle. @@ -603,6 +623,13 @@ check is skipped in CI, when stderr is not a TTY, under `--output json` / failure is silent: the notice can never break or delay a command. This is the only outbound call the CLI makes besides your configured API endpoint. +Separately, the backend advertises its **minimum supported CLI version** on +every `/api/cli/v1` response. When the running CLI is below that floor, a +one-line upgrade advisory is printed to stderr (same opt-outs as the update +notice; it never changes the exit status). A backend may also reject a +too-old client outright with HTTP 426 — surfaced as `CLIENT_TOO_OLD`, +exit `14`, non-retriable, with upgrade guidance. + ### Scopes API-key scopes gate the write and run surfaces: @@ -613,8 +640,8 @@ API-key scopes gate the write and run surfaces: | `read:projects` | `project list / get` | | `read:tests` | every `test *` read command | | `write:tests` | `test create / create-batch / update / delete / code put / plan put` | -| `write:projects` | `project create / update / credential / auto-auth` | -| `run:tests` | `test run / rerun / flaky / wait / artifact get` | +| `write:projects` | `project create / update / delete / credential / auto-auth` | +| `run:tests` | `test run / rerun / flaky / wait / cancel / artifact get` | New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. @@ -632,25 +659,26 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not ## Exit codes -| Code | Meaning | -| --------------------- | --------------------------------------------------------------------------- | -| `0` | Success | -| `1` | Generic failure / non-passed run status | -| `2` | Not yet implemented | -| `3` | Auth error | -| `4` | Not found | -| `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | -| `7` | Timeout / unsupported | -| `10` | Service unavailable | -| `11` | Rate limited (retriable) | -| `12` | Insufficient credits (non-retriable) | -| `13` | Feature gated (paid plan required) | -| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | +| Code | Meaning | +| --------------------- | ------------------------------------------------------------------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | +| `14` | Client too old — the backend requires a newer CLI (HTTP 426 `CLIENT_TOO_OLD`); upgrade to proceed | +| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | ### Signals & pipes -On SIGINT (Ctrl-C), SIGTERM, or SIGHUP the CLI prints `Interrupted (). Any run already started keeps executing on the server; check it with 'testsprite test list' or 'testsprite test wait '.` and exits `128 + signal`. **Ctrl-C does not cancel the server-side run** — execution (and any credit spend) continues; there is no cancel command today, so re-attach with `test wait ` instead of re-triggering. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. +During any `--wait`, SIGINT (Ctrl-C), SIGTERM, or SIGHUP triggers a **graceful detach**: the in-flight request aborts immediately, stdout gets the same partial `{ runId, status: "running" }` envelope as the request-timeout path (under `--output json`, stderr carries an `INTERRUPTED` envelope naming the signal), and stderr states the truth — the server-side run keeps executing, and any credit spend continues — with a re-attach hint (`test wait `) and a `test cancel ` pointer. The exit code is `128 + signal` (130 / 143 / 129). A second signal forces an immediate hard exit. Outside a `--wait` (prompts, one-shot commands), signals keep the pre-existing immediate-exit behavior. **Ctrl-C never cancels the server-side run** — `test cancel ` is the explicit stop. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. ## Design principles diff --git a/README.md b/README.md index 78d293a..3bb465e 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude > **Pointing a coding agent (Claude Code, Cursor, Codex, Cline, …) at TestSprite?** Have it run `testsprite setup` first — that installs the verification skill, so the agent knows how to create, run, and triage tests on its own (instead of guessing from this README). New here? Start with the **[getting-started overview](https://docs.testsprite.com/cli/getting-started/overview)**. -> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). +> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. The backend also advertises its minimum supported CLI version — a below-floor CLI prints a one-line upgrade advisory on stderr, and a too-old client may be rejected with exit 14 (`CLIENT_TOO_OLD`). Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). From there, the loop runs on its own — an example session, typed by the coding agent: @@ -91,34 +91,35 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | -| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | -| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / permanently delete (no restore window; `--confirm` required) | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | -| | `test wait` | Block on one or more `runId`s until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | +| Group | Command | What it does | +| --------- | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | +| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | +| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata and BE dependency declarations (`--produces` / `--needs` / `--category`) / permanently delete (no restore window; `--confirm` required) | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` / `project delete` | Manage projects; `delete` removes a project and everything under it (`--confirm` required, no restore window) | +| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | +| | `test wait` | Block on one or more `runId`s until terminal | +| | `test cancel` | Cancel one or more in-flight runs (Ctrl-C during `--wait` only detaches — `cancel` is the real stop) | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). diff --git a/package-lock.json b/package-lock.json index b254e0d..4461016 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.2.0", + "version": "0.3.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", diff --git a/package.json b/package.json index e78ce17..18336cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.4.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..7e17420 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,29 @@ +# `scripts/` — what runs where + +Every file here is classified below so nobody has to guess whether it's +safe/expected to run on a laptop, and so a Windows contributor knows exactly +what (if anything) they're missing. Per DEV-356 ("Windows-proof the +toolchain"): **the release/backport shell scripts are CI-only** — nothing in +this directory requires a human to run bash or perl locally. + +| File | Classification | Runs on | Notes | +| ------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `make-public-snapshot.sh` | **CI-ONLY** | the release pipeline's build job (`release-build.yml`) and the nightly divergence sentinel (dry-run mode) | Builds the scrubbed public snapshot. Never ships to the public repo (it's in its own DROP list). Not meant to be run by hand — see [`docs/internal/cli-oss/release-pipeline-ops.md`](../docs/internal/cli-oss/release-pipeline-ops.md) for the operator-facing flow. | +| `backport-public-pr.sh` | **CI-ONLY** (human-runnable for conflict recovery) | the auto-backport workflow, invoked per merged community PR | Also directly runnable by an operator resolving a cherry-pick conflict (`--record-only` after a manual fix) — that's an escape hatch, not the steady-state path. Requires `bash` + `gh` + `jq`; a Windows operator resolving a conflict does it via Git Bash/WSL, or asks another maintainer — this one script is the sole remaining bash dependency in the whole release flow. | +| `generate-version.mjs` | **HUMAN-RUN** (Node) | `npm run prebuild` / `npm run generate:version`, any OS | Pure Node — no shell-out, no bash/perl/BSD-vs-GNU assumptions. | +| `postbuild.mjs` | **HUMAN-RUN** (Node) | `npm run build`, any OS | Pure Node `fs` calls (sets the executable bit on `dist/index.js`; a no-op on Windows, which has no POSIX exec bit). | +| `p0-status-coverage.sql` | **DROPPED-INTERNAL** | ad hoc, by an operator, against Athena | Not part of any npm script or CI workflow. Never ships to the public repo (internal AWS resource references). | + +## The upshot for a Windows contributor / operator + +- **Local dev loop** (`npm ci`, `npm test`, `npm run build`, `npm run lint`, + `npm run typecheck`) needs only **git + Node ≥ 20** — nothing in this + directory runs as part of that loop. See CONTRIBUTING.md's "Developing on + Windows" section. +- **Releasing** does not require running `make-public-snapshot.sh` (or any + bash) on your own machine at all — it's a `workflow_dispatch` you trigger + and approve from a browser. See + [`docs/internal/cli-oss/release-pipeline-ops.md`](../docs/internal/cli-oss/release-pipeline-ops.md). +- The **only** scenario that still touches bash directly is resolving a rare + backport cherry-pick conflict by hand — everything else in the pipeline is + either Node or runs unattended in Actions. diff --git a/skills/testsprite-verify.codex.md b/skills/testsprite-verify.codex.md index f2c2456..5d28f16 100644 --- a/skills/testsprite-verify.codex.md +++ b/skills/testsprite-verify.codex.md @@ -10,6 +10,12 @@ docs/config and is about to be reported complete. Run after a feature or fix lands. Skip only for: docs-only edits, pure build/config changes, or when the repo has no TestSprite project linked. +The CLI only tests a reachable deployed URL (it rejects localhost). If the +change is only running locally, hand off to the TestSprite MCP when it's +available — it tunnels your local server; otherwise report the change as +unverified-because-undeployed and stop. If the user explicitly named a tool +(the CLI or the MCP), honor that over this reachability heuristic. + ## Core loop ### 1. Preflight diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index fe7c661..5020e42 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -21,6 +21,17 @@ Run the loop only once the change is live somewhere reachable (e.g. open the PR, let CI deploy the preview/staging environment) and pass that URL as `--target-url`. Running earlier verifies the previous build, not your change. +This CLI only tests a reachable deployed URL (it rejects localhost). If the +change is only running locally and isn't deployed anywhere reachable yet: + +- if the TestSprite MCP is available in this environment, hand off to it — it + tunnels your local server and tests the running app; +- otherwise report the change as unverified-because-undeployed and stop — don't + run against a stale deployment to manufacture a verdict. + +If the user explicitly named a tool (the CLI or the MCP), honor that over this +reachability heuristic. + ## When to skip The skip list is narrow: @@ -225,10 +236,10 @@ testsprite test create --type backend --project --name "fixture user ordering: trigger the set with `test run --all` (§4) and producers run before consumers, `teardown` last. Chaining `run A --wait && run B --wait` yourself loses the engine's variable passing and conflicts with concurrent runs. -- **Declarations are currently create-only.** `test update` cannot amend them and - `test get` / `test list` don't echo them back, so note what each test - produces/needs as you create it; changing the graph later means delete + - recreate. +- **Declarations are editable.** `test update` amends them (`--produces` / + `--needs` / `--category`, repeatable) and `test get` echoes them back — + still declare the graph at create time when you can, so wave ordering is + right on the first run. **Show the user the drafted plan / code before creating it** — creating writes to their project. One short confirmation; let them edit the tempfile first. diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 24d8ec6..7326aeb 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -1108,75 +1108,87 @@ describe('runInstall — default AgentFs (real disk)', () => { expect(readFileSync(abs, 'utf8')).toBe(content); }); - it('refuses to write through a symlinked parent dir (real disk) — exit 5', async () => { - const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-parent-')); - const outside = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-')); - // `.claude` is a real symlink to a directory outside the project root. - symlinkSync(outside, path.join(tmpRoot, '.claude'), 'dir'); - const { deps } = makeCapture(); - - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: ['claude'], - skills: ['testsprite-verify'], - force: false, - dir: tmpRoot, - }, - { ...deps }, - ); - } catch (err) { - thrown = err; - } - - expect(thrown).toBeInstanceOf(CLIError); - expect((thrown as CLIError).exitCode).toBe(5); - // Nothing was created through the symlink, outside --dir. - expect(existsSync(path.join(outside, 'skills'))).toBe(false); - }); - - it('refuses to overwrite a symlinked target file (real disk) with --force — exit 5', async () => { - const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-target-')); - const outsideDir = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-target-')); - const { path: relPath } = renderForTarget('claude', 'testsprite-verify'); - const abs = path.resolve(tmpRoot, relPath); - const nodeFs = await import('node:fs/promises'); - await nodeFs.mkdir(path.dirname(abs), { recursive: true }); - // SKILL.md is a real symlink to a file outside the project root. - const outsideFile = path.join(outsideDir, 'secret.txt'); - await nodeFs.writeFile(outsideFile, 'SECRET', 'utf8'); - symlinkSync(outsideFile, abs, 'file'); - const { deps } = makeCapture(); + // `fs.symlinkSync` needs elevated privileges or Developer Mode on Windows + // (EPERM otherwise) — not guaranteed on hosted CI runners. The underlying + // guard (`inspectTargetPath` fail-closing via `lstat`) is exercised on + // POSIX runners; TODO(DEV-356): revisit if/when a reliable Windows + // symlink-creation path (junctions, or an elevated runner) is available. + it.skipIf(process.platform === 'win32')( + 'refuses to write through a symlinked parent dir (real disk) — exit 5', + async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-parent-')); + const outside = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-')); + // `.claude` is a real symlink to a directory outside the project root. + symlinkSync(outside, path.join(tmpRoot, '.claude'), 'dir'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + skills: ['testsprite-verify'], + force: false, + dir: tmpRoot, + }, + { ...deps }, + ); + } catch (err) { + thrown = err; + } - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: ['claude'], - skills: ['testsprite-verify'], - force: true, - dir: tmpRoot, - }, - { ...deps }, - ); - } catch (err) { - thrown = err; - } + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + // Nothing was created through the symlink, outside --dir. + expect(existsSync(path.join(outside, 'skills'))).toBe(false); + }, + ); + + // Same Windows symlink-privilege caveat as the test above. + it.skipIf(process.platform === 'win32')( + 'refuses to overwrite a symlinked target file (real disk) with --force — exit 5', + async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-target-')); + const outsideDir = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-target-')); + const { path: relPath } = renderForTarget('claude', 'testsprite-verify'); + const abs = path.resolve(tmpRoot, relPath); + const nodeFs = await import('node:fs/promises'); + await nodeFs.mkdir(path.dirname(abs), { recursive: true }); + // SKILL.md is a real symlink to a file outside the project root. + const outsideFile = path.join(outsideDir, 'secret.txt'); + await nodeFs.writeFile(outsideFile, 'SECRET', 'utf8'); + symlinkSync(outsideFile, abs, 'file'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + skills: ['testsprite-verify'], + force: true, + dir: tmpRoot, + }, + { ...deps }, + ); + } catch (err) { + thrown = err; + } - expect(thrown).toBeInstanceOf(CLIError); - expect((thrown as CLIError).exitCode).toBe(5); - // The outside file was NOT overwritten (nor clobbered via the .bak path). - expect(readFileSync(outsideFile, 'utf8')).toBe('SECRET'); - }); + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + // The outside file was NOT overwritten (nor clobbered via the .bak path). + expect(readFileSync(outsideFile, 'utf8')).toBe('SECRET'); + }, + ); }); // --------------------------------------------------------------------------- @@ -2036,12 +2048,12 @@ describe('[P3 round-2] codex --dry-run: composed-size precision + read-failure s const { capture, deps } = makeCapture(); const agentsAbs = path.resolve(CWD, TARGETS.codex.path); - // Old managed section with a 6 KiB body + 26 KiB of user prose. - // existing (~32.9 KiB) + new section (~4.8 KiB) > 32 KiB → the OLD formula - // would warn; the composed replace result (26 KiB + new section) is - // comfortably under budget → no warn expected. + // Old managed section with a 6 KiB body + 25 KiB of user prose. + // existing (~31.9 KiB) + new section (~6.2 KiB: verify codex body + + // onboard aggregate) > 32 KiB → the OLD formula would warn; the composed + // replace result (25 KiB + new section) is under budget → no warn expected. const oldSection = `${MANAGED_SECTION_BEGIN}\n${'o'.repeat(6 * 1024)}\n${MANAGED_SECTION_END}\n`; - const userProse = `# My own AGENTS.md\n${'u'.repeat(26 * 1024)}\n`; + const userProse = `# My own AGENTS.md\n${'u'.repeat(25 * 1024)}\n`; seedFile(agentsAbs, `${userProse}\n${oldSection}`); await runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }); diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index c0b4e00..6f8ce94 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -850,6 +850,63 @@ describe('runWhoami', () => { expect(printed).toEqual(sampleMe); }); + it('renders routing: v3 and the gap advisory when v3Enabled is true', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV3 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV3) }, + ); + expect(capture.stdout.join('\n')).toContain('routing: v3'); + expect(capture.stderr.join('\n')).toContain('[advisory]'); + expect(capture.stderr.join('\n')).toContain('test cancel'); + }); + + it('renders routing: v2 and NO advisory when v3Enabled is false', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV2 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: false }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV2) }, + ); + expect(capture.stdout.join('\n')).toContain('routing: v2'); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('omits the routing line when the backend does not return v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ); + expect(capture.stdout.join('\n')).not.toContain('routing:'); + }); + + it('does not emit the advisory in JSON mode even when v3Enabled is true', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV3 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV3) }, + ); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + const parsed = JSON.parse(capture.stdout.join('')) as MeResponse; + expect(parsed.v3Enabled).toBe(true); + }); + it('dry-run: whitespace-only TESTSPRITE_API_URL falls through to prod default endpoint', async () => { const { capture, deps } = makeCapture(); await runWhoami( diff --git a/src/commands/auth.ts b/src/commands/auth.ts index 59cd43f..e594066 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -22,6 +22,7 @@ import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptSecret } from '../lib/prompt.js'; +import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; export interface MeResponse { userId: string; @@ -37,6 +38,11 @@ export interface MeResponse { email?: string; /** Human-readable display name for the bound account. Absent-safe (dogfood L1866). */ displayName?: string; + /** + * Authoritative per-user V3 routing bit. Absent-safe: older backends omit it, + * so it is only rendered when present. + */ + v3Enabled?: boolean; } export interface AuthDeps { @@ -201,6 +207,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promise { const out = makeOutput(opts.output, deps); const env = deps.env ?? process.env; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); // Resolve the endpoint URL so it can be surfaced in text output. // Dry-run uses the flag/env/default chain without touching credentials. @@ -244,6 +251,8 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi `endpoint: ${resolvedEndpoint}`, `env: ${m.env}`, `scopes: ${m.scopes.join(', ')}`, + // Authoritative routing mode, rendered only when the backend supplies it. + ...(m.v3Enabled !== undefined ? [`routing: ${routingLabel(m.v3Enabled)}`] : []), ]; // C2: warn in text mode when key cannot write/run const missingScopes = (['write:tests', 'run:tests'] as const).filter( @@ -256,6 +265,11 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi } return lines.join('\n'); }); + // When V3 routing is on, warn (text mode only) about the still-open behavior + // gaps. JSON consumers read `v3Enabled` directly; stdout stays pure. + if (opts.output !== 'json' && me.v3Enabled === true) { + emitV3RoutingAdvisory(stderr); + } return me; } diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index 230398a..7a508dd 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -78,6 +78,51 @@ describe('runDoctor — healthy environment', () => { expect(out).toContain('reached GET /me'); }); + it('adds a Routing check (v3) and the gap advisory when /me reports v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { + ...healthyDeps(credentialsPath, { + fetchImpl: makeFetch({ ...OK_ME, v3Enabled: true }), + }), + ...deps, + }, + ); + expect(report.failures).toBe(0); + expect(report.checks.some(c => c.name === 'Routing' && c.detail.includes('v3'))).toBe(true); + expect(capture.stderr.join('\n')).toContain('[advisory]'); + expect(capture.stderr.join('\n')).toContain('test cancel'); + }); + + it('shows Routing v2 and no advisory when v3Enabled is false', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { + ...healthyDeps(credentialsPath, { + fetchImpl: makeFetch({ ...OK_ME, v3Enabled: false }), + }), + ...deps, + }, + ); + expect(report.checks.some(c => c.name === 'Routing' && c.detail.includes('v2'))).toBe(true); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('omits the Routing check when /me does not report v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, // OK_ME has no v3Enabled + ); + expect(report.checks.some(c => c.name === 'Routing')).toBe(false); + expect(report.warnings).toBe(0); + }); + it('never prints the API key anywhere in the report', async () => { writeProfile('default', { apiKey: 'sk-super-secret-value' }, { path: credentialsPath }); const { capture, deps } = makeCapture(); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 2fa0c57..cceeb13 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -26,6 +26,7 @@ import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; +import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; import { VERSION } from '../version.js'; import { MIN_SUPPORTED_NODE_MAJOR, shouldRejectNodeVersion } from '../version-guard.js'; @@ -49,6 +50,7 @@ export interface DoctorReport { interface MeIdentity { userId?: string; keyId?: string; + v3Enabled?: boolean; } export interface DoctorDeps { @@ -81,6 +83,12 @@ export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Pro }); const endpointCheck = checkEndpoint(config.apiUrl); const hasKey = Boolean(config.apiKey); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + const connectivity = await checkConnectivity(opts, deps, { + hasKey, + endpointOk: endpointCheck.status === 'ok', + }); const checks: DoctorCheck[] = [ { name: 'CLI version', status: 'ok', detail: VERSION }, @@ -88,19 +96,34 @@ export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Pro { name: 'Profile', status: 'ok', detail: config.profile }, endpointCheck, checkCredentials(hasKey, config.profile, opts.dryRun ?? false), - await checkConnectivity(opts, deps, { - hasKey, - endpointOk: endpointCheck.status === 'ok', - }), - checkSkill(cwd, deps), + connectivity.check, ]; + // Informational routing line, only when the backend reported it (no new call). + if (connectivity.v3Enabled !== undefined) { + const label = routingLabel(connectivity.v3Enabled); + checks.push({ + name: 'Routing', + status: 'ok', + detail: + connectivity.v3Enabled === true + ? `${label} (V3 execution routing is ON)` + : `${label} (default routing)`, + }); + } + + checks.push(checkSkill(cwd, deps)); + const failures = checks.filter(check => check.status === 'fail').length; const warnings = checks.filter(check => check.status === 'warn').length; const report: DoctorReport = { checks, failures, warnings }; out.print(report, () => renderDoctor(report)); + if (connectivity.v3Enabled === true) { + emitV3RoutingAdvisory(stderr); + } + if (failures > 0) { // Non-zero exit so `testsprite doctor && ...` gates a CI step or an agent // preflight. The full report already printed above; this line is the stderr @@ -175,11 +198,13 @@ async function checkConnectivity( opts: CommonOptions, deps: DoctorDeps, ctx: { hasKey: boolean; endpointOk: boolean }, -): Promise { +): Promise<{ check: DoctorCheck; v3Enabled?: boolean }> { const name = 'Connectivity'; - if (opts.dryRun) return { name, status: 'warn', detail: 'skipped under --dry-run' }; - if (!ctx.hasKey) return { name, status: 'warn', detail: 'skipped; no API key to test with' }; - if (!ctx.endpointOk) return { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' }; + if (opts.dryRun) return { check: { name, status: 'warn', detail: 'skipped under --dry-run' } }; + if (!ctx.hasKey) + return { check: { name, status: 'warn', detail: 'skipped; no API key to test with' } }; + if (!ctx.endpointOk) + return { check: { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' } }; try { const client = makeHttpClient(opts, { @@ -190,7 +215,10 @@ async function checkConnectivity( }); const me = await client.get('/me'); const who = me.userId ? ` (userId ${me.userId})` : ''; - return { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }; + return { + check: { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }, + v3Enabled: me.v3Enabled, + }; } catch (error) { if (error instanceof ApiError) { if ( @@ -198,14 +226,16 @@ async function checkConnectivity( error.code === 'AUTH_INVALID' || error.code === 'AUTH_FORBIDDEN' ) { - return { name, status: 'fail', detail: `API key rejected (${error.code})` }; + return { check: { name, status: 'fail', detail: `API key rejected (${error.code})` } }; } - return { name, status: 'fail', detail: `GET /me failed (${error.code})` }; + return { check: { name, status: 'fail', detail: `GET /me failed (${error.code})` } }; } return { - name, - status: 'fail', - detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, + check: { + name, + status: 'fail', + detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, + }, }; } } diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index daea851..0f8c51c 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -6,11 +6,13 @@ import { ApiError } from '../lib/errors.js'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; import { type CliProject, + type CliDeleteProjectResponse, type CliUpdateProjectResponse, createProjectCommand, runAutoAuth, runCreate, runCredential, + runDelete, runGet, runList, runUpdate, @@ -74,10 +76,10 @@ describe('createProjectCommand', () => { errorSpy.mockRestore(); }); - it('exposes list, get, create, update, credential and auto-auth subcommands', () => { + it('exposes list, get, create, update, delete, credential and auto-auth subcommands', () => { const project = createProjectCommand(); const names = project.commands.map(c => c.name()).sort(); - expect(names).toEqual(['auto-auth', 'create', 'credential', 'get', 'list', 'update']); + expect(names).toEqual(['auto-auth', 'create', 'credential', 'delete', 'get', 'list', 'update']); }); it('list exposes the pagination flags from the design contract', () => { @@ -681,6 +683,34 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + + it('rejects --description with VALIDATION_ERROR (exit 5), no network — projects have no description', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network — validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Desc Project', + targetUrl: 'https://example.com', + description: 'a human description', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); }); // --------------------------------------------------------------------------- @@ -946,6 +976,140 @@ describe('runUpdate', () => { }); }); +describe('runDelete', () => { + it('refuses without --confirm and never hits the network (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: {} }; + }); + await expect( + runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'confirm' }), + }); + expect(called).toBe(0); + }); + + it('DELETEs /projects/{id} with a minted idempotency-key when --confirm is set', async () => { + const { credentialsPath } = makeCreds(); + const deleteResponse: CliDeleteProjectResponse = { + projectId: 'proj_alpha', + deletedAt: '2026-05-16T10:00:00.000Z', + }; + let seenUrl = ''; + let seenMethod = ''; + let seenIdemKey: string | null = null; + const fetchImpl = (async (input: Parameters[0], init: RequestInit = {}) => { + seenUrl = typeof input === 'string' ? input : (input as { url: string }).url; + seenMethod = init.method ?? 'GET'; + seenIdemKey = new Headers(init.headers).get('idempotency-key'); + return new Response(JSON.stringify(deleteResponse), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + + const result = await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: true, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(seenMethod).toBe('DELETE'); + expect(seenUrl).toContain('/api/cli/v1/projects/proj_alpha'); + expect(seenIdemKey).toMatch(/^cli-delete-[0-9a-f-]{36}$/); + expect(result.projectId).toBe('proj_alpha'); + expect(result.deletedAt).toBe('2026-05-16T10:00:00.000Z'); + }); + + it('forwards a caller-supplied --idempotency-key verbatim', async () => { + const { credentialsPath } = makeCreds(); + let seenIdemKey: string | null = null; + const fetchImpl = (async (_input: Parameters[0], init: RequestInit = {}) => { + seenIdemKey = new Headers(init.headers).get('idempotency-key'); + return new Response( + JSON.stringify({ projectId: 'proj_alpha', deletedAt: '2026-05-16T10:00:00.000Z' }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof fetch; + + await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: true, + idempotencyKey: 'idem-del-001', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(seenIdemKey).toBe('idem-del-001'); + }); + + it('--dry-run bypasses --confirm and returns the canned sample without network', async () => { + resetDryRunBannerForTesting(); + const { credentialsPath } = makeCreds(); + const err: string[] = []; + // No fetchImpl → the client-factory dry-run fetch serves the samples.ts value. + const result = await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_b3c91efa', + confirm: false, + }, + { credentialsPath, stdout: () => {}, stderr: line => err.push(line) }, + ); + + expect(result.projectId).toBe('project_b3c91efa'); + expect(result.deletedAt).toBe('2026-05-16T00:00:00.000Z'); + expect(err).toContain(DRY_RUN_BANNER); + }); + + it('renders text mode with projectId and deletedAt', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { projectId: 'proj_text', deletedAt: '2026-05-16T10:00:00.000Z' }, + })); + const out: string[] = []; + await runDelete( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_text', + confirm: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, + ); + const block = out.join('\n'); + expect(block).toContain('projectId proj_text'); + expect(block).toContain('deletedAt 2026-05-16T10:00:00.000Z'); + }); +}); + describe('runCredential', () => { interface Captured { url: string; diff --git a/src/commands/project.ts b/src/commands/project.ts index 96f2fc8..0a843e5 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -110,7 +110,8 @@ export interface CliCreateProjectRequest { type: 'frontend' | 'backend'; name: string; targetUrl?: string; - description?: string; + // `description` is intentionally not part of the wire request — projects have + // no description field. The `--description` flag is rejected client-side. username?: string; password?: string; instruction?: string; @@ -154,8 +155,13 @@ export async function runCreate( if (opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } - if (opts.description !== undefined && opts.description.length > 2000) { - throw localValidationError('--description must be at most 2000 characters'); + // `--description` is not supported on projects — no project entity stores a + // description, and the backend rejects it with a 422. Fail fast client-side + // with an actionable message instead of a wasted round trip. + if (opts.description !== undefined) { + throw localValidationError( + '--description is not supported for projects; omit it (test-level descriptions are set on `test create`)', + ); } // P2-7: guard --url against localhost/RFC1918/non-http(s) (same rules as @@ -210,7 +216,6 @@ export async function runCreate( type: opts.type, name: opts.name, ...(opts.targetUrl !== undefined ? { targetUrl: opts.targetUrl } : {}), - ...(opts.description !== undefined ? { description: opts.description } : {}), ...(opts.username !== undefined ? { username: opts.username } : {}), ...(password !== undefined ? { password } : {}), ...(opts.instruction !== undefined ? { instruction: opts.instruction } : {}), @@ -346,6 +351,80 @@ export async function runUpdate( return updated; } +// --------------------------------------------------------------------------- +// project delete +// --------------------------------------------------------------------------- + +export interface CliDeleteProjectResponse { + projectId: string; + deletedAt: string; +} + +interface DeleteOptions extends CommonOptions { + projectId: string; + /** Hard gate — required (unless `--dry-run` is set). No interactive prompts. */ + confirm: boolean; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; +} + +/** + * `project delete --confirm` — permanent cascade delete via + * DELETE /projects/{id}. + * + * The server deletes the project together with everything under it — its + * frontend/backend sub-projects, all their tests, and backend fixtures — + * matching the Portal's own delete behavior. There is no restore window. + * + * **`--confirm` is required** (unless `--dry-run`). Without either, the CLI + * exits 5 `VALIDATION_ERROR` with a typed envelope explaining the convention. + * The CLI never prompts interactively (CI-friendly contract). Re-delete on an already-deleted (or missing) project returns 404 from + * the server; the CLI surfaces the envelope as-is (exit 4), no client branching. + */ +export async function runDelete( + opts: DeleteOptions, + deps: ProjectDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + if (opts.projectId === undefined || opts.projectId.trim().length === 0) { + throw localValidationError(' is required'); + } + + if (!opts.confirm && !opts.dryRun) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Refusing to delete without --confirm.', + nextAction: + 'This permanently deletes the project and everything under it — its ' + + 'sub-projects, all their tests, and backend fixtures (no restore window). ' + + 'The CLI convention is explicit confirmation for destructive operations. ' + + 'Re-run with --confirm. (--dry-run also works without --confirm.)', + requestId: 'local', + details: { field: 'confirm', reason: 'required for destructive operation' }, + }, + }); + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-delete-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + const response = await client.delete( + `/projects/${encodeURIComponent(opts.projectId)}`, + { + headers: { 'idempotency-key': idempotencyKey }, + }, + ); + + out.print(response, data => renderDeleteText(data as CliDeleteProjectResponse)); + return response; +} + // --------------------------------------------------------------------------- // project credential — set the static backend credential // --------------------------------------------------------------------------- @@ -616,7 +695,10 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--type ', 'project type (required)') .option('--name ', 'project name (required)') .option('--url ', 'target URL (required for frontend)') - .option('--description ', 'optional human description') + .option( + '--description ', + 'not supported — projects have no description (test-level descriptions are set on `test create`)', + ) .option('--username ', 'optional auth username') .option('--password ', 'optional auth password (use --password-file for non-interactive)') .option('--password-file ', 'read password from file instead of inline flag') @@ -684,6 +766,35 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { ); }); + project + .command('delete ') + .description( + 'Permanently delete a project and everything under it (sub-projects,\n' + + 'their tests, and backend fixtures). Requires --confirm.\n' + + '\nExit codes:\n' + + ' 0 success\n' + + ' 3 auth error\n' + + ' 4 project not found (or already deleted)\n' + + ' 5 validation error (e.g., missing --confirm)', + ) + .option('--confirm', 'required: explicit confirmation for the destructive operation', false) + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: DeleteFlagOpts, command: Command) => { + await runDelete( + { + ...resolveCommonOptions(command), + projectId, + confirm: cmdOpts.confirm === true, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + project .command('credential ') .description( @@ -808,6 +919,11 @@ interface UpdateFlagOpts { idempotencyKey?: string; } +interface DeleteFlagOpts { + confirm?: boolean; + idempotencyKey?: string; +} + interface CredentialFlagOpts { type: string; credential?: string; @@ -952,6 +1068,10 @@ function renderUpdateText(r: CliUpdateProjectResponse): string { ].join('\n'); } +function renderDeleteText(r: CliDeleteProjectResponse): string { + return [`projectId ${r.projectId}`, `deletedAt ${r.deletedAt}`].join('\n'); +} + function localValidationError(message: string): ApiError { return ApiError.fromEnvelope({ error: { diff --git a/src/commands/test.cancel.spec.ts b/src/commands/test.cancel.spec.ts new file mode 100644 index 0000000..72496b0 --- /dev/null +++ b/src/commands/test.cancel.spec.ts @@ -0,0 +1,309 @@ +/** + * Unit tests for `test cancel `. + * + * Covers: single-id happy path (text + json), `alreadyCancelled` advisory, + * multi-id mixed summary + exit precedence, 404, 409, dry-run. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; +import { ApiError } from '../lib/errors.js'; +import type { CancelRunResponse } from '../lib/runs.types.js'; +import { runTestCancel, type CliCancelSummary } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13503', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-cancel-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +function makeCancelResponse( + runId: string, + overrides: Partial = {}, +): CancelRunResponse { + return { + runId, + testId: 'test_xyz', + projectId: 'project_1', + userId: 'user_1', + status: 'cancelled', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 2, passedCount: 2, failedCount: 0 }, + alreadyCancelled: false, + ...overrides, + }; +} + +function errorBody(code: string, details: Record = {}) { + const statusMap: Record = { + NOT_FOUND: 404, + CONFLICT: 409, + AUTH_FORBIDDEN: 403, + }; + return { + status: statusMap[code] ?? 400, + body: { + error: { + code, + message: `Error: ${code}`, + nextAction: 'do something', + requestId: 'req_test', + details, + }, + }, + }; +} + +/** Extract the runId embedded in a `POST /runs/{runId}/cancel` URL. */ +function runIdFromCancelUrl(url: string): string | undefined { + const match = /\/runs\/([^/]+)\/cancel/.exec(url); + return match?.[1]; +} + +// --------------------------------------------------------------------------- +// Single id — happy path +// --------------------------------------------------------------------------- + +describe('runTestCancel — single id happy path', () => { + it('CXL-1: fresh cancel of a queued/running run → 200, alreadyCancelled:false, no advisory', async () => { + const { credentialsPath } = makeCreds(); + let seenUrl = ''; + let seenMethod = ''; + const fetchImpl = makeFetch((url, init) => { + seenUrl = url; + seenMethod = init.method ?? 'GET'; + return { body: makeCancelResponse('run_abc') }; + }); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + + expect(seenMethod).toBe('POST'); + expect(runIdFromCancelUrl(seenUrl)).toBe('run_abc'); + expect(result.status).toBe('cancelled'); + expect(result.alreadyCancelled).toBe(false); + expect(stderrLines.some(l => l.includes('already cancelled'))).toBe(false); + }); + + it('renders the run card in text mode with status cancelled', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeCancelResponse('run_abc') })); + const stdoutLines: string[] = []; + await runTestCancel( + { profile: 'default', output: 'text', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => {} }, + ); + const block = stdoutLines.join('\n'); + expect(block).toContain('run_abc'); + expect(block).toContain('status'); + expect(block).toContain('cancelled'); + }); + + it('CXL-5: alreadyCancelled:true → [advisory] stderr line, still exit 0 (no throw)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: makeCancelResponse('run_abc', { alreadyCancelled: true }), + })); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + expect(result.alreadyCancelled).toBe(true); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('already cancelled'))).toBe( + true, + ); + }); + + it('CXL-6: unknown/cross-tenant runId → 404 propagates as ApiError exit 4', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('NOT_FOUND')); + const err = await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_ghost'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(4); + }); + + it('CXL-4: already-terminal run → 409 propagates as ApiError exit 6', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('CONFLICT', { status: 'passed' })); + const err = await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_done'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(6); + }); +}); + +// --------------------------------------------------------------------------- +// Multi-id — summary + exit precedence (CXL-11) +// --------------------------------------------------------------------------- + +describe('runTestCancel — multi-id summary + exit precedence (CXL-11)', () => { + it('all cancelled/alreadyCancelled → exit 0, summary buckets correct', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_2') { + return { body: makeCancelResponse(runId, { alreadyCancelled: true }) }; + } + return { body: makeCancelResponse(runId) }; + }); + const result = (await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_1', 'run_2'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + )) as CliCancelSummary; + expect(result.cancelled).toEqual(['run_1']); + expect(result.alreadyCancelled).toEqual(['run_2']); + expect(result.conflicts).toEqual([]); + expect(result.notFound).toEqual([]); + // Stable machine shape (codex finding 2): errors is ALWAYS present, + // an empty array on full success — never an absent key. + expect(result.errors).toEqual([]); + }); + + it('mixed: cancelled + conflict + notFound → notFound wins (exit 4), all buckets populated', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'failed' }); + if (runId === 'run_ghost') return errorBody('NOT_FOUND'); + return { body: makeCancelResponse(runId) }; + }); + const err = await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict', 'run_ghost'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err.exitCode).toBe(4); // notFound outranks conflict + expect(err.message).toContain('run_ghost'); + }); + + it('cancelled + conflict, no notFound → conflict wins (exit 6)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'blocked' }); + return { body: makeCancelResponse(runId) }; + }); + const err = await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err.exitCode).toBe(6); + expect(err.message).toContain('run_conflict'); + expect(err.message).toContain('blocked'); + }); + + it('JSON summary shape carries the conflicting run status', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'passed' }); + return { body: makeCancelResponse(runId) }; + }); + const stdoutLines: string[] = []; + await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict'], + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + }, + ).catch(() => {}); + const summary = JSON.parse(stdoutLines.join('\n')) as CliCancelSummary; + expect(summary.cancelled).toEqual(['run_ok']); + expect(summary.conflicts).toEqual([{ runId: 'run_conflict', status: 'passed' }]); + }); +}); + +// --------------------------------------------------------------------------- +// Dry-run +// --------------------------------------------------------------------------- + +describe('runTestCancel — dry-run', () => { + it('single id: prints the dry-run banner and a canned cancelled envelope, no real network', async () => { + resetDryRunBannerForTesting(); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: true, runIds: ['run_dry'] }, + { stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + expect(stderrLines.some(l => l.includes(DRY_RUN_BANNER))).toBe(true); + expect(result.status).toBe('cancelled'); + expect(result.alreadyCancelled).toBe(false); + }); +}); diff --git a/src/commands/test.quickwins.spec.ts b/src/commands/test.quickwins.spec.ts index c781300..322000c 100644 --- a/src/commands/test.quickwins.spec.ts +++ b/src/commands/test.quickwins.spec.ts @@ -937,3 +937,113 @@ describe('runDeleteBatch (dogfood L1796)', () => { expect(flagNames).toContain('--status'); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 (codex finding 2) — create-batch --run --wait interrupt partial +// carries the dispatched runIds, not empty placeholders +// --------------------------------------------------------------------------- + +describe('create-batch --run --wait — InterruptError partial names dispatched runIds (DEV-331)', () => { + it('interrupt mid-poll → partial rows carry the runIds recorded at trigger time', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-dev331-cbrun-')); + for (let i = 0; i < 2; i++) { + writeFileSync( + join(dir, `plan_${i}.json`), + JSON.stringify({ ...PLAN_SPEC, name: `Plan ${i}` }), + 'utf8', + ); + } + + const { ShutdownController } = await import('../lib/interrupt.js'); + const { InterruptError } = await import('../lib/errors.js'); + const shutdown = new ShutdownController(); + + // batch create resolves; each trigger POST resolves with a per-test runId; + // every run poll hangs until the composed signal aborts. + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/batch') && init.method === 'POST') { + return new Response(JSON.stringify(batchCreateResp(2)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (init.method === 'POST' && /\/tests\/[^/]+\/runs$/.test(url)) { + const testId = /\/tests\/([^/]+)\/runs$/.exec(url)![1]!; + return new Response( + JSON.stringify({ + runId: `run_${testId}`, + status: 'queued', + enqueuedAt: '2026-07-09T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + } + // GET /runs/{id} long-poll: hang until aborted. + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const pending = runCreateBatch( + { + plans: '', + planFromDir: dir, + run: true, + wait: true, + timeoutSeconds: 600, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + fetchImpl, + stdout: (l: string) => stdoutLines.push(l), + stderr: (l: string) => stderrLines.push(l), + sleep: () => Promise.resolve(), + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 25); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + + // The partial must name the real runIds recorded at trigger time — + // members mid-poll have no settled result, but their runId is known. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + results: Array<{ testId: string; runId: string; status: string }>; + }; + const byTestId = new Map(stdoutJson.results.map(r => [r.testId, r])); + expect(byTestId.get('test_batch_0')?.runId).toBe('run_test_batch_0'); + expect(byTestId.get('test_batch_0')?.status).toBe('running'); + expect(byTestId.get('test_batch_1')?.runId).toBe('run_test_batch_1'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('run_test_batch_0'); + expect(stderrBlock).toContain('run_test_batch_1'); + }); +}); diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 9f16c4b..3797f14 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -9,7 +9,8 @@ import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; import type { FetchImpl } from '../lib/http.js'; import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; @@ -4881,3 +4882,98 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach during batch rerun --wait (SIG-6) +// --------------------------------------------------------------------------- + +describe('R-BAT: batch rerun --wait — InterruptError partial lists all dispatched runIds (DEV-331)', () => { + it('interrupt mid fan-out → stdout partial covers every accepted runId, honest stderr, exit 130', async () => { + const creds = makeCreds(); + const shutdown = new ShutdownController(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + // Batch trigger resolves; every run poll hangs until the composed signal aborts. + const fetchImpl: FetchImpl = (async (input: unknown, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/batch/rerun')) { + return new Response(JSON.stringify(batchResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as FetchImpl; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const pending = runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 10); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + + // SIG-6: the partial lists ALL dispatched runIds, marked running. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ runId: string; status: string }>; + }; + const byRunId = new Map(stdoutJson.accepted.map(r => [r.runId, r.status])); + expect(byRunId.get('run_b1')).toBe('running'); + expect(byRunId.get('run_b2')).toBe('running'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('run_b1'); + expect(stderrBlock).toContain('run_b2'); + }); +}); diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index c0ff083..51b527f 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -10,7 +10,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; import type { FetchImpl } from '../lib/http.js'; import type { RunResponse, TriggerRunResponse, BatchRunFreshResponse } from '../lib/runs.types.js'; @@ -1699,6 +1700,88 @@ describe('C2 — backend run renders steps: n/a (backend) in text mode', () => { expect(out).toContain('steps n/a (backend)'); expect(out).not.toContain('0/0'); }); + + it('standalone BE run --wait: text probes /tests/{id} → n/a (backend); JSON never probes (DEV-282)', async () => { + // Standalone `test run ` supplies NO type hint, and the run row is + // terminal on the first poll (BE rows finalize server-side now), so + // `beFallbackUsed` stays false. In TEXT mode the card must still read + // `n/a (backend)`, resolved via a one-time `GET /tests/{id}` probe; in JSON + // mode the output is already correct and must NOT pay for that round-trip. + const { credentialsPath } = makeCreds(); + const passedBeRun: RunResponse = { + runId: 'run_282', + testId: 'be_282', + projectId: 'p1', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:02.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 0, completed: 0, passedCount: 0, failedCount: 0 }, + }; + const makeHandler = (urls: string[]) => (url: string) => { + urls.push(url); + if (url.includes('/tests/be_282/runs')) { + return { + body: { + runId: 'run_282', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + if (url.includes('/runs/run_282')) return { body: passedBeRun }; + // Bare type probe: GET /tests/be_282 (no /runs, no /result suffix). + if (/\/tests\/be_282$/.test(url)) return { body: { id: 'be_282', type: 'backend' } }; + return { status: 404, body: {} }; + }; + const runOnce = async (output: 'text' | 'json') => { + const urls: string[] = []; + const stdoutLines: string[] = []; + await runTestRun( + { + profile: 'default', + output, + debug: false, + dryRun: false, + testId: 'be_282', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(makeHandler(urls)), + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + return { urls, out: stdoutLines.join('\n') }; + }; + + // Text mode: probes the type and renders the honest backend placeholder. + const text = await runOnce('text'); + expect(text.out).toContain('steps n/a (backend)'); + expect(text.out).not.toContain('0/0'); + expect(text.urls.some(u => /\/tests\/be_282$/.test(u))).toBe(true); + + // JSON mode: no extra probe; the wire envelope ships stepSummary verbatim. + const json = await runOnce('json'); + expect(json.urls.some(u => /\/tests\/be_282$/.test(u))).toBe(false); + const parsed = JSON.parse(json.out); + expect(parsed.status).toBe('passed'); + expect(parsed.stepSummary).toEqual({ total: 0, completed: 0, passedCount: 0, failedCount: 0 }); + }); }); // --------------------------------------------------------------------------- @@ -3748,3 +3831,80 @@ describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out p expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach on SIGINT during test run --wait +// --------------------------------------------------------------------------- + +describe('runTestRun --wait — InterruptError graceful detach (DEV-331)', () => { + it('SIG-1: trigger succeeds, poll interrupted → partial to stdout + honest stderr + exit 130', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + // Trigger POST resolves; the subsequent GET /runs/{id} long-poll hangs + // until the composed signal aborts (real-fetch contract). + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (init.method === 'POST') { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + void url; + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + + const pending = runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 5); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('testsprite test wait run_abc'); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index f7c7602..9493908 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -120,6 +120,7 @@ describe('createTestCommand — surface', () => { const names = test.commands.map(c => c.name()).sort(); expect(names).toEqual([ 'artifact', + 'cancel', 'code', 'create', 'create-batch', @@ -834,6 +835,40 @@ describe('runGet', () => { expect(out.join('\n')).not.toContain('planSteps:'); }); + it('renders produces/consumes/category when the facade ships them', async () => { + const withDeps: CliTest = { + ...FE_TEST, + produces: ['user_id', 'order_id'], + consumes: ['session_token'], + category: 'teardown', + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: withDeps })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('produces: user_id, order_id'); + expect(block).toContain('consumes: session_token'); + expect(block).toContain('category: teardown'); + }); + + it('omits produces/consumes/category lines when absent', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: FE_TEST })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).not.toContain('produces:'); + expect(block).not.toContain('consumes:'); + expect(block).not.toContain('category:'); + }); + it('NOT_FOUND envelope from server propagates as ApiError exit 4', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ @@ -6275,6 +6310,74 @@ describe('runUpdate', () => { expect(seenBody).toEqual({ priority: 'p2' }); }); + it('threads --produces/--needs/--category into the PUT body with wire names', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + produces: ['user_id', 'order_id'], + needs: ['session_token'], + category: 'teardown', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ + produces: ['user_id', 'order_id'], + consumes: ['session_token'], + category: 'teardown', + }); + }); + + it('accepts a dependency-only update (not rejected as a no-op)', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + category: 'teardown', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ category: 'teardown' }); + }); + + it('omits empty dependency arrays from the body', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: 'renamed', + produces: [], + needs: [], + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ name: 'renamed' }); + }); + it('respects a caller-supplied --idempotency-key', async () => { const { credentialsPath } = makeCreds(); let seenKey: string | null = null; diff --git a/src/commands/test.ts b/src/commands/test.ts index 0602afc..837c170 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -38,10 +38,12 @@ import { import { ApiError, CLIError, + InterruptError, RequestTimeoutError, TransportError, localValidationError, } from '../lib/errors.js'; +import { globalShutdown, type ShutdownHandle } from '../lib/interrupt.js'; import { assertIdempotencyKey, requireArrayLength, @@ -75,6 +77,7 @@ import type { RunSource, BatchRunFreshResponse, BatchRunFreshAccepted, + CancelRunResponse, } from '../lib/runs.types.js'; import { RUN_SOURCES } from '../lib/runs.types.js'; import { assertNotLocal } from '../lib/target-url.js'; @@ -153,6 +156,14 @@ export interface CliTest { * no priority has been set. Text mode surfaces it only when truthy. */ priority?: string | null; + /** + * Backend-only dependency declarations that drive wave ordering. + * Optional on the wire so older facades that don't ship them still + * type-check; text mode surfaces them only when present/non-empty. + */ + produces?: string[] | null; + consumes?: string[] | null; + category?: string | null; } export type CliPublicStatus = @@ -292,6 +303,18 @@ export interface CliLatestResult { * former `{passed,failed,skipped}` count object). */ summary: string; + /** + * Captured stdout (`api_output`) from a backend-test execution. + * Present (possibly null) only for backend tests; omitted for FE/MCP and on + * older backends that omit them. Capped at 50 KB UTF-8 by the server. + */ + apiOutput?: string | null; + /** + * Python traceback from a backend-test execution (ends with the + * `ExceptionType: message` line). Present (possibly null) only for backend + * tests; omitted for FE/MCP and on older backends. + */ + trace?: string | null; /** * §6.5.1 (M2.1 piece 3) — inline failure analysis. Present when the * caller passed `--include-analysis` (`?includeAnalysis=true` on @@ -382,6 +405,14 @@ export interface CliFailureBlock { */ recommendedFixTarget: CliFixTarget | null; evidence: CliEvidence[]; + /** + * Backend-test execution artifacts, surfaced on the failure block + * so triage has stdout + traceback next to the hypothesis (mirrors + * `result.apiOutput` / `result.trace`). Present (possibly null) only for + * backend failures; omitted for FE and on older backends. + */ + apiOutput?: string | null; + trace?: string | null; } /** §6.7 wire shape — one atomic snapshot of the latest failing run. */ @@ -413,6 +444,36 @@ export interface TestDeps { * no-op in tests to avoid real delays. */ sleep?: (ms: number) => Promise; + /** + * Graceful-detach coordinator for the `--wait` paths (DEV-331 piece 1). + * Defaults to the process-wide `globalShutdown`; tests inject their own + * controller and abort it to simulate Ctrl-C deterministically (same + * pattern as `sleep` / `fetchImpl`). + */ + shutdown?: ShutdownHandle; +} + +/** The effective shutdown handle for a command invocation (DEV-331). */ +function shutdownOf(deps: TestDeps): ShutdownHandle { + return deps.shutdown ?? globalShutdown; +} + +/** + * The honest-detach stderr line (DEV-331 D1 — the heart of the ticket): + * a Ctrl-C detaches the local wait only; the server-side run keeps executing + * AND billing. Names both re-attach (`test wait`) and the real cancel + * (`test cancel`, piece 3) so the user always has a path to actually stop it. + */ +function interruptDetachMessage(err: InterruptError, runIds: string[]): string { + const subject = + runIds.length === 1 + ? `Run ${runIds[0]} is still executing on the server and will keep running (and billing) until it finishes.` + : `${runIds.length} runs are still executing on the server and will keep running (and billing) until they finish.`; + return ( + `Interrupted (${err.signal}). ${subject}\n` + + ` Re-attach with: testsprite test wait ${runIds.join(' ')}\n` + + ` Cancel with: testsprite test cancel ${runIds.join(' ')}` + ); } type CommonOptions = FactoryCommonOptions; @@ -1274,6 +1335,12 @@ interface UpdateOptions extends CommonOptions { description?: string; /** Optional new priority. Enum-validated CLI-side. */ priority?: CliCreatePriority; + /** Backend-only: variable names this test produces (repeatable --produces). */ + produces?: string[]; + /** Backend-only: variable names this test consumes (repeatable --needs); wire field `consumes`. */ + needs?: string[]; + /** Backend-only: free-text wave category (--category). */ + category?: string; /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ idempotencyKey?: string; } @@ -1330,11 +1397,14 @@ export async function runUpdate( const hasName = opts.name !== undefined; const hasDescription = opts.description !== undefined; const hasPriority = opts.priority !== undefined; - if (!hasName && !hasDescription && !hasPriority) { + const hasProduces = opts.produces !== undefined && opts.produces.length > 0; + const hasNeeds = opts.needs !== undefined && opts.needs.length > 0; + const hasCategory = opts.category !== undefined; + if (!hasName && !hasDescription && !hasPriority && !hasProduces && !hasNeeds && !hasCategory) { throw localValidationError( 'fields', - 'at least one of --name / --description / --priority must be set', - ['name', 'description', 'priority'], + 'at least one of --name / --description / --priority / --produces / --needs / --category must be set', + ['name', 'description', 'priority', 'produces', 'needs', 'category'], ); } @@ -1349,10 +1419,13 @@ export async function runUpdate( // is the intended wire shape — but we build the body deliberately // so the contract is auditable rather than dependent on // JSON.stringify undefined-skipping. - const body: Record = {}; + const body: Record = {}; if (hasName) body.name = opts.name!; if (hasDescription) body.description = opts.description!; if (hasPriority) body.priority = opts.priority!; + if (hasProduces) body.produces = opts.produces!; + if (hasNeeds) body.consumes = opts.needs!; + if (hasCategory) body.category = opts.category!; const client = makeClient(opts, deps); const out = makeOutput(opts.output, deps); @@ -2583,7 +2656,14 @@ async function runBatchRun( * * Returns a CliBatchRunResult. Never throws — errors are captured * into the result's `error` field so one failure doesn't abort siblings. + * (Exception: InterruptError rethrows so the collect point can print the + * partial — DEV-331.) */ + // testId → dispatched runId for members still mid-poll; the interrupt + // partial reads it because a member's runId is local to triggerOne until + // the poll settles (DEV-331, codex finding 2). + const dispatchedRunIds = new Map(); + async function triggerOne(testId: string): Promise { // Mint a fresh idempotency key per run — MUST NOT reuse the create key. const runIdempotencyKey = `cli-batch-run-${randomUUID()}`; @@ -2673,6 +2753,10 @@ async function runBatchRun( triggerResponse = result.body; break; // success — exit the outer retry loop } catch (err) { + // Interrupt must reject the fan-out (the collect point prints the + // partial for every spec), never flatten into a per-member outcome + // that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; // RATE_LIMITED outer retry. Since the HTTP layer no longer retries // RATE_LIMITED (retryOnRateLimit: false above), every 429 reaches here // on the first attempt. @@ -2809,6 +2893,12 @@ async function runBatchRun( } } + // Record the dispatched runId (fresh trigger OR conflict-resume) so the + // interrupt partial at the collect point can name every in-flight run — + // a member's runId is otherwise local until its poll settles (DEV-331, + // codex finding 2). + if (triggerResponse.runId) dispatchedRunIds.set(testId, triggerResponse.runId); + if (!opts.wait) { // No-wait path: return the trigger response as-is. if (opts.output !== 'json') { @@ -2850,11 +2940,14 @@ async function runBatchRun( finalRun = await pollRunUntilTerminal(client, triggerResponse.runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[batch-run][verbose] ${testId}: ${msg}`) : undefined, }); } catch (err) { + // Interrupt rejects the fan-out — see the trigger-stage catch above. + if (err instanceof InterruptError) throw err; if (err instanceof TimeoutError) { if (opts.output !== 'json') { stderrFn( @@ -2920,24 +3013,59 @@ async function runBatchRun( let nextIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && nextIdx < testIds.length) { - const testId = testIds[nextIdx++]!; - inFlight++; - triggerOne(testId) - .then(result => { - batchRunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && nextIdx >= testIds.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && nextIdx < testIds.length) { + const testId = testIds[nextIdx++]!; + inFlight++; + triggerOne(testId) + .then(result => { + batchRunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= testIds.length) resolve(); + }) + .catch(reject); + } + } + startNext(); + if (testIds.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): leave stdout parseable — settled members keep + // their real status, unfinished ones are marked running — then rethrow so + // index.ts exits 128+signum. + if (fanOutErr instanceof InterruptError) { + const settled = new Map(batchRunResults.map(r => [r.testId, r] as const)); + // Members mid-poll have no settled result yet — their runId comes from + // the dispatchedRunIds map recorded at trigger time (codex finding 2). + const partialResults = testIds.map( + (testId): CliBatchRunResult => + settled.get(testId) ?? { + testId, + runId: dispatchedRunIds.get(testId) ?? '', + status: dispatchedRunIds.has(testId) ? 'running' : 'not_dispatched', + codeVersion: '', + }, + ); + out.print({ results: partialResults }, () => + partialResults.map(r => `${r.testId} ${r.runId || '-'} ${r.status}`).join('\n'), + ); + const unfinished = partialResults + .filter(r => r.status === 'running' && r.runId) + .map(r => r.runId); + if (unfinished.length > 0) { + stderrFn(interruptDetachMessage(fanOutErr, unfinished)); + } else { + stderrFn( + `Interrupted (${fanOutErr.signal}). Already-triggered runs keep executing (and billing) server-side; ` + + `check them with: testsprite test list`, + ); } } - startNext(); - if (testIds.length === 0) resolve(); - }); + throw fanOutErr; + } // Sort by testId order (same as input order for stable output). batchRunResults.sort((a, b) => testIds.indexOf(a.testId) - testIds.indexOf(b.testId)); @@ -5100,6 +5228,36 @@ function renderRunResponseText( return lines.join('\n'); } +/** + * Best-effort resolve whether a finished run's test is a backend test, for the + * text run-card's step line + failure hint only (DEV-282). Returns true with no + * network call when already known (create-chain `--type`, or the BE wait + * fallback fired). Otherwise, in TEXT mode only, issues one `GET /tests/{id}`; + * any error → false (render the numeric step summary as before). JSON mode + * never probes — its envelope carries `stepSummary` verbatim and has no + * "n/a (backend)" concept, so it is already correct. + * + * This closes the standalone-path gap: `test run ` / `test wait ` + * never supply a type hint, and now that BE run rows finalize server-side + * (backend-v2.0 #551/#555) the wait fallback rarely fires — so a backend run + * card would otherwise show a misleading `steps 0/0 (passed=0, failed=0)`. + */ +async function resolveRunCardIsBackend( + client: ResultReadClient, + testId: string | undefined, + output: string, + alreadyKnown: boolean, +): Promise { + if (alreadyKnown) return true; + if (output === 'json' || !testId) return false; + try { + const test = await client.get(`/tests/${encodeURIComponent(testId)}`); + return test.type === 'backend'; + } catch { + return false; // best-effort — fall back to the numeric step summary. + } +} + /** * Render a `TriggerRunResponse` (no-wait path) to human-readable text. */ @@ -5259,7 +5417,8 @@ export async function runTestRun( stderrFn( `[advisory] Run already in flight (runId: ${currentRunId}, ` + `target: ${inFlightRun.targetUrl}). ` + - `Attaching to that run's --wait poll instead of creating a new one.`, + `Attaching to that run's --wait poll instead of creating a new one. ` + + `To stop it instead: testsprite test cancel ${currentRunId}`, ); triggerResponse = { runId: currentRunId, @@ -5285,12 +5444,16 @@ export async function runTestRun( // Auto-resume but emit a stronger advisory so the caller is aware // they are attaching to the project default. + // SIG-9 (DEV-331 final): the in-flight run is NOT auto-cancelled — + // name the real `test cancel` command instead of the old (false) + // "cancel with Ctrl-C" claim. stderrFn( `[advisory] Run already in flight (runId: ${currentRunId}` + (inFlightTargetUrl ? `, target: ${inFlightTargetUrl}` : '') + `). Auto-resuming wait on in-flight run. ` + - `If you needed a specific target URL, cancel with Ctrl-C and ` + - `re-trigger with --target-url.`, + `If you needed a specific target URL, cancel it with ` + + `testsprite test cancel ${currentRunId}, or re-trigger with ` + + `--target-url after it finishes.`, ); triggerResponse = { runId: currentRunId, @@ -5359,6 +5522,7 @@ export async function runTestRun( finalRun = await pollRunUntilTerminal(client, triggerResponse.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -5390,13 +5554,14 @@ export async function runTestRun( ]; if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); return lines.join('\n'); }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${triggerResponse.runId}.`, - nextAction: `Resume polling: testsprite test wait ${triggerResponse.runId}`, + nextAction: `Resume polling: testsprite test wait ${triggerResponse.runId}, or cancel it: testsprite test cancel ${triggerResponse.runId}`, requestId: 'local', details: { runId: triggerResponse.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -5423,14 +5588,39 @@ export async function runTestRun( const lines = [`runId ${p.runId}`, `status ${p.status} (request timed out)`]; if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); return lines.join('\n'); }); stderrFn( `Run ${triggerResponse.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${triggerResponse.runId}`, + `Re-attach with: testsprite test wait ${triggerResponse.runId}, or cancel with: testsprite test cancel ${triggerResponse.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1): same partial- + // envelope shape as the timeout paths so stdout stays parseable, plus the + // honest "keeps running and billing" stderr line. Rethrow → index.ts + // renders the INTERRUPTED envelope and exits 128+signum. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${triggerResponse.runId} — interrupted (${err.signal})`); + const partial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, partial, opts.createContext, data => { + const p = data as typeof partial; + const lines = [`runId ${p.runId}`, `status ${p.status} (interrupted)`]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); + return lines.join('\n'); + }); + stderrFn(interruptDetachMessage(err, [triggerResponse.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -5441,16 +5631,21 @@ export async function runTestRun( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, ); + // BE detection: type hint (create-chain) OR beFallbackUsed (slow runs); on + // the standalone `test run ` path neither is set, so probe the test type + // once (text mode only, best-effort) — DEV-282. + const isBackend = await resolveRunCardIsBackend( + client, + opts.testId, + opts.output, + beFallbackUsed || opts.type === 'backend', + ); + printRunOrChain( out, withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), opts.createContext, - data => - renderRunResponseText(data as RunResponse, { - // BE detection: type hint (create-chain) OR beFallbackUsed (slow runs). - // This ensures fast BE runs terminal on first poll still render n/a. - isBackend: beFallbackUsed || opts.type === 'backend', - }), + data => renderRunResponseText(data as RunResponse, { isBackend }), ); // Surface the trigger requestId under --verbose/--debug or JSON mode so @@ -5460,10 +5655,10 @@ export async function runTestRun( stderrFn(`requestId: ${triggerRequestId}`); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { - // BE runs (resolved via the testId fallback) have no run-scoped artifact - // bundle — their failure bundle is addressed by testId, not runId. + // BE runs have no run-scoped artifact bundle — their failure bundle is + // addressed by testId, not runId. stderrFn( - beFallbackUsed + isBackend ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${finalRun.testId}' to download the bundle.` : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); @@ -5559,6 +5754,7 @@ export async function runTestWaitMany( const run = await pollRunUntilTerminal(client, runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -5570,6 +5766,10 @@ export async function runTestWaitMany( } catch (err) { if (err instanceof TimeoutError) return { kind: 'timeout' }; if (err instanceof RequestTimeoutError) throw err; + // Interrupt must reject the fan-out (handled at the collect point), not + // be flattened into a per-member 'error' outcome that would swallow the + // 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; if (err instanceof ApiError) return { kind: 'error', code: err.code, exitCode: err.exitCode }; return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; } @@ -5599,12 +5799,16 @@ export async function runTestWaitMany( if (opts.runIds.length === 0) resolve(); }); } catch (fanOutErr) { - if (fanOutErr instanceof RequestTimeoutError) { + if (fanOutErr instanceof RequestTimeoutError || fanOutErr instanceof InterruptError) { // Same contract as the batch pollers: leave stdout parseable before - // exiting 7. Members that already settled keep their real status; only + // exiting. Members that already settled keep their real status; only // the still-unfinished ids are marked running and named in the hint // (re-attaching to an already-terminal run would be a wasted command). - ticker.finalize('Multi-run wait — request timed out'); + ticker.finalize( + fanOutErr instanceof InterruptError + ? `Multi-run wait — interrupted (${fanOutErr.signal})` + : 'Multi-run wait — request timed out', + ); const partial = { results: opts.runIds.map((runId): CliMultiWaitResult => { const outcome = outcomes.get(runId); @@ -5620,7 +5824,13 @@ export async function runTestWaitMany( .filter(r => r.status === 'running' || r.status === 'timeout') .map(r => r.runId); if (unfinished.length > 0) { - stderrFn(`Re-attach with: testsprite test wait ${unfinished.join(' ')}`); + if (fanOutErr instanceof InterruptError) { + stderrFn(interruptDetachMessage(fanOutErr, unfinished)); + } else { + stderrFn( + `Re-attach with: testsprite test wait ${unfinished.join(' ')}, or cancel with: testsprite test cancel ${unfinished.join(' ')}`, + ); + } } } throw fanOutErr; @@ -5656,7 +5866,9 @@ export async function runTestWaitMany( .filter(r => r.status === 'timeout' || r.status.startsWith('error:')) .map(r => r.runId); if (unfinishedIds.length > 0) { - stderrFn(`Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}`); + stderrFn( + `Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}, or cancel with: testsprite test cancel ${unfinishedIds.join(' ')}`, + ); } // Worst-status exit: auth escalates (a rejected key fails every member the @@ -5684,6 +5896,190 @@ export async function runTestWaitMany( return payload; } +// --------------------------------------------------------------------------- +// DEV-331 piece 3 — `test cancel ` +// --------------------------------------------------------------------------- + +export interface RunTestCancelOptions extends CommonOptions { + runIds: string[]; +} + +/** One run's outcome in a multi-id `test cancel` summary. */ +export interface CliCancelResultRow { + runId: string; + status: 'cancelled' | 'alreadyCancelled' | 'conflict' | 'notFound' | 'error'; + /** Terminal status the run was already in, for a `conflict` row. */ + conflictStatus?: string; + error?: string; +} + +/** JSON payload for a multi-id `test cancel` — per piece-3 CXL-11. */ +export interface CliCancelSummary { + cancelled: string[]; + alreadyCancelled: string[]; + conflicts: Array<{ runId: string; status: string }>; + notFound: string[]; + /** + * Runs that errored for a reason other than 404/409 (e.g. auth, transport). + * Always present — an empty array on full success — so machine consumers + * can rely on a stable shape (DEV-331 codex finding 2). + */ + errors: Array<{ runId: string; message: string }>; +} + +/** + * `test cancel ` — DEV-331 piece 3. + * + * User-initiated cancel of one or more queued/running runs via + * `POST /api/cli/v1/runs/{runId}/cancel`. Naturally idempotent (D10): a + * repeat cancel of the same run is a 200 `alreadyCancelled` success, not an + * error. Dispatches serially (per-id volume is tiny — no batch endpoint, + * D8 "Batch-level cancel endpoint... YAGNI"). + * + * Single id: renders the returned run card (`status cancelled`); an + * `alreadyCancelled` response adds an `[advisory]` line. Exit code mirrors + * the server response directly — 0 on success (fresh or already-cancelled), + * 4 on 404 (unknown/cross-tenant), 6 on 409 (already terminal). + * + * Multi-id: prints a `{cancelled, alreadyCancelled, conflicts, notFound}` + * summary. Exit precedence (CXL-11): any `notFound` → 4 (outranks conflict — + * it signals a caller bug, wrong id/tenant); else any `conflicts` → 6; else 0. + */ +export async function runTestCancel( + opts: RunTestCancelOptions, + deps: TestDeps = {}, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + } + + if (opts.runIds.length === 1) { + const runId = opts.runIds[0]!; + const result = await client.cancelRun(runId); + out.print(result, data => { + const r = data as CancelRunResponse; + return renderRunResponseText(r); + }); + if (result.alreadyCancelled) { + stderrFn(`[advisory] run ${runId} was already cancelled`); + } + return result; + } + + const rows: CliCancelResultRow[] = []; + for (const runId of opts.runIds) { + try { + const result = await client.cancelRun(runId); + rows.push({ + runId, + status: result.alreadyCancelled ? 'alreadyCancelled' : 'cancelled', + }); + } catch (err) { + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + rows.push({ runId, status: 'notFound' }); + continue; + } + if (err instanceof ApiError && err.code === 'CONFLICT') { + const conflictStatus = + err.getDetail('status', (v): v is string => typeof v === 'string') ?? 'unknown'; + rows.push({ runId, status: 'conflict', conflictStatus }); + continue; + } + const message = err instanceof Error ? err.message : String(err); + rows.push({ runId, status: 'error', error: message }); + } + } + + const errorRows = rows.filter(r => r.status === 'error'); + const summary: CliCancelSummary = { + cancelled: rows.filter(r => r.status === 'cancelled').map(r => r.runId), + alreadyCancelled: rows.filter(r => r.status === 'alreadyCancelled').map(r => r.runId), + conflicts: rows + .filter(r => r.status === 'conflict') + .map(r => ({ runId: r.runId, status: r.conflictStatus ?? 'unknown' })), + notFound: rows.filter(r => r.status === 'notFound').map(r => r.runId), + errors: errorRows.map(r => ({ runId: r.runId, message: r.error ?? 'unknown error' })), + }; + + out.print(summary, data => renderCancelSummaryText(data as CliCancelSummary)); + + const parts = [ + `${summary.cancelled.length} cancelled`, + `${summary.alreadyCancelled.length} already cancelled`, + ]; + if (summary.conflicts.length > 0) parts.push(`${summary.conflicts.length} conflict`); + if (summary.notFound.length > 0) parts.push(`${summary.notFound.length} not found`); + if (errorRows.length > 0) parts.push(`${errorRows.length} error`); + stderrFn(`Cancel summary: ${parts.join(', ')}.`); + + // Exit precedence (CXL-11): notFound outranks conflict — a caller-side bug + // (wrong id / wrong tenant) is more actionable to surface than "it already + // finished". A bare transport/auth error on any member also fails loudly + // rather than being silently absorbed into a 0 exit. + if (summary.notFound.length > 0) { + throw new CLIError( + `${summary.notFound.length} run id${summary.notFound.length !== 1 ? 's' : ''} not found: ${summary.notFound.join(' ')}`, + 4, + ); + } + if (errorRows.length > 0) { + throw new CLIError( + `${errorRows.length} cancel request${errorRows.length !== 1 ? 's' : ''} failed: ${errorRows.map(r => r.runId).join(' ')}`, + 1, + ); + } + if (summary.conflicts.length > 0) { + throw new CLIError( + `${summary.conflicts.length} run${summary.conflicts.length !== 1 ? 's' : ''} already terminal: ${summary.conflicts.map(c => `${c.runId} (${c.status})`).join(', ')}`, + 6, + ); + } + return summary; +} + +function renderCancelSummaryText(summary: CliCancelSummary): string { + const lines: string[] = []; + for (const runId of summary.cancelled) lines.push(`${runId} cancelled`); + for (const runId of summary.alreadyCancelled) lines.push(`${runId} alreadyCancelled`); + for (const c of summary.conflicts) lines.push(`${c.runId} conflict (${c.status})`); + for (const runId of summary.notFound) lines.push(`${runId} notFound`); + for (const e of summary.errors ?? []) lines.push(`${e.runId} error (${e.message})`); + return lines.join('\n'); +} + +export function createTestCancelCommand(deps: TestDeps): Command { + const cancel = new Command('cancel'); + cancel + .argument('', 'one or more run ids to cancel') + .description( + 'Cancel one or more queued/running runs.\n' + + '\nCtrl-C during --wait only detaches — it does NOT cancel the server-side\n' + + 'run. This is the real stop button. No refund is issued for the credits\n' + + 'already charged at trigger time (D3); an in-flight Lambda finishes on its\n' + + 'own and its result is discarded once cancelled.\n' + + '\nExit codes:\n' + + ' 0 cancelled (fresh or already-cancelled — naturally idempotent)\n' + + ' 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict)\n' + + ' 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict\n' + + '\nMulti-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (runIds: string[], _cmdOpts: unknown, command: Command) => { + await runTestCancel( + { + ...resolveCommonOptions(command), + runIds, + }, + deps, + ); + }); + return cancel; +} + /** * `test wait ` — M3.3 piece-3. * @@ -5744,6 +6140,7 @@ export async function runTestWait( finalRun = await pollRunUntilTerminal(client, opts.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -5767,13 +6164,14 @@ export async function runTestWait( `runId ${p.runId}`, `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, ].join('\n'); }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${opts.runId}.`, - nextAction: `Resume polling: testsprite test wait ${opts.runId}`, + nextAction: `Resume polling: testsprite test wait ${opts.runId}, or cancel it: testsprite test cancel ${opts.runId}`, requestId: 'local', details: { runId: opts.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -5791,14 +6189,31 @@ export async function runTestWait( `runId ${p.runId}`, `status ${p.status} (request timed out)`, `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, ].join('\n'); }); stderrFn( `Run ${opts.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${opts.runId}`, + `Re-attach with: testsprite test wait ${opts.runId}, or cancel with: testsprite test cancel ${opts.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1) — see runTestRun. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${opts.runId} — interrupted (${err.signal})`); + const partial = { runId: opts.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (interrupted)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); + stderrFn(interruptDetachMessage(err, [opts.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -5809,15 +6224,24 @@ export async function runTestWait( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, ); + // `test wait` has no type hint; probe the test type once (text mode only, + // best-effort) so a backend run's card reads `n/a (backend)` — DEV-282. + const isBackend = await resolveRunCardIsBackend( + client, + finalRun.testId, + opts.output, + beFallbackUsed, + ); + out.print(withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), data => - renderRunResponseText(data as RunResponse, { isBackend: beFallbackUsed }), + renderRunResponseText(data as RunResponse, { isBackend }), ); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { - // BE runs (resolved via the testId fallback) have no run-scoped artifact - // bundle — their failure bundle is addressed by testId, not runId. + // BE runs have no run-scoped artifact bundle — their failure bundle is + // addressed by testId, not runId. stderrFn( - beFallbackUsed + isBackend ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${finalRun.testId}' to download the bundle.` : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); @@ -6276,6 +6700,7 @@ export async function runTestRunAll( const finalRun = await pollRunUntilTerminal(client, runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -6305,6 +6730,10 @@ export async function runTestRunAll( }, }; } + // Interrupt must reject the fan-out (the collect point below prints the + // partial for every dispatched run), never flatten into a per-member + // outcome that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; if (err instanceof RequestTimeoutError) { // Client-side per-request timeout during polling — classify as timeout // (exit 7) so the fan-out completes and stdout carries every runId. @@ -6340,24 +6769,45 @@ export async function runTestRunAll( let pollIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && pollIdx < pollable.length) { - const entry = pollable[pollIdx++]!; - inFlight++; - pollFreshAccepted(entry) - .then(result => { - freshRunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && pollIdx >= pollable.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && pollIdx < pollable.length) { + const entry = pollable[pollIdx++]!; + inFlight++; + pollFreshAccepted(entry) + .then(result => { + freshRunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && pollIdx >= pollable.length) resolve(); + }) + .catch(reject); + } } + startNext(); + if (pollable.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): stdout stays parseable — settled members + // keep their real status, unfinished ones are marked running — and the + // honest stderr line names every runId still executing (and billing). + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Batch run — interrupted (${fanOutErr.signal})`); + const settled = new Map(freshRunResults.map(r => [r.runId, r] as const)); + const partialResults = pollable.map( + (e): CliBatchRunFreshResult => + settled.get(e.runId) ?? { testId: e.testId, runId: e.runId, status: 'running' }, + ); + out.print( + { accepted: partialResults, conflicts, deferred, skippedFrontend, skippedIntegration }, + () => partialResults.map(r => `${r.runId} ${r.status}`).join('\n'), + ); + const unfinished = pollable.filter(e => !settled.has(e.runId)).map(e => e.runId); + if (unfinished.length > 0) stderrFn(interruptDetachMessage(fanOutErr, unfinished)); } - startNext(); - if (pollable.length === 0) resolve(); - }); + throw fanOutErr; + } ticker.finalize(); @@ -6411,7 +6861,10 @@ export async function runTestRunAll( error: { code: 'UNSUPPORTED', message: `${timedOut} run${timedOut !== 1 ? 's' : ''} timed out.`, - nextAction: timedOutRunIds.map(rid => `Resume: testsprite test wait ${rid}`).join('\n'), + nextAction: [ + ...timedOutRunIds.map(rid => `Resume: testsprite test wait ${rid}`), + ...timedOutRunIds.map(rid => `Cancel: testsprite test cancel ${rid}`), + ].join('\n'), requestId: 'local', details: { timedOutRunIds, timeoutSeconds: opts.timeoutSeconds }, }, @@ -6799,6 +7252,7 @@ export async function runTestRerun( return await pollRunUntilTerminal(client, member.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -6894,8 +7348,35 @@ export async function runTestRerun( const reattachHints = closureMembers .map(m => `testsprite test wait ${m.runId}`) .join('\n'); + const cancelHints = closureMembers + .map(m => `testsprite test cancel ${m.runId}`) + .join('\n'); stderrFn( - `Closure members are still in progress (request timed out). Re-attach with:\n${reattachHints}`, + `Closure members are still in progress (request timed out). Re-attach with:\n${reattachHints}\n` + + `Or cancel with:\n${cancelHints}`, + ); + throw fanOutErr; + } + // Graceful detach (DEV-331): same partial shape as the timeout path — + // SIG-6 requires the partial to list ALL dispatched runIds. + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Closure fan-out — interrupted (${fanOutErr.signal})`); + const dispatchedRunIds = closureMembers.map(m => ({ + runId: m.runId, + testId: m.testId, + role: m.role, + status: 'running' as const, + })); + out.print({ runId: namedRunId, status: 'running', closure: dispatchedRunIds }, () => + dispatchedRunIds + .map(m => `${m.role.padEnd(9)} ${m.testId} (runId: ${m.runId}) — running`) + .join('\n'), + ); + stderrFn( + interruptDetachMessage( + fanOutErr, + closureMembers.map(m => m.runId), + ), ); throw fanOutErr; } @@ -6937,7 +7418,7 @@ export async function runTestRerun( error: { code: 'UNSUPPORTED', message: `Timed out after ${opts.timeoutSeconds}s waiting for rerun ${namedRunId}.`, - nextAction: `Resume polling: testsprite test wait ${namedRunId}`, + nextAction: `Resume polling: testsprite test wait ${namedRunId}, or cancel it: testsprite test cancel ${namedRunId}`, requestId: 'local', details: { runId: namedRunId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -6957,7 +7438,10 @@ export async function runTestRerun( // as a whole was never observed to reach terminal. const timedOutMembers = closureFailures.filter(f => f.status === 'timeout'); if (timedOutMembers.length > 0) { - const resumeHints = timedOutMembers.map(f => `testsprite test wait ${f.runId}`).join('\n'); + const timedOutIds = timedOutMembers.map(f => f.runId); + const resumeHints = + timedOutIds.map(runId => `testsprite test wait ${runId}`).join('\n') + + `\nCancel instead: testsprite test cancel ${timedOutIds.join(' ')}`; throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', @@ -6995,6 +7479,7 @@ export async function runTestRerun( finalRun = await pollRunUntilTerminal(client, rerunResp.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -7012,7 +7497,7 @@ export async function runTestRerun( error: { code: 'UNSUPPORTED', message: `Timed out after ${opts.timeoutSeconds}s waiting for rerun ${rerunResp.runId}.`, - nextAction: `Resume polling: testsprite test wait ${rerunResp.runId}`, + nextAction: `Resume polling: testsprite test wait ${rerunResp.runId}, or cancel it: testsprite test cancel ${rerunResp.runId}`, requestId: 'local', details: { runId: rerunResp.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -7029,14 +7514,31 @@ export async function runTestRerun( `runId ${p.runId}`, `status ${p.status} (request timed out)`, `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, ].join('\n'); }); stderrFn( `Run ${rerunResp.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${rerunResp.runId}`, + `Re-attach with: testsprite test wait ${rerunResp.runId}, or cancel with: testsprite test cancel ${rerunResp.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1) — see runTestRun. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${rerunResp.runId} — interrupted (${err.signal})`); + const partial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (interrupted)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); + stderrFn(interruptDetachMessage(err, [rerunResp.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -7048,13 +7550,21 @@ export async function runTestRerun( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps replay)`, ); + // Probe the test type once (text mode only, best-effort) so a backend + // rerun's card reads `n/a (backend)` even when the fallback never fired + // (BE run rows finalize server-side now) — DEV-282. + const isBackend = await resolveRunCardIsBackend(client, testId, opts.output, beFallbackUsed); + out.print(withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), data => - renderRunResponseText(data as RunResponse, { isBackend: beFallbackUsed }), + renderRunResponseText(data as RunResponse, { isBackend }), ); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { + // BE reruns have no run-scoped artifact bundle — address by testId. stderrFn( - `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, + isBackend + ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${testId}' to download the bundle.` + : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); } @@ -7501,6 +8011,7 @@ export async function runTestRerun( const finalRun = await pollRunUntilTerminal(client, entry.runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -7530,6 +8041,10 @@ export async function runTestRerun( }, }; } + // Interrupt must reject the fan-out (the collect point below prints the + // partial for every dispatched run), never flatten into a per-member + // outcome that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; if (err instanceof RequestTimeoutError) { // Client-side per-request timeout during polling — classify as timeout // (exit 7) so the fan-out completes and stdout carries every runId. @@ -7565,24 +8080,44 @@ export async function runTestRerun( let acceptedIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && acceptedIdx < accepted.length) { - const entry = accepted[acceptedIdx++]!; - inFlight++; - pollAccepted(entry) - .then(result => { - rerunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && acceptedIdx >= accepted.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && acceptedIdx < accepted.length) { + const entry = accepted[acceptedIdx++]!; + inFlight++; + pollAccepted(entry) + .then(result => { + rerunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && acceptedIdx >= accepted.length) resolve(); + }) + .catch(reject); + } } + startNext(); + if (accepted.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): stdout stays parseable — settled members + // keep their real status, unfinished ones are marked running — and the + // honest stderr line names every runId still executing (and billing). + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Batch rerun — interrupted (${fanOutErr.signal})`); + const settled = new Map(rerunResults.map(r => [r.runId, r] as const)); + const partialResults = accepted.map( + (e): CliRerunResult => + settled.get(e.runId) ?? { testId: e.testId, runId: e.runId, status: 'running' }, + ); + out.print({ accepted: partialResults, deferred, conflicts, notFound }, () => + partialResults.map(r => `${r.runId} ${r.status}`).join('\n'), + ); + const unfinished = accepted.filter(e => !settled.has(e.runId)).map(e => e.runId); + if (unfinished.length > 0) stderrFn(interruptDetachMessage(fanOutErr, unfinished)); } - startNext(); - if (accepted.length === 0) resolve(); - }); + throw fanOutErr; + } ticker.finalize(); @@ -7646,6 +8181,7 @@ export async function runTestRerun( // `test wait` accepts exactly one run id — emit one command per // timed-out run so the hint is always valid. ...(timedOut > 0 ? stillRunning.map(rid => `Resume: testsprite test wait ${rid}`) : []), + ...(timedOut > 0 ? stillRunning.map(rid => `Cancel: testsprite test cancel ${rid}`) : []), ] .filter(Boolean) .join('\n'), @@ -8312,6 +8848,22 @@ export function createTestCommand(deps: TestDeps = {}): Command { .option('--name ', 'new human-readable test name') .option('--description ', 'new human description (≤ 2000 chars)') .option('--priority ', 'new priority — one of: p0, p1, p2, p3') + .option( + '--produces ', + 'BE only: variable name this test captures (repeatable). Drives dependency-aware wave ordering.', + (val: string, prev: string[]) => [...(prev ?? []), val], + [] as string[], + ) + .option( + '--needs ', + 'BE only: variable name this test consumes (repeatable). Declares an upstream producer dependency.', + (val: string, prev: string[]) => [...(prev ?? []), val], + [] as string[], + ) + .option( + '--category ', + "BE only: test category. Use 'teardown' or 'cleanup' to mark a final-wave cleanup test.", + ) .option( '--idempotency-key ', 'opaque idempotency token (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', @@ -8327,6 +8879,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { priority: parseEnumFlag(cmdOpts.priority, 'priority', CLI_CREATE_PRIORITIES) as | CliCreatePriority | undefined, + produces: cmdOpts.produces, + needs: cmdOpts.needs, + category: cmdOpts.category, idempotencyKey: cmdOpts.idempotencyKey, }, deps, @@ -8411,10 +8966,13 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' 4 test not found\n' + ' 5 validation error (e.g., bad --target-url, or positional + --all both set)\n' + ' 6 conflict (already running — see nextAction for the active runId)\n' + - ' 7 timeout — resume with: testsprite test wait \n' + + ' 7 timeout — resume with: testsprite test wait , ' + + 'or stop it with: testsprite test cancel \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + ' 11 rate limited — honor Retry-After\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C during --wait detaches only (the run keeps executing and billing);\n' + + 'stop it for real with: testsprite test cancel ', ) .option( '--target-url ', @@ -8574,7 +9132,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' is recorded as error: in its row and folded into exit 7)\n' + ' 7 timeout or per-member poll error — resume with: testsprite test wait \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C detaches only (the run keeps executing and billing); stop it for\n' + + 'real with: testsprite test cancel ', ) .option('--timeout ', `max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`) .option( @@ -8626,9 +9186,12 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' 4 test not found\n' + ' 5 validation error\n' + ' 6 conflict (already running — see nextAction for the active runId)\n' + - ' 7 timeout or deferred — resume with: testsprite test wait \n' + + ' 7 timeout or deferred — resume with: testsprite test wait , ' + + 'or stop it with: testsprite test cancel \n' + ' 11 rate limited — honor Retry-After\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C during --wait detaches only (the run keeps executing and billing);\n' + + 'stop it for real with: testsprite test cancel ', ) .option('--all', 'rerun all tests in the resolved project (requires --project)', false) .option( @@ -8802,6 +9365,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { test.addCommand(createTestPlanCommand(deps)); test.addCommand(createTestFailureCommand(deps)); test.addCommand(createTestArtifactCommand(deps)); + test.addCommand(createTestCancelCommand(deps)); return test; } @@ -8937,12 +9501,20 @@ export async function runFlaky( const finalRun = await pollRunUntilTerminal(client, runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, resolveAlternate, }); outcome = finalRun.status as FlakyOutcome; failureKind = finalRun.failureKind; } catch (err) { + // Graceful detach (DEV-331): clean up the ticker line, name the run + // still executing server-side, and let index.ts exit 128+signum. + if (err instanceof InterruptError) { + ticker.finalize(`Attempt ${i}/${opts.runs} — interrupted (${err.signal})`); + stderrFn(interruptDetachMessage(err, [runId])); + throw err; + } // A per-attempt deadline (poll TimeoutError) or a client-side request // timeout both count as a non-passing "timeout" outcome for this attempt. if (err instanceof TimeoutError || err instanceof RequestTimeoutError) { @@ -9015,6 +9587,9 @@ interface UpdateFlagOpts { name?: string; description?: string; priority?: string; + produces?: string[]; + needs?: string[]; + category?: string; idempotencyKey?: string; } @@ -9199,6 +9774,7 @@ function makeClient(opts: CommonOptions, deps: TestDeps): HttpClient { credentialsPath: deps.credentialsPath, fetchImpl: deps.fetchImpl, stderr: deps.stderr, + shutdownSignal: shutdownOf(deps).signal, }); } @@ -9506,6 +10082,16 @@ function renderTestText(t: CliTest): string { if (typeof t.planStepCount === 'number') { lines.push(`planSteps: ${t.planStepCount}`); } + // Surface backend dependency declarations when present. + if (Array.isArray(t.produces) && t.produces.length > 0) { + lines.push(`produces: ${t.produces.join(', ')}`); + } + if (Array.isArray(t.consumes) && t.consumes.length > 0) { + lines.push(`consumes: ${t.consumes.join(', ')}`); + } + if (t.category) { + lines.push(`category: ${t.category}`); + } lines.push(`createdAt: ${t.createdAt}`, `updatedAt: ${t.updatedAt}`); return lines.join('\n'); } @@ -9639,6 +10225,14 @@ function renderFailureContextText(ctx: CliFailureContext): string { lines.push(`evidence: ${ctx.failure.evidence.length} items (${breakdown})`); } if (ctx.result.videoUrl !== null) lines.push(`videoUrl: ${ctx.result.videoUrl}`); + // backend stdout + traceback (prefer the failure block, falling + // back to the embedded result; identical values). Bounded tail; full content + // in --output json / the written failure.json. + appendBackendArtifactLines( + lines, + ctx.failure.apiOutput ?? ctx.result.apiOutput, + ctx.failure.trace ?? ctx.result.trace, + ); return lines.join('\n'); } @@ -9760,6 +10354,9 @@ function renderResultText(r: CliLatestResult): string { lines.push(`summary: ${r.summary}`); if (r.videoUrl !== null) lines.push(`videoUrl: ${r.videoUrl}`); if (r.failureAnalysisUrl !== null) lines.push(`failureAnalysisUrl: ${r.failureAnalysisUrl}`); + // backend stdout + traceback (null/absent for FE, passed, and + // older backends). Full content always available via `--output json`. + appendBackendArtifactLines(lines, r.apiOutput, r.trace); if (r.analysis !== undefined) { // §6.5.1 (M2.1 piece 3) — render the inline analysis block under // the result summary. Only fires when the caller passed @@ -9783,6 +10380,44 @@ function renderResultText(r: CliLatestResult): string { * level null. Non-null wrappers render `kind=...` plus an optional * reference and indented rationale. */ +/** + * Render backend-test stdout + traceback in text mode. Shared by + * `renderResultText` and `renderFailureContextText`. Both are bounded to a + * tail (last {@link BACKEND_ARTIFACT_TAIL_LINES} lines) so a large stdout can't + * flood the terminal; the full, untruncated content is always in the + * `--output json` envelope. No-op when both are null/absent (FE / passed / + * older backends), so non-backend output stays byte-identical. + */ +const BACKEND_ARTIFACT_TAIL_LINES = 20; + +function appendArtifactTail( + lines: string[], + label: string, + value: string | null | undefined, +): void { + if (value == null || value === '') return; + const allLines = value.replace(/\n+$/, '').split('\n'); + const dropped = Math.max(0, allLines.length - BACKEND_ARTIFACT_TAIL_LINES); + const tail = dropped > 0 ? allLines.slice(-BACKEND_ARTIFACT_TAIL_LINES) : allLines; + const bytes = Buffer.byteLength(value, 'utf8'); + lines.push(''); + lines.push( + `${label} (${bytes} bytes${dropped > 0 ? `, showing last ${tail.length} lines` : ''}):`, + ); + for (const l of tail) lines.push(` ${l}`); + if (dropped > 0) + lines.push(` … ${dropped} earlier line(s) omitted — full content in --output json`); +} + +function appendBackendArtifactLines( + lines: string[], + apiOutput: string | null | undefined, + trace: string | null | undefined, +): void { + appendArtifactTail(lines, 'stdout', apiOutput); + appendArtifactTail(lines, 'trace', trace); +} + function appendFixTargetLines(lines: string[], fix: CliFixTarget | null, label: string): void { if (fix === null) { lines.push(`${label}— (analysis pipeline did not propose one)`); diff --git a/src/commands/test.wait.spec.ts b/src/commands/test.wait.spec.ts index c83195d..4830e50 100644 --- a/src/commands/test.wait.spec.ts +++ b/src/commands/test.wait.spec.ts @@ -10,7 +10,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import type { RunResponse } from '../lib/runs.types.js'; import { runTestWait } from './test.js'; @@ -1305,3 +1306,122 @@ describe('runTestWait — dashboardUrl on terminal output', () => { expect(printed.dashboardUrl).toBeUndefined(); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach on SIGINT/SIGTERM during test wait +// --------------------------------------------------------------------------- + +describe('runTestWait — InterruptError graceful detach (DEV-331)', () => { + function hangingFetch(): typeof globalThis.fetch { + return (async (_input: FetchInput, init: RequestInit = {}) => { + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + } + + it('SIG-2/SIG-4: emits partial JSON to stdout, honest billing line to stderr, rethrows exit 130', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const pending = runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: hangingFetch(), + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + shutdown, + }, + ); + // Simulate Ctrl-C while the long-poll fetch is in flight. + setTimeout(() => shutdown.interrupt('SIGINT'), 5); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + expect((err as InterruptError).signal).toBe('SIGINT'); + + // Stdout stays parseable and carries the runId for re-attach. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + + // The honest line: run keeps executing AND billing; re-attach hint. + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('testsprite test wait run_abc'); + }); + + it('SIG-3: SIGTERM maps to exit 143', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const pending = runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: hangingFetch(), + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGTERM'), 5); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(143); + }); + + it('arms the shutdown scope during the wait and disarms after a terminal result', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const fetchImpl = makeFetch(() => ({ body: makeRun('passed') })); + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + shutdown, + }, + ); + expect(shutdown.isArmed).toBe(false); // disarmed after the poll completes + }); +}); diff --git a/src/index.ts b/src/index.ts index bf9a9aa..ae3f944 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,7 @@ import { import { createProjectCommand } from './commands/project.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; -import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; +import { ApiError, CLIError, InterruptError, RequestTimeoutError } from './lib/errors.js'; import { installBrokenPipeGuard, installSignalHandlers } from './lib/interrupt.js'; import { Output, isOutputMode } from './lib/output.js'; import { maybeInstallProxyAgent } from './lib/proxy.js'; @@ -164,9 +164,12 @@ program.hook('preAction', (_thisCommand, actionCommand) => { } }); -// Clean process lifecycle: a clear message + conventional exit code on SIGINT / -// SIGTERM / SIGHUP (instead of Node's silent abrupt kill) so an interrupted -// `test run --wait` explains the run continues server-side; plus an EPIPE guard +// Clean process lifecycle (DEV-331 piece 1, errors.md §8.1): during a `--wait` +// poll the scope is armed — the first SIGINT/SIGTERM/SIGHUP aborts gracefully +// and the wait path prints an honest partial (the run KEEPS executing and +// billing server-side) + re-attach hint before exiting 128+signum; a second +// signal hard-exits. Outside an armed scope: a clear one-line message + +// immediate exit (instead of Node's silent abrupt kill). Plus an EPIPE guard // so piping to a reader that closes early (`| head`) exits cleanly instead of // dumping a raw `write EPIPE` stack. installSignalHandlers(); @@ -209,10 +212,44 @@ try { process.stderr.write(` granted: ${(granted as string[]).join(', ')}\n`); } } + // Surface the version gap on CLIENT_TOO_OLD so the user sees exactly what + // moved without parsing the message string. (No "latest" line — the npm + // update-notice is the single source of truth for the newest release.) + if (err.code === 'CLIENT_TOO_OLD') { + const your = err.getDetail('yourVersion'); + const min = err.getDetail('minVersion'); + if (typeof your === 'string' && typeof min === 'string') { + process.stderr.write(` your version: ${your}, minimum supported: ${min}\n`); + } + } } process.exit(err.exitCode); } const output = new Output(mode); + if (err instanceof InterruptError) { + // Graceful detach (DEV-331 piece 1, errors.md §8.1): the wait-path catch + // block already printed the honest partial + re-attach hint. Exit with + // the conventional 128+signum code; `INTERRUPTED` is deliberately outside + // the error catalog. Note: Ctrl-C does NOT cancel the server-side run. + if (mode === 'json') { + const envelope = { + error: { + code: 'INTERRUPTED', + message: err.message, + nextAction: + 'The server-side run (if any) keeps executing and billing. ' + + 'Re-attach with: testsprite test wait , or stop it with: testsprite test cancel ' + + '(runId is in the partial JSON on stdout).', + requestId: 'local', + details: { signal: err.signal }, + }, + }; + process.stderr.write(`${JSON.stringify(envelope, null, 2)}\n`); + } else { + process.stderr.write(`Error: ${err.message}\n`); + } + process.exit(err.exitCode); + } if (err instanceof RequestTimeoutError) { // Structured rendering for per-request timeouts: JSON mode emits a // machine-readable envelope; text mode emits the message with a hint. diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index ff20e8c..d954381 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -601,6 +601,25 @@ describe('resolveBundleDir', () => { const out = resolveBundleDir('/tmp/x/'); expect(out).toBe('/tmp/x'); }); + + // `C:\...` is only recognized as absolute by node:path when the process + // itself is running on win32 (path.isAbsolute/resolve are platform-native, + // not path.win32.* explicitly) — these two assertions are only meaningful + // under an actual Windows runtime, hence gated rather than run everywhere. + it.runIf(process.platform === 'win32')( + 'strips a trailing backslash (native Windows path)', + () => { + const out = resolveBundleDir('C:\\Users\\me\\bundle\\'); + expect(out).toBe('C:\\Users\\me\\bundle'); + }, + ); + + it.runIf(process.platform === 'win32')( + 'preserves a bare Windows drive root instead of truncating it to a drive-relative path', + () => { + expect(resolveBundleDir('C:\\')).toBe('C:\\'); + }, + ); }); describe('streamUrlToFile retry', () => { diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index a3c0808..f44549e 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -306,6 +306,23 @@ export function applyFailedOnly(ctx: CliFailureContext): CliFailureContext { }; } +/** + * Strip trailing path separators, tolerating both `/` and `\` since a + * native Windows `--out` value (typed or pasted from Explorer) commonly + * ends in a backslash. Preserves a bare drive root (`C:\`) — stripping + * its separator would turn it into `C:`, which Windows resolves as + * "current directory on drive C", not the drive root. + */ +function stripTrailingSeparators(rawPath: string): string { + if (rawPath.length <= 1) return rawPath; + let end = rawPath.length; + while (end > 1 && (rawPath[end - 1] === '/' || rawPath[end - 1] === '\\')) { + if (end === 3 && rawPath[1] === ':' && /[A-Za-z]/.test(rawPath[0]!)) break; + end--; + } + return rawPath.slice(0, end); +} + /** * Resolve the user-supplied `--out` path into an absolute directory. * Empty strings are rejected with `VALIDATION_ERROR` for consistency @@ -325,7 +342,7 @@ export function resolveBundleDir(rawPath: string): string { }, }); } - const trimmed = rawPath.endsWith('/') ? rawPath.slice(0, -1) : rawPath; + const trimmed = stripTrailingSeparators(rawPath); return isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed); } diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index 829410d..96d9dc0 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -24,8 +24,11 @@ import { REQUEST_TIMEOUT_MAX_MS, REQUEST_TIMEOUT_MIN_MS, } from './http.js'; +import { globalShutdown } from './interrupt.js'; import type { OutputMode } from './output.js'; import { createDryRunFetch } from './dry-run/fetch.js'; +import { noteServerVersion } from './version-notice.js'; +import { VERSION } from '../version.js'; export interface CommonOptions { profile: string; @@ -68,6 +71,12 @@ export interface ClientFactoryDeps { credentialsPath?: string; fetchImpl?: FetchImpl; stderr?: (line: string) => void; + /** + * Shutdown signal composed into every outgoing fetch (DEV-331 piece 1). + * Defaults to `globalShutdown.signal` so an armed SIGINT/SIGTERM aborts an + * in-flight request; tests inject their own controller's signal. + */ + shutdownSignal?: AbortSignal; } /** @@ -252,6 +261,7 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} onDebug: opts.debug ? (event: DebugEvent) => stderr(formatDryRunDebug(event)) : undefined, onTransition: opts.verbose ? (msg: string) => stderr(`[verbose] ${msg}`) : undefined, requestTimeoutMs, + shutdownSignal: deps.shutdownSignal ?? globalShutdown.signal, }); } @@ -274,7 +284,20 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} fetchImpl: deps.fetchImpl, onDebug: opts.debug ? (event: DebugEvent) => stderr(formatDebug(event)) : undefined, onTransition: opts.verbose ? (msg: string) => stderr(`[verbose] ${msg}`) : undefined, + // Warn once if the backend advertises a minimum supported version above + // this binary's. Gating (opt-out env, TTY, output mode, dry-run) lives in + // noteServerVersion; the client just forwards the observed headers. + onServerVersion: info => + noteServerVersion(info, { + currentVersion: VERSION, + env, + isTTY: process.stderr.isTTY === true, + outputMode: opts.output, + dryRun: opts.dryRun, + stderr, + }), requestTimeoutMs, + shutdownSignal: deps.shutdownSignal ?? globalShutdown.signal, }); } diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index a1838d6..078c653 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -333,6 +333,23 @@ describe('findSample', () => { updatedAt: expect.any(String), }); break; + case 'cancelRun': + // DEV-331 piece 3 — POST /runs/{runId}/cancel → CancelRunResponse + // (RunResponse shape + alreadyCancelled). + expect(body).toMatchObject({ + runId: expect.any(String), + testId: expect.any(String), + status: 'cancelled', + alreadyCancelled: false, + }); + break; + case 'deleteProject': + // DELETE /projects/{id} → CliDeleteProjectResponse shape. + expect(body).toMatchObject({ + projectId: expect.any(String), + deletedAt: expect.any(String), + }); + break; default: throw new Error(`Unexpected operationId in samples: ${e.operationId}`); } @@ -423,6 +440,22 @@ describe('findSample', () => { expect(body.stepSummary.failedCount).toBe(0); }); + // DEV-331 piece 3: POST /runs/{runId}/cancel must resolve to `cancelRun`, + // never fall through to the GET-only `getRun` entry despite sharing the + // `/runs/{runId}` path prefix — findSample filters by method first. + it('POST /runs/{runId}/cancel resolves cancelRun (not getRun)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz/cancel'); + expect(e?.operationId).toBe('cancelRun'); + const body = e?.body() as { status: string; alreadyCancelled: boolean; runId: string }; + expect(body.status).toBe('cancelled'); + expect(body.alreadyCancelled).toBe(false); + }); + + it('GET /runs/{runId}/cancel has no sample (cancel is POST-only)', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz/cancel'); + expect(e).toBeUndefined(); + }); + it('getTest sample carries priority field (G1a)', () => { const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_abc'); expect(e?.operationId).toBe('getTest'); diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 4c7cbb0..f6f5ec2 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -16,7 +16,11 @@ * If the CLI OpenAPI spec changes, both this file AND * `test/mock-backend/fixtures.ts` must be updated in the same PR. */ -import type { CliProject, CliUpdateProjectResponse } from '../../commands/project.js'; +import type { + CliProject, + CliUpdateProjectResponse, + CliDeleteProjectResponse, +} from '../../commands/project.js'; import type { CliBulkDeleteSummary, CliFailureContext, @@ -36,6 +40,7 @@ import type { BatchRerunResponse, BatchRunFreshResponse, ListRunsResponse, + CancelRunResponse, } from '../runs.types.js'; const SAMPLE_USER_ID = '11111111-1111-4111-8111-111111111111'; @@ -104,6 +109,7 @@ const me: MeResponse = { keyId: SAMPLE_KEY_ID, scopes: ['read:projects', 'read:tests', 'write:tests', 'run:tests'], env: 'development', + v3Enabled: true, }; const projects: CliProject[] = [ @@ -397,6 +403,11 @@ const ENTRIES: DryRunSampleEntry[] = [ updatedFields: ['name'], updatedAt: '2026-05-16T00:00:00.000Z', } satisfies CliUpdateProjectResponse), + // DELETE /projects/{id} (cascade delete project + tests + fixtures). + entry('deleteProject', 'DELETE', '/projects/{projectId}', { + projectId: SAMPLE_PROJECT_ID, + deletedAt: '2026-05-16T00:00:00.000Z', + } satisfies CliDeleteProjectResponse), entry('listTests', 'GET', '/tests', pageOf(tests)), entry('getTestCode', 'GET', '/tests/{testId}/code', testCode), entry('listTestSteps', 'GET', '/tests/{testId}/steps', pageOf(testSteps)), @@ -747,6 +758,36 @@ const ENTRIES: DryRunSampleEntry[] = [ }, ], } satisfies RunResponse), + // DEV-331 piece 3 — POST /runs/{runId}/cancel. Method-guarded in + // `findSample` (POST vs `getRun`'s GET), so this can't be shadowed by the + // broader `/runs/{runId}` pattern above despite sharing its path prefix. + // `alreadyCancelled: false` — a fresh cancel is the more instructive shape + // for a dry-run learner than the idempotent no-op. + entry('cancelRun', 'POST', '/runs/{runId}/cancel', { + runId: SAMPLE_RUN_ID, + testId: SAMPLE_TEST_ID_PASSED, + projectId: SAMPLE_PROJECT_ID, + userId: SAMPLE_USER_ID, + status: 'cancelled', + source: 'cli', + createdAt: '2026-05-15T19:32:00.000Z', + startedAt: '2026-05-15T19:32:05.000Z', + finishedAt: '2026-05-15T19:33:12.000Z', + codeVersion: 'v1', + targetUrl: SAMPLE_TARGET_URL, + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 8, + completed: 3, + passedCount: 3, + failedCount: 0, + }, + alreadyCancelled: false, + } satisfies CancelRunResponse), ]; function entry( diff --git a/src/lib/errors.test.ts b/src/lib/errors.test.ts index aa3926b..5c5ba66 100644 --- a/src/lib/errors.test.ts +++ b/src/lib/errors.test.ts @@ -59,6 +59,8 @@ describe('exitCodeFor', () => { ['UNAVAILABLE', 10], ['RATE_LIMITED', 11], ['INSUFFICIENT_CREDITS', 12], + ['FEATURE_GATED', 13], + ['CLIENT_TOO_OLD', 14], ['INTERNAL', 1], ] as const)('%s → exit %d', (code, expected) => { expect(exitCodeFor(code)).toBe(expected); @@ -138,6 +140,7 @@ describe('ApiError.fromEnvelope status fallback', () => { [403, 'AUTH_FORBIDDEN' as const], [404, 'NOT_FOUND' as const], [409, 'CONFLICT' as const], + [426, 'CLIENT_TOO_OLD' as const], [429, 'RATE_LIMITED' as const], [501, 'UNSUPPORTED' as const], [503, 'UNAVAILABLE' as const], @@ -157,6 +160,25 @@ describe('ApiError.fromEnvelope status fallback', () => { expect(err.exitCode).toBe(10); }); + it('recognizes a well-formed CLIENT_TOO_OLD (426) envelope and echoes the version details', () => { + const err = ApiError.fromEnvelope( + { + error: { + code: 'CLIENT_TOO_OLD', + message: 'Your CLI is older than the minimum supported version.', + nextAction: 'Upgrade with npm i -g @testsprite/testsprite-cli@latest.', + requestId: 'req_old', + details: { minVersion: '1.0.0', yourVersion: '0.9.0' }, + }, + }, + 426, + ); + // The code is in the union, so it is trusted directly — NOT remapped. + expect(err.code).toBe('CLIENT_TOO_OLD'); + expect(err.exitCode).toBe(14); + expect(err.details).toEqual({ minVersion: '1.0.0', yourVersion: '0.9.0' }); + }); + // Track A dogfood: NestJS raw 404 (route not registered) has the // shape `{ message, error: "Not Found", statusCode }`. Previously the // parser saw `obj.error = "Not Found"` (a string) and fell through to diff --git a/src/lib/errors.ts b/src/lib/errors.ts index f64d8e2..40b6c05 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -45,6 +45,12 @@ export const ERROR_CODES = [ // so the CLI can drop the client-side detection heuristic. 'FEATURE_GATED', 'UNSUPPORTED', + // The running CLI is older than the backend's minimum supported version. + // Backend emits this as HTTP 426 when version enforcement is enabled. Exit + // 14, non-retriable — a too-old client cannot self-heal by retrying; the + // user must upgrade. `nextAction` carries the upgrade instruction and + // `details` echoes { minVersion, latestVersion, yourVersion }. + 'CLIENT_TOO_OLD', 'INTERNAL', 'UNAVAILABLE', ] as const; @@ -77,6 +83,8 @@ export function exitCodeFor(code: ErrorCode): number { return 6; case 'UNSUPPORTED': return 7; + case 'CLIENT_TOO_OLD': + return 14; case 'UNAVAILABLE': return 10; case 'RATE_LIMITED': @@ -100,6 +108,41 @@ export class CLIError extends Error { } } +/** + * Termination signals with graceful handling, mapped to their conventional + * `128 + signum` exit code. sourceRef: POSIX signal numbers (SIGHUP=1, + * SIGINT=2, SIGTERM=15). Lives here (not interrupt.ts) so `InterruptError` + * needs no import cycle; `interrupt.ts` re-exports for back-compat. + */ +export const TERMINATION_EXIT_CODES = { + SIGINT: 130, // 128 + 2 + SIGTERM: 143, // 128 + 15 + SIGHUP: 129, // 128 + 1 +} as const; + +export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; + +/** + * User-initiated interrupt (SIGINT/SIGTERM/SIGHUP) observed while a + * graceful-detach scope (the `--wait` polling window) was armed — see + * `interrupt.ts::ShutdownController`. The `--wait` catch blocks render the + * honest partial envelope + re-attach hint, then rethrow to `index.ts`. + * + * Deliberately NOT in `ERROR_CODES` — on a signal the CLI + * exits 130/143 without consulting the error catalog. The JSON-mode stderr + * envelope uses the out-of-catalog code `"INTERRUPTED"`. Same client-local + * synthetic category as `RequestTimeoutError`. + */ +export class InterruptError extends CLIError { + readonly signal: TerminationSignal; + + constructor(signal: TerminationSignal) { + super(`Interrupted by ${signal}.`, TERMINATION_EXIT_CODES[signal]); + this.name = 'InterruptError'; + this.signal = signal; + } +} + export class NotImplementedError extends CLIError { constructor(commandPath: string) { super(`Command not yet implemented: ${commandPath}`, 2); @@ -314,6 +357,8 @@ function codeFromHttpStatus(status: number | undefined): ErrorCode { return 'PRECONDITION_FAILED'; case 413: return 'PAYLOAD_TOO_LARGE'; + case 426: + return 'CLIENT_TOO_OLD'; case 429: return 'RATE_LIMITED'; case 501: diff --git a/src/lib/failing-fe-resolver.spec.ts b/src/lib/failing-fe-resolver.spec.ts index d6524e7..1fe05a9 100644 --- a/src/lib/failing-fe-resolver.spec.ts +++ b/src/lib/failing-fe-resolver.spec.ts @@ -43,6 +43,40 @@ function makeItem( return { id, type, status, updatedAt }; } +/** + * URL-routing fetch stub for the preferredId direct-probe path: + * `GET /tests/{id}` (no query string) is answered from `preferred`; + * `GET /tests?...` list calls are answered from `pages` in order. + * Call counts are exposed so tests can assert which endpoints were hit. + */ +function makeRoutedFetch(opts: { + preferred?: { status: number; body: unknown }; + preferredThrows?: boolean; + pages: Array<{ items: TestListItem[]; nextToken: string | null }>; +}) { + let listCalls = 0; + let preferredCalls = 0; + const impl = (async (url: string | URL | Request) => { + const u = String(url); + if (/\/tests\/[^/?]+$/.test(u)) { + preferredCalls++; + if (opts.preferredThrows) throw new Error('ECONNRESET (direct probe)'); + const p = opts.preferred ?? { status: 404, body: { error: 'not found' } }; + return new Response(JSON.stringify(p.body), { + status: p.status, + headers: { 'Content-Type': 'application/json' }, + }); + } + const page = opts.pages[listCalls] ?? { items: [], nextToken: null }; + listCalls++; + return new Response(JSON.stringify(page), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof fetch; + return { impl, counts: () => ({ listCalls, preferredCalls }) }; +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -180,4 +214,163 @@ describe('resolveFailingFrontendTestId', () => { expect(result.testId).toBe('test_abc'); expect(result.reason).toContain('2026-05-27T09:00:00Z'); }); + + // DEV-393: dedicated pinned fixture must win over a fresher incidental + // failure — live-reproduced 2026-07-16 (a passingFrontendTestId fresh-run + // flip immediately outranked a dedicated "always red" fixture by + // updatedAt). See ResolverOptions.preferredId doc comment. + describe('preferredId (DEV-393 pinned-fixture preference)', () => { + test('prefers the pinned id over a fresher candidate when the direct probe is non-ok', async () => { + // Probe answers HTTP 500 (not ok, no throw) → falls through to the list + // scan, where the in-candidates preference must still pick the pinned id. + const routed = makeRoutedFetch({ + preferred: { status: 500, body: { error: 'internal' } }, + pages: [ + { + items: [ + makeItem('test_pinned', '2026-05-01T00:00:00Z'), + makeItem('test_incidental', '2026-05-20T00:00:00Z'), + ], + nextToken: null, + }, + ], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('Preferred pinned'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('falls back to freshest when the pinned id is not in the failed set', async () => { + // Probe returns 200 with a malformed body (no type/status fields) → + // falls through; pinned is absent from the list → freshest wins. + const routed = makeRoutedFetch({ + preferred: { status: 200, body: {} }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned_but_now_passing', + }); + expect(result.testId).toBe('test_incidental'); + }); + + test('behaves exactly as before when preferredId is not supplied', async () => { + const older = makeItem('test_old', '2026-05-01T00:00:00Z'); + const newer = makeItem('test_new', '2026-05-20T00:00:00Z'); + const fetchImpl = makeFetchReturning([{ items: [older, newer], nextToken: null }]); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: fetchImpl as typeof fetch, + }); + expect(result.testId).toBe('test_new'); + }); + + // Codex F3: the direct probe makes the preference immune to the maxPages + // list-scan cap — a pinned fixture beyond the last scanned page must + // still win, without any list call at all on the happy path. + test('direct probe wins without any list call when the pinned test is failed', async () => { + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { + ...makeItem('test_pinned', '2026-05-01T00:00:00Z'), + projectId: BASE_OPTS.projectId, + }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + maxPages: 1, + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('direct GET'); + expect(routed.counts()).toEqual({ listCalls: 0, preferredCalls: 1 }); + }); + + test('direct probe on a now-passing pinned test falls through to freshest', async () => { + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { + ...makeItem('test_pinned', '2026-05-25T00:00:00Z', 'frontend', 'passed'), + projectId: BASE_OPTS.projectId, + }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe error falls through to the list scan, where the pinned id still wins', async () => { + const routed = makeRoutedFetch({ + preferredThrows: true, + pages: [ + { + items: [ + makeItem('test_pinned', '2026-05-01T00:00:00Z'), + makeItem('test_incidental', '2026-05-20T00:00:00Z'), + ], + nextToken: null, + }, + ], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('Preferred pinned'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe on a pinned test from a DIFFERENT project falls through (codex round 2)', async () => { + // The pin points at a genuinely failed FE test — but in another + // project. The old list scan (projectId-filtered server-side) would + // never have returned it, so the direct probe must not let it win. + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { ...makeItem('test_pinned', '2026-05-01T00:00:00Z'), projectId: 'proj_OTHER' }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe 404 falls through and freshest wins when pinned is absent everywhere', async () => { + const routed = makeRoutedFetch({ + preferred: { status: 404, body: { error: 'not found' } }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned_deleted', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + }); }); diff --git a/src/lib/failing-fe-resolver.ts b/src/lib/failing-fe-resolver.ts index 14df49f..f8d6b5a 100644 --- a/src/lib/failing-fe-resolver.ts +++ b/src/lib/failing-fe-resolver.ts @@ -50,6 +50,21 @@ export interface ResolverOptions { * Each page uses pageSize=50, so 5 pages = 250 tests scanned. */ maxPages?: number; + /** + * DEV-393: the operator's pinned `failingFrontendTestId` (the static value + * from fixtures.local.json). "Freshest failed" alone is fragile in a shared + * project: any OTHER FE test that fails more recently than a dedicated, + * permanently-red fixture silently steals the slot (live-reproduced + * 2026-07-16 — a `passingFrontendTestId` fresh-run flip immediately + * outranked the dedicated fixture on `updatedAt`). When set, the pinned id + * is probed directly first (one GET /tests/{id} — immune to the maxPages + * list-scan cap); if it is currently `status:failed` it wins outright, + * regardless of timestamp. If the probe errors, the list scan below still + * prefers the pinned id when it appears among the candidates. Falls through + * to the existing freshest-wins behavior when unset or no longer failing — + * fully backward compatible. + */ + preferredId?: string; } export interface ResolverResult { @@ -67,9 +82,46 @@ export interface ResolverResult { * when no Failed test exists — never throws. */ export async function resolveFailingFrontendTestId(opts: ResolverOptions): Promise { - const { baseUrl, apiKey, projectId, maxPages = 5 } = opts; + const { baseUrl, apiKey, projectId, maxPages = 5, preferredId } = opts; const fetchImpl = opts.fetchImpl ?? fetch; + // DEV-393 (codex F3): probe the pinned fixture directly first, so the + // preference cannot be defeated by the maxPages list-scan cap (a pinned + // fixture beyond page `maxPages` would otherwise silently lose the slot to + // an incidental fresher failure). Any probe error or non-failed status + // falls through to the list scan; the in-candidates preference there + // remains as a second chance after a transient probe failure. + if (preferredId) { + try { + const resp = await fetchImpl(`${baseUrl}/tests/${encodeURIComponent(preferredId)}`, { + headers: { + 'x-api-key': apiKey, + 'x-request-id': `dev-e2e-resolver-preferred-${Date.now()}`, + accept: 'application/json', + }, + signal: AbortSignal.timeout(10_000), + }); + if (resp.ok) { + const test = (await resp.json()) as Partial & { projectId?: string }; + // The probe must honor the resolver's project scope (codex round 2): + // the list path filters by projectId server-side, so a pin that + // points at a failed FE test in a DIFFERENT project must not win + // here either. A response without a matching projectId (absent or + // mismatched) falls through to the project-scoped list scan. + if (test.type === 'frontend' && test.status === 'failed' && test.projectId === projectId) { + return { + testId: preferredId, + reason: + `Preferred pinned failingFrontendTestId (${preferredId}) confirmed status:failed ` + + `via direct GET (updatedAt=${test.updatedAt ?? 'unknown'}); list scan skipped`, + }; + } + } + } catch { + // Transient probe failure — fall through to the list scan below. + } + } + const candidates: TestListItem[] = []; let cursor: string | undefined; let pagesFetched = 0; @@ -134,6 +186,19 @@ export async function resolveFailingFrontendTestId(opts: ResolverOptions): Promi }; } + // DEV-393: a pinned, dedicated "always red" fixture wins over freshest + // whenever it is still in the failed set — see the doc comment on + // ResolverOptions.preferredId for why "freshest" alone is not enough. + if (preferredId) { + const pinned = candidates.find(c => c.id === preferredId); + if (pinned) { + return { + testId: pinned.id, + reason: `Preferred pinned failingFrontendTestId (${pinned.id}) is still status:failed (updatedAt=${pinned.updatedAt}); took priority over freshest-updatedAt candidate`, + }; + } + } + // Pick the most recently updated candidate — freshest failure is the // most likely to have a valid failure bundle attached. candidates.sort((a, b) => { diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index a9c1193..2baff2c 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { ApiError, RequestTimeoutError, TransportError } from './errors.js'; +import { ApiError, InterruptError, RequestTimeoutError, TransportError } from './errors.js'; import type { DebugEvent } from './http.js'; import { HttpClient, REQUEST_TIMEOUT_DEFAULT_MS, buildUrl, parseRetryAfter } from './http.js'; import { VERSION } from '../version.js'; @@ -29,7 +29,11 @@ function errorEnvelopeResponse(status: number, code: string, init: ResponseInit function makeClient( fetchImpl: typeof fetch, - options: { apiKey?: string | null; onDebug?: (e: DebugEvent) => void } = {}, + options: { + apiKey?: string | null; + onDebug?: (e: DebugEvent) => void; + onServerVersion?: (info: { minVersion?: string }) => void; + } = {}, ): HttpClient { const apiKey = 'apiKey' in options ? (options.apiKey ?? undefined) : 'sk-test'; return new HttpClient({ @@ -39,9 +43,73 @@ function makeClient( sleep: () => Promise.resolve(), random: () => 0, onDebug: options.onDebug, + onServerVersion: options.onServerVersion, }); } +describe('CLIENT_TOO_OLD (426)', () => { + it('is not retried — fails fast with the typed error', async () => { + const fetchImpl = vi.fn().mockResolvedValue(errorEnvelopeResponse(426, 'CLIENT_TOO_OLD')); + const client = makeClient(fetchImpl as unknown as typeof fetch); + + const err = await client.get('/me').catch((e: unknown) => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CLIENT_TOO_OLD'); + expect((err as ApiError).exitCode).toBe(14); + expect(fetchImpl).toHaveBeenCalledTimes(1); // no retry + }); +}); + +describe('onServerVersion hook', () => { + it('fires with the parsed floor header on a 2xx response', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue( + jsonResponse( + { ok: true }, + { + headers: { + 'content-type': 'application/json', + 'x-testsprite-cli-min-version': '1.0.0', + }, + }, + ), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/me'); + + expect(onServerVersion).toHaveBeenCalledWith({ minVersion: '1.0.0' }); + }); + + it('fires on a non-2xx response too (the floor header rides on every response)', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue( + errorEnvelopeResponse(404, 'NOT_FOUND', { + headers: { + 'content-type': 'application/json', + 'x-testsprite-cli-min-version': '1.0.0', + }, + }), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/tests/missing').catch(() => undefined); + + expect(onServerVersion).toHaveBeenCalledWith({ minVersion: '1.0.0' }); + }); + + it('does not fire when the floor header is absent', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue(jsonResponse({ ok: true })); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/me'); + + expect(onServerVersion).not.toHaveBeenCalled(); + }); +}); + describe('buildUrl', () => { it('handles trailing slashes and absolute paths', () => { expect(buildUrl('https://api.example.com/api/cli/v1', '/me')).toBe( @@ -641,3 +709,177 @@ describe('HttpClient per-request timeout', () => { expect(err).toBeInstanceOf(RequestTimeoutError); }); }); + +describe('HttpClient shutdown signal (DEV-331 graceful detach)', () => { + /** Stalled fetch that rejects with the effective signal's reason on abort. */ + function stalledFetch(callCounter?: { count: number }): typeof fetch { + return vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + if (callCounter) callCounter.count += 1; + return new Promise((_resolve, reject) => { + const signal = init?.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + if (reason instanceof Error) { + reject(reason); + return; + } + const err = new Error('aborted'); + err.name = 'AbortError'; + reject(err); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as unknown as typeof fetch; + } + + function makeShutdownClient( + shutdownSignal: AbortSignal, + callCounter?: { count: number }, + ): HttpClient { + return new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: stalledFetch(callCounter), + sleep: () => Promise.resolve(), + random: () => 0, + shutdownSignal, + }); + } + + it('aborts an in-flight fetch with the InterruptError reason (no retry, no re-wrap)', async () => { + const counter = { count: 0 }; + const shutdown = new AbortController(); + const client = makeShutdownClient(shutdown.signal, counter); + const pending = client.get('/runs/run_1'); + queueMicrotask(() => shutdown.abort(new InterruptError('SIGINT'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + expect((err as InterruptError).exitCode).toBe(130); + expect(counter.count).toBe(1); // never retried + }); + + it('classifies an anonymous AbortError as the interrupt when the shutdown signal fired', async () => { + // Some runtimes reject with a bare AbortError instead of the abort reason; + // rethrowIfAbort must still classify shutdown-first (never TransportError, + // never RequestTimeoutError). + const shutdown = new AbortController(); + const fetchImpl = vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => { + const err = new Error('aborted'); + err.name = 'AbortError'; + reject(err); + }, + { once: true }, + ); + }); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => Promise.resolve(), + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/runs/run_1'); + queueMicrotask(() => shutdown.abort(new InterruptError('SIGTERM'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + }); + + it('skips dispatch entirely when the shutdown signal is already aborted', async () => { + const counter = { count: 0 }; + const shutdown = new AbortController(); + shutdown.abort(new InterruptError('SIGINT')); + const client = makeShutdownClient(shutdown.signal, counter); + const err = await client.get('/me').catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(counter.count).toBe(0); + }); + + it('passes an InterruptError from a caller-composed signal through untouched (poll path, no shutdownSignal configured)', async () => { + // The polling loop composes the shutdown signal into its per-iteration + // caller signal; the fetch then rejects with the InterruptError reason + // even though the client itself has no shutdownSignal. + const counter = { count: 0 }; + const fetchImpl = stalledFetch(counter); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => Promise.resolve(), + random: () => 0, + }); + const caller = new AbortController(); + const pending = client.get('/runs/run_1', { signal: caller.signal }); + queueMicrotask(() => caller.abort(new InterruptError('SIGINT'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(counter.count).toBe(1); // no transport retry + }); +}); + +describe('HttpClient retry-delay sleep bails on shutdown (DEV-331 codex finding 1)', () => { + it('a shutdown during a transport-retry sleep rejects with InterruptError immediately', async () => { + // First fetch throws a transport error → the client schedules a retry + // sleep. The injected sleep NEVER resolves, so only the shutdown race can + // end it — without the bail, the interrupt would wait out the delay + // (e.g. a Retry-After: 60) before surfacing. + const shutdown = new AbortController(); + let calls = 0; + const fetchImpl = vi.fn(async () => { + calls += 1; + throw new Error('socket hang up'); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => new Promise(() => {}), // never resolves + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/me'); + // Let the first attempt fail and the retry sleep begin, then interrupt. + await new Promise(resolve => setTimeout(resolve, 10)); + shutdown.abort(new InterruptError('SIGINT')); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(calls).toBe(1); // the retry never dispatched + }); + + it('a shutdown during a RATE_LIMITED Retry-After sleep rejects with InterruptError immediately', async () => { + const shutdown = new AbortController(); + let calls = 0; + const fetchImpl = vi.fn(async () => { + calls += 1; + return errorEnvelopeResponse(429, 'RATE_LIMITED', { + headers: { 'content-type': 'application/json', 'retry-after': '60' }, + }); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => new Promise(() => {}), // never resolves + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/me'); + await new Promise(resolve => setTimeout(resolve, 10)); + shutdown.abort(new InterruptError('SIGTERM')); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + expect(calls).toBe(1); + }); +}); diff --git a/src/lib/http.ts b/src/lib/http.ts index 7a9051a..f0139ca 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; import type { ErrorCode } from './errors.js'; -import { ApiError, RequestTimeoutError, TransportError } from './errors.js'; +import { ApiError, InterruptError, RequestTimeoutError, TransportError } from './errors.js'; import { VERSION } from '../version.js'; import type { TriggerRunBody, @@ -14,6 +14,7 @@ import type { BatchRunFreshResponse, ListRunsQuery, ListRunsResponse, + CancelRunResponse, } from './runs.types.js'; export type FetchImpl = typeof globalThis.fetch; @@ -74,6 +75,14 @@ export interface HttpClientOptions { * Stays silent when absent; wired to stderr at `--verbose` level. */ onTransition?: (msg: string) => void; + /** + * Optional callback fired with the backend's supported-floor header + * (`X-TestSprite-CLI-Min-Version`) when present on a response. Fires on both + * success and error responses. The client forwards the value verbatim — it + * applies no policy itself; the caller (client-factory) decides whether to + * warn the user. + */ + onServerVersion?: (info: { minVersion?: string }) => void; /** * Per-request wall-clock timeout in milliseconds applied to every outgoing * fetch. The signal fires independently of any caller-supplied signal — the @@ -87,6 +96,16 @@ export interface HttpClientOptions { * request is well within the 120s default. */ requestTimeoutMs?: number; + /** + * Process-lifetime shutdown signal (DEV-331 piece 1). Composed into every + * outgoing fetch so an armed SIGINT/SIGTERM aborts an in-flight request + * (a `--wait` long-poll can sit inside a single fetch for minutes) instead + * of waiting out its window. Aborts with an `InterruptError` reason, which + * is classified before the timeout-vs-caller logic and is never retried or + * re-wrapped as `TransportError`. Defaults off; production wiring passes + * `globalShutdown.signal` via the client factory. + */ + shutdownSignal?: AbortSignal; } export interface RequestOptions { @@ -167,19 +186,42 @@ export class HttpClient { private readonly random: () => number; private readonly onDebug?: (event: DebugEvent) => void; private readonly onTransition?: (msg: string) => void; + private readonly onServerVersion?: (info: { minVersion?: string }) => void; private readonly requestTimeoutMs: number; + private readonly shutdownSignal?: AbortSignal; constructor(options: HttpClientOptions) { this.baseUrl = trimTrailingSlash(options.baseUrl); this.apiKey = options.apiKey; + this.shutdownSignal = options.shutdownSignal; this.fetchImpl = options.fetchImpl ?? globalThis.fetch.bind(globalThis); this.sleep = options.sleep ?? defaultSleep; this.random = options.random ?? Math.random; this.onDebug = options.onDebug; this.onTransition = options.onTransition; + this.onServerVersion = options.onServerVersion; this.requestTimeoutMs = options.requestTimeoutMs ?? REQUEST_TIMEOUT_DEFAULT_MS; } + /** + * Read the backend's supported-floor header off a response and forward it to + * `onServerVersion` when present. Never throws — a bad header must not break + * the request. Called on every response (success or error) so the advisory + * covers all paths. (The backend advertises only the floor; "latest" is + * resolved client-side via the npm update-notice.) + */ + private captureServerVersion(response: Response): void { + if (!this.onServerVersion) return; + try { + const minVersion = response.headers.get('x-testsprite-cli-min-version') ?? undefined; + if (minVersion !== undefined) { + this.onServerVersion({ minVersion }); + } + } catch { + // Header parsing is best-effort; never let it affect the request. + } + } + async get(path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta('GET', path, options).then(r => r.body); } @@ -382,6 +424,24 @@ export class HttpClient { }); } + /** + * POST /api/cli/v1/runs/{runId}/cancel + * User-initiated cancel of a queued/running run (DEV-331 piece 3). No + * body, no `Idempotency-Key` — the endpoint is naturally idempotent (D10): + * re-cancel → 200 `alreadyCancelled: true`; already-terminal (passed/ + * failed/blocked) → 409 CONFLICT; unknown/cross-tenant runId → 404. + * + * `retryOnConflict: false` — same rationale as `triggerRun`: a 409 here is + * a truthful terminal answer ("already finished"), not a transient + * snapshot conflict to paper over with a retry. + */ + async cancelRun(runId: string, options?: { signal?: AbortSignal }): Promise { + return this.post(`/runs/${encodeURIComponent(runId)}/cancel`, { + signal: options?.signal, + retryOnConflict: false, + }); + } + /** * Classify an error thrown while issuing OR reading a request. When it is an * abort/timeout and our per-request timeout signal fired (and the caller had @@ -397,6 +457,14 @@ export class HttpClient { requestId: string, effectiveSignal: AbortSignal = timeoutSignal, ): void { + // A user interrupt (DEV-331) outranks every other classification: once the + // shutdown signal fired, whatever error surfaced from the aborted fetch or + // body read is the interrupt. Throw its InterruptError reason so the wait + // paths can render the honest detach UX — never a RequestTimeoutError, and + // never fall through to the transport retry loop. + if (this.shutdownSignal?.aborted) { + throw this.shutdownSignal.reason; + } if (isAbortError(err) || isTimeoutError(err)) { const timeoutWon = timeoutSignal.aborted && @@ -422,6 +490,10 @@ export class HttpClient { let attempt = 0; while (true) { + // Bail before issuing (or re-issuing after a retry sleep) a request the + // user has already interrupted; the composed signal below would abort it + // immediately anyway, this just skips the wasted dispatch. + if (this.shutdownSignal?.aborted) throw this.shutdownSignal.reason; attempt += 1; this.debug({ kind: 'request', method, url, attempt, requestId }); const startedAt = Date.now(); @@ -436,8 +508,14 @@ export class HttpClient { // long-poll window (<=25s via ?waitSeconds), so it never bites polling. const requestTimeout = createRequestTimeout(this.requestTimeoutMs); const timeoutSignal = requestTimeout.signal; + const composedSignals = [timeoutSignal]; + if (options.signal != null) composedSignals.push(options.signal); + // Shutdown composition (DEV-331): an armed SIGINT/SIGTERM aborts the + // in-flight fetch immediately (reason: InterruptError) instead of + // letting a long-poll drain its window before the interrupt surfaces. + if (this.shutdownSignal != null) composedSignals.push(this.shutdownSignal); const effectiveSignal = - options.signal != null ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal; + composedSignals.length > 1 ? AbortSignal.any(composedSignals) : timeoutSignal; try { try { @@ -455,6 +533,13 @@ export class HttpClient { // A caller-supplied abort sets `name === 'AbortError'`. // We treat both abort variants together: if the timeout signal fired and // the caller hadn't already aborted, surface a clear RequestTimeoutError. + // A user interrupt is never retried and never re-wrapped as a + // TransportError (same passthrough discipline as RequestTimeoutError). + // The instanceof check matters even without `this.shutdownSignal`: + // the polling path composes the shutdown signal into its per- + // iteration caller signal, so the fetch can reject with the + // InterruptError reason directly (DEV-331). + if (err instanceof InterruptError) throw err; // A timeout/abort during the fetch itself: classify it (RequestTimeoutError // when our deadline fired; otherwise rethrow the caller's abort unmodified). this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); @@ -487,11 +572,15 @@ export class HttpClient { delayMs: decision.delayMs, }); requestTimeout.clear(); - await this.sleep(decision.delayMs); + await this.sleepBeforeRetry(decision.delayMs); continue; } const durationMs = Date.now() - startedAt; + // Surface the backend version-compatibility headers on every response + // (success or error) before branching, so the caller's advisory covers + // all paths. + this.captureServerVersion(response); if (response.ok) { this.debug({ kind: 'response', @@ -505,6 +594,8 @@ export class HttpClient { try { return { body: (await response.json()) as T, requestId, status: response.status }; } catch (err) { + // Interrupt passthrough (see the fetch catch above). + if (err instanceof InterruptError) throw err; // A timeout/abort can fire mid-body-read (headers received, stream stalls). this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); // Otherwise the successful response body was not valid JSON — a @@ -521,6 +612,8 @@ export class HttpClient { try { rawBody = await safeReadJson(response); } catch (err) { + // Interrupt passthrough (see the fetch catch above). + if (err instanceof InterruptError) throw err; // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse // errors), so a timeout fired mid-body-read on a non-OK response lands here. this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); @@ -559,7 +652,7 @@ export class HttpClient { delayMs: decision.delayMs, }); requestTimeout.clear(); - await this.sleep(decision.delayMs); + await this.sleepBeforeRetry(decision.delayMs); continue; } @@ -624,7 +717,7 @@ export class HttpClient { delayMs: decision.delayMs, }); requestTimeout.clear(); - await this.sleep(decision.delayMs); + await this.sleepBeforeRetry(decision.delayMs); } finally { requestTimeout.clear(); } @@ -652,6 +745,33 @@ export class HttpClient { return headers; } + /** + * Retry-delay sleep that bails the moment the shutdown signal fires + * (DEV-331, codex finding 1): a RATE_LIMITED `Retry-After: 60` or a + * transport backoff must not delay the honest-detach exit by up to a + * minute — reject with the InterruptError reason immediately. Mirrors + * poll.ts::sleepUnlessInterrupted. + */ + private sleepBeforeRetry(ms: number): Promise { + const signal = this.shutdownSignal; + if (signal == null) return this.sleep(ms); + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + this.sleep(ms).then( + () => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, + err => { + signal.removeEventListener('abort', onAbort); + reject(err instanceof Error ? err : new Error(String(err))); + }, + ); + }); + } + private debug(event: DebugEvent): void { if (this.onDebug) this.onDebug(event); } @@ -856,6 +976,9 @@ function apiRetryDecision( case 'UNSUPPORTED': case 'INSUFFICIENT_CREDITS': case 'FEATURE_GATED': + case 'CLIENT_TOO_OLD': + // CLIENT_TOO_OLD: retrying re-sends the same too-old client — it can only + // self-heal by upgrading, so fail fast with the upgrade guidance. return { retry: false, delayMs: 0 }; case 'CONFLICT': // Read paths (e.g. GET /failure) retry once: 409 = mid-mutation snapshot. diff --git a/src/lib/interrupt.test.ts b/src/lib/interrupt.test.ts index 1fcf8eb..bd20bba 100644 --- a/src/lib/interrupt.test.ts +++ b/src/lib/interrupt.test.ts @@ -1,8 +1,10 @@ import { EventEmitter } from 'node:events'; import { writeSync } from 'node:fs'; import { describe, expect, it, vi } from 'vitest'; +import { InterruptError } from './errors.js'; import { SIGINT_EXIT_CODE, + ShutdownController, TERMINATION_EXIT_CODES, formatInterruptMessage, installBrokenPipeGuard, @@ -30,32 +32,37 @@ describe('formatInterruptMessage', () => { }); }); +/** Fresh handler map + controller per case: the disarmed path exits on the + * FIRST signal, so sequential signals on one install take the second-signal + * hard-exit branch (by design — DEV-331 SIG-5). */ +function install(shutdown = new ShutdownController()) { + const handlers = new Map void>(); + const stderr: string[] = []; + const exit = vi.fn(); + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + stderr: line => stderr.push(line), + exit, + shutdown, + }); + return { handlers, stderr, exit, shutdown }; +} + describe('installSignalHandlers', () => { it('registers SIGINT, SIGTERM and SIGHUP with the conventional 128+signum exit codes', () => { - const handlers = new Map void>(); - const stderr: string[] = []; - const exit = vi.fn(); - - installSignalHandlers({ - on: (signal, handler) => handlers.set(signal, handler), - stderr: line => stderr.push(line), - exit, - }); - - expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); - - handlers.get('SIGINT')!(); - expect(exit).toHaveBeenLastCalledWith(130); - handlers.get('SIGTERM')!(); - expect(exit).toHaveBeenLastCalledWith(143); - handlers.get('SIGHUP')!(); - expect(exit).toHaveBeenLastCalledWith(129); - - // Each handler emits a leading blank line then the explanation. - expect(stderr[0]).toBe(''); - expect(stderr.join('\n')).toContain('Interrupted (SIGINT)'); - expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); - expect(stderr.join('\n')).toContain('Interrupted (SIGHUP)'); + for (const [signal, code] of [ + ['SIGINT', 130], + ['SIGTERM', 143], + ['SIGHUP', 129], + ] as const) { + const { handlers, stderr, exit } = install(); + expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); + handlers.get(signal)!(); + expect(exit).toHaveBeenLastCalledWith(code); + // Disarmed handler emits a leading blank line then the explanation. + expect(stderr[0]).toBe(''); + expect(stderr.join('\n')).toContain(`Interrupted (${signal})`); + } expect(SIGINT_EXIT_CODE).toBe(130); expect(TERMINATION_EXIT_CODES.SIGTERM).toBe(143); expect(TERMINATION_EXIT_CODES.SIGHUP).toBe(129); @@ -69,6 +76,7 @@ describe('installSignalHandlers', () => { installSignalHandlers({ on: (signal, handler) => handlers.set(signal, handler), exit, + shutdown: new ShutdownController(), }); handlers.get('SIGINT')!(); expect(exit).toHaveBeenCalledWith(130); @@ -78,6 +86,51 @@ describe('installSignalHandlers', () => { .join(''); expect(written).toContain('Interrupted (SIGINT)'); }); + + it('armed scope: first signal aborts with InterruptError and does NOT exit or print', () => { + const { handlers, stderr, exit, shutdown } = install(); + const disarm = shutdown.arm(); + handlers.get('SIGINT')!(); + + expect(exit).not.toHaveBeenCalled(); + expect(stderr).toEqual([]); + expect(shutdown.signal.aborted).toBe(true); + expect(shutdown.received).toBe('SIGINT'); + const reason = shutdown.signal.reason as InterruptError; + expect(reason).toBeInstanceOf(InterruptError); + expect(reason.signal).toBe('SIGINT'); + expect(reason.exitCode).toBe(130); + disarm(); + }); + + it('second signal during armed cleanup hard-exits with the second signal code (SIG-5)', () => { + const { handlers, exit, shutdown } = install(); + shutdown.arm(); + handlers.get('SIGINT')!(); + expect(exit).not.toHaveBeenCalled(); + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenCalledWith(143); + }); + + it('disposed scope reverts to the disarmed immediate-exit behavior', () => { + const { handlers, stderr, exit, shutdown } = install(); + const disarm = shutdown.arm(); + disarm(); + disarm(); // idempotent — double dispose must not underflow the counter + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenCalledWith(143); + expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); + }); + + it('nested arms (fan-out members) stay armed until the last disposer runs', () => { + const shutdown = new ShutdownController(); + const a = shutdown.arm(); + const b = shutdown.arm(); + a(); + expect(shutdown.isArmed).toBe(true); + b(); + expect(shutdown.isArmed).toBe(false); + }); }); describe('installBrokenPipeGuard', () => { diff --git a/src/lib/interrupt.ts b/src/lib/interrupt.ts index cc5b4d5..19edb09 100644 --- a/src/lib/interrupt.ts +++ b/src/lib/interrupt.ts @@ -19,22 +19,100 @@ * without spawning a subprocess or sending a real signal. */ +import { setMaxListeners } from 'node:events'; import { writeSync } from 'node:fs'; +import { InterruptError, TERMINATION_EXIT_CODES, type TerminationSignal } from './errors.js'; + +export { TERMINATION_EXIT_CODES, type TerminationSignal } from './errors.js'; + +/** Back-compat alias: SIGINT's conventional exit code. */ +export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; /** - * Termination signals handled, mapped to their conventional `128 + signum` - * exit code. sourceRef: POSIX signal numbers (SIGHUP=1, SIGINT=2, SIGTERM=15). + * Structural view of {@link ShutdownController} threaded through the DI + * surfaces (`TestDeps`, `PollOptions`) — commands and the polling loop need + * only these members, and tests can supply a lightweight fake. */ -export const TERMINATION_EXIT_CODES = { - SIGINT: 130, // 128 + 2 - SIGTERM: 143, // 128 + 15 - SIGHUP: 129, // 128 + 1 -} as const; +export interface ShutdownHandle { + /** Aborts (reason: `InterruptError`) when a termination signal arrives while armed. */ + readonly signal: AbortSignal; + /** Enter a graceful-detach scope. Returns the disposer that leaves it. */ + arm(): () => void; +} -export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; +/** + * Process-lifetime coordinator between the signal handler and the `--wait` + * polling paths (DEV-331 piece 1). + * + * Two modes, chosen by whether a graceful-detach scope is armed when the + * signal arrives: + * + * - **Armed** (inside `pollRunUntilTerminal`): the handler only aborts + * `signal` with an `InterruptError` — no I/O, no exit. The in-flight fetch + * and every backoff sleep bail immediately; the `--wait` catch blocks own + * the cleanup (finalize the ticker, print the honest partial envelope + + * re-attach hint, rethrow to `index.ts` → exit 130/143/129). + * - **Disarmed** (no wait in progress — prompts, one-shot commands, local + * FS work): the handler prints the generic explanation and exits + * immediately, preserving the pre-DEV-331 behavior. An abort nobody + * observes must never leave the process hanging at e.g. a readline prompt. + * + * A second signal while the armed cleanup is in flight is the documented + * escape hatch: immediate hard exit. + */ +export class ShutdownController { + private readonly controller = new AbortController(); + private armedCount = 0; + private receivedSignal: TerminationSignal | null = null; -/** Back-compat alias: SIGINT's conventional exit code. */ -export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; + constructor() { + // Every fetch and every poll iteration composes this signal via + // AbortSignal.any — a 50-run batch fan-out legitimately holds >10 + // concurrent listeners, so silence Node's MaxListeners warning. + setMaxListeners(0, this.controller.signal); + } + + get signal(): AbortSignal { + return this.controller.signal; + } + + /** The first termination signal received, or null if none yet. */ + get received(): TerminationSignal | null { + return this.receivedSignal; + } + + get isArmed(): boolean { + return this.armedCount > 0; + } + + /** + * Enter a graceful-detach scope (re-entrant: fan-out members overlap). + * Returns an idempotent disposer. + */ + arm(): () => void { + this.armedCount += 1; + let disposed = false; + return () => { + if (disposed) return; + disposed = true; + this.armedCount -= 1; + }; + } + + /** Record the signal and abort with an `InterruptError` carrying it. */ + interrupt(signal: TerminationSignal): void { + this.receivedSignal = signal; + this.controller.abort(new InterruptError(signal)); + } +} + +/** + * The process-wide instance: `index.ts` hands it to `installSignalHandlers`, + * and it is the default `shutdown` for `TestDeps` / `PollOptions` / + * `ClientFactoryDeps`, so production wiring is automatic. Tests inject their + * own `ShutdownController` (or a `ShutdownHandle` fake) instead. + */ +export const globalShutdown = new ShutdownController(); export function formatInterruptMessage(signal: TerminationSignal = 'SIGINT'): string { return ( @@ -50,11 +128,19 @@ export interface InterruptDeps { stderr?: (line: string) => void; /** Process exit. Defaults to `process.exit`. */ exit?: (code: number) => void; + /** Shutdown coordinator. Defaults to {@link globalShutdown}. */ + shutdown?: ShutdownController; } /** * Register handlers for SIGINT, SIGTERM and SIGHUP. Idempotent enough for a * single top-level call in `index.ts`; not designed to be installed twice. + * + * First signal, armed scope: abort-only — the `--wait` catch paths own the + * honest-detach UX and the exit (DEV-331 D1: Ctrl-C = detach, never cancel). + * First signal, disarmed: print the generic explanation + exit `128+signum`. + * Second signal (any mode): immediate hard exit — the escape hatch when the + * graceful cleanup itself wedges. */ export function installSignalHandlers(deps: InterruptDeps = {}): void { const on = @@ -75,9 +161,27 @@ export function installSignalHandlers(deps: InterruptDeps = {}): void { } }); const exit = deps.exit ?? ((code: number) => process.exit(code)); + const shutdown = deps.shutdown ?? globalShutdown; for (const signal of Object.keys(TERMINATION_EXIT_CODES) as TerminationSignal[]) { on(signal, () => { + if (shutdown.received !== null) { + // Second signal while graceful cleanup is in flight: hard exit now. + exit(TERMINATION_EXIT_CODES[signal]); + return; + } + if (shutdown.isArmed) { + // Graceful detach: abort only (sync, signal-safe — no I/O here so a + // pending stdout `drain` wait can settle); the armed catch paths + // finalize the ticker, print the partial + re-attach hint, and exit + // via index.ts with this signal's code. + shutdown.interrupt(signal); + return; + } + // Disarmed (no --wait in progress): legacy immediate exit. Record the + // signal first so a second one takes the hard-exit branch even when + // `exit` is injected and does not terminate (unit tests). + shutdown.interrupt(signal); // Blank line first so the message starts on its own row rather than // trailing the progress ticker's in-place line. stderr(''); diff --git a/src/lib/poll.spec.ts b/src/lib/poll.spec.ts index e6a14a1..93889eb 100644 --- a/src/lib/poll.spec.ts +++ b/src/lib/poll.spec.ts @@ -7,7 +7,8 @@ */ import { describe, expect, it } from 'vitest'; -import { ApiError } from './errors.js'; +import { ApiError, InterruptError } from './errors.js'; +import { ShutdownController } from './interrupt.js'; import { pollRunUntilTerminal, TimeoutError } from './poll.js'; import type { RunClient } from './poll.js'; import type { RunResponse } from './runs.types.js'; @@ -794,3 +795,104 @@ describe('pollRunUntilTerminal — resolveAlternate hook', () => { } }); }); + +// --------------------------------------------------------------------------- +// Graceful detach — shutdown handle (DEV-331 piece 1) +// --------------------------------------------------------------------------- + +describe('pollRunUntilTerminal — shutdown (SIGINT/SIGTERM graceful detach)', () => { + it('throws the InterruptError when the shutdown signal was already aborted (beats the deadline)', async () => { + const shutdown = new ShutdownController(); + shutdown.interrupt('SIGINT'); + // timeoutSeconds 0 → the deadline has also passed; the interrupt must win. + const err = await pollRunUntilTerminal(makeClient([makeRun('passed')]), RUN_ID, { + timeoutSeconds: 0, + sleep: instantSleep, + shutdown, + }).catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + }); + + it('aborts an in-flight long-poll fetch and surfaces InterruptError (not TimeoutError)', async () => { + // getRun hangs until the composed per-iteration signal aborts — the same + // contract as a real fetch. Flag-checking between iterations would never + // notice; only the signal composition can interrupt this. + const client: RunClient = { + getRun: (_runId, opts) => + new Promise((_resolve, reject) => { + opts?.signal?.addEventListener('abort', () => reject(opts.signal!.reason), { + once: true, + }); + }), + }; + const shutdown = new ShutdownController(); + const poll = pollRunUntilTerminal(client, RUN_ID, { + timeoutSeconds: 30, + sleep: instantSleep, + shutdown, + }); + queueMicrotask(() => shutdown.interrupt('SIGINT')); + const err = await poll.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + expect((err as InterruptError).exitCode).toBe(130); + }); + + it('bails out of a retryAfterSeconds sleep immediately on interrupt', async () => { + // The injected sleep never resolves — only the shutdown race can end it. + const neverSleep = () => new Promise(() => {}); + const client = makeClient([makeRun('running', { retryAfterSeconds: 60 }), makeRun('running')]); + const shutdown = new ShutdownController(); + const poll = pollRunUntilTerminal(client, RUN_ID, { + timeoutSeconds: 300, + sleep: neverSleep, + shutdown, + }); + await new Promise(resolve => setTimeout(resolve, 10)); // let the loop reach the sleep + shutdown.interrupt('SIGTERM'); + const err = await poll.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + expect((err as InterruptError).exitCode).toBe(143); + }); + + it('arms the graceful-detach scope for the poll duration and disarms after', async () => { + let armed = 0; + let disposedCount = 0; + const handle = { + signal: new AbortController().signal, + arm: () => { + armed += 1; + return () => { + disposedCount += 1; + }; + }, + }; + const run = await pollRunUntilTerminal(makeClient([makeRun('passed')]), RUN_ID, { + timeoutSeconds: 5, + sleep: instantSleep, + shutdown: handle, + }); + expect(run.status).toBe('passed'); + expect(armed).toBe(1); + expect(disposedCount).toBe(1); + }); + + it('disarms even when the poll throws (TimeoutError path)', async () => { + let disposedCount = 0; + const handle = { + signal: new AbortController().signal, + arm: () => () => { + disposedCount += 1; + }, + }; + const err = await pollRunUntilTerminal(makeClient([makeRun('running')]), RUN_ID, { + timeoutSeconds: 0, + sleep: instantSleep, + shutdown: handle, + }).catch(e => e); + expect(err).toBeInstanceOf(TimeoutError); + expect(disposedCount).toBe(1); + }); +}); diff --git a/src/lib/poll.ts b/src/lib/poll.ts index 0dfb0ac..92498e2 100644 --- a/src/lib/poll.ts +++ b/src/lib/poll.ts @@ -21,7 +21,8 @@ * Deadline exceeded → throw `TimeoutError`. */ -import { ApiError } from './errors.js'; +import { ApiError, InterruptError } from './errors.js'; +import type { ShutdownHandle } from './interrupt.js'; import type { RunResponse } from './runs.types.js'; import { isTerminalStatus } from './runs.types.js'; @@ -89,6 +90,16 @@ export interface PollOptions { elapsedMs: number, signal: AbortSignal, ) => Promise; + /** + * Graceful-detach coordinator (DEV-331 piece 1). While the poll runs, the + * scope is armed: a SIGINT/SIGTERM aborts `shutdown.signal` with an + * `InterruptError` instead of killing the process, and this loop surfaces + * it immediately — the in-flight long-poll fetch aborts (composed into the + * per-iteration signal) and every backoff/retry sleep bails early. The + * caller's catch block renders the honest partial + re-attach hint. + * Absent (tests, non-wait callers): behavior is unchanged. + */ + shutdown?: ShutdownHandle; } const LONG_POLL_WAIT_SECONDS = 25; @@ -108,9 +119,30 @@ export async function pollRunUntilTerminal( client: RunClient, runId: string, options: PollOptions, +): Promise { + // Arm the graceful-detach scope for the duration of the poll (DEV-331): + // while armed, a termination signal aborts instead of hard-killing the + // process, and the wait-path catch blocks own the honest detach UX. + const disarm = options.shutdown?.arm(); + try { + return await pollLoop(client, runId, options); + } finally { + disarm?.(); + } +} + +async function pollLoop( + client: RunClient, + runId: string, + options: PollOptions, ): Promise { const { timeoutSeconds, onTick, onTransition, resolveAlternate } = options; - const sleep = options.sleep ?? defaultSleep; + const shutdownSignal = options.shutdown?.signal; + const rawSleep = options.sleep ?? defaultSleep; + // Every sleep site (retryAfterSeconds, backoff schedule, not_yet_visible, + // 5xx retry) bails immediately on interrupt — a Ctrl-C must not sit out a + // 15s backoff before it is noticed. + const sleep = (ms: number): Promise => sleepUnlessInterrupted(rawSleep, ms, shutdownSignal); const startMs = Date.now(); const deadlineMs = startMs + timeoutSeconds * 1000; @@ -123,6 +155,9 @@ export async function pollRunUntilTerminal( let notYetVisibleRetries = 0; while (true) { + // Interrupt outranks the deadline: a Ctrl-C that raced the timeout must + // surface as the honest detach, not as a generic TimeoutError. + if (shutdownSignal?.aborted) throw shutdownSignal.reason; const now = Date.now(); if (now >= deadlineMs) { throw new TimeoutError(runId, timeoutSeconds); @@ -139,20 +174,33 @@ export async function pollRunUntilTerminal( const abortTimer = setTimeout(() => { abortController.abort(); }, remainingMs + TRANSPORT_CUSHION_MS); + // Compose the interrupt into the per-iteration signal: a `--wait` can sit + // inside one <=25s long-poll fetch (and the auto-raised per-request + // timeout means even longer for slow backends) — checking the flag + // between iterations is not enough, the in-flight fetch must abort. + const iterationSignal = + shutdownSignal != null + ? AbortSignal.any([abortController.signal, shutdownSignal]) + : abortController.signal; let run: RunResponse; try { if (useBackoff) { - run = await client.getRun(runId, { signal: abortController.signal }); + run = await client.getRun(runId, { signal: iterationSignal }); } else { const waitSeconds = Math.min(remainingSeconds, LONG_POLL_WAIT_SECONDS); - run = await client.getRun(runId, { waitSeconds, signal: abortController.signal }); + run = await client.getRun(runId, { waitSeconds, signal: iterationSignal }); } // Successful GET resets the consecutive-error counter. consecutiveErrors = 0; notYetVisibleRetries = 0; } catch (err) { clearTimeout(abortTimer); + // Interrupt classification precedes the timeout mapping: the composed + // signal makes the fetch reject on Ctrl-C, and that abort must surface + // as the InterruptError — not as a spurious TimeoutError. + if (err instanceof InterruptError) throw err; + if (shutdownSignal?.aborted) throw shutdownSignal.reason; // An AbortError from our per-iteration controller means the deadline // passed while the fetch was in flight — surface as TimeoutError. if (isAbortError(err)) { @@ -239,9 +287,16 @@ export async function pollRunUntilTerminal( } const altAbort = new AbortController(); const altTimer = setTimeout(() => altAbort.abort(), altRemainingMs); + // The alternate lookup aborts on interrupt too; its errors are swallowed + // by the fallback (best-effort), so the loop-top interrupt check above + // surfaces the InterruptError on the next iteration. + const altSignal = + shutdownSignal != null + ? AbortSignal.any([altAbort.signal, shutdownSignal]) + : altAbort.signal; let alternate: RunResponse | null = null; try { - alternate = await resolveAlternate(run, elapsedMs, altAbort.signal); + alternate = await resolveAlternate(run, elapsedMs, altSignal); } finally { clearTimeout(altTimer); } @@ -294,6 +349,36 @@ function defaultSleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } +/** + * Race a sleep against the shutdown signal: rejects with the signal's + * `InterruptError` reason the moment it fires, so no backoff/retry wait can + * delay the honest-detach UX. Wraps the injected `sleep` (tests keep their + * deterministic fakes); the underlying timer is left to fire harmlessly — + * the interrupt path exits the process long before it matters. + */ +function sleepUnlessInterrupted( + sleep: (ms: number) => Promise, + ms: number, + signal: AbortSignal | undefined, +): Promise { + if (signal == null) return sleep(ms); + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + sleep(ms).then( + () => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, + err => { + signal.removeEventListener('abort', onAbort); + reject(err instanceof Error ? err : new Error(String(err))); + }, + ); + }); +} + /** * Detects an AbortError thrown when an AbortSignal fires. * Works for native fetch AbortErrors as well as `AbortController.abort()` diff --git a/src/lib/runs.types.ts b/src/lib/runs.types.ts index 7e556fe..15501c9 100644 --- a/src/lib/runs.types.ts +++ b/src/lib/runs.types.ts @@ -225,6 +225,20 @@ export interface RunResponse { steps?: RunStepDto[] | null; } +// --------------------------------------------------------------------------- +// DEV-331 piece 3 — cancel wire types +// --------------------------------------------------------------------------- + +/** + * Response from `POST /api/cli/v1/runs/{runId}/cancel`. + * Same shape as `GET /runs/{runId}` (`status: "cancelled"`, verdict + * untouched) plus `alreadyCancelled` distinguishing a fresh cancel + * (naturally idempotent) from a no-op re-cancel. + */ +export interface CancelRunResponse extends RunResponse { + alreadyCancelled: boolean; +} + /** Terminal states from the RunStatus union. */ export const TERMINAL_RUN_STATUSES: ReadonlySet = new Set([ 'passed', diff --git a/src/lib/v3-advisory.test.ts b/src/lib/v3-advisory.test.ts new file mode 100644 index 0000000..124720f --- /dev/null +++ b/src/lib/v3-advisory.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; +import { routingLabel, V3_ROUTING_ADVISORY, emitV3RoutingAdvisory } from './v3-advisory.js'; + +describe('routingLabel', () => { + it('maps the boolean to v3 / v2', () => { + expect(routingLabel(true)).toBe('v3'); + expect(routingLabel(false)).toBe('v2'); + }); +}); + +describe('V3 routing advisory', () => { + it('names each open behavior gap (cancel, delete, target-url)', () => { + const text = V3_ROUTING_ADVISORY.join('\n'); + expect(text).toContain('test cancel'); + expect(text).toContain('test delete'); + expect(text).toContain('--target-url'); + }); + + it('emitV3RoutingAdvisory writes every line to the sink', () => { + const lines: string[] = []; + emitV3RoutingAdvisory(l => lines.push(l)); + expect(lines).toEqual(V3_ROUTING_ADVISORY); + }); +}); diff --git a/src/lib/v3-advisory.ts b/src/lib/v3-advisory.ts new file mode 100644 index 0000000..a8b75ed --- /dev/null +++ b/src/lib/v3-advisory.ts @@ -0,0 +1,25 @@ +/** + * Shared V3-routing text surfaces for `auth status` and `doctor`. + * + * `v3Enabled` on the `/me` response is the authoritative routing bit. When it + * is on, some commands behave differently while the V3 gaps stay open — the + * advisory names them. Copy lives here so both commands stay in sync. + */ + +/** One-word routing label for the text card. */ +export function routingLabel(v3Enabled: boolean): 'v3' | 'v2' { + return v3Enabled ? 'v3' : 'v2'; +} + +/** Consolidated advisory (stderr) emitted when V3 routing is on. */ +export const V3_ROUTING_ADVISORY: string[] = [ + '[advisory] V3 routing is on for this account. While these gaps are open:', + ' - `test cancel` may return 404', + ' - `test delete` may leave a zombie run', + ' - `--target-url` is ignored on frontend runs', +]; + +/** Write the advisory to a stderr sink, one line per call. */ +export function emitV3RoutingAdvisory(stderr: (line: string) => void): void { + for (const line of V3_ROUTING_ADVISORY) stderr(line); +} diff --git a/src/lib/version-notice.test.ts b/src/lib/version-notice.test.ts new file mode 100644 index 0000000..1ce473e --- /dev/null +++ b/src/lib/version-notice.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + formatBelowFloorNotice, + noteServerVersion, + resetBelowFloorNoticeState, + shouldWarnBelowFloor, + type VersionNoticeDeps, +} from './version-notice.js'; + +/** Baseline deps: all gates open, running version below the floor. */ +function baseDeps(overrides: Partial = {}): VersionNoticeDeps { + return { + currentVersion: '0.9.0', + env: {}, + isTTY: true, + outputMode: 'text', + dryRun: false, + ...overrides, + }; +} + +afterEach(() => { + resetBelowFloorNoticeState(); + vi.restoreAllMocks(); +}); + +describe('shouldWarnBelowFloor', () => { + it('warns when the running version is strictly below the floor', () => { + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps())).toBe(true); + }); + + it('does not warn when at or above the floor', () => { + expect(shouldWarnBelowFloor({ minVersion: '0.9.0' }, baseDeps())).toBe(false); // equal + expect(shouldWarnBelowFloor({ minVersion: '0.8.0' }, baseDeps())).toBe(false); // above + }); + + it('does not warn when no minVersion header was present', () => { + expect(shouldWarnBelowFloor({}, baseDeps())).toBe(false); + expect(shouldWarnBelowFloor({ minVersion: undefined }, baseDeps())).toBe(false); + }); + + it('does not warn on unparseable versions (garbage never warns)', () => { + expect(shouldWarnBelowFloor({ minVersion: 'not-a-version' }, baseDeps())).toBe(false); + expect( + shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ currentVersion: 'nope' })), + ).toBe(false); + }); + + it('is gated off by the opt-out env (any non-empty value)', () => { + expect( + shouldWarnBelowFloor( + { minVersion: '1.0.0' }, + baseDeps({ env: { TESTSPRITE_NO_UPDATE_NOTIFIER: '1' } }), + ), + ).toBe(false); + expect( + shouldWarnBelowFloor( + { minVersion: '1.0.0' }, + baseDeps({ env: { TESTSPRITE_NO_UPDATE_NOTIFIER: '0' } }), + ), + ).toBe(false); + }); + + it('is gated off under --output json, --dry-run, and non-TTY', () => { + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ outputMode: 'json' }))).toBe( + false, + ); + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ dryRun: true }))).toBe(false); + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ isTTY: false }))).toBe(false); + }); +}); + +describe('formatBelowFloorNotice', () => { + it('names the current version, the floor, and the npm upgrade command', () => { + const line = formatBelowFloorNotice('0.9.0', '1.0.0'); + expect(line).toContain('0.9.0'); + expect(line).toContain('minimum supported version 1.0.0'); + expect(line).toContain('npm install -g @testsprite/testsprite-cli'); + expect(line).toContain('TESTSPRITE_NO_UPDATE_NOTIFIER=1'); + }); + + it('does not name a target release (npm update-notice owns "latest")', () => { + const line = formatBelowFloorNotice('0.9.0', '1.0.0'); + expect(line).not.toContain('Upgrade to 1.'); + }); +}); + +describe('noteServerVersion', () => { + it('emits exactly one advisory line when below the floor', () => { + const stderr = vi.fn(); + noteServerVersion({ minVersion: '1.0.0' }, baseDeps({ stderr })); + expect(stderr).toHaveBeenCalledTimes(1); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('below the minimum supported')); + }); + + it('warns at most once per process', () => { + const stderr = vi.fn(); + const deps = baseDeps({ stderr }); + noteServerVersion({ minVersion: '1.0.0' }, deps); + noteServerVersion({ minVersion: '1.0.0' }, deps); + noteServerVersion({ minVersion: '1.0.0' }, deps); + expect(stderr).toHaveBeenCalledTimes(1); + }); + + it('stays silent when not below the floor', () => { + const stderr = vi.fn(); + noteServerVersion({ minVersion: '0.5.0' }, baseDeps({ stderr })); + expect(stderr).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/version-notice.ts b/src/lib/version-notice.ts new file mode 100644 index 0000000..0e492dc --- /dev/null +++ b/src/lib/version-notice.ts @@ -0,0 +1,111 @@ +/** + * Inline "your CLI is below the backend's minimum supported version" advisory. + * + * The backend advertises its supported floor on every `/api/cli/v1` response via + * the `X-TestSprite-CLI-Min-Version` header. Unlike the npm-registry update + * notice (`update-check.ts`, which runs in a Commander `preAction` hook before + * any HTTP request), this advisory reacts to a header observed *during* the + * request, so it is emitted from the HTTP layer's `onServerVersion` hook (wired + * in `client-factory.ts`) rather than the pre-request notifier — no cross-run + * cache needed. + * + * Distinct from the update notice on purpose, and non-redundant: this names the + * backend FLOOR and fires only when the running version is strictly below it (a + * serious "you will be rejected once enforcement is on" state); the update + * notice names the npm LATEST and fires whenever a newer release exists. Each + * owns its own number — the backend never advertises "latest". + * + * Reuses `compareSemver` and the `TESTSPRITE_NO_UPDATE_NOTIFIER` opt-out from + * `update-check.ts` so the gating and semver semantics stay consistent. + */ +import { compareSemver, UPDATE_CHECK_OPT_OUT_ENV } from './update-check.js'; +import { VERSION } from '../version.js'; + +/** Version-compatibility signal observed on a backend response (the floor). */ +export interface ServerVersionInfo { + minVersion?: string; +} + +export interface VersionNoticeDeps { + /** Version the running binary reports. Defaults to the built-in `VERSION`. */ + currentVersion?: string; + env?: NodeJS.ProcessEnv; + /** Whether stderr is an interactive terminal. */ + isTTY?: boolean; + /** The command's `--output` mode; `'json'` suppresses the advisory. */ + outputMode?: string; + /** Suppress under `--dry-run` (the dry-run fetch returns no real headers). */ + dryRun?: boolean; + /** Sink for the single advisory line. */ + stderr?: (line: string) => void; +} + +/** + * True when every gate passes and the running version is strictly below the + * advertised floor. Gates mirror the update notice: opt-out env, JSON output, + * dry-run, and non-TTY all suppress. Pure — no side effects, no process state. + */ +export function shouldWarnBelowFloor( + info: ServerVersionInfo, + deps: VersionNoticeDeps = {}, +): boolean { + const env = deps.env ?? process.env; + const currentVersion = deps.currentVersion ?? VERSION; + const isTTY = deps.isTTY ?? process.stderr.isTTY === true; + + const optOut = env[UPDATE_CHECK_OPT_OUT_ENV]; + if (optOut !== undefined && optOut !== '') return false; + if (deps.outputMode === 'json') return false; + if (deps.dryRun === true) return false; + if (!isTTY) return false; + + const minVersion = info.minVersion; + if (!minVersion) return false; + + // compareSemver returns -1 when the first arg is OLDER than the second. + // Unparseable input on either side compares as 0, so garbage never warns. + return compareSemver(currentVersion, minVersion) === -1; +} + +/** + * The single advisory line. Names the floor (the backend's authority) and the + * upgrade command — it deliberately does NOT name a target release; the npm + * update-notice is the single source of truth for the newest version. + */ +export function formatBelowFloorNotice(currentVersion: string, minVersion: string): string { + return ( + `Your testsprite-cli (${currentVersion}) is below the minimum supported version ${minVersion}. ` + + `Upgrade: npm install -g @testsprite/testsprite-cli ` + + `(disable this notice with ${UPDATE_CHECK_OPT_OUT_ENV}=1).` + ); +} + +/** Module-level guard: at most one below-floor advisory per process. */ +let warnedThisProcess = false; + +/** Test-only: reset the once-per-process guard between cases. */ +export function resetBelowFloorNoticeState(): void { + warnedThisProcess = false; +} + +/** + * Emit the below-floor advisory at most once per process. Wired to the HTTP + * client's `onServerVersion` hook. Never throws — an advisory must not break or + * delay the command it rides along with. + */ +export function noteServerVersion(info: ServerVersionInfo, deps: VersionNoticeDeps = {}): void { + try { + if (warnedThisProcess) return; + if (!shouldWarnBelowFloor(info, deps)) return; + + const minVersion = info.minVersion; + if (!minVersion) return; + + const currentVersion = deps.currentVersion ?? VERSION; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(formatBelowFloorNotice(currentVersion, minVersion)); + warnedThisProcess = true; + } catch { + // Advisory is best-effort; never surface its failures to the command. + } +} diff --git a/src/version.ts b/src/version.ts index f60fdfd..5b301a6 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.3.0'; +export const VERSION = '0.4.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 62efe5d..b0681c5 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -146,6 +146,14 @@ Commands: get Get a project by id create [options] Create a new project update [options] Update project metadata + delete [options] Permanently delete a project and everything under it (sub-projects, + their tests, and backend fixtures). Requires --confirm. + + Exit codes: + 0 success + 3 auth error + 4 project not found (or already deleted) + 5 validation error (e.g., missing --confirm) credential [options] Set the static backend credential injected into every backend test (Bearer token / API key / Basic token / @@ -260,11 +268,14 @@ Commands: 4 test not found 5 validation error (e.g., bad --target-url, or positional + --all both set) 6 conflict (already running — see nextAction for the active runId) - 7 timeout — resume with: testsprite test wait + 7 timeout — resume with: testsprite test wait , or stop it with: testsprite test cancel 10 transport/network failure (UNAVAILABLE) — retry the command 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get + + Ctrl-C during --wait detaches only (the run keeps executing and billing); + stop it for real with: testsprite test cancel wait [options] Wait for one or more runs to reach a terminal status. With several run-ids the runs are polled concurrently under one shared @@ -282,6 +293,9 @@ Commands: 10 transport/network failure (UNAVAILABLE) — retry the command On failure/blocked/cancelled, run: testsprite test artifact get + + Ctrl-C detaches only (the run keeps executing and billing); stop it for + real with: testsprite test cancel rerun [options] [test-ids...] Re-execute a test (or multiple) as a cheap replay — FE replays the saved script (no credit), BE re-runs the dependency closure. Exit codes: @@ -291,10 +305,13 @@ Commands: 4 test not found 5 validation error 6 conflict (already running — see nextAction for the active runId) - 7 timeout or deferred — resume with: testsprite test wait + 7 timeout or deferred — resume with: testsprite test wait , or stop it with: testsprite test cancel 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get + + Ctrl-C during --wait detaches only (the run keeps executing and billing); + stop it for real with: testsprite test cancel flaky [options] Repeatedly replay a test to measure stability and surface flakiness. Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. @@ -309,10 +326,51 @@ Commands: failure Export the latest-failure agent bundle artifact Download run-scoped artifact bundles (M3.3 piece-4) + cancel Cancel one or more queued/running runs. + + Ctrl-C during --wait only detaches — it does NOT cancel the server-side + run. This is the real stop button. No refund is issued for the credits + already charged at trigger time (D3); an in-flight Lambda finishes on its + own and its result is discarded once cancelled. + + Exit codes: + 0 cancelled (fresh or already-cancelled — naturally idempotent) + 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict) + 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict + + Multi-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}. help [command] display help for command " `; +exports[`--help snapshots > test cancel 1`] = ` +"Usage: testsprite test cancel [options] + +Cancel one or more queued/running runs. + +Ctrl-C during --wait only detaches — it does NOT cancel the server-side +run. This is the real stop button. No refund is issued for the credits +already charged at trigger time (D3); an in-flight Lambda finishes on its +own and its result is discarded once cancelled. + +Exit codes: + 0 cancelled (fresh or already-cancelled — naturally idempotent) + 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict) + 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict + +Multi-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}. + +Arguments: + run-id one or more run ids to cancel + +Options: + -h, --help display help for command + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test code get 1`] = ` "Usage: testsprite test code get [options] @@ -463,11 +521,14 @@ Exit codes: 4 test not found 5 validation error 6 conflict (already running — see nextAction for the active runId) - 7 timeout or deferred — resume with: testsprite test wait + 7 timeout or deferred — resume with: testsprite test wait , or stop it with: testsprite test cancel 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get +Ctrl-C during --wait detaches only (the run keeps executing and billing); +stop it for real with: testsprite test cancel + Options: --all rerun all tests in the resolved project (requires --project) (default: false) @@ -568,12 +629,15 @@ Exit codes: 4 test not found 5 validation error (e.g., bad --target-url, or positional + --all both set) 6 conflict (already running — see nextAction for the active runId) - 7 timeout — resume with: testsprite test wait + 7 timeout — resume with: testsprite test wait , or stop it with: testsprite test cancel 10 transport/network failure (UNAVAILABLE) — retry the command 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get +Ctrl-C during --wait detaches only (the run keeps executing and billing); +stop it for real with: testsprite test cancel + Options: --target-url override the project default env URL for this run (http/https only, no localhost/private IPs) diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 959f212..3b04712 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -7,7 +7,7 @@ * and runs `auth whoami` against the mock." */ -import { execFileSync, spawn } from 'node:child_process'; +import { spawn } from 'node:child_process'; import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { createServer } from 'node:http'; @@ -15,6 +15,7 @@ import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execNpm } from './helpers/execNpm.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, '..'); @@ -37,7 +38,7 @@ beforeAll(async () => { // existsSync skip we used to do here let `dist` rot under // refactors and gave false-green on `project list` once // already. - execFileSync('npm', ['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); server = createServer((req: IncomingMessage, res: ServerResponse) => { const url = req.url ?? '/'; if (url.startsWith('/api/cli/v1/projects/')) { diff --git a/test/e2e/signal.e2e.test.ts b/test/e2e/signal.e2e.test.ts new file mode 100644 index 0000000..6e03743 --- /dev/null +++ b/test/e2e/signal.e2e.test.ts @@ -0,0 +1,188 @@ +/** + * Local e2e tests for SIGINT/SIGTERM graceful detach during `--wait` + * (exit 130/143/129 per the documented signal contract). + * + * Spawns the real built binary (`dist/index.js`) against a local HTTP stub + * whose `GET /runs/{id}` long-poll hangs forever, sends a real signal to the + * child, and asserts the honest-detach contract: + * + * - stdout: parseable partial `{runId, status:"running"}` (JSON mode) + * - stderr: "keeps running (and billing)" + re-attach hint (+ INTERRUPTED + * envelope in JSON mode) + * - exit code 130 (SIGINT) / 143 (SIGTERM) + * + * Run via: `npm run test:e2e` (builds first). Excluded from `npm test`. + */ + +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { createServer, type Server } from 'node:http'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, '../..'); +const BIN_PATH = join(REPO_ROOT, 'dist', 'index.js'); + +const RUN_ID = 'run_sig_e2e_01'; + +let server: Server; +let baseUrl = ''; +/** Resolvers waiting for the next hanging /runs request to arrive. */ +const runRequestWaiters: Array<() => void> = []; + +beforeAll(async () => { + if (!existsSync(BIN_PATH)) { + throw new Error('dist/index.js not found — run `npm run test:e2e` which builds first.'); + } + server = createServer((req, res) => { + // Hang every request (long-poll / stalled-backend simulation): the CLI's + // abort must cut it. Signal any test waiting for the request to arrive. + runRequestWaiters.splice(0).forEach(fn => fn()); + req.on('close', () => res.destroy()); + }); + await new Promise(resolveListen => { + server.listen(0, '127.0.0.1', () => resolveListen()); + }); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('no server address'); + baseUrl = `http://127.0.0.1:${address.port}`; +}); + +afterAll(async () => { + await new Promise(resolveClose => { + server.close(() => resolveClose()); + server.closeAllConnections(); + }); +}); + +/** Resolves when the stub receives the next hanging GET /runs request. */ +function nextRunRequest(): Promise { + return new Promise(resolveWait => runRequestWaiters.push(resolveWait)); +} + +interface SpawnResult { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +} + +/** + * Spawn `testsprite test wait` against the hanging stub, deliver `signal` + * once the long-poll request is in flight, and collect the outcome. + */ +async function waitAndInterrupt( + signal: NodeJS.Signals, + extraArgs: string[] = [], +): Promise { + const child = spawn( + process.execPath, + [BIN_PATH, 'test', 'wait', RUN_ID, '--timeout', '120', ...extraArgs], + { + env: { + ...process.env, + TESTSPRITE_API_KEY: 'sk-e2e-signal', + TESTSPRITE_API_URL: baseUrl, + TESTSPRITE_NO_SKILL_WARNING: '1', + TESTSPRITE_NO_UPDATE_NOTIFIER: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString())); + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString())); + + const arrived = nextRunRequest(); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + resolveExit => { + child.on('exit', (code, exitSignal) => resolveExit({ code, signal: exitSignal })); + }, + ); + + await arrived; // the long-poll fetch is in flight — the armed window is open + await new Promise(r => setTimeout(r, 150)); // let the request settle into the poll loop + child.kill(signal); + + const { code, signal: exitSignal } = await exited; + return { code, signal: exitSignal, stdout, stderr }; +} + +describe('signal e2e — graceful detach during test wait (DEV-331)', () => { + it('SIG-1/SIG-2: SIGINT → exit 130, partial JSON on stdout, honest stderr hint', async () => { + const result = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(result.code).toBe(130); + + // stdout is a parseable partial naming the runId (file redirects never 0-byte). + const partial = JSON.parse(result.stdout) as { runId: string; status: string }; + expect(partial.runId).toBe(RUN_ID); + expect(partial.status).toBe('running'); + + // stderr: honest detach line + machine-readable INTERRUPTED envelope. + expect(result.stderr).toContain('Interrupted (SIGINT)'); + expect(result.stderr).toContain('billing'); + expect(result.stderr).toContain(`testsprite test wait ${RUN_ID}`); + expect(result.stderr).toContain('"code": "INTERRUPTED"'); + expect(result.stderr).toContain('"signal": "SIGINT"'); + }, 30_000); + + it('SIG-1 (text mode): SIGINT → exit 130, human-readable partial + hint', async () => { + const result = await waitAndInterrupt('SIGINT'); + expect(result.code).toBe(130); + expect(result.stdout).toContain(RUN_ID); + expect(result.stdout).toContain('running (interrupted)'); + expect(result.stderr).toContain('Interrupted (SIGINT)'); + expect(result.stderr).toContain('Error: Interrupted by SIGINT.'); + }, 30_000); + + it('SIG-3: SIGTERM → exit 143', async () => { + const result = await waitAndInterrupt('SIGTERM', ['--output', 'json']); + expect(result.code).toBe(143); + expect(result.stderr).toContain('Interrupted (SIGTERM)'); + expect(result.stderr).toContain('"signal": "SIGTERM"'); + }, 30_000); + + it('SIG-7: SIGINT during a non-wait command → immediate exit 130 with the generic explanation', async () => { + // `test list` is outside any armed --wait scope. The stub hangs its fetch; + // the disarmed handler must exit immediately with the generic explanation + // (no partial envelope — there is no runId to re-attach to). + const child = spawn(process.execPath, [BIN_PATH, 'test', 'list', '--project', 'p1'], { + env: { + ...process.env, + TESTSPRITE_API_KEY: 'sk-e2e-signal', + TESTSPRITE_API_URL: baseUrl, + TESTSPRITE_NO_SKILL_WARNING: '1', + TESTSPRITE_NO_UPDATE_NOTIFIER: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString())); + const exited = new Promise<{ code: number | null }>(resolveExit => { + child.on('exit', code => resolveExit({ code })); + }); + const arrived = nextRunRequest(); + await arrived; // the list fetch is in flight (disarmed — no poll running) + child.kill('SIGINT'); + const { code } = await exited; + expect(code).toBe(130); + expect(stderr).toContain('Interrupted (SIGINT)'); + expect(stderr).toContain('test wait'); + expect(stderr).not.toContain(' at '); // no stack trace / corrupted output + }, 30_000); + + it('SIG-8: detach then re-attach — the same runId can be waited on again (server unaffected)', async () => { + // First wait: interrupted. + const first = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(first.code).toBe(130); + // Re-attach: the stub receives a fresh long-poll for the SAME runId — + // proof the detach was client-side only. (We interrupt again to end it.) + const second = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(second.code).toBe(130); + const partial = JSON.parse(second.stdout) as { runId: string }; + expect(partial.runId).toBe(RUN_ID); + }, 60_000); +}); diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index ee89b3b..b6c23b4 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -13,6 +13,7 @@ import { execFileSync } from 'node:child_process'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { beforeAll, describe, expect, it } from 'vitest'; +import { execNpm } from './helpers/execNpm.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, '..'); @@ -43,11 +44,13 @@ const cases: Array<[string, string[]]> = [ // R5: regression guard for commands that gained new flag wording ['test create-batch', ['test', 'create-batch', '--help']], ['test run', ['test', 'run', '--help']], + // DEV-331 piece 3 + ['test cancel', ['test', 'cancel', '--help']], ]; describe('--help snapshots', () => { beforeAll(() => { - execFileSync('npm', ['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); }); for (const [name, args] of cases) { diff --git a/test/helpers/execNpm.ts b/test/helpers/execNpm.ts new file mode 100644 index 0000000..0535112 --- /dev/null +++ b/test/helpers/execNpm.ts @@ -0,0 +1,17 @@ +import { execFileSync } from 'node:child_process'; + +/** + * Cross-platform `npm` invocation for test `beforeAll` build steps. + * On Windows, `npm` is a `.cmd` shim rather than a directly executable + * binary — `execFileSync('npm', ...)` fails with `ENOENT` there unless + * `shell: true` lets the OS resolve the shim through PATHEXT. + */ +export function execNpm( + args: string[], + options: { cwd: string; stdio?: 'pipe' | 'inherit' | 'ignore' }, +): Buffer | string { + return execFileSync('npm', args, { + ...options, + shell: process.platform === 'win32', + }); +} From d2f0357ec1f9f08419768a40bd89beead8bc7cb3 Mon Sep 17 00:00:00 2001 From: zeshi-du Date: Fri, 17 Jul 2026 16:44:17 -0700 Subject: [PATCH 064/102] =?UTF-8?q?fix(ci):=20migrate=20github-script=20to?= =?UTF-8?q?=20v8=20=E2=80=94=20repair=20the=20repo-wide=20broken=20gate=20?= =?UTF-8?q?check=20(#260)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub's runner infra force-upgrades actions/github-script@v7 to Node 24, where the script-side require('@actions/github') can no longer resolve. Every recent PR shows a red 'gate' check with 'Cannot find module @actions/github' regardless of whether it links an issue. - Pin actions/github-script to ed597411 (v8.0.0) in pr-triage, ci-nudge, issue-triage. - Replace require('@actions/github').getOctokit(APP_TOKEN) with new (github.constructor)({ auth: APP_TOKEN }) — v8.0.0 injects no getOctokit helper (verified against src/async-function.ts at the pinned SHA), and the wrapped require cannot resolve modules without a checkout. - Bump actions/stale to 1e223db (v10.4.0). Diagnosis first surfaced by @sandman-sh in #257 — thank you! This lands separately with a SHA pin and a require-free client construction. Co-authored-by: Zeshi Du Co-authored-by: Claude Fable 5 --- .github/workflows/ci-nudge.yml | 4 ++-- .github/workflows/issue-triage.yml | 2 +- .github/workflows/pr-triage.yml | 4 ++-- .github/workflows/stale.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml index f14461f..bd6f9ad 100644 --- a/.github/workflows/ci-nudge.yml +++ b/.github/workflows/ci-nudge.yml @@ -55,7 +55,7 @@ jobs: # back to github-actions[bot], defeating this bot's own purpose. permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: APP_TOKEN: ${{ steps.app-token.outputs.token }} with: @@ -65,7 +65,7 @@ jobs: const marker = process.env.MARKER; const appClient = process.env.APP_TOKEN - ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + ? new (github.constructor)({ auth: process.env.APP_TOKEN }) : null; async function write(fn) { if (appClient) { diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 0da78c8..53bd570 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -57,7 +57,7 @@ jobs: # of inheriting the App's full installation permissions. permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: # App token when available (→ testsprite-hob[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml index cfd114d..d505f0b 100644 --- a/.github/workflows/pr-triage.yml +++ b/.github/workflows/pr-triage.yml @@ -71,7 +71,7 @@ jobs: # fallback path (unused by this specific App-token mint). permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: APP_TOKEN: ${{ steps.app-token.outputs.token }} with: @@ -86,7 +86,7 @@ jobs: const author = pr.user.login; const appClient = process.env.APP_TOKEN - ? require('@actions/github').getOctokit(process.env.APP_TOKEN) + ? new (github.constructor)({ auth: process.env.APP_TOKEN }) : null; // Prefer the App identity; fall back to github-actions[bot] when the // App is absent or lacks the Pull-requests permission (403/404). diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fbaa607..b670166 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -26,7 +26,7 @@ jobs: issues: write # comment + (un)label + close stale issues pull-requests: write # comment + (un)label + close stale PRs steps: - - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 + - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 with: # ── issues ────────────────────────────────────────────────────── days-before-issue-stale: 60 From 3abcf2d501533caa2e803b922825ab0d5239caa4 Mon Sep 17 00:00:00 2001 From: Joseph Jang Date: Fri, 17 Jul 2026 16:52:41 -0700 Subject: [PATCH 065/102] fix(doctor): validate output mode (#251) Co-authored-by: Yazan-O --- src/commands/doctor.test.ts | 43 ++++++++++++++++++++++++++++++++++++- src/commands/doctor.ts | 4 ++-- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index 7a508dd..7974947 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -9,8 +9,9 @@ import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { Command } from 'commander'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { CLIError } from '../lib/errors.js'; +import { ApiError, CLIError } from '../lib/errors.js'; import { writeProfile } from '../lib/credentials.js'; import type { DoctorDeps, DoctorReport } from './doctor.js'; import { createDoctorCommand, runDoctor } from './doctor.js'; @@ -56,6 +57,14 @@ function healthyDeps(credentialsPath: string, extra: Partial = {}): }; } +function makeDoctorProgram(deps: DoctorDeps = {}): Command { + const program = new Command(); + program.exitOverride(); + program.option('--output ', 'output', 'text'); + program.addCommand(createDoctorCommand(deps)); + return program; +} + let credentialsPath: string; beforeEach(() => { @@ -271,4 +280,36 @@ describe('createDoctorCommand wiring', () => { it('--help describes the diagnostic', () => { expect(createDoctorCommand().helpInformation()).toContain('Diagnose'); }); + + it('rejects invalid --output with the shared VALIDATION_ERROR', async () => { + const rejection = await makeDoctorProgram() + .parseAsync(['node', 'ts', '--output', 'yaml', 'doctor']) + .catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(ApiError); + expect(rejection).toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + nextAction: 'Flag `--output` is invalid: must be one of: json, text.', + }); + }); + + it('accepts valid --output modes through command wiring', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + for (const mode of ['text', 'json'] as const) { + const { capture, deps } = makeCapture(); + await makeDoctorProgram({ ...healthyDeps(credentialsPath), ...deps }).parseAsync([ + 'node', + 'ts', + '--output', + mode, + 'doctor', + ]); + const raw = capture.stdout.join(''); + if (mode === 'json') { + expect((JSON.parse(raw) as DoctorReport).failures).toBe(0); + } else { + expect(raw).toContain('All checks passed.'); + } + } + }); }); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index cceeb13..40c1ac8 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -24,7 +24,7 @@ import { import { loadConfig } from '../lib/config.js'; import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; import { VERSION } from '../version.js'; @@ -287,7 +287,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, From 3c3a2e05b976abc7f4e5f823a55a78c9cf16ad25 Mon Sep 17 00:00:00 2001 From: Joseph Jang Date: Fri, 17 Jul 2026 16:53:25 -0700 Subject: [PATCH 066/102] Recovered: fix(bundle): atomic re-commit via per-entry aside (#196 by @SahilRakhaiya05) (#246) * fix(bundle): atomic re-commit via per-entry aside with rollback * fix(bundle): move meta.json aside before other bundle entries during commit --------- Co-authored-by: SahilRakhaiya05 --- src/lib/bundle.commit.test.ts | 124 ++++++++++++++++++++++++++ src/lib/bundle.ts | 160 +++++++++++++++++++--------------- 2 files changed, 212 insertions(+), 72 deletions(-) create mode 100644 src/lib/bundle.commit.test.ts diff --git a/src/lib/bundle.commit.test.ts b/src/lib/bundle.commit.test.ts new file mode 100644 index 0000000..83cbce8 --- /dev/null +++ b/src/lib/bundle.commit.test.ts @@ -0,0 +1,124 @@ +import type * as NodeFsPromises from 'node:fs/promises'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const renameMock = vi.hoisted(() => vi.fn()); +const rmMock = vi.hoisted(() => vi.fn()); + +vi.mock('node:fs/promises', async importOriginal => { + const actual = (await importOriginal()) as typeof NodeFsPromises; + return { + ...actual, + rename: renameMock, + rm: rmMock, + }; +}); + +const { commitBundle } = await import('./bundle.js'); + +describe('commitBundle', () => { + let realRename: typeof NodeFsPromises.rename; + let realRm: typeof NodeFsPromises.rm; + + beforeEach(async () => { + const actual = (await vi.importActual('node:fs/promises')) as typeof NodeFsPromises; + realRename = actual.rename; + realRm = actual.rm; + renameMock.mockImplementation(realRename); + rmMock.mockImplementation(realRm); + }); + + afterEach(() => { + renameMock.mockReset(); + rmMock.mockReset(); + }); + + async function withTempParent(run: (parent: string) => Promise): Promise { + const parent = mkdtempSync(join(tmpdir(), 'bundle-commit-parent-')); + try { + await run(parent); + } finally { + await realRm(parent, { recursive: true, force: true }).catch(() => undefined); + } + } + + function seedBundleDirs(parent: string): { dir: string; tmpDir: string; files: string[] } { + const dir = join(parent, 'bundle'); + const tmpDir = join(dir, '.tmp'); + + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'notes.txt'), 'foreign notes\n', 'utf8'); + mkdirSync(join(dir, 'steps'), { recursive: true }); + writeFileSync(join(dir, 'meta.json'), '{"snapshotId":"snap_old"}\n', 'utf8'); + writeFileSync(join(dir, 'steps', '01-evidence.json'), '{"step":1}\n', 'utf8'); + + mkdirSync(join(tmpDir, 'steps'), { recursive: true }); + writeFileSync(join(tmpDir, 'meta.json'), '{"snapshotId":"snap_new"}\n', 'utf8'); + writeFileSync(join(tmpDir, 'result.json'), '{}\n', 'utf8'); + writeFileSync(join(tmpDir, 'steps', '01-evidence.json'), '{"step":9}\n', 'utf8'); + + return { dir, tmpDir, files: ['result.json', 'meta.json', 'steps/01-evidence.json'] }; + } + + it('rolls back to the prior complete bundle when a staged rename fails', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + renameMock.mockImplementation(async (oldPath, newPath) => { + const dest = String(newPath); + if (dest.endsWith('result.json') && !dest.includes('.aside.')) { + throw Object.assign(new Error('simulated install failure'), { code: 'EACCES' }); + } + return realRename(oldPath, newPath); + }); + + await expect(commitBundle(tmpDir, dir, files)).rejects.toThrow('simulated install failure'); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_old"}\n'); + expect(readFileSync(join(dir, 'steps', '01-evidence.json'), 'utf8')).toBe('{"step":1}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + const leftovers = readdirSync(parent).filter(name => name.includes('.aside.')); + expect(leftovers).toEqual([]); + }); + }); + + it('preserves foreign files while installing the new bundle on success', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + await expect(commitBundle(tmpDir, dir, files)).resolves.toBeUndefined(); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_new"}\n'); + expect(readFileSync(join(dir, 'steps', '01-evidence.json'), 'utf8')).toBe('{"step":9}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + expect(existsSync(join(dir, 'result.json'))).toBe(true); + }); + }); + + it('keeps the new bundle when post-commit aside cleanup fails', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + rmMock.mockImplementation(async (path, options) => { + if (String(path).includes('.aside.')) { + throw Object.assign(new Error('simulated aside cleanup failure'), { code: 'EACCES' }); + } + return realRm(path, options); + }); + + await expect(commitBundle(tmpDir, dir, files)).resolves.toBeUndefined(); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_new"}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + }); + }); +}); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index f44549e..0685fd7 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -33,12 +33,13 @@ * when the agent re-runs. M3 may add resume. */ +import { randomUUID } from 'node:crypto'; import { mkdir, mkdtemp, readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'; import type { Writable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import type { ReadableStream as NodeReadableStream } from 'node:stream/web'; import { createWriteStream } from 'node:fs'; -import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'; import type { CliFailureContext, CliTestStep } from '../commands/test.js'; import { ApiError, TransportError, localValidationError } from './errors.js'; import { requireEnum } from './validate.js'; @@ -536,30 +537,6 @@ async function freshTmpDir(dir: string): Promise { return tmpDir; } -/** - * Rename `/` → `/` for every file in `files`. - * - * Critical ordering for atomicity (the §3 "agent-safe" contract): - * - * 1. **Remove the OLD `meta.json` first.** The bundle's completion - * signal is `meta.json`'s presence; an agent reading `` while - * we're mutating it must see "no meta → bundle absent or - * mid-write" rather than "meta points at a snapshot that's already - * been partially overwritten." Removing the old meta is what - * makes the rest of the swap safe to do in place. - * 2. Wipe stale top-level files (e.g. an old `video.mp4` when the new - * bundle has no video). Without this, a fresh bundle could ship - * with a stale video lingering at the top level. - * 3. Replace `/steps/` wholesale. - * 4. Rename top-level files into place. - * 5. **Rename `meta.json` LAST.** Its visible presence is the atomic - * completion signal; until step 5 lands, agents see "incomplete." - * - * The window between (1) and (5) is bounded by a handful of `rename` - * syscalls — small enough that a SIGKILL there is rare, and any agent - * caught reading the dir during it sees no meta and refuses to consume - * (per §7.3). That's what we want. - */ /** * Whether a top-level directory entry belongs to the bundle format — * i.e. something a prior `writeBundle` could have produced and this @@ -584,62 +561,101 @@ export function isBundleOwnedEntry(entry: string): boolean { return /^code\.[A-Za-z0-9]+$/.test(entry); } -async function commitBundle( +/** + * Atomically install the complete bundle staged in `tmpDir` into `dir`. + * + * Compatible with the #162 data-loss guard: only `isBundleOwnedEntry` + * names are moved aside or replaced — foreign files in `--out` survive. + * + * Re-commit safety: bundle-owned entries are renamed to a sibling + * aside directory before the new artifacts land. The prior `meta.json` + * is moved aside first (§3/§7.3 — no meta ⇒ refuse to consume) so a + * concurrent reader never sees meta pointing at steps already gone. + * The new `meta.json` is installed last. On failure, aside entries are + * restored so a failed re-commit never leaves the directory unusable. + */ +export async function commitBundle( tmpDir: string, dir: string, files: ReadonlyArray, ): Promise { - // (1) Remove the prior bundle's completion signal FIRST. - await unlink(join(dir, 'meta.json')).catch(() => undefined); - - // (2) Sweep stale top-level files that the new bundle won't write. - // If the prior run wrote `video.mp4` and the new run has no video, - // an in-place rename leaves the old video lingering. Only entries the - // bundle format OWNS are candidates: `--out` may point at a directory - // that also holds the user's unrelated files, and those must survive - // the commit (deleting them would be silent data loss). - const topLevel = files.filter(f => !f.startsWith('steps/')); - const newTopLevelSet = new Set(topLevel); - newTopLevelSet.add('meta.json'); // about to land last, do not delete - const existing = await readdir(dir).catch(() => [] as string[]); - for (const entry of existing) { - // Preserve the writer's own scratch dir + the .partial marker - // (we'll re-evaluate .partial at the end of commit). Any other - // bundle-owned entry not-listed in the new bundle is stale. - if (entry === '.tmp' || entry === '.partial') continue; - if (newTopLevelSet.has(entry)) continue; - if (entry === 'steps') continue; // handled below - if (!isBundleOwnedEntry(entry)) continue; // foreign file — never touch - await rm(join(dir, entry), { recursive: true, force: true }); - } + const parent = dirname(dir); + const base = basename(dir); + const asideDir = join(parent, `.${base}.aside.${randomUUID()}`); + const asideLog: Array<{ asidePath: string; restorePath: string }> = []; + + const asideIfPresent = async (entry: string): Promise => { + const restorePath = join(dir, entry); + if (!(await pathExists(restorePath))) return; + const asidePath = join(asideDir, entry); + await mkdir(dirname(asidePath), { recursive: true }); + await rename(restorePath, asidePath); + asideLog.push({ asidePath, restorePath }); + }; - // (3) Replace `/steps/` with `/steps/`. - const stepsTmp = join(tmpDir, 'steps'); - const stepsDir = join(dir, 'steps'); - await rm(stepsDir, { recursive: true, force: true }); - if (await dirExists(stepsTmp)) { - await rename(stepsTmp, stepsDir); - } + const rollback = async (): Promise => { + for (const { asidePath, restorePath } of [...asideLog].reverse()) { + await rm(restorePath, { recursive: true, force: true }).catch(() => undefined); + await rename(asidePath, restorePath).catch(() => undefined); + } + await rm(asideDir, { recursive: true, force: true }).catch(() => undefined); + }; - // (4) Top-level files (result/failure/code/video). meta.json renames - // LAST; track it separately. - const metaIdx = topLevel.indexOf('meta.json'); - const beforeMeta = metaIdx >= 0 ? topLevel.filter((_, i) => i !== metaIdx) : topLevel; - for (const file of beforeMeta) { - await rename(join(tmpDir, file), join(dir, file)); - } + try { + const topLevel = files.filter(f => !f.startsWith('steps/')); + const newTopLevelSet = new Set(topLevel); + newTopLevelSet.add('meta.json'); + + // meta.json first — its presence is the completion signal; do not + // move steps (or anything else) aside while a stale meta is still visible. + await asideIfPresent('meta.json'); + + const existing = await readdir(dir).catch(() => [] as string[]); + for (const entry of existing) { + if (entry === '.tmp' || entry === 'meta.json') continue; + if (!isBundleOwnedEntry(entry)) continue; + + const isStale = entry !== 'steps' && entry !== '.partial' && !newTopLevelSet.has(entry); + const willReplace = entry === 'steps' || newTopLevelSet.has(entry); + if (isStale || willReplace) { + await asideIfPresent(entry); + } + } - // (5) meta.json LAST → atomic completion signal. - if (metaIdx >= 0) { - await rename(join(tmpDir, 'meta.json'), join(dir, 'meta.json')); - } + const stepsTmp = join(tmpDir, 'steps'); + const stepsDir = join(dir, 'steps'); + if (await dirExists(stepsTmp)) { + await rename(stepsTmp, stepsDir); + } - // .partial from a prior aborted run is now stale. Remove it so an - // agent inspecting the dir sees only the fresh bundle. - await unlink(join(dir, '.partial')).catch(() => undefined); + const metaIdx = topLevel.indexOf('meta.json'); + const beforeMeta = metaIdx >= 0 ? topLevel.filter((_, i) => i !== metaIdx) : topLevel; + for (const file of beforeMeta) { + await rename(join(tmpDir, file), join(dir, file)); + } - // Clean up the now-empty tmp dir. - await rm(tmpDir, { recursive: true, force: true }); + if (metaIdx >= 0) { + await rename(join(tmpDir, 'meta.json'), join(dir, 'meta.json')); + } + + await unlink(join(dir, '.partial')).catch(() => undefined); + await rm(tmpDir, { recursive: true, force: true }); + // Best-effort aside cleanup — failures must not roll back a committed bundle. + await rm(asideDir, { recursive: true, force: true }).catch(() => undefined); + } catch (err) { + await rollback(); + await rm(tmpDir, { recursive: true, force: true }).catch(() => undefined); + throw err; + } +} + +async function pathExists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } } async function dirExists(path: string): Promise { From 36e5b14fd1e2983bdacadd895313e0ef5881b9cd Mon Sep 17 00:00:00 2001 From: Lex Date: Sat, 18 Jul 2026 06:53:36 +0700 Subject: [PATCH 067/102] fix(init): debug log whoami fallback (#223) Co-authored-by: Lexiie <28455136+Lexiie@users.noreply.github.com> --- src/commands/init.test.ts | 39 +++++++++++++++++++++++++++++++++++++++ src/commands/init.ts | 6 +++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 617fd6f..0c1bd8e 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -237,6 +237,45 @@ describe('runInit — happy path (interactive)', () => { expect(agent.skills).toContain('testsprite-verify'); expect(agent.skills).toContain('testsprite-onboard'); }); + + it('debug mode reports a display-only whoami lookup failure without corrupting JSON stdout', async () => { + const { captured, deps } = makeCapture(); + let callCount = 0; + const fetchMock = vi.fn(async () => { + callCount += 1; + if (callCount === 1) { + return new Response(JSON.stringify(ME), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'Invalid API key', + nextAction: 'Provide a valid key.', + requestId: 'r-whoami', + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ); + }) as unknown as InitDeps['fetchImpl']; + + await runInit( + makeBaseOpts({ apiKey: 'sk-json-test', debug: true, noAgent: true, output: 'json' }), + { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + }, + ); + + const parsed = JSON.parse(captured.stdout.join('\n')) as Record; + expect(parsed.status).toBe('initialized'); + expect(captured.stderr.some(line => line.includes('setup identity lookup failed'))).toBe(true); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/init.ts b/src/commands/init.ts index c2d5678..6357e3b 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -291,9 +291,13 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise Date: Sat, 18 Jul 2026 06:53:51 +0700 Subject: [PATCH 068/102] fix(auth): tighten Windows credentials ACL (#104) (#253) * fix(auth): tighten Windows credentials ACL * docs(auth): document credential permission tightening --- src/lib/credentials.test.ts | 41 +++++++++++++++++++++++- src/lib/credentials.ts | 64 ++++++++++++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index d50ad52..194c1ff 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -1,7 +1,7 @@ import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DEFAULT_PROFILE, assertValidProfileName, @@ -191,6 +191,45 @@ describe('ensureRestrictiveMode', () => { expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); }); + it('tightens the Windows ACL with icacls instead of POSIX chmod', () => { + mkdirSync(tmpRoot, { recursive: true }); + writeFileSync(credentialsPath, 'data', { mode: 0o666 }); + const spawn = vi.fn(() => ({ status: 0, signal: null, output: [], pid: 123 })) as never; + + ensureRestrictiveMode(credentialsPath, { + platform: 'win32', + env: { USERNAME: 'alice' } as NodeJS.ProcessEnv, + spawnSync: spawn, + }); + + expect(spawn).toHaveBeenCalledWith( + 'icacls', + [credentialsPath, '/inheritance:r', '/grant:r', 'alice:F'], + { + shell: false, + stdio: 'ignore', + windowsHide: true, + }, + ); + }); + + it('warns on Windows when credentials ACL tightening cannot run', () => { + mkdirSync(tmpRoot, { recursive: true }); + writeFileSync(credentialsPath, 'data'); + const warnings: string[] = []; + const spawn = vi.fn(() => ({ status: 0, signal: null, output: [], pid: 123 })) as never; + + ensureRestrictiveMode(credentialsPath, { + platform: 'win32', + env: {} as NodeJS.ProcessEnv, + spawnSync: spawn, + warn: line => warnings.push(line), + }); + + expect(spawn).not.toHaveBeenCalled(); + expect(warnings.join('\n')).toContain('credentials file permissions were not tightened'); + }); + // POSIX-only premise: Windows has no 0644/0600 distinction to downgrade. it.skipIf(process.platform === 'win32')('downgrades over-permissive modes', () => { mkdirSync(tmpRoot, { recursive: true }); diff --git a/src/lib/credentials.ts b/src/lib/credentials.ts index 21e2ac0..6c20b1f 100644 --- a/src/lib/credentials.ts +++ b/src/lib/credentials.ts @@ -7,6 +7,7 @@ import { statSync, writeFileSync, } from 'node:fs'; +import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { localValidationError } from './errors.js'; @@ -61,6 +62,17 @@ export interface CredentialsOptions { path?: string; } +interface RestrictiveModeOptions { + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; + spawnSync?: ( + command: string, + args: readonly string[], + options: { shell: false; stdio: 'ignore'; windowsHide: true }, + ) => SpawnSyncReturns; + warn?: (line: string) => void; +} + const FILE_KEY_TO_FIELD: Record = { api_key: 'apiKey', api_url: 'apiUrl', @@ -172,12 +184,62 @@ export function deleteProfile(profile: string, options: CredentialsOptions = {}) return true; } -export function ensureRestrictiveMode(path: string): void { +/** + * Enforce restrictive access on the credentials file after atomic writes. + * POSIX hosts use chmod(0600); Windows hosts use ACL tightening via icacls. + */ +export function ensureRestrictiveMode(path: string, options: RestrictiveModeOptions = {}): void { if (!existsSync(path)) return; + if ((options.platform ?? process.platform) === 'win32') { + ensureWindowsRestrictiveAcl(path, options); + return; + } const overpermissive = (statSync(path).mode & 0o077) !== 0; if (overpermissive) chmodSync(path, 0o600); } +/** + * Restrict a Windows credentials file to the current user using icacls. + * The command is invoked with an args array so credential paths are never shell-interpreted. + */ +function ensureWindowsRestrictiveAcl(path: string, options: RestrictiveModeOptions): void { + const username = (options.env ?? process.env).USERNAME?.trim(); + if (!username) { + warnWindowsAcl( + 'could not determine the Windows username; credentials file permissions were not tightened', + options, + ); + return; + } + + const run = options.spawnSync ?? spawnSync; + const result = run('icacls', [path, '/inheritance:r', '/grant:r', `${username}:F`], { + shell: false, + stdio: 'ignore', + windowsHide: true, + }); + + if (result.error) { + warnWindowsAcl( + `icacls failed while tightening credentials file permissions: ${result.error.message}`, + options, + ); + return; + } + if (result.status !== 0) { + warnWindowsAcl( + `icacls exited with status ${result.status ?? 'unknown'}; credentials file permissions may be too broad`, + options, + ); + } +} + +/** Emit an explicit warning when Windows ACL tightening cannot be completed. */ +function warnWindowsAcl(message: string, options: RestrictiveModeOptions): void { + const warn = options.warn ?? ((line: string) => process.stderr.write(`${line}\n`)); + warn(`[warning] ${message}`); +} + function resolvePath(options: CredentialsOptions): string { return options.path ?? defaultCredentialsPath(); } From 34f5d430797437ab2f67910fe356c3732ea0be57 Mon Sep 17 00:00:00 2001 From: nopp Date: Sat, 18 Jul 2026 06:54:04 +0700 Subject: [PATCH 069/102] fix(http): avoid retrying keyless writes (#256) --- src/lib/http.test.ts | 46 ++++++++++++++++++++++++++++++++++++++++++++ src/lib/http.ts | 23 ++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 2baff2c..e8f4c35 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -328,6 +328,31 @@ describe('HttpClient error mapping', () => { expect(fetchImpl).toHaveBeenCalledTimes(4); }); + it('does not retry transport errors for keyless writes', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('ECONNRESET'); + }); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.post('/projects', { body: { name: 'Checkout' } })).rejects.toBeInstanceOf( + TransportError, + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('retries transport errors for writes with an idempotency key', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('ECONNRESET'); + }); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect( + client.post('/projects', { + body: { name: 'Checkout' }, + headers: { 'Idempotency-Key': 'op_123' }, + }), + ).rejects.toBeInstanceOf(TransportError); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + it('does not retry AbortError', async () => { const fetchImpl = vi.fn(async () => { const err = new Error('aborted'); @@ -399,6 +424,27 @@ describe('HttpClient transport-edge statuses', () => { }, ); + it('does not retry bare transport-edge responses for keyless writes', async () => { + const fetchImpl = vi.fn(async () => new Response('proxy gateway html', { status: 502 })); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.post('/projects', { body: { name: 'Checkout' } })).rejects.toBeInstanceOf( + TransportError, + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('retries bare transport-edge responses for writes with an idempotency key', async () => { + const fetchImpl = vi.fn(async () => new Response('proxy gateway html', { status: 502 })); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect( + client.post('/projects', { + body: { name: 'Checkout' }, + headers: { 'idempotency-key': 'op_123' }, + }), + ).rejects.toBeInstanceOf(TransportError); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + it('502 carrying our envelope still maps to its catalog code', async () => { const body = { error: { diff --git a/src/lib/http.ts b/src/lib/http.ts index f0139ca..21369de 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -487,6 +487,7 @@ export class HttpClient { const url = buildUrl(this.baseUrl, path, options.query); const requestId = options.requestId ?? newRequestId(); + const allowTransportRetry = canRetryTransport(method, options); let attempt = 0; while (true) { @@ -557,7 +558,9 @@ export class HttpClient { errorCode: 'TRANSPORT', durationMs: Date.now() - startedAt, }); - const decision = transportRetryDecision(attempt, this.random); + const decision = allowTransportRetry + ? transportRetryDecision(attempt, this.random) + : { retry: false, delayMs: 0 }; if (!decision.retry) throw new TransportError(message, requestId); this.transition( `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, @@ -635,7 +638,9 @@ export class HttpClient { errorCode: 'TRANSPORT', durationMs, }); - const decision = transportRetryDecision(attempt, this.random); + const decision = allowTransportRetry + ? transportRetryDecision(attempt, this.random) + : { retry: false, delayMs: 0 }; if (!decision.retry) { throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); } @@ -945,6 +950,20 @@ function transportRetryDecision(attempt: number, random: () => number): RetryDec return { retry: true, delayMs: backoffDelay(attempt, random) }; } +function canRetryTransport(method: string, options: RequestOptions): boolean { + return isIdempotentMethod(method) || hasIdempotencyKey(options.headers); +} + +function isIdempotentMethod(method: string): boolean { + const normalized = method.toUpperCase(); + return normalized === 'GET' || normalized === 'HEAD'; +} + +function hasIdempotencyKey(headers: Record | undefined): boolean { + if (!headers) return false; + return Object.keys(headers).some(name => name.toLowerCase() === 'idempotency-key'); +} + function apiRetryDecision( code: ErrorCode, attempt: number, From 759a1c7df81b926c70ab125b16754460e47055dd Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:54:18 -0400 Subject: [PATCH 070/102] feat(cli): add "testsprite completion" for bash/zsh/fish (#227) Emit a shell completion script for bash, zsh, or fish. Command names, subcommands, and global flags are derived from the fully-assembled Commander tree at call time (buildCompletionSpec walks program.commands), so the script can never drift from the real command surface. The shell auto-detects from $SHELL when the argument is omitted. Fixes #74 --- src/commands/completion.test.ts | 91 ++++++++++ src/commands/completion.ts | 164 ++++++++++++++++++ src/index.ts | 23 +++ test/__snapshots__/help.snapshot.test.ts.snap | 1 + 4 files changed, 279 insertions(+) create mode 100644 src/commands/completion.test.ts create mode 100644 src/commands/completion.ts diff --git a/src/commands/completion.test.ts b/src/commands/completion.test.ts new file mode 100644 index 0000000..8388a09 --- /dev/null +++ b/src/commands/completion.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import type { CompletionSpec } from './completion.js'; +import { createCompletionCommand, detectShell, isShell, renderCompletion } from './completion.js'; + +const SPEC: CompletionSpec = { + program: 'testsprite', + commands: ['setup', 'auth', 'test', 'doctor', 'completion', 'help'], + subcommands: { auth: ['status', 'remove'], test: ['run', 'wait'] }, + globalFlags: ['--output', '--profile', '--help'], +}; + +describe('isShell / detectShell', () => { + it('recognizes the three supported shells', () => { + expect(isShell('bash')).toBe(true); + expect(isShell('zsh')).toBe(true); + expect(isShell('fish')).toBe(true); + expect(isShell('powershell')).toBe(false); + }); + + it('detects the shell from a $SHELL path', () => { + expect(detectShell({ SHELL: '/bin/bash' })).toBe('bash'); + expect(detectShell({ SHELL: '/usr/bin/zsh' })).toBe('zsh'); + expect(detectShell({ SHELL: '/usr/local/bin/fish' })).toBe('fish'); + }); + + it('returns undefined for an unknown or missing shell', () => { + expect(detectShell({ SHELL: '/bin/sh' })).toBeUndefined(); + expect(detectShell({})).toBeUndefined(); + }); +}); + +describe('renderCompletion', () => { + it('bash script wires a completion function and lists commands, subcommands, flags', () => { + const script = renderCompletion('bash', SPEC); + expect(script).toContain('complete -F _testsprite_completion testsprite'); + expect(script).toContain('setup'); + expect(script).toContain('auth) COMPREPLY'); + expect(script).toContain('status remove'); + expect(script).toContain('--output'); + }); + + it('zsh script declares #compdef and per-group subcommands', () => { + const script = renderCompletion('zsh', SPEC); + expect(script.startsWith('#compdef testsprite')).toBe(true); + expect(script).toContain('compdef _testsprite testsprite'); + expect(script).toContain('run wait'); + }); + + it('fish script uses complete -c with subcommand conditions and flags', () => { + const script = renderCompletion('fish', SPEC); + expect(script).toContain('complete -c testsprite -f'); + expect(script).toContain('__fish_seen_subcommand_from auth'); + expect(script).toContain('-l output'); + }); +}); + +describe('createCompletionCommand', () => { + function run(args: string[], env: NodeJS.ProcessEnv): Promise { + const out: string[] = []; + const cmd = createCompletionCommand(() => SPEC, { env, stdout: line => out.push(line) }); + return cmd.parseAsync(args, { from: 'user' }).then(() => out); + } + + it('prints the requested shell script from an explicit argument', async () => { + const out = await run(['bash'], {}); + expect(out.join('\n')).toContain('complete -F'); + }); + + it('auto-detects the shell from $SHELL when no argument is given', async () => { + const out = await run([], { SHELL: '/usr/bin/zsh' }); + expect(out.join('\n')).toContain('#compdef testsprite'); + }); + + it('rejects an unsupported shell with VALIDATION_ERROR (exit 5)', async () => { + const cmd = createCompletionCommand(() => SPEC, { env: {}, stdout: () => undefined }); + await expect(cmd.parseAsync(['powershell'], { from: 'user' })).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + }); + }); + + it('errors when the shell cannot be detected and none is given', async () => { + const cmd = createCompletionCommand(() => SPEC, { env: {}, stdout: () => undefined }); + await expect(cmd.parseAsync([], { from: 'user' })).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + }); + }); + + it('is named "completion"', () => { + expect(createCompletionCommand(() => SPEC).name()).toBe('completion'); + }); +}); diff --git a/src/commands/completion.ts b/src/commands/completion.ts new file mode 100644 index 0000000..15cf7a4 --- /dev/null +++ b/src/commands/completion.ts @@ -0,0 +1,164 @@ +/** + * `testsprite completion [bash|zsh|fish]` — emit a shell completion script. + * + * The command names, per-group subcommands, and global flags are NOT hardcoded: + * `index.ts` builds a {@link CompletionSpec} by walking the fully-assembled + * Commander program and passes it in, so the generated script can never drift + * from the real command tree. `renderCompletion` is a pure function of the spec, + * which keeps it unit-testable without a live program. + * + * Usage: + * bash: eval "$(testsprite completion bash)" (add to ~/.bashrc) + * zsh: testsprite completion zsh > ~/.zsh/_testsprite (on your fpath) + * fish: testsprite completion fish | source (add to config.fish) + */ + +import { Command } from 'commander'; +import { localValidationError } from '../lib/errors.js'; + +export const SUPPORTED_SHELLS = ['bash', 'zsh', 'fish'] as const; +export type Shell = (typeof SUPPORTED_SHELLS)[number]; + +export interface CompletionSpec { + /** Binary name, e.g. "testsprite". */ + program: string; + /** Top-level command names. */ + commands: string[]; + /** command name -> its subcommand names (only groups that have subcommands). */ + subcommands: Record; + /** Global long option flags (e.g. "--output"). */ + globalFlags: string[]; +} + +export interface CompletionDeps { + env?: NodeJS.ProcessEnv; + stdout?: (line: string) => void; +} + +export function isShell(value: string): value is Shell { + return (SUPPORTED_SHELLS as readonly string[]).includes(value); +} + +/** Best-effort shell detection from `$SHELL` (e.g. "/bin/zsh" -> "zsh"). */ +export function detectShell(env: NodeJS.ProcessEnv): Shell | undefined { + const shellPath = env.SHELL ?? ''; + const base = shellPath.slice(shellPath.lastIndexOf('/') + 1); + return isShell(base) ? base : undefined; +} + +export function renderCompletion(shell: Shell, spec: CompletionSpec): string { + switch (shell) { + case 'bash': + return renderBash(spec); + case 'zsh': + return renderZsh(spec); + case 'fish': + return renderFish(spec); + } +} + +function renderBash(spec: CompletionSpec): string { + const fn = `_${spec.program}_completion`; + const lines = [ + `# ${spec.program} bash completion. Enable with: eval "$(${spec.program} completion bash)"`, + `${fn}() {`, + ' local cur prev', + ' cur="${COMP_WORDS[COMP_CWORD]}"', + ' prev="${COMP_WORDS[COMP_CWORD-1]}"', + ` local commands="${spec.commands.join(' ')}"`, + ` local global_flags="${spec.globalFlags.join(' ')}"`, + ' case "$prev" in', + ...Object.entries(spec.subcommands).map( + ([group, subs]) => + ` ${group}) COMPREPLY=( $(compgen -W "${subs.join(' ')}" -- "$cur") ); return;;`, + ), + ' esac', + ' if [[ "$cur" == -* ]]; then', + ' COMPREPLY=( $(compgen -W "$global_flags" -- "$cur") ); return', + ' fi', + ' COMPREPLY=( $(compgen -W "$commands" -- "$cur") )', + '}', + `complete -F ${fn} ${spec.program}`, + ]; + return lines.join('\n'); +} + +function renderZsh(spec: CompletionSpec): string { + const fn = `_${spec.program}`; + const lines = [ + `#compdef ${spec.program}`, + `# ${spec.program} zsh completion. Enable with: ${spec.program} completion zsh > "$fpath[1]/_${spec.program}"`, + `${fn}() {`, + ' local -a commands', + ` commands=(${spec.commands.join(' ')})`, + ' if (( CURRENT == 2 )); then', + " _describe 'command' commands", + ' return', + ' fi', + ' case "${words[2]}" in', + ...Object.entries(spec.subcommands).map( + ([group, subs]) => + ` ${group}) local -a subs; subs=(${subs.join(' ')}); _describe 'subcommand' subs;;`, + ), + ' esac', + '}', + `compdef ${fn} ${spec.program}`, + ]; + return lines.join('\n'); +} + +function renderFish(spec: CompletionSpec): string { + const lines = [ + `# ${spec.program} fish completion. Enable with: ${spec.program} completion fish | source`, + `complete -c ${spec.program} -f`, + ...spec.commands.map( + command => `complete -c ${spec.program} -n '__fish_use_subcommand' -a '${command}'`, + ), + ...Object.entries(spec.subcommands).flatMap(([group, subs]) => + subs.map( + sub => `complete -c ${spec.program} -n '__fish_seen_subcommand_from ${group}' -a '${sub}'`, + ), + ), + ...spec.globalFlags.map(flag => `complete -c ${spec.program} -l ${flag.replace(/^--/, '')}`), + ]; + return lines.join('\n'); +} + +export function createCompletionCommand( + getSpec: () => CompletionSpec, + deps: CompletionDeps = {}, +): Command { + return new Command('completion') + .description('Print a shell completion script (bash|zsh|fish)') + .argument( + '[shell]', + 'Shell to generate for (bash|zsh|fish); auto-detected from $SHELL when omitted', + ) + .addHelpText( + 'after', + '\nExamples:\n' + + ' eval "$(testsprite completion bash)" # bash, current session\n' + + ' testsprite completion zsh > ~/.zsh/_testsprite\n' + + ' testsprite completion fish | source # fish, current session', + ) + .action((shellArg: string | undefined, _cmdOpts: unknown) => { + const env = deps.env ?? process.env; + const shell = shellArg ?? detectShell(env); + if (shell === undefined) { + throw localValidationError( + 'shell', + `could not detect the shell from $SHELL; pass one explicitly (${SUPPORTED_SHELLS.join(', ')})`, + [...SUPPORTED_SHELLS], + ); + } + if (!isShell(shell)) { + throw localValidationError( + 'shell', + `unsupported shell "${shell}"; use one of: ${SUPPORTED_SHELLS.join(', ')}`, + [...SUPPORTED_SHELLS], + ); + } + const write = deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)); + write(renderCompletion(shell, getSpec())); + }); +} diff --git a/src/index.ts b/src/index.ts index ae3f944..31d3399 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; +import { createCompletionCommand, type CompletionSpec } from './commands/completion.js'; import { createDoctorCommand } from './commands/doctor.js'; import { createDeprecatedInitCommand, @@ -92,6 +93,28 @@ program.addCommand(createTestCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); program.addCommand(createDoctorCommand()); +program.addCommand(createCompletionCommand(() => buildCompletionSpec())); + +// Derive the shell-completion spec from the fully-assembled command tree at call +// time (not module-load), so `testsprite completion` can never drift from the +// real commands, subcommands, and global flags. +function buildCompletionSpec(): CompletionSpec { + const subcommands: Record = {}; + for (const command of program.commands) { + const subs = command.commands.map(sub => sub.name()).filter(name => name !== 'help'); + if (subs.length > 0) subcommands[command.name()] = subs; + } + const flags = program.options + .map(option => option.long) + .filter((long): long is string => typeof long === 'string'); + if (!flags.includes('--help')) flags.push('--help'); + return { + program: 'testsprite', + commands: [...new Set([...program.commands.map(command => command.name()), 'help'])], + subcommands, + globalFlags: flags, + }; +} // Buffer Commander error messages instead of writing immediately. The catch // block re-emits in the correct format (JSON or text) once the requested diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index b0681c5..6f3e290 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -749,6 +749,7 @@ Commands: (proactive pre-flight before a large test run) doctor Diagnose CLI setup: version, Node, profile, endpoint, credentials, connectivity, skill + completion [shell] Print a shell completion script (bash|zsh|fish) help [command] display help for command " `; From 45dcd0895f10b406646333e537c658f2b3cd0b41 Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Fri, 17 Jul 2026 18:54:29 -0500 Subject: [PATCH 071/102] fix(agent): update verify skill auth commands (#216) --- docs/cli-v1-agent-install/skill-template.md | 6 +++--- skills/testsprite-verify.codex.md | 4 ++-- skills/testsprite-verify.skill.md | 6 +++--- src/lib/agent-targets.test.ts | 21 +++++++++++++++++++++ 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/cli-v1-agent-install/skill-template.md b/docs/cli-v1-agent-install/skill-template.md index d051a39..764e04e 100644 --- a/docs/cli-v1-agent-install/skill-template.md +++ b/docs/cli-v1-agent-install/skill-template.md @@ -61,13 +61,13 @@ because . Treat this as unverified until that's resolved." Don't claim done. ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials configured? +testsprite auth status # credentials configured? ``` - `--version` fails → the CLI isn't installed. Tell the user to install the TestSprite CLI (see the TestSprite docs) and stop; don't install it for them. -- `auth whoami` fails → no credentials. Tell the user they can run - `testsprite auth configure`, then stop. +- `auth status` fails → no credentials. Tell the user they can run + `testsprite setup`, then stop. ## 2. Find the project diff --git a/skills/testsprite-verify.codex.md b/skills/testsprite-verify.codex.md index 5d28f16..d3c5373 100644 --- a/skills/testsprite-verify.codex.md +++ b/skills/testsprite-verify.codex.md @@ -22,11 +22,11 @@ unverified-because-undeployed and stop. If the user explicitly named a tool ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials valid? +testsprite auth status # credentials valid? ``` If `--version` fails, tell the user to install the CLI and stop. -If `auth whoami` fails, tell the user to run `testsprite auth configure` and stop. +If `auth status` fails, tell the user to run `testsprite setup` and stop. ### 2. Find the project diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index 5020e42..40d60f2 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -67,13 +67,13 @@ because . Treat this as unverified until that's resolved." Don't claim done. ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials configured? +testsprite auth status # credentials configured? ``` - `--version` fails → the CLI isn't installed. Tell the user to install the TestSprite CLI (see the TestSprite docs) and stop; don't install it for them. -- `auth whoami` fails → no credentials. Tell the user they can run - `testsprite auth configure`, then stop. +- `auth status` fails → no credentials. Tell the user they can run + `testsprite setup`, then stop. ## 2. Find the project diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index 2ac9158..73feee3 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -449,6 +449,27 @@ describe('loadCodexSkillBody', () => { }); }); +// --------------------------------------------------------------------------- +// content integrity — current auth command names +// --------------------------------------------------------------------------- + +describe('content integrity — testsprite-verify auth commands', () => { + it('uses current auth/setup commands in every canonical verify skill asset', () => { + const assets = [ + ['docs template', templateRaw], + ['own-file skill body', loadSkillBodyFor('testsprite-verify')], + ['codex skill body', codexContentFor('testsprite-verify')], + ] as const; + + for (const [name, content] of assets) { + expect(content, name).toContain('testsprite auth status'); + expect(content, name).toContain('testsprite setup'); + expect(content, name).not.toContain('auth whoami'); + expect(content, name).not.toContain('auth configure'); + } + }); +}); + // --------------------------------------------------------------------------- // MANAGED_SECTION sentinels // --------------------------------------------------------------------------- From eaf0585dd29b960161fe3d160adb23ce0e6040ad Mon Sep 17 00:00:00 2001 From: JerryNee <37407632+JerryNee@users.noreply.github.com> Date: Fri, 17 Jul 2026 19:00:05 -0500 Subject: [PATCH 072/102] fix(flaky): propagate auth trigger errors (#217) --- src/commands/test.flaky.spec.ts | 42 +++++++++++++++++++++++++++++++++ src/commands/test.ts | 15 ++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/commands/test.flaky.spec.ts b/src/commands/test.flaky.spec.ts index 0c4450e..8076f3e 100644 --- a/src/commands/test.flaky.spec.ts +++ b/src/commands/test.flaky.spec.ts @@ -18,6 +18,7 @@ import { runFlaky } from './test.js'; type FetchInput = Parameters[0]; type RunStatus = 'passed' | 'failed' | 'blocked' | 'cancelled'; +type TriggerAuthErrorCode = 'AUTH_REQUIRED' | 'AUTH_INVALID' | 'AUTH_FORBIDDEN'; function urlOf(input: FetchInput): string { return typeof input === 'string' @@ -46,6 +47,7 @@ function makeFlakyFetch(opts: { statuses: RunStatus[]; testType?: 'frontend' | 'backend'; notFoundOnTrigger?: boolean; + triggerAuthError?: TriggerAuthErrorCode; }): { fetchImpl: FetchImpl; triggerCount: () => number } { let triggers = 0; const testType = opts.testType ?? 'frontend'; @@ -67,6 +69,17 @@ function makeFlakyFetch(opts: { } if (method === 'POST' && url.includes('/runs/rerun')) { + if (opts.triggerAuthError) { + return jsonResponse(opts.triggerAuthError === 'AUTH_FORBIDDEN' ? 403 : 401, { + error: { + code: opts.triggerAuthError, + message: 'auth failed', + nextAction: 'run setup', + requestId: 'req_auth', + details: {}, + }, + }); + } if (opts.notFoundOnTrigger) { return jsonResponse(404, { error: { @@ -345,6 +358,35 @@ describe('runFlaky', () => { expect((err as ApiError).code).toBe('NOT_FOUND'); }); + it.each(['AUTH_REQUIRED', 'AUTH_INVALID', 'AUTH_FORBIDDEN'] as const)( + 'propagates %s during trigger instead of scoring an error attempt', + async code => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: [], + triggerAuthError: code, + }); + const { deps, stdout } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect(err).toMatchObject({ code, exitCode: 3 }); + expect(stdout).toEqual([]); + expect(triggerCount()).toBe(0); + }, + ); + it('rejects --runs below the range (0) with a validation error (exit 5)', async () => { const { fetchImpl } = makeFlakyFetch({ statuses: [] }); const { deps } = makeDeps(fetchImpl); diff --git a/src/commands/test.ts b/src/commands/test.ts index 837c170..e82c5f4 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -9379,6 +9379,18 @@ const MAX_FLAKY_RUNS = 10; /** Default replay count when `--runs` is omitted. */ const DEFAULT_FLAKY_RUNS = 5; +function isFlakyFatalTriggerError(err: unknown): boolean { + if (!(err instanceof ApiError)) return false; + switch (err.code) { + case 'AUTH_REQUIRED': + case 'AUTH_INVALID': + case 'AUTH_FORBIDDEN': + return true; + default: + return false; + } +} + interface RunTestFlakyOptions extends CommonOptions { testId: string; /** Number of replays to run (1..MAX_FLAKY_RUNS). */ @@ -9477,6 +9489,9 @@ export async function runFlaky( }, }); } + if (isFlakyFatalTriggerError(err)) { + throw err; + } // Any other trigger error is recorded as an errored attempt so a single // transient blip doesn't abort a long stability probe. const code = err instanceof ApiError ? err.code : 'ERROR'; From 8db1882964b93db3dcd9b7de526cc7d179cd6619 Mon Sep 17 00:00:00 2001 From: Andy <89641810+Andy00L@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:00:18 -0400 Subject: [PATCH 073/102] feat(test): add "test open " to jump from the terminal to the dashboard (#226) * feat(test): add "test open " to jump from the terminal to the dashboard * fix(open): derive the dry-run URL via resolvePortalUrl and handle async spawn ENOENT * fix(browser): map non-http(s) URL rejection to exit 5 (validation error) openInBrowser threw a plain Error for a non-http(s) URL, which fell through the top-level handler and exited 1. Route it through localValidationError so it is classified as VALIDATION_ERROR and maps to the documented exit code 5, matching every other bad-argument path. Test asserts the exit code, not the message string. --- src/commands/test.test.ts | 90 ++++++++++++++++ src/commands/test.ts | 80 ++++++++++++++ src/lib/browser.test.ts | 100 ++++++++++++++++++ src/lib/browser.ts | 63 +++++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 4 + 5 files changed, 337 insertions(+) create mode 100644 src/lib/browser.test.ts create mode 100644 src/lib/browser.ts diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 9493908..bd238d9 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -33,6 +33,7 @@ import { runGet, runLint, runList, + runOpen, runPlanPut, runResult, runScaffold, @@ -132,6 +133,7 @@ describe('createTestCommand — surface', () => { 'get', 'lint', 'list', + 'open', 'plan', 'rerun', 'result', @@ -2450,6 +2452,94 @@ describe('runScaffold', () => { }); }); +describe('runOpen', () => { + // The mock endpoint host has no portal mapping; the operator override is the + // supported escape hatch and gives the tests a deterministic base. + beforeEach(() => { + process.env.TESTSPRITE_PORTAL_URL = 'https://portal.example.com'; + }); + afterEach(() => { + delete process.env.TESTSPRITE_PORTAL_URL; + }); + + const TEST_ROW = { + id: 'test_open_me', + projectId: 'project_alice', + projectName: 'Alice', + name: 'Checkout', + type: 'frontend', + createdFrom: 'cli', + status: 'ready', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }; + + it('prints the dashboard URL and spawns the opener with it', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const out: string[] = []; + const opened: string[] = []; + const result = await runOpen( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_open_me', + noBrowser: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + url => opened.push(url), + ); + expect(result.dashboardUrl).toContain('/dashboard/tests/project_alice/test/test_open_me'); + expect(out.join('\n')).toContain(result.dashboardUrl); + expect(opened).toEqual([result.dashboardUrl]); + }); + + it('--no-browser prints the URL but never spawns', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const out: string[] = []; + const opened: string[] = []; + await runOpen( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_open_me', + noBrowser: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + url => opened.push(url), + ); + expect(opened).toEqual([]); + expect((JSON.parse(out.join('')) as { dashboardUrl: string }).dashboardUrl).toContain( + 'test_open_me', + ); + }); + + it('a broken opener downgrades to a stderr hint, never a failure', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const errs: string[] = []; + await expect( + runOpen( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_open_me', + noBrowser: false, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) }, + () => { + throw new Error('no display'); + }, + ), + ).resolves.toBeDefined(); + expect(errs.join('\n')).toContain('could not launch a browser'); + }); +}); + describe('runSteps', () => { it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => { const { credentialsPath } = makeCreds(); diff --git a/src/commands/test.ts b/src/commands/test.ts index e82c5f4..4a594dd 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -10,6 +10,7 @@ import { rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path'; import { randomUUID } from 'node:crypto'; import { Command } from 'commander'; +import { openInBrowser } from '../lib/browser.js'; import { emitDryRunBanner, makeHttpClient, @@ -4276,6 +4277,67 @@ export async function runScaffold( return payload; } +export interface OpenOptions extends CommonOptions { + testId: string; + /** Print the URL only; never spawn a browser (SSH/headless/CI/agents). */ + noBrowser: boolean; +} + +/** + * `test open ` (issue #121): jump from the terminal to the test's + * dashboard page. The CLI already computes this deep-link and prints it as + * text on other commands; this closes the last inch (the `gh browse` / + * `cypress open` hop). The URL is ALWAYS printed to stdout (so `--no-browser` + * and headless use still compose), then the OS browser is spawned unless + * --no-browser. An endpoint with no known portal mapping is a hard error + * rather than a silent no-op. + */ +export async function runOpen( + opts: OpenOptions, + deps: TestDeps = {}, + opener: (url: string) => void = openInBrowser, +): Promise<{ dashboardUrl: string }> { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + // Derive the sample through the SAME resolver as the live path (against + // the canonical prod endpoint and the dry-run project id) so the two can + // never drift; the ?? arm is unreachable for the prod mapping but keeps + // the type total. + const sample = { + dashboardUrl: + resolvePortalUrl('https://api.testsprite.com', 'p_dryrun_2026', opts.testId) ?? + `https://www.testsprite.com/dashboard/tests/p_dryrun_2026/test/${encodeURIComponent(opts.testId)}`, + }; + out.print(sample, data => (data as { dashboardUrl: string }).dashboardUrl); + return sample; + } + + const client = makeClient(opts, deps); + // The deep-link needs the projectId; the test record is the source of truth. + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + const dashboardUrl = resolvePortalUrl(resolveApiUrl(opts, deps), test.projectId, opts.testId); + if (dashboardUrl === undefined) { + throw new CLIError( + `no dashboard mapping for this API endpoint; set TESTSPRITE_PORTAL_URL to your Portal origin`, + 1, + ); + } + out.print({ dashboardUrl }, () => dashboardUrl); + if (!opts.noBrowser) { + try { + opener(dashboardUrl); + } catch { + // The URL is already on stdout; a missing opener (containers, minimal + // hosts) downgrades to "open it yourself" instead of a hard failure. + stderrFn('could not launch a browser; open the URL above manually (or use --no-browser)'); + } + } + return { dashboardUrl }; +} + export async function runSteps( opts: StepsOptions, deps: TestDeps = {}, @@ -8720,6 +8782,24 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('open ') + .description( + 'Open the test in the TestSprite dashboard: prints the deep-link URL, then spawns your default browser unless --no-browser.', + ) + .option('--no-browser', 'print the URL only (SSH, headless, CI, agents)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, cmdOpts: { browser?: boolean }, command: Command) => { + await runOpen( + { + ...resolveCommonOptions(command), + testId, + noBrowser: cmdOpts.browser === false, + }, + deps, + ); + }); + test .command('steps ') .description( diff --git a/src/lib/browser.test.ts b/src/lib/browser.test.ts new file mode 100644 index 0000000..3d6717b --- /dev/null +++ b/src/lib/browser.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Mock the real spawner so the default `exec` path (detached spawn + unref) +// is exercised without launching a real process. +const spawnMock = vi.fn(); +vi.mock('node:child_process', () => ({ + spawn: (command: string, args: readonly string[], opts: unknown) => + spawnMock(command, args, opts), +})); + +import { openInBrowser } from './browser.js'; +import { ApiError } from './errors.js'; + +describe('openInBrowser', () => { + const url = 'https://portal.example.com/tests/t_123'; + + it('uses `open ` on darwin', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'darwin', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'open', args: [url] }]); + }); + + it('uses rundll32 FileProtocolHandler on win32', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'win32', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'rundll32', args: ['url.dll,FileProtocolHandler', url] }]); + }); + + it('uses xdg-open on other platforms', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'linux', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'xdg-open', args: [url] }]); + }); + + it('refuses a non-http(s) URL with exit 5 before spawning', () => { + const exec = vi.fn(); + let error: unknown; + try { + openInBrowser('file:///etc/passwd', { platform: 'linux', exec }); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiError); + expect((error as ApiError).exitCode).toBe(5); + expect(exec).not.toHaveBeenCalled(); + }); + + it('throws on a malformed URL', () => { + const exec = vi.fn(); + expect(() => openInBrowser('not a url', { exec })).toThrow(); + expect(exec).not.toHaveBeenCalled(); + }); + + describe('default spawner', () => { + beforeEach(() => { + spawnMock.mockReset(); + }); + + it('spawns detached, ignores stdio, and unrefs the child', () => { + const unref = vi.fn(); + spawnMock.mockReturnValue({ unref, on: vi.fn() }); + openInBrowser(url, { platform: 'darwin' }); + expect(spawnMock).toHaveBeenCalledWith('open', [url], { detached: true, stdio: 'ignore' }); + expect(unref).toHaveBeenCalledTimes(1); + }); + + it("handles the child's async 'error' (missing binary) with a stderr hint, not a crash", () => { + // spawn() reports ENOENT asynchronously on the child; an unhandled + // 'error' event would crash the CLI. The default spawner must register + // a listener that degrades to the manual-open hint. + const listeners = new Map void>(); + spawnMock.mockReturnValue({ + unref: vi.fn(), + on: (event: string, listener: (err: Error) => void) => { + listeners.set(event, listener); + }, + }); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + try { + openInBrowser(url, { platform: 'linux' }); + const onError = listeners.get('error'); + expect(onError).toBeDefined(); + // Firing the listener must not throw and must print the hint. + expect(() => onError!(new Error('spawn xdg-open ENOENT'))).not.toThrow(); + expect(String(stderrSpy.mock.calls.at(-1)?.[0])).toContain('could not launch a browser'); + } finally { + stderrSpy.mockRestore(); + } + }); + }); +}); diff --git a/src/lib/browser.ts b/src/lib/browser.ts new file mode 100644 index 0000000..ccd2b8a --- /dev/null +++ b/src/lib/browser.ts @@ -0,0 +1,63 @@ +/** + * Cross-platform "open this URL in the default browser" helper for + * `test open` (issue #121). Spawns the platform opener with an argv array + * (never a shell string) so a URL can never be shell-injected, and refuses + * anything that is not http(s) before any process is spawned. + * + * Platform openers: + * darwin open + * win32 rundll32 url.dll,FileProtocolHandler (avoids `cmd /c start`, + * whose re-parsing would mangle `&` and other metachars in the URL) + * other xdg-open + * + * The child is detached and unref'd so the CLI exits immediately; failures + * are the caller's to surface (it already printed the URL as the fallback). + */ +import { spawn } from 'node:child_process'; +import { localValidationError } from './errors.js'; + +export interface OpenInBrowserDeps { + platform?: NodeJS.Platform; + /** Process spawner taking an argv array. Defaults to a detached spawn. */ + exec?: (command: string, args: readonly string[]) => void; +} + +export function openInBrowser(url: string, deps: OpenInBrowserDeps = {}): void { + const parsed = new URL(url); + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { + // User-input error, not an internal failure: classify as VALIDATION_ERROR + // so it maps to exit 5 (like every other bad-argument path), not exit 1. + throw localValidationError( + 'url', + `must be an http(s) URL (got ${parsed.protocol})`, + undefined, + 'field', + ); + } + const platform = deps.platform ?? process.platform; + const exec = + deps.exec ?? + ((command: string, args: readonly string[]) => { + const child = spawn(command, [...args], { detached: true, stdio: 'ignore' }); + // spawn() reports a missing binary (ENOENT) ASYNCHRONOUSLY on the child, + // so the caller's try/catch cannot see it — and an unhandled 'error' + // event would crash the whole CLI. The URL is already on stdout, so + // degrade to the same manual-open hint the sync failure path prints. + child.on('error', () => { + process.stderr.write( + 'could not launch a browser; open the URL above manually (or use --no-browser)\n', + ); + }); + child.unref(); + }); + + if (platform === 'darwin') { + exec('open', [url]); + return; + } + if (platform === 'win32') { + exec('rundll32', ['url.dll,FileProtocolHandler', url]); + return; + } + exec('xdg-open', [url]); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6f3e290..441e700 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -222,6 +222,10 @@ Commands: definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials. + open [options] Open the test in the TestSprite + dashboard: prints the deep-link URL, + then spawns your default browser unless + --no-browser. steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) From 9d4af09af285ad9863c9c1bc9106d968d3d1bf05 Mon Sep 17 00:00:00 2001 From: nopp Date: Sat, 18 Jul 2026 07:01:37 +0700 Subject: [PATCH 074/102] feat(cli): add text list column controls (#165) (#252) * feat(cli): add text list column controls * fix(cli): address list column review feedback --- src/commands/project.test.ts | 68 +++++++ src/commands/project.ts | 79 ++++---- src/commands/test.result.history.spec.ts | 126 ++++++++++++ src/commands/test.test.ts | 71 +++++++ src/commands/test.ts | 188 +++++++++--------- src/lib/text-table.ts | 94 +++++++++ test/__snapshots__/help.snapshot.test.ts.snap | 8 + 7 files changed, 504 insertions(+), 130 deletions(-) create mode 100644 src/lib/text-table.ts diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 0f8c51c..ae98070 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -89,6 +89,8 @@ describe('createProjectCommand', () => { expect(flagNames).toContain('--page-size'); expect(flagNames).toContain('--starting-token'); expect(flagNames).toContain('--max-items'); + expect(flagNames).toContain('--columns'); + expect(flagNames).toContain('--no-header'); }); }); @@ -285,6 +287,72 @@ describe('runList', () => { expect(block).toContain('nextToken: next-please'); }); + it('text output selects/reorders columns and suppresses the header', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + pageSize: 25, + columns: 'name,id', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const block = out.join('\n'); + expect(block).toMatch(/^Checkout\s+project_b3c91efa$/); + expect(block).not.toContain('NAME'); + expect(block).not.toContain('CREATED'); + }); + + it('text output rejects unknown columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runList( + { + profile: 'default', + output: 'text', + debug: false, + pageSize: 25, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json output ignores text-only column flags', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'json', + debug: false, + pageSize: 25, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(JSON.parse(out.join('\n')).items[0].id).toBe('project_b3c91efa'); + }); + it('text output reads "No projects." when items is empty and nextToken is null', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); diff --git a/src/commands/project.ts b/src/commands/project.ts index 0a843e5..6a0b332 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -12,6 +12,7 @@ import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { assertNotLocal } from '../lib/target-url.js'; +import { renderTextTable, resolveTextColumns, type TextTableColumn } from '../lib/text-table.js'; import { assertIdempotencyKey } from '../lib/validate.js'; import { fetchSinglePage, @@ -44,6 +45,8 @@ interface ListOptions extends CommonOptions { pageSize?: number; startingToken?: string; maxItems?: number; + columns?: string; + noHeader?: boolean; } export async function runList( @@ -57,6 +60,9 @@ export async function runList( startingToken: opts.startingToken, maxItems: opts.maxItems, }); + if (opts.output === 'text') { + resolveTextColumns(opts.columns, PROJECT_LIST_COLUMNS); + } const client = makeClient(opts, deps); // When the user explicitly passed a page-size flag and did NOT ask @@ -84,7 +90,7 @@ export async function runList( out.print(page, data => { const p = data as Page; - return renderProjectListText(p); + return renderProjectListText(p, { columns: opts.columns, noHeader: opts.noHeader }); }); return page; } @@ -663,6 +669,8 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--page-size ', 'service page-size hint (1-100, default 25)') .option('--starting-token ', 'opaque cursor from a previous list response') .option('--max-items ', 'stop after this many items across auto-paged pages') + .option('--columns ', 'select/reorder text table columns (comma-separated keys)') + .option('--no-header', 'suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (cmdOpts: ListFlagOpts, command: Command) => { // Don't parse numeric flags via Commander — its parser throws a @@ -676,6 +684,8 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { pageSize: parseFlag(cmdOpts.pageSize, 'page-size'), startingToken: cmdOpts.startingToken, maxItems: parseFlag(cmdOpts.maxItems, 'max-items'), + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -895,6 +905,8 @@ interface ListFlagOpts { pageSize?: string; startingToken?: string; maxItems?: string; + columns?: string; + header?: boolean; } interface CreateFlagOpts { @@ -1001,45 +1013,37 @@ function makeOutput(mode: OutputMode, deps: ProjectDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } -function renderProjectListText(page: Page): string { +const PROJECT_LIST_COLUMNS: ReadonlyArray> = [ + { + header: 'ID', + width: rows => Math.max(2, ...rows.map(project => project.id.length)), + render: project => project.id, + }, + { + header: 'NAME', + width: rows => Math.max(4, ...rows.map(project => project.name.length)), + render: project => project.name, + }, + { header: 'TYPE', width: 8, render: project => project.type }, + { header: 'FROM', width: 6, render: project => project.createdFrom }, + { header: 'CREATED', width: 0, render: project => project.createdAt }, +]; + +function renderProjectListText( + page: Page, + options: { columns?: string; noHeader?: boolean } = {}, +): string { if (page.items.length === 0) { return page.nextToken ? `No projects on this page.\nnextToken: ${page.nextToken}` : 'No projects.'; } - // Compact, AWS-CLI-grade columnar output. Column widths are computed - // per-call so a single absurdly long project name doesn't push the - // whole table off-screen. - const idWidth = Math.max(2, ...page.items.map(p => p.id.length)); - const nameWidth = Math.max(4, ...page.items.map(p => p.name.length)); - const typeWidth = 8; - const fromWidth = 6; - - const header = - pad('ID', idWidth) + - ' ' + - pad('NAME', nameWidth) + - ' ' + - pad('TYPE', typeWidth) + - ' ' + - pad('FROM', fromWidth) + - ' ' + - 'CREATED'; - - const rows = page.items.map( - p => - pad(p.id, idWidth) + - ' ' + - pad(p.name, nameWidth) + - ' ' + - pad(p.type, typeWidth) + - ' ' + - pad(p.createdFrom, fromWidth) + - ' ' + - p.createdAt, - ); - - const lines = [header, ...rows]; + const lines = [ + renderTextTable(page.items, PROJECT_LIST_COLUMNS, { + columns: options.columns, + noHeader: options.noHeader, + }), + ]; if (page.nextToken) lines.push('', `nextToken: ${page.nextToken}`); return lines.join('\n'); } @@ -1055,11 +1059,6 @@ function renderProjectText(p: CliProject): string { ].join('\n'); } -function pad(s: string, width: number): string { - if (s.length >= width) return s; - return s + ' '.repeat(width - s.length); -} - function renderUpdateText(r: CliUpdateProjectResponse): string { return [ `id: ${r.id}`, diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index a2d2018..87df3e0 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -225,6 +225,132 @@ describe('runResultHistory — text mode', () => { expect(output).toMatch(/DURATION/); }); + it('selects/reorders columns and suppresses header/separator/detail sub-lines', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_url_001', + targetUrl: 'https://staging.example.com/checkout', + targetUrlSource: 'run', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'status,runid', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output.split('\n')[0]).toMatch(/^passed\s+run_url_001$/); + expect(output).not.toMatch(/^RUN ID/m); + expect(output).not.toMatch(/^-+$/m); + expect(output).not.toContain('targetUrl:'); + }); + + it('no-header suppresses only the history header and separator', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_url_001', + targetUrl: 'https://staging.example.com/checkout', + targetUrlSource: 'run', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).not.toMatch(/^RUN ID/m); + expect(output).not.toMatch(/^-+$/m); + expect(output).toContain('run_url_001'); + expect(output).toContain('targetUrl: https://staging.example.com/checkout'); + }); + + it('rejects unknown history columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json mode ignores text-only history column flags', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: makeHistoryResp() }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const parsed = JSON.parse(lines.join('')) as { runs: Array<{ runId: string }> }; + expect(parsed.runs[0]?.runId).toBe('run_hist_001'); + }); + it('renders run_hist_001 row with status passed and source cli', async () => { const { credentialsPath } = makeCreds(); const lines: string[] = []; diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index bd238d9..b6455a2 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -202,6 +202,7 @@ describe('createTestCommand — surface', () => { it('result exposes --include-analysis (M2.1) + M3.4 piece-5 --history flags', () => { // M2.1 piece 3 adds `--include-analysis` to `test result`. // M3.4 piece 5 adds `--history`, `--source`, `--since`, `--page-size`, `--cursor`. + // Issue #165 adds text-table shaping via `--columns` and `--no-header`. // Pinning the surface so a future flag-consolidation sweep keeps every // option intentional. Back-compat: bare `test result ` (no --history) // still calls runResult and returns the M2 CliLatestResult shape. @@ -215,6 +216,8 @@ describe('createTestCommand — surface', () => { '--since', '--page-size', '--cursor', + '--columns', + '--no-header', ]); }); @@ -577,6 +580,74 @@ describe('runList', () => { expect(block).toContain('mcp'); }); + it('text mode selects/reorders columns and suppresses the header', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [FE_TEST, BE_TEST], nextToken: null }, + })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'status,id', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const lines = out.join('\n').split('\n'); + expect(lines[0]).toMatch(/^failed\s+test_fe$/); + expect(lines[1]).toMatch(/^passed\s+test_be$/); + expect(out.join('\n')).not.toContain('STATUS'); + expect(out.join('\n')).not.toContain('UPDATED'); + }); + + it('text mode rejects unknown columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json mode ignores text-only column flags', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [FE_TEST], nextToken: null }, + })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(JSON.parse(out.join('\n')).items[0].id).toBe('test_fe'); + }); + it('text mode reads "No tests." when items is empty and nextToken is null', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); diff --git a/src/commands/test.ts b/src/commands/test.ts index 4a594dd..7839ca7 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -82,6 +82,13 @@ import type { } from '../lib/runs.types.js'; import { RUN_SOURCES } from '../lib/runs.types.js'; import { assertNotLocal } from '../lib/target-url.js'; +import { + formatTextTableRow, + measureTextColumns, + renderTextTable, + resolveTextColumns, + type TextTableColumn, +} from '../lib/text-table.js'; import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; @@ -493,6 +500,8 @@ interface ListOptions extends CommonOptions { pageSize?: number; startingToken?: string; maxItems?: number; + columns?: string; + noHeader?: boolean; } const TEST_TYPES: ReadonlyArray<'frontend' | 'backend'> = ['frontend', 'backend']; @@ -558,6 +567,9 @@ export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise

renderTestListText(data as Page)); + out.print(page, data => + renderTestListText(data as Page, { columns: opts.columns, noHeader: opts.noHeader }), + ); return page; } @@ -4543,6 +4557,8 @@ interface ResultHistoryOptions extends CommonOptions { pageSize?: number; /** Opaque cursor from a prior page's `nextCursor`. */ cursor?: string; + columns?: string; + noHeader?: boolean; } /** @@ -4571,6 +4587,9 @@ export async function runResultHistory( throw localValidationError('page-size', 'must be between 1 and 100'); } } + if (opts.output === 'text') { + resolveTextColumns(opts.columns, RUN_HISTORY_TABLE_COLUMNS); + } const client = makeClient(opts, deps); const pageSize = opts.pageSize ?? 20; @@ -4583,7 +4602,7 @@ export async function runResultHistory( since: sinceIso, }); - if (opts.output === 'json') { + if (opts.output !== 'text') { out.print({ runs: resp.runs, nextCursor: resp.nextCursor }, data => JSON.stringify(data)); return resp; } @@ -4620,7 +4639,7 @@ export async function runResultHistory( } const lines: string[] = []; - lines.push(renderRunHistoryTable(resp.runs)); + lines.push(renderRunHistoryTable(resp.runs, { columns: opts.columns, noHeader: opts.noHeader })); // Footer: pointer to per-run detail commands. lines.push(''); @@ -4647,13 +4666,18 @@ export async function runResultHistory( return resp; } -const RUN_HISTORY_TABLE_COL_WIDTHS = { - runId: 36, - status: 10, - source: 18, - rerun: 6, - when: 25, -}; +const RUN_HISTORY_TABLE_COLUMNS: ReadonlyArray> = [ + { header: 'RUN ID', width: 36, render: run => run.runId }, + { header: 'STATUS', width: 10, render: run => run.status }, + { header: 'SOURCE', width: 18, render: run => run.source }, + { header: 'RERUN?', width: 6, render: run => (run.isRerun ? 'yes' : 'no') }, + { header: 'WHEN', width: 25, render: run => run.createdAt }, + { + header: 'DURATION', + width: 0, + render: run => formatDurationMs(run.startedAt ?? run.createdAt, run.finishedAt), + }, +]; /** * Max width of the `test steps` DESCRIPTION column in text mode. Long / @@ -4689,60 +4713,41 @@ const HISTORY_TARGET_URL_MAX = 80; * run row (truncated to `HISTORY_TARGET_URL_MAX` chars). The table columns * are left intact to avoid width blow-out on terminals. */ -function renderRunHistoryTable(runs: RunHistoryItem[]): string { - const cols = RUN_HISTORY_TABLE_COL_WIDTHS; - const header = [ - padEnd('RUN ID', cols.runId), - padEnd('STATUS', cols.status), - padEnd('SOURCE', cols.source), - padEnd('RERUN?', cols.rerun), - padEnd('WHEN', cols.when), - 'DURATION', - ].join(' '); - const sep = '-'.repeat(header.length); - - const rows = runs.flatMap(r => { - // FE runs never populate `startedAt` today — the RUNNING heartbeat - // that would set it doesn't fire on the legacy/sync execution path - // (dogfood 2026-06-04), so without a fallback DURATION was always - // "—" for every FE run. Fall back to `createdAt` so the column shows - // wall-clock from trigger to finish; on sync dev the queue gap is - // ~0, and `--output json` still exposes raw startedAt/finishedAt for - // consumers that need to exclude queue time. - const duration = formatDurationMs(r.startedAt ?? r.createdAt, r.finishedAt); - const mainRow = [ - padEnd(r.runId, cols.runId), - padEnd(r.status, cols.status), - padEnd(r.source, cols.source), - padEnd(r.isRerun ? 'yes' : 'no', cols.rerun), - padEnd(r.createdAt, cols.when), - duration, - ].join(' '); +function renderRunHistoryTable( + runs: RunHistoryItem[], + options: { columns?: string; noHeader?: boolean } = {}, +): string { + const selectedColumns = resolveTextColumns(options.columns, RUN_HISTORY_TABLE_COLUMNS); + const widths = measureTextColumns(runs, selectedColumns); + const customColumns = options.columns !== undefined && options.columns.trim() !== ''; + const includeDetailLines = !customColumns; + const header = formatTextTableRow( + selectedColumns.map(column => column.header), + widths, + ); + + const rows = runs.flatMap(run => { + const mainRow = formatTextTableRow( + selectedColumns.map(column => column.render(run)), + widths, + ); - // G1b: surface per-run targetUrl as an indented sub-line. - // Render only when truthy (skip null, undefined, empty) and when the - // source is not 'unresolved' (that would mean "backend couldn't resolve - // a URL" — printing "—" is less informative than omitting the line). const lines: string[] = [mainRow]; - if (r.targetUrl && r.targetUrlSource !== 'unresolved') { + if (includeDetailLines && run.targetUrl && run.targetUrlSource !== 'unresolved') { const url = - r.targetUrl.length > HISTORY_TARGET_URL_MAX - ? `${r.targetUrl.slice(0, HISTORY_TARGET_URL_MAX - 1)}…` - : r.targetUrl; + run.targetUrl.length > HISTORY_TARGET_URL_MAX + ? `${run.targetUrl.slice(0, HISTORY_TARGET_URL_MAX - 1)}…` + : run.targetUrl; lines.push(` targetUrl: ${url}`); - } else if (r.targetUrlSource === 'unresolved') { + } else if (includeDetailLines && run.targetUrlSource === 'unresolved') { lines.push(` targetUrl: —`); } return lines; }); - return [header, sep, ...rows].join('\n'); -} - -function padEnd(s: string, width: number): string { - if (s.length >= width) return s; - return s + ' '.repeat(width - s.length); + if (options.noHeader === true) return rows.join('\n'); + return [header, '-'.repeat(header.length), ...rows].join('\n'); } function formatDurationMs(startedAt: string | null, finishedAt: string | null): string { @@ -8543,6 +8548,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { 'alias for --starting-token; accepted for parity with `test result --history`', ) .option('--max-items ', 'stop after this many items across auto-paged pages') + .option('--columns ', 'select/reorder text table columns (comma-separated keys)') + .option('--no-header', 'suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (cmdOpts: ListFlagOpts, command: Command) => { // Same parser strategy as `project list`: skip Commander's number @@ -8563,6 +8570,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { pageSize: parseNumericFlag(cmdOpts.pageSize, 'page-size'), startingToken: cmdOpts.startingToken ?? cmdOpts.cursor, maxItems: parseNumericFlag(cmdOpts.maxItems, 'max-items'), + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -8891,6 +8900,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option('--page-size ', 'with --history: number of runs per page (1–100, default 20)') .option('--cursor ', 'with --history: opaque cursor from a prior page') + .option('--columns ', 'with --history: select/reorder text table columns') + .option('--no-header', 'with --history: suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testId: string, cmdOpts: ResultFlagOpts, command: Command) => { if (cmdOpts.history) { @@ -8906,6 +8917,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { ? parseNumericFlag(cmdOpts.pageSize, 'page-size') : undefined, cursor: cmdOpts.cursor, + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -9712,6 +9725,8 @@ interface ResultFlagOpts { pageSize?: string; /** Opaque pagination cursor from a prior page's nextCursor. */ cursor?: string; + columns?: string; + header?: boolean; } interface CreateFlagOpts { @@ -9759,6 +9774,8 @@ interface ListFlagOpts { */ cursor?: string; maxItems?: string; + columns?: string; + header?: boolean; } interface StepsFlagOpts { @@ -10106,45 +10123,36 @@ async function streamPresignedBody(url: string, out: Output, deps: TestDeps): Pr } } -function renderTestListText(page: Page): string { +const TEST_LIST_COLUMNS: ReadonlyArray> = [ + { + header: 'ID', + width: rows => Math.max(2, ...rows.map(test => test.id.length)), + render: test => test.id, + }, + { + header: 'NAME', + width: rows => Math.max(4, ...rows.map(test => test.name.length)), + render: test => test.name, + }, + { header: 'TYPE', width: 8, render: test => test.type }, + { header: 'FROM', width: 6, render: test => test.createdFrom }, + { header: 'STATUS', width: 9, render: test => test.status }, + { header: 'UPDATED', width: 0, render: test => test.updatedAt }, +]; + +function renderTestListText( + page: Page, + options: { columns?: string; noHeader?: boolean } = {}, +): string { if (page.items.length === 0) { return page.nextToken ? `No tests on this page.\nnextToken: ${page.nextToken}` : 'No tests.'; } - const idWidth = Math.max(2, ...page.items.map(t => t.id.length)); - const nameWidth = Math.max(4, ...page.items.map(t => t.name.length)); - const typeWidth = 8; - const fromWidth = 6; - const statusWidth = 9; - - const header = - pad('ID', idWidth) + - ' ' + - pad('NAME', nameWidth) + - ' ' + - pad('TYPE', typeWidth) + - ' ' + - pad('FROM', fromWidth) + - ' ' + - pad('STATUS', statusWidth) + - ' ' + - 'UPDATED'; - - const rows = page.items.map( - t => - pad(t.id, idWidth) + - ' ' + - pad(t.name, nameWidth) + - ' ' + - pad(t.type, typeWidth) + - ' ' + - pad(t.createdFrom, fromWidth) + - ' ' + - pad(t.status, statusWidth) + - ' ' + - t.updatedAt, - ); - - const lines = [header, ...rows]; + const lines = [ + renderTextTable(page.items, TEST_LIST_COLUMNS, { + columns: options.columns, + noHeader: options.noHeader, + }), + ]; if (page.nextToken) lines.push('', `nextToken: ${page.nextToken}`); return lines.join('\n'); } diff --git a/src/lib/text-table.ts b/src/lib/text-table.ts new file mode 100644 index 0000000..bd66b5d --- /dev/null +++ b/src/lib/text-table.ts @@ -0,0 +1,94 @@ +import { localValidationError } from './errors.js'; + +export interface TextTableColumn { + header: string; + width: number | ((rows: readonly T[]) => number); + render: (row: T) => string; +} + +export interface TextTableOptions { + columns?: string; + noHeader?: boolean; + separator?: boolean; +} + +export function renderTextTable( + rows: readonly T[], + columns: readonly TextTableColumn[], + options: TextTableOptions = {}, +): string { + const selected = resolveTextColumns(options.columns, columns); + const widths = measureTextColumns(rows, selected); + const body = rows.map(row => + formatTextTableRow( + selected.map(column => column.render(row)), + widths, + ), + ); + + if (options.noHeader === true) return body.join('\n'); + + const header = formatTextTableRow( + selected.map(column => column.header), + widths, + ); + return [header, ...(options.separator === true ? ['-'.repeat(header.length)] : []), ...body].join( + '\n', + ); +} + +export function resolveTextColumns( + raw: string | undefined, + columns: readonly TextTableColumn[], +): readonly TextTableColumn[] { + if (raw === undefined || raw.trim() === '') return columns; + + const byKey = new Map(columns.map(column => [textColumnKey(column.header), column])); + const validKeys = columns.map(column => textColumnKey(column.header)); + const requested = raw.split(',').map(token => token.trim()); + + if (requested.some(token => token.length === 0)) { + throw localValidationError( + 'columns', + `must be a comma-separated list of: ${validKeys.join(', ')}`, + validKeys, + ); + } + + return requested.map(token => { + const key = textColumnKey(token); + const column = byKey.get(key); + if (column === undefined) { + throw localValidationError( + 'columns', + `unknown column "${token}"; must be one of: ${validKeys.join(', ')}`, + validKeys, + ); + } + return column; + }); +} + +export function measureTextColumns( + rows: readonly T[], + columns: readonly TextTableColumn[], +): number[] { + return columns.map(column => + typeof column.width === 'function' ? column.width(rows) : column.width, + ); +} + +export function formatTextTableRow(values: readonly string[], widths: readonly number[]): string { + return values + .map((value, index) => (index === values.length - 1 ? value : pad(value, widths[index] ?? 0))) + .join(' '); +} + +function textColumnKey(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9]/g, ''); +} + +function pad(value: string, width: number): string { + if (value.length >= width) return value; + return value + ' '.repeat(width - value.length); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 441e700..9d364c1 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -195,6 +195,9 @@ Options: --page-size service page-size hint (1-100, default 25) --starting-token opaque cursor from a previous list response --max-items stop after this many items across auto-paged pages + --columns select/reorder text table columns (comma-separated + keys) + --no-header suppress the text table header row -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -506,6 +509,9 @@ Options: --cursor alias for --starting-token; accepted for parity with \`test result --history\` --max-items stop after this many items across auto-paged pages + --columns select/reorder text table columns (comma-separated + keys) + --no-header suppress the text table header row -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): @@ -613,6 +619,8 @@ Options: --page-size with --history: number of runs per page (1–100, default 20) --cursor with --history: opaque cursor from a prior page + --columns with --history: select/reorder text table columns + --no-header with --history: suppress the text table header row -h, --help display help for command Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): From 326fa97aae538a3069395f7aa06e51a0e5d6df3c Mon Sep 17 00:00:00 2001 From: Lex Date: Sat, 18 Jul 2026 07:02:21 +0700 Subject: [PATCH 075/102] test(test): guard status filter validation order (#222) Co-authored-by: Lexiie <28455136+Lexiie@users.noreply.github.com> --- src/commands/test.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index b6455a2..c8fe5a5 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -470,6 +470,14 @@ describe('runList', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); }); + it('rejects --status=junk before reading credentials', async () => { + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['list', '--project', 'project_alice', '--status', 'junk'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + it('rejects --page-size=0 locally with VALIDATION_ERROR (no network call)', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => { From 5fe0b0012c7a16209caaeee2bb0771ea0da4d39c Mon Sep 17 00:00:00 2001 From: Shalahuddin Al-Ayyubi <147135570+0xshalah@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:03:49 +0700 Subject: [PATCH 076/102] fix(config): normalize empty/whitespace TESTSPRITE_PROFILE to unset (#233) --- src/lib/config.test.ts | 17 +++++++++++++++++ src/lib/config.ts | 10 +++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/lib/config.test.ts b/src/lib/config.test.ts index cf56eaf..2a58fc8 100644 --- a/src/lib/config.test.ts +++ b/src/lib/config.test.ts @@ -55,6 +55,23 @@ describe('loadConfig', () => { ).toBe('option-profile'); }); + it('treats empty TESTSPRITE_PROFILE as unset (falls back to default)', () => { + expect(loadConfig({ env: { TESTSPRITE_PROFILE: '' }, credentialsPath }).profile).toBe( + 'default', + ); + }); + + it('treats whitespace-only TESTSPRITE_PROFILE as unset (falls back to default)', () => { + const config = loadConfig({ env: { TESTSPRITE_PROFILE: ' ' }, credentialsPath }); + expect(config.profile).toBe('default'); + }); + + it('reads credentials from the default profile when TESTSPRITE_PROFILE is blank', () => { + writeProfile('default', { apiKey: 'sk-default' }, { path: credentialsPath }); + const config = loadConfig({ env: { TESTSPRITE_PROFILE: ' ' }, credentialsPath }); + expect(config.apiKey).toBe('sk-default'); + }); + it('option.endpointUrl overrides everything', () => { writeProfile('default', { apiUrl: 'https://file' }, { path: credentialsPath }); const config = loadConfig({ diff --git a/src/lib/config.ts b/src/lib/config.ts index 7f8065f..a69f2e4 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -39,13 +39,17 @@ export function defaultConfigPath(): string { */ export function loadConfig(options: LoadConfigOptions = {}): Config { const env = options.env ?? process.env; - const profile = options.profile ?? env.TESTSPRITE_PROFILE ?? DEFAULT_PROFILE; + const profile = options.profile ?? normalizeEnvVar(env.TESTSPRITE_PROFILE) ?? DEFAULT_PROFILE; const credentialsPath = options.credentialsPath ?? defaultCredentialsPath(); const fileEntry = readProfile(profile, { path: credentialsPath }); // Empty / whitespace-only env vars are treated as unset so they do not - // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` in a shell - // profile). Matches the normalization in auth configure and init/setup. + // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` or + // `export TESTSPRITE_PROFILE=` in a shell profile). For the profile this + // also avoids a confusing VALIDATION_ERROR: an empty name fails the INI + // section-name guard, so without normalization a blank env var would break + // every command instead of falling back to the default profile. Matches the + // normalization in auth configure and init/setup. const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); const envApiKey = normalizeEnvVar(env.TESTSPRITE_API_KEY); From 2708a4069fba72822f94906d05001b4d19e66d9c Mon Sep 17 00:00:00 2001 From: Tasfia Chowdhury Date: Sat, 18 Jul 2026 06:04:01 +0600 Subject: [PATCH 077/102] fix(init): skip API key prompt when credentials already exist (#234) Add --skip-if-configured to testsprite setup (and the deprecated init alias). When the active profile already has a saved API key and no explicit key source (--api-key or --from-env) is given, the interactive prompt is skipped and the existing key is reused. Motivation: re-running setup to refresh the agent skill -- a common pattern in dotfiles, onboarding scripts, and CI bootstraps -- currently always re-prompts for the key, even when credentials were already configured. The flag makes setup idempotent for the credential step. Behaviour: - runConfigure short-circuits with status:already_configured when skipIfConfigured is true and existingProfile.apiKey is present. - runInit relaxes the non-interactive guard and the --output json guard when skipWillApply is true, so the flag works in CI (isTTY=false) and JSON mode without requiring a separate --api-key. - --api-key always overwrites, regardless of --skip-if-configured. - --from-env always overwrites, regardless of --skip-if-configured. - --dry-run is unaffected (no network, no writes; preview only). New tests (8 cases across auth.test.ts and init.test.ts): - skips prompt and returns early when credentials exist (text + JSON) - proceeds to prompt when no credentials exist - allows isTTY=false CI runs when skip applies - --api-key overwrites despite skip flag - --from-env overwrites despite skip flag Closes #206 --- src/commands/auth.test.ts | 87 +++++++++++++++++ src/commands/auth.ts | 27 +++++- src/commands/init.test.ts | 97 +++++++++++++++++++ src/commands/init.ts | 40 +++++++- test/__snapshots__/help.snapshot.test.ts.snap | 28 +++--- 5 files changed, 261 insertions(+), 18 deletions(-) diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index 6f8ce94..0ce0bfb 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -1281,3 +1281,90 @@ describe('createAuthCommand surface', () => { expect(err.exitCode).toBe(3); }); }); + +// --------------------------------------------------------------------------- +// runConfigure -- skipIfConfigured +// --------------------------------------------------------------------------- + +describe('runConfigure -- skipIfConfigured', () => { + it('skips the prompt and returns early when credentials already exist', async () => { + const { capture, deps } = makeCapture(); + // Write a saved key first. + writeProfile('default', { apiKey: 'sk-existing' }, { path: credentialsPath }); + const prompt = { secret: vi.fn(async () => 'sk-new') }; + const fetchImpl = vi.fn(); + + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false, skipIfConfigured: true }, + { + ...deps, + credentialsPath, + prompt, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ); + + // Prompt must never have fired. + expect(prompt.secret).not.toHaveBeenCalled(); + // No network call -- we never validated or wrote a key. + expect(fetchImpl).not.toHaveBeenCalled(); + // The saved key must be untouched. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-existing'); + // Output indicates already_configured. + expect(capture.stdout.join('\n')).toContain('already configured'); + }); + + it('emits already_configured status in JSON mode', async () => { + const { capture, deps } = makeCapture(); + writeProfile('default', { apiKey: 'sk-saved' }, { path: credentialsPath }); + const fetchImpl = vi.fn(); + + await runConfigure( + { profile: 'default', output: 'json', debug: false, fromEnv: false, skipIfConfigured: true }, + { ...deps, credentialsPath, fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'] }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + const parsed = JSON.parse(capture.stdout.join('')); + expect(parsed).toMatchObject({ profile: 'default', status: 'already_configured' }); + }); + + it('proceeds normally when no credentials exist and skipIfConfigured is true', async () => { + const { deps } = makeCapture(); + // No pre-existing profile -- skip has no effect, should fall through to prompt. + const prompt = { secret: vi.fn(async () => 'sk-new') }; + + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false, skipIfConfigured: true }, + { ...deps, credentialsPath, prompt, fetchImpl: meOkFetch }, + ); + + expect(prompt.secret).toHaveBeenCalledTimes(1); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-new'); + }); + + it('ignores skipIfConfigured when --from-env is set', async () => { + const { deps } = makeCapture(); + // Pre-existing key -- but fromEnv should override and write a new one. + writeProfile('default', { apiKey: 'sk-old' }, { path: credentialsPath }); + + await runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + skipIfConfigured: true, + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-from-env' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + + // The env key must overwrite the saved key. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-from-env'); + }); +}); diff --git a/src/commands/auth.ts b/src/commands/auth.ts index e594066..7e60c05 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -71,6 +71,17 @@ type CommonOptions = FactoryCommonOptions; interface ConfigureOptions extends CommonOptions { fromEnv: boolean; + /** + * When true and the active profile already has a saved API key, skip the + * interactive key prompt and proceed directly to the skill-install step. + * A CI-safe flag: lets `setup` run idempotently without prompting on + * machines that already have credentials (e.g. re-running setup to + * refresh the agent skill without re-entering the key). + * + * Ignored when an explicit `--api-key` or `--from-env` key source is + * provided -- those paths always overwrite, regardless of existing state. + */ + skipIfConfigured?: boolean; } const DEFAULT_API_URL = 'https://api.testsprite.com'; @@ -131,9 +142,23 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): apiKey = env.TESTSPRITE_API_KEY?.trim(); if (!apiKey) throw validationError('TESTSPRITE_API_KEY', FROM_ENV_MISSING_KEY); } else { + // --skip-if-configured: when a non-empty API key is already saved for + // this profile, skip the interactive prompt and return early. The + // key is NOT re-validated via GET /me on this path -- the caller + // (runInit) only reaches this when no explicit key source was given, + // and the subsequent whoami call in runInit will surface an expired + // key to the user anyway. + if (opts.skipIfConfigured && existingProfile?.apiKey) { + out.print({ profile: opts.profile, apiUrl, status: 'already_configured' }, data => { + const d = data as { profile: string; apiUrl: string }; + return `Profile "${d.profile}" already configured. Endpoint: ${d.apiUrl}`; + }); + return; + } + const promptApi = deps.prompt ?? { secret: (q: string) => promptSecret(q) }; prelude(`Configuring profile "${opts.profile}".\n`); - // Only the API key is prompted — the endpoint defaults to prod (see above). + // Only the API key is prompted -- the endpoint defaults to prod (see above). apiKey = (await promptApi.secret('TestSprite API key: ')).trim(); if (!apiKey) throw new CLIError('No API key provided.', 5); } diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index 0c1bd8e..92e2095 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -9,6 +9,7 @@ import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ApiError, CLIError } from '../lib/errors.js'; import { resetDryRunBannerForTesting } from '../lib/client-factory.js'; +import { readProfile, writeProfile } from '../lib/credentials.js'; import type { MeResponse } from './auth.js'; import type { AgentFs } from './agent.js'; import type { InitDeps } from './init.js'; @@ -1045,3 +1046,99 @@ describe('runInit — telemetry attribution (X-CLI-Command)', () => { expect(initTagged).toHaveLength(1); }); }); + +// --------------------------------------------------------------------------- +// runInit -- skipIfConfigured +// --------------------------------------------------------------------------- + +describe('runInit -- skipIfConfigured', () => { + it('skips the API key prompt and reuses saved credentials when the profile exists', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + // Write a saved key before running setup. + writeProfile('default', { apiKey: 'sk-saved' }, { path: credentialsPath }); + // Provide a mock fetch that accepts /me so runWhoami (identity banner) succeeds. + const fetchMock = makeOkFetch(); + const prompt = { secret: vi.fn(async () => 'sk-should-never-be-asked') }; + + await runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'json' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + prompt, + }); + + // The prompt must never have fired. + expect(prompt.secret).not.toHaveBeenCalled(); + // The saved key must be untouched. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-saved'); + // The summary must still be emitted. + const parsed = JSON.parse(captured.stdout.join('')) as { status: string }; + expect(parsed.status).toBe('initialized'); + }); + + it('proceeds to prompt when skipIfConfigured is true but no credentials exist', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + // No pre-existing credentials -- skip has no effect. + const fetchMock = makeOkFetch(); + const prompt = { secret: vi.fn(async () => 'sk-fresh') }; + + await runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'text' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: true, + prompt, + }); + + // With no saved key, the prompt should fire. + expect(prompt.secret).toHaveBeenCalledTimes(1); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-fresh'); + expect(captured.stdout.join('')).toContain('initialized'); + }); + + it('allows non-interactive (isTTY=false) when skipIfConfigured is true and credentials exist', async () => { + const { deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + writeProfile('default', { apiKey: 'sk-ci' }, { path: credentialsPath }); + const fetchMock = makeOkFetch(); + + // Must not throw exit 5 for "non-interactive mode, no key source". + await expect( + runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'json' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + }), + ).resolves.toBeUndefined(); + + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-ci'); + }); + + it('--api-key takes precedence over skipIfConfigured and overwrites the saved key', async () => { + const { deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + writeProfile('default', { apiKey: 'sk-old' }, { path: credentialsPath }); + const fetchMock = makeOkFetch(); + + await runInit( + makeBaseOpts({ apiKey: 'sk-new', skipIfConfigured: true, noAgent: true, output: 'text' }), + { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + }, + ); + + // Explicit --api-key must overwrite regardless of skipIfConfigured. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-new'); + }); +}); diff --git a/src/commands/init.ts b/src/commands/init.ts index 6357e3b..659945f 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -90,6 +90,12 @@ interface InitOptions extends CommonOptions { force: boolean; dir?: string; yes: boolean; + /** + * When true and the active profile already has a saved API key, skip the + * interactive key prompt. Forwarded verbatim to runConfigure. Has no + * effect when --api-key or --from-env is also given. + */ + skipIfConfigured?: boolean; /** Set by the command action when both --agent and --no-agent appear in rawArgs. */ rawArgConflict?: boolean; } @@ -196,9 +202,16 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise, --from-env (reads TESTSPRITE_API_KEY), or run interactively.', @@ -208,7 +221,7 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise or --from-env.', @@ -223,7 +236,13 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise', 'Project root for the skill install (default: current directory)') - .option('-y, --yes', 'Non-interactive: accept all defaults, never prompt'); + .option('-y, --yes', 'Non-interactive: accept all defaults, never prompt') + .option( + '--skip-if-configured', + 'Skip the API key prompt when credentials already exist for this profile (CI-safe idempotent re-run)', + ); } /** Build {@link InitOptions} from raw Commander opts + globals. */ @@ -543,6 +572,7 @@ function buildSetupOptions( force: Boolean(cmdOpts.force), dir: cmdOpts.dir, yes: Boolean(cmdOpts.yes), + skipIfConfigured: Boolean(cmdOpts.skipIfConfigured), rawArgConflict, }; } diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 9d364c1..4724064 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -112,18 +112,22 @@ exports[`--help snapshots > init 1`] = ` (deprecated) alias for \`setup\` Options: - --api-key API key to configure (skips the interactive prompt) - --from-env Read TESTSPRITE_API_KEY from the environment instead of - prompting (default: false) - --agent Coding-agent target to install: claude, antigravity, - cursor, cline, kiro, windsurf, copilot, codex (default: - claude) (default: "claude") - --no-agent Skip the agent skill install (configure credentials only) - --force Overwrite an existing skill file (a .bak backup is kept) - --dir Project root for the skill install (default: current - directory) - -y, --yes Non-interactive: accept all defaults, never prompt - -h, --help display help for command + --api-key API key to configure (skips the interactive prompt) + --from-env Read TESTSPRITE_API_KEY from the environment instead of + prompting (default: false) + --agent Coding-agent target to install: claude, antigravity, + cursor, cline, kiro, windsurf, copilot, codex (default: + claude) (default: "claude") + --no-agent Skip the agent skill install (configure credentials + only) + --force Overwrite an existing skill file (a .bak backup is + kept) + --dir Project root for the skill install (default: current + directory) + -y, --yes Non-interactive: accept all defaults, never prompt + --skip-if-configured Skip the API key prompt when credentials already exist + for this profile (CI-safe idempotent re-run) + -h, --help display help for command " `; From 7975f6ff510b47ad0d5006bb91fc99a839798f16 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 04:52:53 +0200 Subject: [PATCH 078/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index a6f68d3..ba0d84f 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4819,6 +4819,48 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { }); }); +// --------------------------------------------------------------------------- +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty +// --------------------------------------------------------------------------- +describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } + return errorBody('NOT_FOUND'); + }); + + const stdoutLines: string[] = []; + const err = await runTestRerun( + { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false }, + fetchImpl, + (line) => stdoutLines.push(line) + ); + + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + }); +}); + // --------------------------------------------------------------------------- // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- From 51af9393d0671acc6824cb5dbecafd4115060fc8 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 04:58:55 +0200 Subject: [PATCH 079/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index ba0d84f..e275b48 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4824,6 +4824,7 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { // --------------------------------------------------------------------------- describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const creds = makeCreds(); const batchResp: BatchRerunResponse = { accepted: [ { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, @@ -4864,7 +4865,6 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol // --------------------------------------------------------------------------- // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- - describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); From 7dd2576a1f07bc9fe0370b000b7aca3792cd6c9d Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:05:11 +0300 Subject: [PATCH 080/102] Add newline at end of test.rerun.spec.ts Fix missing newline at end of file. From 15ee2dccc53280b269eda2ceefebd1539865858d Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:09:33 +0300 Subject: [PATCH 081/102] Add newline at end of test.rerun.spec.ts Fix missing newline at end of file in test.rerun.spec.ts From 770cd4788de000559689f39a4ca8c3ce8e6c1911 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:16:06 +0300 Subject: [PATCH 082/102] Refactor runTestRerun call with new parameters --- src/commands/test.rerun.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index e275b48..672d270 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4848,9 +4848,8 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const stdoutLines: string[] = []; const err = await runTestRerun( { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false }, - fetchImpl, - (line) => stdoutLines.push(line) - ); + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (line) => stdoutLines.push(line), stderr: () => undefined } + ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { From f7120c25f34a54fda7f487f43e3533d98b0ca3d3 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:22:08 +0300 Subject: [PATCH 083/102] Add newline at end of test.rerun.spec.ts Fix missing newline at end of file in test.rerun.spec.ts From 0b34b1adc1d984d04cfb37bcf1d8b08b83b585c1 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:32:14 +0300 Subject: [PATCH 084/102] Add newline at end of test.rerun.spec.ts Fix missing newline at end of file in test.rerun.spec.ts From 5e2bf88cbed0b214736ac7da152bc25298f849a0 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:26:01 +0200 Subject: [PATCH 085/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 672d270..4428f8f 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4847,7 +4847,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const stdoutLines: string[] = []; const err = await runTestRerun( - { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false }, + { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false maxConcurrency: 1, profile: 'default', output: 'json'}, { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (line) => stdoutLines.push(line), stderr: () => undefined } ).catch(e => e); From 98f9393df3d16c0998ca43bddc15349319008627 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:35:05 +0300 Subject: [PATCH 086/102] Refactor test parameters for runTestRerun function --- src/commands/test.rerun.spec.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 4428f8f..8958606 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4847,8 +4847,26 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const stdoutLines: string[] = []; const err = await runTestRerun( - { testIds: ['test_1', 'test_2'], all: false, wait: true, timeoutSeconds: 60, autoHeal: false, autoHealExplicit: false, skipDependencies: false maxConcurrency: 1, profile: 'default', output: 'json'}, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (line) => stdoutLines.push(line), stderr: () => undefined } + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 60, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 1, + profile: 'default', + output: 'json', + debug: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); From c4bef009e228fe810e832d2e922749568463de09 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:44:11 +0300 Subject: [PATCH 087/102] Fix formatting and comments in test.rerun.spec.ts Fixed formatting issues by adding a missing comma and clarifying comments. --- src/commands/test.rerun.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 8958606..3ff9859 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4855,7 +4855,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 1, + maxConcurrency: 1, // Fixed: Added comma and missing fields profile: 'default', output: 'json', debug: false, @@ -4864,7 +4864,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), + stdout: line => stdoutLines.push(line), stderr: () => undefined, }, ).catch(e => e); From 0a3bfe352c8ffc47c603cd4b8fda7ba312e0d0b2 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:50:13 +0200 Subject: [PATCH 088/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 3ff9859..0fdde9f 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4855,18 +4855,18 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 1, // Fixed: Added comma and missing fields + maxConcurrency: 1, // تأكدت من وجود الفواصل هنا profile: 'default', output: 'json', - debug: false, + debug: false }, { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: line => stdoutLines.push(line), - stderr: () => undefined, - }, + stderr: () => undefined + } ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); From 9a6a48dec23600fb964d9f7ef69d73a61cf9e2f0 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:54:04 +0200 Subject: [PATCH 089/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 0fdde9f..2e97da3 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4855,7 +4855,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 1, // تأكدت من وجود الفواصل هنا + maxConcurrency: 1, profile: 'default', output: 'json', debug: false From 19376059eba9f303a15ff094b4b33e6de07774fc Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:58:11 +0200 Subject: [PATCH 090/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 55 +++++++++------------------------ 1 file changed, 14 insertions(+), 41 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 2e97da3..fd39488 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4855,7 +4855,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol autoHeal: false, autoHealExplicit: false, skipDependencies: false, - maxConcurrency: 1, + maxConcurrency: 1, profile: 'default', output: 'json', debug: false @@ -4870,12 +4870,9 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')) as { - accepted: Array<{ testId: string; runId: string; status: string }>; - }; + const parsed = JSON.parse(stdoutLines.join('\n')); expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + expect(parsed.accepted.map((r: any) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); }); }); @@ -4885,39 +4882,20 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); - const rerunResp = makeFeRerunResp(); + const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - let fetchCallCount = 0; - const fetchImpl: typeof globalThis.fetch = async (input, _init) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL - ? input.toString() - : (input as { url: string }).url; - fetchCallCount++; - if (url.includes('/tests/test_fe_01/runs/rerun')) { - return new Response(JSON.stringify(rerunResp), { - status: 202, - headers: { 'content-type': 'application/json' }, - }); + const fetchImpl: any = async (input: any) => { + const url = typeof input === 'string' ? input : input.url; + if (url.includes('/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { status: 202 }); } if (url.includes('/runs/')) { - const runningRun: RunResponse = { - ...makeTerminalRun(rerunResp.runId, 'passed'), - status: 'running', - finishedAt: null, - }; - return new Response(JSON.stringify(runningRun), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); + return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); } return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); }; const stdoutLines: string[] = []; - const err = await runTestRerun( { testIds: ['test_fe_01'], @@ -4930,25 +4908,20 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t maxConcurrency: 10, output: 'json', profile: 'default', - dryRun: false, - debug: false, - verbose: false, + debug: false }, { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: line => stdoutLines.push(line), - stderr: () => undefined, - }, + stderr: () => undefined + } ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); - expect(stdoutLines.length).toBeGreaterThan(0); - const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + const parsed = JSON.parse(stdoutLines.join('\n')); expect(parsed.runId).toBe(rerunResp.runId); expect(parsed.status).toBe('running'); - - void fetchCallCount; }); -}); +}); \ No newline at end of file From f6aea0e2d948f15a6c3d30c6d90f9ad10e0e6aee Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:23:29 +0200 Subject: [PATCH 091/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 137 +++----------------------------- 1 file changed, 12 insertions(+), 125 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index e08b850..396e84b 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4819,7 +4819,6 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); - // --------------------------------------------------------------------------- // Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- @@ -4829,23 +4828,17 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const batchResp: BatchRerunResponse = { accepted: [ { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' } ], deferred: [], conflicts: [], - closure: { byProject: [] }, + closure: { byProject: [] } }; - - const fetchImpl = makeFetch(url => { - if (url.includes('/tests/batch/rerun')) { - return { status: 202, body: batchResp }; - } - if (url.includes('/runs/')) { - throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); - } + const fetchImpl = makeFetch((url) => { + if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; + if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch'); return errorBody('NOT_FOUND'); }); - const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4861,19 +4854,9 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol output: 'json', debug: false }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: line => stdoutLines.push(line), - stderr: () => undefined - } - ).catch(e => e); - + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as any, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')); - expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map((r: any) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); }); }); @@ -4884,18 +4867,12 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - const fetchImpl: any = async (input: any) => { const url = typeof input === 'string' ? input : input.url; - if (url.includes('/runs/rerun')) { - return new Response(JSON.stringify(rerunResp), { status: 202 }); - } - if (url.includes('/runs/')) { - return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); - } + if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); + if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); }; - const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4903,59 +4880,6 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t all: false, wait: true, timeoutSeconds: 0, -// DEV-331 piece 1 — graceful detach during batch rerun --wait (SIG-6) -// --------------------------------------------------------------------------- - -describe('R-BAT: batch rerun --wait — InterruptError partial lists all dispatched runIds (DEV-331)', () => { - it('interrupt mid fan-out → stdout partial covers every accepted runId, honest stderr, exit 130', async () => { - const creds = makeCreds(); - const shutdown = new ShutdownController(); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - // Batch trigger resolves; every run poll hangs until the composed signal aborts. - const fetchImpl: FetchImpl = (async (input: unknown, init: RequestInit = {}) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL - ? input.toString() - : (input as { url: string }).url; - if (url.includes('/tests/batch/rerun')) { - return new Response(JSON.stringify(batchResp), { - status: 202, - headers: { 'content-type': 'application/json' }, - }); - } - return new Promise((_resolve, reject) => { - const signal = init.signal; - const rejectWithReason = (): void => { - const reason: unknown = signal?.reason; - reject(reason instanceof Error ? reason : new Error('aborted')); - }; - if (signal?.aborted) { - rejectWithReason(); - return; - } - signal?.addEventListener('abort', rejectWithReason, { once: true }); - }); - }) as FetchImpl; - - const stdoutLines: string[] = []; - const stderrLines: string[] = []; - const pending = runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 600, autoHeal: false, autoHealExplicit: false, skipDependencies: false, @@ -4963,49 +4887,12 @@ describe('R-BAT: batch rerun --wait — InterruptError partial lists all dispatc output: 'json', profile: 'default', debug: false - dryRun: false, - debug: false, - verbose: false, }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: line => stdoutLines.push(line), - stderr: () => undefined - } - ).catch(e => e); - + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as any, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')); expect(parsed.runId).toBe(rerunResp.runId); expect(parsed.status).toBe('running'); }); -}); - fetchImpl, - stdout: line => stdoutLines.push(line), - stderr: line => stderrLines.push(line), - shutdown, - }, - ); - setTimeout(() => shutdown.interrupt('SIGINT'), 10); - - const err = await pending.catch(e => e); - expect(err).toBeInstanceOf(InterruptError); - expect((err as InterruptError).exitCode).toBe(130); - - // SIG-6: the partial lists ALL dispatched runIds, marked running. - const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { - accepted: Array<{ runId: string; status: string }>; - }; - const byRunId = new Map(stdoutJson.accepted.map(r => [r.runId, r.status])); - expect(byRunId.get('run_b1')).toBe('running'); - expect(byRunId.get('run_b2')).toBe('running'); - - const stderrBlock = stderrLines.join('\n'); - expect(stderrBlock).toContain('Interrupted (SIGINT)'); - expect(stderrBlock).toContain('billing'); - expect(stderrBlock).toContain('run_b1'); - expect(stderrBlock).toContain('run_b2'); - }); -}); +}); \ No newline at end of file From ad0dd10d31706135185314654ecf242085df0959 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:32:16 +0200 Subject: [PATCH 092/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 396e84b..2c1b7ba 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4819,6 +4819,7 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); + // --------------------------------------------------------------------------- // Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- @@ -4854,7 +4855,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol output: 'json', debug: false }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as any, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); }); @@ -4867,8 +4868,8 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - const fetchImpl: any = async (input: any) => { - const url = typeof input === 'string' ? input : input.url; + const fetchImpl: FetchImpl = async (input) => { + const url = typeof input === 'string' ? input : (input as Request).url; if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); @@ -4888,10 +4889,10 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t profile: 'default', debug: false }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as any, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; expect(parsed.runId).toBe(rerunResp.runId); expect(parsed.status).toBe('running'); }); From 817781274d798d0bdf3a15b822367c897eb4c42d Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:35:15 +0200 Subject: [PATCH 093/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 58 ++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 2c1b7ba..f569b61 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4829,17 +4829,19 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const batchResp: BatchRerunResponse = { accepted: [ { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' } + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, ], deferred: [], conflicts: [], - closure: { byProject: [] } + closure: { byProject: [] }, }; + const fetchImpl = makeFetch((url) => { if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch'); + if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); return errorBody('NOT_FOUND'); }); + const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4853,11 +4855,24 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol maxConcurrency: 1, profile: 'default', output: 'json', - debug: false + debug: false, }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + } ).catch((e) => e); + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted?: Array<{ testId?: string; runId: string; status?: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); }); }); @@ -4865,15 +4880,27 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { - it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + it('exit 7 AND stdout contains {runId, status:\"running\"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - const fetchImpl: FetchImpl = async (input) => { - const url = typeof input === 'string' ? input : (input as Request).url; - if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); - if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); + + const fetchImpl: FetchImpl = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + + if (url.includes('/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { status: 202 }); + } + if (url.includes('/runs/')) { + return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); + } return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); }; + const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4887,10 +4914,17 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t maxConcurrency: 10, output: 'json', profile: 'default', - debug: false + debug: false, }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + } ).catch((e) => e); + expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; expect(parsed.runId).toBe(rerunResp.runId); From bdec093a27edfe1eb75e8f7c1be0186ee11af3ca Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:40:33 +0200 Subject: [PATCH 094/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 58 +++++++-------------------------- 1 file changed, 12 insertions(+), 46 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index f569b61..2c1b7ba 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4829,19 +4829,17 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol const batchResp: BatchRerunResponse = { accepted: [ { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' } ], deferred: [], conflicts: [], - closure: { byProject: [] }, + closure: { byProject: [] } }; - const fetchImpl = makeFetch((url) => { if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch'); return errorBody('NOT_FOUND'); }); - const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4855,24 +4853,11 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol maxConcurrency: 1, profile: 'default', output: 'json', - debug: false, + debug: false }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), - stderr: () => undefined, - } + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } ).catch((e) => e); - expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')) as { - accepted?: Array<{ testId?: string; runId: string; status?: string }>; - }; - expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); }); }); @@ -4880,27 +4865,15 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { - it('exit 7 AND stdout contains {runId, status:\"running\"} when --timeout polling deadline is exceeded', async () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { const creds = makeCreds(); const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - - const fetchImpl: FetchImpl = async (input, _init) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL - ? input.toString() - : (input as { url: string }).url; - - if (url.includes('/runs/rerun')) { - return new Response(JSON.stringify(rerunResp), { status: 202 }); - } - if (url.includes('/runs/')) { - return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); - } + const fetchImpl: FetchImpl = async (input) => { + const url = typeof input === 'string' ? input : (input as Request).url; + if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); + if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); }; - const stdoutLines: string[] = []; const err = await runTestRerun( { @@ -4914,17 +4887,10 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t maxConcurrency: 10, output: 'json', profile: 'default', - debug: false, + debug: false }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), - stderr: () => undefined, - } + { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } ).catch((e) => e); - expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; expect(parsed.runId).toBe(rerunResp.runId); From 6b080ac62e829b848c36b6eeb12618aae20b064e Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 07:44:37 +0200 Subject: [PATCH 095/102] Update test.rerun.spec.ts --- src/commands/test.rerun.spec.ts | 199 +++++++++++++++++++++----------- 1 file changed, 132 insertions(+), 67 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 2c1b7ba..966d16b 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4823,77 +4823,142 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { // --------------------------------------------------------------------------- // Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- -describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { - it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { - const creds = makeCreds(); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' } - ], - deferred: [], - conflicts: [], - closure: { byProject: [] } - }; - const fetchImpl = makeFetch((url) => { - if (url.includes('/tests/batch/rerun')) return { status: 202, body: batchResp }; - if (url.includes('/runs/')) throw new RequestTimeoutError(120000, 'req_timeout_batch'); - return errorBody('NOT_FOUND'); - }); - const stdoutLines: string[] = []; - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 60, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 1, - profile: 'default', - output: 'json', - debug: false +describe( + '[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', + () => { + it( + 'stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', + async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch((url) => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } + return errorBody('NOT_FOUND'); + }); + + const stdoutLines: string[] = []; + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 60, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 1, + profile: 'default', + output: 'json', + debug: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch((e) => e); + + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } - ).catch((e) => e); - expect(err).toMatchObject({ exitCode: 7 }); - }); -}); + ); + }, +); // --------------------------------------------------------------------------- // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { - it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { - const creds = makeCreds(); - const rerunResp = { runId: 'run_fe_01', status: 'accepted' }; - const fetchImpl: FetchImpl = async (input) => { - const url = typeof input === 'string' ? input : (input as Request).url; - if (url.includes('/runs/rerun')) return new Response(JSON.stringify(rerunResp), { status: 202 }); - if (url.includes('/runs/')) return new Response(JSON.stringify({ runId: rerunResp.runId, status: 'running', finishedAt: null }), { status: 200 }); - return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); - }; - const stdoutLines: string[] = []; - const err = await runTestRerun( - { - testIds: ['test_fe_01'], - all: false, - wait: true, - timeoutSeconds: 0, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 10, - output: 'json', - profile: 'default', - debug: false - }, - { ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, stdout: (l) => stdoutLines.push(l), stderr: () => undefined } - ).catch((e) => e); - expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; - expect(parsed.runId).toBe(rerunResp.runId); - expect(parsed.status).toBe('running'); - }); + it( + 'exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', + async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + let fetchCallCount = 0; + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + fetchCallCount++; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + const runningRun: RunResponse = { + ...makeTerminalRun(rerunResp.runId, 'passed'), + status: 'running', + finishedAt: null, + }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 0, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch((e) => e); + + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(parsed.runId).toBe(rerunResp.runId); + expect(parsed.status).toBe('running'); + + void fetchCallCount; + }, + ); }); \ No newline at end of file From 44501e068a1c0bb655f47e316147bed1fd203ac9 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 08:51:58 +0300 Subject: [PATCH 096/102] Refactor batch rerun tests for RequestTimeoutError --- src/commands/test.rerun.spec.ts | 241 +++++++++++++++----------------- 1 file changed, 116 insertions(+), 125 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 966d16b..ea05c9a 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4823,142 +4823,133 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { // --------------------------------------------------------------------------- // Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty // --------------------------------------------------------------------------- -describe( - '[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', - () => { - it( - 'stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', - async () => { - const creds = makeCreds(); - const batchResp: BatchRerunResponse = { - accepted: [ - { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, - ], - deferred: [], - conflicts: [], - closure: { byProject: [] }, - }; - - const fetchImpl = makeFetch((url) => { - if (url.includes('/tests/batch/rerun')) { - return { status: 202, body: batchResp }; - } - if (url.includes('/runs/')) { - throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); - } - return errorBody('NOT_FOUND'); - }); +describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; - const stdoutLines: string[] = []; - const err = await runTestRerun( - { - testIds: ['test_1', 'test_2'], - all: false, - wait: true, - timeoutSeconds: 60, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 1, - profile: 'default', - output: 'json', - debug: false, - }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), - stderr: () => undefined, - }, - ).catch((e) => e); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } + return errorBody('NOT_FOUND'); + }); - expect(err).toMatchObject({ exitCode: 7 }); - const parsed = JSON.parse(stdoutLines.join('\n')) as { - accepted: Array<{ testId: string; runId: string; status: string }>; - }; - expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); + const stdoutLines: string[] = []; + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 60, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 1, + profile: 'default', + output: 'json', + debug: false, }, - ); - }, -); + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch(e => e); + + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + }); +}); // --------------------------------------------------------------------------- // TimeoutError on single FE rerun --wait: partial stdout + exit 7 // --------------------------------------------------------------------------- describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { - it( - 'exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', - async () => { - const creds = makeCreds(); - const rerunResp = makeFeRerunResp(); - - let fetchCallCount = 0; - const fetchImpl: typeof globalThis.fetch = async (input, _init) => { - const url = - typeof input === 'string' - ? input - : input instanceof URL + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + let fetchCallCount = 0; + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL ? input.toString() : (input as { url: string }).url; - fetchCallCount++; - if (url.includes('/tests/test_fe_01/runs/rerun')) { - return new Response(JSON.stringify(rerunResp), { - status: 202, - headers: { 'content-type': 'application/json' }, - }); - } - if (url.includes('/runs/')) { - const runningRun: RunResponse = { - ...makeTerminalRun(rerunResp.runId, 'passed'), - status: 'running', - finishedAt: null, - }; - return new Response(JSON.stringify(runningRun), { - status: 200, - headers: { 'content-type': 'application/json' }, - }); - } - return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); - }; + fetchCallCount++; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + const runningRun: RunResponse = { + ...makeTerminalRun(rerunResp.runId, 'passed'), + status: 'running', + finishedAt: null, + }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; - const stdoutLines: string[] = []; + const stdoutLines: string[] = []; - const err = await runTestRerun( - { - testIds: ['test_fe_01'], - all: false, - wait: true, - timeoutSeconds: 0, - autoHeal: false, - autoHealExplicit: false, - skipDependencies: false, - maxConcurrency: 10, - output: 'json', - profile: 'default', - dryRun: false, - debug: false, - verbose: false, - }, - { - ...creds, - sleep: instantSleep, - fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), - stderr: () => undefined, - }, - ).catch((e) => e); + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 0, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch(e => e); - expect(err).toMatchObject({ exitCode: 7 }); - expect(stdoutLines.length).toBeGreaterThan(0); - const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; - expect(parsed.runId).toBe(rerunResp.runId); - expect(parsed.status).toBe('running'); + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(parsed.runId).toBe(rerunResp.runId); + expect(parsed.status).toBe('running'); - void fetchCallCount; - }, - ); -}); \ No newline at end of file + void fetchCallCount; + }); +}); From 7774c29dff80bfc0edc94fa89176500bdcfe1dc1 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 09:02:38 +0300 Subject: [PATCH 097/102] Refactor test.rerun.spec.ts imports Removed InterruptError and ShutdownController imports. --- src/commands/test.rerun.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index ea05c9a..a35cfd0 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -9,8 +9,8 @@ import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; -import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; -import { ShutdownController } from '../lib/interrupt.js'; +import { ApiError, RequestTimeoutError } from '../lib/errors.js'; + import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; import type { FetchImpl } from '../lib/http.js'; import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; From 73441e43e4b54c56bec0ca7671bd925a8f27826b Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 08:27:38 +0200 Subject: [PATCH 098/102] Refactor fetch implementation and error handling --- src/commands/test.rerun.spec.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index a35cfd0..81283d9 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -10,7 +10,6 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { ApiError, RequestTimeoutError } from '../lib/errors.js'; - import type { RunResponse, RerunResponse, BatchRerunResponse } from '../lib/runs.types.js'; import type { FetchImpl } from '../lib/http.js'; import { runTestRerun, resolveWaitRequestTimeoutMs } from './test.js'; @@ -4836,7 +4835,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol closure: { byProject: [] }, }; - const fetchImpl = makeFetch(url => { + const fetchImpl = makeFetch((url) => { if (url.includes('/tests/batch/rerun')) { return { status: 202, body: batchResp }; } @@ -4868,15 +4867,15 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol stdout: (line) => stdoutLines.push(line), stderr: () => undefined, }, - ).catch(e => e); + ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { accepted: Array<{ testId: string; runId: string; status: string }>; }; expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); }); }); @@ -4939,10 +4938,10 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: line => stdoutLines.push(line), + stdout: (line) => stdoutLines.push(line), stderr: () => undefined, }, - ).catch(e => e); + ).catch((e) => e); expect(err).toMatchObject({ exitCode: 7 }); expect(stdoutLines.length).toBeGreaterThan(0); From 0081f3ff4f0a4820b7a3a4f9324f7fd15db7f509 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 08:41:26 +0200 Subject: [PATCH 099/102] Fix formatting --- src/commands/test.rerun.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 81283d9..e018e88 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -2889,7 +2889,7 @@ describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', ...creds, sleep: instantSleep, fetchImpl, - stdout: () => {}, + stdout: () => { }, stderr: line => stderrLines.push(line), }, ); @@ -2945,7 +2945,7 @@ describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', ...creds, sleep: instantSleep, fetchImpl, - stdout: () => {}, + stdout: () => { }, stderr: line => stderrLines.push(line), }, ); @@ -3177,7 +3177,7 @@ describe('runTestRerun --all --skip-terminal (dogfood L1796)', () => { ...creds, sleep: instantSleep, fetchImpl: makeFilterFetch(dispatched), - stderr: () => {}, + stderr: () => { }, }, ); From c286766a346e65fa7cd327f0392901607ed954fc Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 06:48:05 +0000 Subject: [PATCH 100/102] Fix Prettier formatting strictly --- package-lock.json | 4 ++-- src/commands/test.rerun.spec.ts | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4461016..f9afe03 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.3.0", + "version": "0.4.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index e018e88..c4e865a 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -2889,7 +2889,7 @@ describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', ...creds, sleep: instantSleep, fetchImpl, - stdout: () => { }, + stdout: () => {}, stderr: line => stderrLines.push(line), }, ); @@ -2945,7 +2945,7 @@ describe('[fix-1] batch rerun: notFound[] ids aggregated and warned on stderr', ...creds, sleep: instantSleep, fetchImpl, - stdout: () => { }, + stdout: () => {}, stderr: line => stderrLines.push(line), }, ); @@ -3177,7 +3177,7 @@ describe('runTestRerun --all --skip-terminal (dogfood L1796)', () => { ...creds, sleep: instantSleep, fetchImpl: makeFilterFetch(dispatched), - stderr: () => { }, + stderr: () => {}, }, ); @@ -4835,7 +4835,7 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol closure: { byProject: [] }, }; - const fetchImpl = makeFetch((url) => { + const fetchImpl = makeFetch(url => { if (url.includes('/tests/batch/rerun')) { return { status: 202, body: batchResp }; } @@ -4864,18 +4864,18 @@ describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out pol ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), + stdout: line => stdoutLines.push(line), stderr: () => undefined, }, - ).catch((e) => e); + ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); const parsed = JSON.parse(stdoutLines.join('\n')) as { accepted: Array<{ testId: string; runId: string; status: string }>; }; expect(parsed.accepted).toHaveLength(2); - expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); - expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); }); }); @@ -4938,10 +4938,10 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t ...creds, sleep: instantSleep, fetchImpl: fetchImpl as unknown as FetchImpl, - stdout: (line) => stdoutLines.push(line), + stdout: line => stdoutLines.push(line), stderr: () => undefined, }, - ).catch((e) => e); + ).catch(e => e); expect(err).toMatchObject({ exitCode: 7 }); expect(stdoutLines.length).toBeGreaterThan(0); From 774db845970a9e4826ff9823c8e6a343a0a0f049 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 09:26:20 +0200 Subject: [PATCH 101/102] Re-run failed jobs --- src/commands/test.rerun.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index c4e865a..9f1f4ef 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4952,3 +4952,4 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t void fetchCallCount; }); }); + \ No newline at end of file From 943b76198ab486b07567b7531ac0daff8d984933 Mon Sep 17 00:00:00 2001 From: Muath Awad <94539921+Awad-de@users.noreply.github.com> Date: Sat, 18 Jul 2026 09:27:37 +0200 Subject: [PATCH 102/102] Re-run failed jobs --- src/commands/test.rerun.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 9f1f4ef..c4e865a 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -4952,4 +4952,3 @@ describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON t void fetchCallCount; }); }); - \ No newline at end of file