diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..468c8ed --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Check out all text files with LF on every platform. Tests compare bytes +# from checked-out files (skill templates, snapshots); a CRLF working tree +# (core.autocrlf on Windows) broke those comparisons. +* text=auto eol=lf + +*.png binary diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c490f9b..1c165f6 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,7 +12,9 @@ agree on the approach before you invest time — see CONTRIBUTING.md. ## Related issue - + ## Type of change diff --git a/.github/workflows/backport-dispatch.yml b/.github/workflows/backport-dispatch.yml new file mode 100644 index 0000000..8b6a99a --- /dev/null +++ b/.github/workflows/backport-dispatch.yml @@ -0,0 +1,83 @@ +# backport-dispatch.yml — OPTIONAL latency optimization for DEV-352's +# auto-backport bot. Authored in atlas (release/public path), ships to the +# PUBLIC repo via the snapshot per invariant I0 — this workflow only ever +# runs on the public repo, never on atlas (see the repo guard below). +# +# NOT YET ARMED (2026-07-15): the atlas-side auto-backport.yml has a +# scheduled SWEEP (every 30 min) that scans merged public PRs anonymously +# and backports anything unabsorbed — the whole pipeline is fully +# functional with ZERO public-repo credentials via that path alone. This +# workflow instantly notifies atlas the moment a PR merges instead of +# waiting for the next sweep (seconds vs. up to 30 minutes) — a nice-to-have, +# not a requirement. +# +# Arming it means placing a GitHub App credential with contents:write on +# the PRIVATE atlas repo into THIS PUBLIC repo's secrets — a real trust +# decision (a compromised public-repo secret store would let an attacker +# push to atlas) that is deliberately left to the operator, not decided by +# this file. Until TESTSPRITE_HOB_APP_ID/TESTSPRITE_HOB_PRIVATE_KEY (or the +# ALFHEIM_AGENT_* fallback) exist on the PUBLIC repo with contents:write on +# atlas, this workflow no-ops cleanly (see the "not armed" step) — it does +# NOT fail loud, so it stays quiet for every contributor watching the +# Actions tab until the operator decides to enable it. +# +# SECURITY: this workflow reads ONLY event metadata (PR number, merge SHA, +# base ref) — it never checks out the merged PR's code. That is what makes +# `pull_request_target` safe here: the base-repo context (secrets, a +# write-capable token) is required because a fork-originated merged PR gets +# ZERO secrets under a plain `pull_request` trigger, even for the `closed` +# activity type — but nothing in this job ever runs untrusted fork code, so +# the classic pull_request_target RCE/secret-exfiltration risk (checking +# out and executing the fork's own head ref under a privileged context) +# does not apply. Do not add actions/checkout to this workflow. +name: Notify atlas of a merged public PR + +on: + pull_request_target: + types: [closed] + +permissions: + contents: read + +jobs: + notify: + if: github.repository == 'TestSprite/testsprite-cli' && github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + owner: TestSprite + repositories: testsprite-cli-atlas + # Minimum required for POST /repos/{owner}/{repo}/dispatches (verified + # against docs.github.com/en/rest/using-the-rest-api/permissions-required-for-github-apps — + # the endpoint is gated on Contents:write, not Contents:read). + permission-contents: write + + - name: Dispatch to atlas + if: steps.app-token.outputs.token != '' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + # These two fields are fork-influenced (merge_commit_sha/base.ref come + # from a merged PR's event payload) — routed through env instead of + # spliced into the run: string to avoid the GHA script-injection class + # (a maliciously-crafted value could otherwise break out of the -F + # argument and inject arbitrary shell). PR_NUMBER is left inline: it is + # a GitHub-assigned integer, not attacker-authorable content. + MERGE_SHA: ${{ github.event.pull_request.merge_commit_sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + gh api repos/TestSprite/testsprite-cli-atlas/dispatches \ + -f event_type=public-pr-merged \ + -F 'client_payload[pr_number]=${{ github.event.pull_request.number }}' \ + -F "client_payload[merge_commit_sha]=$MERGE_SHA" \ + -F "client_payload[base_ref]=$BASE_REF" + + - name: Not armed yet — the atlas sweep will pick this up + if: steps.app-token.outputs.token == '' + run: | + echo "::notice::No cross-repo credential configured on this repo yet — atlas's scheduled sweep (auto-backport.yml, every 30 min) will pick up this merge instead of an instant dispatch. See DEV-352 / DEV-348 / DEV-350 for the operator decision to arm this." diff --git a/.github/workflows/ci-nudge.yml b/.github/workflows/ci-nudge.yml new file mode 100644 index 0000000..bd6f9ad --- /dev/null +++ b/.github/workflows/ci-nudge.yml @@ -0,0 +1,149 @@ +# "Fix this and we merge" nudge (InsForge `agent-zhang-beihai` pattern, part 3). +# When a PR's CI finishes red, posts ONE sticky comment listing exactly which +# jobs failed and the local one-liner that reproduces/fixes each (the automated +# version of the hand-written "run `npm run format` and you're green" review +# comments). The comment flips to a green confirmation once all checks pass. +# +# Runs in base-repo context via workflow_run — no PR code is checked out, so +# fork PRs are safe. State is recomputed from check-runs each time, so the two +# CI workflows (CI + Test Coverage) can complete in any order. +# +# Bot identity: same App-token-first / GITHUB_TOKEN-fallback pattern as +# pr-triage.yml. The App has had the "Pull requests: Read & write" permission +# since 2026-07-02 (corrected 2026-07-15; this comment previously said the +# permission was still pending) — the fallback path stays as defense-in-depth +# for whenever the App/secrets are absent. +name: CI failure nudge + +on: + workflow_run: + workflows: ['CI', 'Test Coverage'] + types: [completed] + +permissions: + checks: read # read the head SHA's check-run state + pull-requests: write + issues: write # PR comments ride the issues API + +env: + MARKER: '' + +jobs: + nudge: + # Public repo only; PR-triggered runs only (pushes to main have no PR to nudge). + if: >- + github.repository == 'TestSprite/testsprite-cli' && + github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # The script below only ever calls the App client (`appClient`) for + # rest.issues.updateComment/createComment — PR comments ride the + # issues API (see the header comment above). All reads (checks, + # pulls, commit->PR lookup) go through the ambient `github` client, + # governed by this workflow's top-level `permissions:` block, not + # this minted token. Scoping to checks:read/pull-requests:write here + # (matching the job's top-level block literally) would risk the mint + # itself failing if the App's installation doesn't hold Checks + # permission — which would silently drop the App-token path + # entirely (continue-on-error swallows the failure) and always fall + # back to github-actions[bot], defeating this bot's own purpose. + permission-issues: write + + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + with: + script: | + const { owner, repo } = context.repo; + const run = context.payload.workflow_run; + const marker = process.env.MARKER; + + const appClient = process.env.APP_TOKEN + ? new (github.constructor)({ auth: process.env.APP_TOKEN }) + : null; + async function write(fn) { + if (appClient) { + try { return await fn(appClient.rest); } + catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } + } + return await fn(github.rest); + } + + // Resolve the PR for this run. `workflow_run.pull_requests` is empty + // for fork PRs, so fall back to the commit→PRs lookup. + let pr = (run.pull_requests || [])[0]; + if (!pr) { + const { data } = await github.rest.repos.listPullRequestsAssociatedWithCommit( + { owner, repo, commit_sha: run.head_sha }); + pr = data.find(p => p.state === 'open'); + } else { + pr = (await github.rest.pulls.get({ owner, repo, pull_number: pr.number })).data; + } + if (!pr || pr.state !== 'open' || pr.user.type === 'Bot') return; + + // Job name → the local command that reproduces/fixes it. Also the + // allowlist of CI jobs this nudge watches — keep in sync with ci.yml. + const FIX = { + 'Lint & Format': 'run `npm run lint:fix && npm run format`, then commit', + 'Typecheck': 'run `npm run typecheck` and fix the reported type errors', + 'Unit Tests': 'run `npm test` and fix the failing tests', + 'Build': 'run `npm run build` and fix the compile errors', + 'Local E2E Tests': 'run `npm run test:e2e` (it builds first)', + 'Coverage (>= 80%)': 'run `npm run test:coverage` — new code needs tests until every metric is back at 80%', + 'Secret scan (gitleaks)': 'run `gitleaks detect --no-git --redact --source .` locally (config: .gitleaks.toml) and remove the secret — or, for a provable false positive, add a narrowly-anchored allowlist regex', + }; + // Matrix jobs publish check runs as "Unit Tests (Node 20)" etc. — + // exact lookup alone would silently skip them (and the nudge would + // say "all green" while they are red). Try exact first, then the + // name with a trailing parenthetical stripped (review round 2). + const fixFor = (name) => FIX[name] ?? FIX[name.replace(/\s*\([^)]*\)$/, '')]; + + // Recompute full CI state from this SHA's check runs, narrowed to the + // CI jobs above — other workflows (e.g. pr-triage) also attach + // github-actions check runs to the PR head and must not count here. + const checks = await github.paginate(github.rest.checks.listForRef, + { owner, repo, ref: run.head_sha, per_page: 100 }); + const ours = checks.filter(c => c.app && c.app.slug === 'github-actions' && fixFor(c.name)); + const failing = ours.filter(c => ['failure', 'timed_out'].includes(c.conclusion)); + const pending = ours.filter(c => c.status !== 'completed'); + + const comments = await github.paginate(github.rest.issues.listComments, + { owner, repo, issue_number: pr.number, per_page: 100 }); + const sticky = comments.find(c => (c.body || '').includes(marker)); + + if (failing.length === 0) { + // Only speak up on success if we previously flagged a failure, and + // only once everything has actually finished. + if (sticky && pending.length === 0 && !sticky.body.includes('all green')) { + await write(rest => rest.issues.updateComment({ owner, repo, comment_id: sticky.id, + body: `${marker}\n✅ CI is **all green** now — thanks, @${pr.user.login}!` })); + } + return; + } + + const lines = failing + .sort((a, b) => a.name.localeCompare(b.name)) + .map(c => { + const fix = fixFor(c.name) || 'see the logs for details'; + return `- **${c.name}** — ${fix} ([logs](${c.html_url}))`; + }); + const body = `${marker}\nThanks, @${pr.user.login}! CI is red on this PR — ` + + `here's what failed and how to reproduce it locally:\n\n${lines.join('\n')}\n\n` + + `Everything runs on Node 22 after \`npm ci\`. Push a fix and this comment ` + + `flips green automatically once all checks pass.`; + + if (sticky) { + if (sticky.body !== body) { + await write(rest => rest.issues.updateComment( + { owner, repo, comment_id: sticky.id, body })); + } + } else { + await write(rest => rest.issues.createComment( + { owner, repo, issue_number: pr.number, body })); + } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 890f9c7..3766b0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -37,6 +39,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -48,10 +52,35 @@ jobs: - run: npm run typecheck test: - name: Unit Tests + name: Unit Tests (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: ${{ matrix.node-version }} + cache: 'npm' + + - run: npm ci + - run: npm run build + - run: npm test + env: + CI: true + + test-windows: + name: Unit Tests (Windows) + runs-on: windows-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -60,20 +89,27 @@ jobs: cache: 'npm' - run: npm ci + - run: npm run build + - run: npm run typecheck - run: npm test env: CI: true build: - name: Build + name: Build (Node ${{ matrix.node-version }}) runs-on: ubuntu-latest + strategy: + matrix: + node-version: [20, 22] steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: - node-version: 22 + node-version: ${{ matrix.node-version }} cache: 'npm' - run: npm ci @@ -89,6 +125,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -100,3 +138,36 @@ jobs: - run: npm run test:e2e env: CI: true + + gitleaks: + name: Secret scan (gitleaks) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false + + # Same pinned version + checksum-verified install as + # divergence-sentinel.yml (the more recent/secure of the two gitleaks + # invocations already in this repo — release-build.yml pins an older + # 8.21.2 with no checksum check). Continuous, per-PR/per-push gate: a + # working-tree scan (--no-git), not a full-history scan — history + # scanning is too slow to run on every PR, and this mirrors the mode + # scripts/make-public-snapshot.sh already uses for the release-time scan + # (`gitleaks detect --no-git --no-banner --redact --source `). + - name: Install gitleaks + env: + GITLEAKS_VERSION: '8.28.0' + run: | + set -euo pipefail + BASE="https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}" + TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -sSL -o "$TARBALL" "${BASE}/${TARBALL}" + curl -sSL -o checksums.txt "${BASE}/gitleaks_${GITLEAKS_VERSION}_checksums.txt" + grep " ${TARBALL}\$" checksums.txt | sha256sum -c - + tar -xzf "$TARBALL" gitleaks + sudo mv gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Scan working tree for secrets + run: gitleaks detect --no-git --no-banner --redact --source . diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 63323c0..53bd570 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -1,13 +1,15 @@ # P1-2 issue auto-assign + 3-slot-cap bot (InsForge `agent-zhang-beihai` pattern). -# Posts as the `alfheim-agent` GitHub App ⇒ `alfheim-agent[bot]`. +# Posts as the `testsprite-hob` GitHub App ⇒ `testsprite-hob[bot]`. # # Setup (one-time, by an org admin): -# 1. Create a GitHub App named `alfheim-agent` (org Settings → Developer settings → +# 1. Create a GitHub App named `testsprite-hob` (org Settings → Developer settings → # GitHub Apps → New). Permissions: Issues = Read & write, Metadata = Read. No webhook. # Generate a private key; install the App on the public testsprite-cli repo. # 2. Add two repo (or org) secrets: -# ALFHEIM_AGENT_APP_ID = the App's numeric App ID -# ALFHEIM_AGENT_PRIVATE_KEY = the App's .pem private key (full contents) +# TESTSPRITE_HOB_APP_ID = the App's numeric App ID +# TESTSPRITE_HOB_PRIVATE_KEY = the App's .pem private key (full contents) +# (The ALFHEIM_AGENT_* fallbacks are this App's original secret names from +# before it was renamed — same App ID + key; either naming works.) # Until those exist, the bot gracefully falls back to github-actions[bot] (still works). # # Fires on each new issue comment; assigns the commenter when they claim an issue @@ -29,24 +31,35 @@ env: jobs: triage: + # Public repo only — this file also lives in the private mirror (atlas), + # where issue-first claiming doesn't apply; the three sibling bots + # (pr-triage.yml, ci-nudge.yml, stale.yml) all carry this same fence and + # this one was missing it (found in the 2026-07-15 OSS-P2 audit — the + # /assign bot was live on atlas too until this fix). # issues only (issue_comment also fires on PRs), and never react to a bot's own - # comment — incl. our own alfheim-agent[bot], which (unlike GITHUB_TOKEN) would + # comment — incl. our own testsprite-hob[bot], which (unlike GITHUB_TOKEN) would # otherwise re-trigger this workflow. `type == 'Bot'` covers both bot identities. - if: ${{ !github.event.issue.pull_request && github.event.comment.user.type != 'Bot' }} + if: >- + github.repository == 'TestSprite/testsprite-cli' && + !github.event.issue.pull_request && + github.event.comment.user.type != 'Bot' runs-on: ubuntu-latest steps: - # Mint a token for the alfheim-agent App so comments post as alfheim-agent[bot]. + # Mint a token for the testsprite-hob App so comments post as testsprite-hob[bot]. # continue-on-error: until the App + secrets exist this no-ops and we fall back below. - id: app-token continue-on-error: true uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - app-id: ${{ secrets.ALFHEIM_AGENT_APP_ID }} - private-key: ${{ secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # Scope the minted token to what this job uses (issues API only) instead + # of inheriting the App's full installation permissions. + permission-issues: write - - uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.0.1 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: - # App token when available (→ alfheim-agent[bot]); else the default + # App token when available (→ testsprite-hob[bot]); else the default # GITHUB_TOKEN (→ github-actions[bot]). Either way the logic is identical. github-token: ${{ steps.app-token.outputs.token || github.token }} script: | diff --git a/.github/workflows/pr-triage.yml b/.github/workflows/pr-triage.yml new file mode 100644 index 0000000..d505f0b --- /dev/null +++ b/.github/workflows/pr-triage.yml @@ -0,0 +1,199 @@ +# PR-side twin of issue-triage.yml (InsForge `agent-zhang-beihai` pattern, part 2). +# Enforces the issue-first workflow on community PRs: every non-docs PR must +# carry a closing link ("Closes #123") to an issue that is ASSIGNED to the PR +# author (claimed via `/assign`, which issue-triage.yml handles). Violations +# get a `needs-issue` label + a sticky comment, and — for PRs opened after +# GATE_SINCE — a failing check, so unclaimed work is visibly not review-ready. +# PRs opened before GATE_SINCE are grandfathered: nudge only, never a red check. +# +# Assignment happens on the ISSUE, so it cannot re-trigger this PR workflow; +# the comment tells the contributor to edit the PR description or push a +# commit (`edited` / `synchronize`) to re-run the gate. +# +# Runs with NO checkout — metadata-only, so it is safe under pull_request_target +# (which is required for fork PRs to get a write-capable token). +# +# Bot identity: posts as `testsprite-hob[bot]` when the App token works (the +# App has had the "Pull requests: Read & write" permission since 2026-07-02 — +# corrected 2026-07-15; this comment previously said the permission was still +# pending). The App-token-first / GITHUB_TOKEN-fallback code path below is +# kept regardless, as defense-in-depth for whenever the App/secrets are +# absent or a future installation loses the permission. +# Secrets: TESTSPRITE_HOB_* preferred; the ALFHEIM_AGENT_* fallbacks are the +# original names from before the App was renamed (same App ID + key). +name: PR triage (issue-link gate) + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +permissions: + pull-requests: write # add/remove the needs-issue label + issues: write # create/update the nudge comment (PR comments ride the issues API) + +env: + LABEL: 'needs-issue' + MARKER: '' + # PRs created before this instant are grandfathered (nudge, no failing check). + GATE_SINCE: '2026-07-04T00:00:00Z' + +jobs: + gate: + # Public repo only — this file also lives in the private mirror, where PRs + # are internal work that never links public issues. Skip bot authors + # (dependabot etc.), maintainers, and docs-only changes (CONTRIBUTING + # exempts docs/small fixes from the issue-first ask). + if: >- + github.repository == 'TestSprite/testsprite-cli' && + github.event.pull_request.state == 'open' && + github.event.pull_request.user.type != 'Bot' && + !contains(fromJSON('["OWNER","MEMBER","COLLABORATOR"]'), github.event.pull_request.author_association) && + !startsWith(github.event.pull_request.title, 'docs') + runs-on: ubuntu-latest + steps: + # Mint an App token so actions post as testsprite-hob[bot]. Until the App + + # secrets + Pull-requests permission exist this no-ops / gets 403 and the + # script below falls back to the default token per-call. + - id: app-token + continue-on-error: true + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.TESTSPRITE_HOB_APP_ID || secrets.ALFHEIM_AGENT_APP_ID }} + private-key: ${{ secrets.TESTSPRITE_HOB_PRIVATE_KEY || secrets.ALFHEIM_AGENT_PRIVATE_KEY }} + # Every write() call below (createComment/updateComment/addLabels/ + # removeLabel) is an `issues.*` Octokit method — GitHub gates all + # four (comments AND labels, even on a PR number) on the "Issues" + # repository permission, not "Pull requests" (verified against + # docs.github.com/en/rest/using-the-rest-api/permissions-required-for-github-apps). + # Scoping to issues:write alone is therefore both sufficient and + # minimal — narrower than this job's top-level `permissions:` block, + # which also carries pull-requests:write for the ambient-token + # fallback path (unused by this specific App-token mint). + permission-issues: write + + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + APP_TOKEN: ${{ steps.app-token.outputs.token }} + with: + # `github` client = default GITHUB_TOKEN: used for all reads (the App + # can't read PRs without the Pull-requests permission) and as the + # write fallback. App client (when mintable) is preferred for writes. + script: | + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + const label = process.env.LABEL; + const marker = process.env.MARKER; + const author = pr.user.login; + + const appClient = process.env.APP_TOKEN + ? new (github.constructor)({ auth: process.env.APP_TOKEN }) + : null; + // Prefer the App identity; fall back to github-actions[bot] when the + // App is absent or lacks the Pull-requests permission (403/404). + async function write(fn) { + if (appClient) { + try { return await fn(appClient.rest); } + catch (e) { if (e.status !== 403 && e.status !== 404) throw e; } + } + return await fn(github.rest); + } + + // Linked issues: GitHub-computed closing references carry number + + // assignees in one query. + const gql = await github.graphql( + `query($owner:String!,$repo:String!,$num:Int!){ + repository(owner:$owner,name:$repo){ + pullRequest(number:$num){ + closingIssuesReferences(first:10){ + nodes{ number assignees(first:10){ nodes{ login } } } + } + } + } + }`, { owner, repo, num: pr.number }); + const linkedIssues = gql.repository.pullRequest.closingIssuesReferences.nodes + .map(n => ({ number: n.number, assignees: n.assignees.nodes.map(a => a.login) })); + + // Body-text fallback for the brief window before GitHub computes the + // link (same-repo bare "#N" references only). + const bodyRefs = [...(pr.body || '') + .matchAll(/(close[sd]?|fix(e[sd])?|resolve[sd]?)\s*:?\s+#(\d+)/gi)] + .map(m => Number(m[3])) + .filter(n => !linkedIssues.some(i => i.number === n)) + .slice(0, 5); + for (const num of bodyRefs) { + try { + const { data } = await github.rest.issues.get({ owner, repo, issue_number: num }); + if (data.pull_request) continue; // "#N" pointed at a PR, not an issue + linkedIssues.push({ number: num, assignees: (data.assignees || []).map(a => a.login) }); + } catch (e) { /* unknown number — ignore */ } + } + + const linked = linkedIssues.length > 0; + const assignedToAuthor = linkedIssues.some(i => i.assignees.includes(author)); + const state = assignedToAuthor ? 'ok' : (linked ? 'unassigned' : 'unlinked'); + + const hasLabel = (pr.labels || []).some(l => l.name === label); + const comments = await github.paginate(github.rest.issues.listComments, + { owner, repo, issue_number: pr.number, per_page: 100 }); + const nudge = comments.find(c => (c.body || '').includes(marker)); + const stateLine = ``; + + const contributingUrl = + `https://github.com/${owner}/${repo}/blob/main/CONTRIBUTING.md#contribution-model`; + const rerunHint = 'After fixing it, edit the PR description or push a commit to re-run this check.'; + + let body; + if (state === 'ok') { + body = `${marker}\n${stateLine}\n` + + `✅ This PR is linked to an issue assigned to @${author} — thanks! ` + + `The \`${label}\` label has been removed.`; + } else if (state === 'unassigned') { + const list = linkedIssues.map(i => { + const holders = i.assignees.filter(a => a !== author); + return `#${i.number}` + (holders.length ? ` (currently assigned to @${holders.join(', @')})` : ' (unassigned)'); + }).join(', '); + body = `${marker}\n${stateLine}\n` + + `Thanks for the PR, @${author}! It links an issue, but that issue isn't assigned to you yet: ${list}. ` + + `Per our workflow, **claim the issue first by commenting \`/assign\` on it** (the triage bot assigns you automatically). ` + + `If it's already assigned to someone else, please coordinate with them or pick another issue — ` + + `unclaimed-issue PRs are not reviewed. ${rerunHint} ` + + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; + } else { + body = `${marker}\n${stateLine}\n` + + `Thanks for the PR, @${author}! A quick note on our workflow: for **features and behavior changes** ` + + `we require contributors to **open an issue first, claim it by commenting \`/assign\` on the issue, ` + + `then submit a PR that links it** (e.g. \`Closes #123\`). This PR isn't linked to any issue yet, ` + + `so it is not review-ready. ${rerunHint} ` + + `See [CONTRIBUTING → Contribution model](${contributingUrl}).`; + } + + // Sticky comment: create once, update when the state changes. + if (!nudge) { + if (state !== 'ok') { + await write(rest => rest.issues.createComment( + { owner, repo, issue_number: pr.number, body })); + } + } else if (!nudge.body.includes(stateLine)) { + await write(rest => rest.issues.updateComment( + { owner, repo, comment_id: nudge.id, body })); + } + + // Label tracks the gate state. + if (state === 'ok' && hasLabel) { + await write(rest => rest.issues.removeLabel( + { owner, repo, issue_number: pr.number, name: label })).catch(() => {}); + } + if (state !== 'ok' && !hasLabel) { + await write(rest => rest.issues.addLabels( + { owner, repo, issue_number: pr.number, labels: [label] })); + } + + // Hard gate for PRs opened after the cutoff; older PRs are nudged only. + const gated = new Date(pr.created_at) >= new Date(process.env.GATE_SINCE); + if (state !== 'ok' && gated) { + core.setFailed(state === 'unlinked' + ? 'No closing-linked issue. Open/claim an issue, add "Closes #" to the PR description, then re-run.' + : 'Linked issue is not assigned to the PR author. Comment /assign on the issue, then re-run.'); + } else if (state !== 'ok') { + core.notice(`Grandfathered PR (opened before ${process.env.GATE_SINCE}): gate not enforced, nudge only.`); + } diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a0f1346..8ef6dc9 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -14,13 +14,33 @@ jobs: with: node-version: 22 registry-url: 'https://registry.npmjs.org' - # npm trusted publishing (OIDC) requires npm >= 11.5.1; Node 22 bundles npm 10.x - - run: npm install -g npm@latest + # npm trusted publishing (OIDC) requires npm >= 11.5.1; Node 22 never bundles + # that (it ships 10.9.x). Pin an EXACT version rather than floating on + # @latest: npm 12.0.0 shipped with a broken workspace symlink that pulled the + # wrong `sigstore` dependency into its own release tarball, so every + # `npm publish --provenance` failed with `Cannot find module 'sigstore'` + # (npm/cli#9722) — this is exactly what our v0.2.0 and v0.3.0 release.yaml + # runs hit, both AFTER `npm install -g npm@latest` reported success. Fixed + # in 12.0.1 (2026-07-10); bump this pin deliberately, never float it again. + - run: npm install -g npm@11.6.2 - run: npm ci - run: npm run lint - run: npm run typecheck - run: npm run format:check - run: npm run test:coverage - run: npm run build - # Auth via npm trusted publishing (OIDC) — no token needed; provenance is implied - - run: npm publish --provenance --access public + # Auth via npm trusted publishing (OIDC) — no token needed; provenance is + # implied. A prerelease tag (e.g. v0.3.0-rc.1, contains "-") must publish + # under an explicit dist-tag: current npm already refuses an implicit + # `latest` for a prerelease version, but failing mid-release is worse than + # not needing the guard, so this still passes --tag itself rather than + # relying on that enforcement alone. + - name: Publish + env: + REF_NAME: ${{ github.ref_name }} + run: | + if [[ "$REF_NAME" == *-* ]]; then + npm publish --provenance --access public --tag next + else + npm publish --provenance --access public + fi diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..b670166 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,64 @@ +# P1-3 — stale-bot: nudge then close inactive issues / PRs (actions/stale). +# +# Generous windows for an open-source repo (first-response SLA is 5 business days, +# see CONTRIBUTING). Activity removes the `stale` label automatically, so a single +# reply resets the clock. Newcomer- and security-relevant work is exempt from +# closing. `good first issue` / `help wanted` stay open for whoever picks them up. +# +# This is a SYNCED asset (ships to the public mirror). It is scheduled, so unlike +# the event-gated triage bot it is fenced to the PUBLIC repo with a repository +# guard — on private atlas it is a no-op (no nagging internal issues/PRs). +name: Stale + +on: + schedule: + - cron: '30 1 * * *' # daily 01:30 UTC + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + stale: + if: ${{ github.repository == 'TestSprite/testsprite-cli' }} + runs-on: ubuntu-latest + permissions: + issues: write # comment + (un)label + close stale issues + pull-requests: write # comment + (un)label + close stale PRs + steps: + - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 + with: + # ── issues ────────────────────────────────────────────────────── + days-before-issue-stale: 60 + days-before-issue-close: 14 + stale-issue-label: stale + stale-issue-message: > + This issue has had no activity for 60 days, so it's been marked + `stale`. If it's still relevant, just leave a comment (or remove the + `stale` label) and we'll keep it open — otherwise it will be closed + in 14 days. Thanks for helping us keep the tracker tidy! + close-issue-message: > + Closing as `stale` after no activity. This isn't a judgement on the + idea — please reopen or open a fresh issue if it's still relevant. + + # ── pull requests (more grace — external contributors may be slow) ─ + days-before-pr-stale: 45 + days-before-pr-close: 21 + stale-pr-label: stale + stale-pr-message: > + This PR has had no activity for 45 days, so it's been marked `stale`. + Push a commit or leave a comment to keep it open — otherwise it will + be closed in 21 days. We'd still love to merge it; ping a maintainer + if you're blocked on a review. + close-pr-message: > + Closing as `stale` after no activity. Reopen any time you can pick it + back up — your work isn't lost. + + # ── shared behaviour ──────────────────────────────────────────── + exempt-issue-labels: 'pinned,security,in-progress,good first issue,help wanted,hackathon' + exempt-pr-labels: 'pinned,security,in-progress,hackathon' + exempt-draft-pr: true + remove-stale-when-updated: true + ascending: true # oldest first — fairest under the per-run op cap + operations-per-run: 60 + enable-statistics: true diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index 4eb15a9..0e20d9e 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -16,6 +16,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 @@ -73,7 +75,7 @@ jobs: fi - name: Add Coverage PR Comment - uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2.9.4 + uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5 if: github.event_name == 'pull_request' && steps.coverage-summary.outputs.coverage_generated == 'true' with: recreate: true diff --git a/.gitleaks.toml b/.gitleaks.toml index b3f68df..b31d49c 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -19,4 +19,23 @@ regexes = [ '''sk-secret-[0-9]+''', # Documented placeholder keys used by dry-run/sample responses '''sk-user-(test|DRY-RUN)''', + # Example values inside internal (never-shipped, dropped-before-publish) + # design docs. These are only ever scanned by ci.yml's continuous gitleaks + # job (new, 2026-07) — the release-time scan + # (scripts/make-public-snapshot.sh) already drops those internal docs + # wholesale before it ever runs gitleaks, so these never previously + # tripped a scan. Deliberately described here without naming the internal + # doc paths themselves (this file ships to the public repo, and the + # snapshot's own internal-doc-reference scan would flag such a path). + # A "Bearer " curl example whose placeholder token literal is the + # word "dev-token" — not a real key. (Allowlist regexes match against the + # extracted secret value, not the surrounding line — verified empirically + # with `gitleaks detect`.) + '''^dev-token$''', + # A literal base64 pagination-cursor EXAMPLE value in an internal OpenAPI + # spec (decodes to {"ek":"project_a47b2c11"} — spec-example data, not a + # credential). Anchored ^…$ so it only ever matches this exact literal — + # an unanchored version would also suppress any REAL secret that happens + # to contain this substring (review round 2, 2026-07-15; reproduced). + '''^eyJlayI6InByb2plY3RfYTQ3YjJjMTEifQ==$''', ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e0d49..45803e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,70 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +## [0.4.0] - 2026-07-16 + +### Added + +- **`test cancel `** — user-initiated cancel of in-flight runs (the real stop button; Ctrl-C only detaches). A single id renders the run card with status `cancelled` (plus an advisory when it was already cancelled); multiple ids print a `{cancelled, alreadyCancelled, conflicts, notFound}` summary. Exit codes: 4 when any id is not found, else 6 on conflicts, else 0. `--dry-run` supported. +- **Graceful Ctrl-C during `--wait`** — SIGINT/SIGTERM now detaches cleanly instead of killing the process mid-poll: the in-flight request aborts immediately, stdout gets the same partial `{runId, status: "running"}` envelope as the request-timeout path, and stderr states the truth — the server-side run keeps executing (and billing) — with a re-attach hint and a `test cancel` pointer. Exit 130/143/129 per the documented signal contract; a second signal forces a hard exit. Interrupting never cancels the server-side run — that's what `test cancel` is for. +- **`project delete --confirm`** — permanently delete a project and everything under it (its frontend/backend sub-projects, all their tests, and backend fixtures), mirroring the Portal's cascade delete. Requires `--confirm` (the CLI never prompts); `--dry-run` previews the response shape without a network call. Standard exit codes: 0 success, 3 auth, 4 not-found (or already-deleted), 5 validation (e.g. missing `--confirm`). +- **Backend stdout and traceback in results** — `test result` and the failure bundle now surface the backend test's captured stdout and Python traceback: full content in `--output json` (and in `result.json` / `failure.json` bundle files), and a bounded 20-line tail with a byte count in text mode. No change for frontend tests, passing runs, or older backends. +- **Backend dependency declarations are now readable and editable** — `test get` surfaces `produces` / `consumes` / `category`, and `test update` accepts `--produces` / `--needs` / `--category` (previously create-only). +- **Version-compatibility handshake** — the CLI reads the backend's advertised minimum-supported-version on every response and prints a one-line upgrade advisory on stderr when the running version is below the floor (honors the same opt-outs as the update notice; never alters exit status). A `CLIENT_TOO_OLD` rejection (HTTP 426) is now a first-class error: exit 14, non-retriable, rendered with upgrade guidance and the version gap. +- **V3 routing visibility** — `auth status` and `doctor` render a `routing: v2|v3` line when the backend reports the account's routing, and V3-routed accounts get one consolidated advisory listing the known V3-path behavior gaps. Text mode only — JSON consumers read `v3Enabled` from the `/me` payload; absent-safe against older backends. + +### Changed + +- **The `testsprite-verify` agent skill routes local-only changes to the TestSprite MCP** — the skill now states the reachability gate explicitly: the CLI verifies reachable deployed URLs only; when the change is only running locally, the skill hands off to the TestSprite MCP when available (an explicitly named tool always wins), instead of failing against localhost. + +### Fixed + +- **`project create --description` now fails fast with a clear validation error** — projects have no description field, so the flag's value was previously dropped silently; the error points at test-level descriptions (`test create --description`) instead. +- **Standalone backend run cards no longer show a misleading step summary** — `test run` / `test wait` / `test rerun` cards for backend tests render `steps: n/a (backend)` instead of `0/0 (passed=0, failed=0)` (backend tests have no per-step storage). + +## [0.3.0] - 2026-07-08 + +### Added + +- **`testsprite doctor`** — one-command environment diagnostic that checks your Node version, credentials, endpoint reachability, and installed agent skills, and reports what's misconfigured. +- **`test scaffold`** — emit a schema-correct starter plan (frontend) or a backend test skeleton to bootstrap a new test without hand-writing the JSON. +- **`test lint`** — offline validator for plan / steps files; catches malformed test definitions before they are sent to the server. +- **`test diff `** — compare two runs of the same test to isolate what changed between a passing and a failing run. +- **`test flaky `** — repeat-run flaky-test detector. Replays a test N times (`--runs`, default 5), aggregates the outcomes, and reports a stability verdict (`stable` / `flaky` / `failing`) plus the `runId` and `failureKind` of every attempt that did not pass. Replays run with auto-heal off (strict verbatim) so a nondeterministic pass/fail can't be masked. Exit code is 0 only when every attempt passed, so CI can gate a merge on flakiness. Flags: `--runs` (1–10), `--until-fail`, `--timeout`, `--output json`. +- **JUnit XML report export for batch runs.** `test run --all` and batch `test rerun` accept `--report junit --report-file ` to write a CI-friendly XML sidecar after `--wait` polling completes. The report is written even when the batch exits non-zero; `--output json` is unchanged; `--dry-run` writes a canned sample without network calls. +- **`test wait` is now variadic** — pass several run ids to attach to and poll multiple runs in a single invocation. +- **`agent status`** — report which TestSprite skills are installed for each agent target and whether they are current; installed skills are now stamped with a version/hash marker. +- **New `agent install` targets:** GitHub Copilot, Windsurf, and Kiro (experimental), alongside the existing Claude / Cursor / Cline / Codex / Antigravity targets. +- **`project credential` / `project auto-auth`** — configure a project's backend credentials (static credential, free) or a recurring auto-auth token (Pro) from the CLI, with surfaced auth warnings and managed-credential guidance. +- **Proxy support** — the CLI now honors `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` for use behind corporate and CI proxies. +- **`NO_COLOR` support** — colored output is suppressed when `NO_COLOR` is set, per no-color.org. +- **"New version available" notice** — a non-blocking, 24h-cached npm version check prints an upgrade hint on stderr. Opt out with the documented env var; automatically silenced in CI and under `--output json`. + +### Changed + +- **Node.js 20.19+, 22.13+, or 24+ is now the minimum supported runtime.** The CLI checks the running Node version at startup and exits with a clear message on an unsupported version; builds and CI run against Node 20 and 22. +- **Graceful shutdown** — the CLI handles termination signals cleanly and guards against broken-pipe (`EPIPE`) errors when its output is piped to a closing consumer (e.g. `| head`). +- **Interactive prompts and preamble now go to stderr**, keeping stdout pure for machine consumers even in interactive mode. +- **Empty environment variables are treated as unset** when resolving config, so `TESTSPRITE_API_URL=` no longer overrides the built-in default with an empty string. +- `agent install` defaults `--target` to `claude` in non-interactive / CI contexts (matching `setup`). +- The `usage` command no longer implies backend test runs are free. +- `setup`'s "Next steps" guidance no longer suggests `test list` before any project exists. + +### Fixed + +- **Timeouts & polling:** `RequestTimeoutError` is now classified as a timeout in the `--all --wait` fan-out; per-attempt timeout timers are cleared so they can't fire late; `run --all --wait` no longer polls still-queued runs past the shared deadline; a partial result is emitted on stdout when `run --wait` / `test wait` times out (so a redirected file is never zero-byte). +- **Batch rerun:** the exit code is preserved and auth errors escalate correctly; explicit ids combined with `--all` — or `--status` / `--skip-terminal` without `--all` — are rejected with a clear validation error; auto-minted idempotency keys are surfaced under `--output json`. +- **HTTP:** non-JSON `200` responses map to a typed error envelope instead of crashing the parser. +- **Failure bundles / artifacts:** artifact downloads retry on transient errors and guard the default run-id path; the `--out` directory no longer sweeps unrelated pre-existing files (data-loss fix); run-scoped per-step error text and step type are surfaced. +- **Input validation (fail fast, before any network call):** malformed API keys, invalid `--request-timeout`, directory `--code-file` / `--out` paths, blank or whitespace-only `--name` (test and project create/update), blank inline project passwords, fractional pagination flags / page sizes, and `--since` overflow are all rejected up front with `VALIDATION_ERROR` rather than crashing or failing late server-side. `--output` is validated uniformly across all command groups. +- **Setup / auth:** the endpoint is validated before the key check; the typed API-error envelope is preserved when key verification fails; the per-request timeout is honored during `configure`. +- **Misc:** cursor pagination no longer drops empty pages; trailing-dot hostnames are treated as loopback by the local-target guard; buffered input is preserved between interactive prompts; the Codex managed-section skill check requires a complete section; `code get` strips a leading BOM and rejects an empty `--out`. + +### Security + +- **INI injection:** CR/LF characters are stripped from credential values before they are written to `~/.testsprite/credentials`. +- **Symlink fail-close:** the own-file `agent install` path applies its symlink containment guard under `--dry-run` as well, so a planted symlink cannot place or clobber files outside `--dir`. + ## [0.2.0] - 2026-06-29 ### Added @@ -144,6 +208,9 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for - Commander `help [command]` exits 0 (previously exited 5 on `test help` / `project help`). -[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...HEAD +[Unreleased]: https://github.com/TestSprite/testsprite-cli/compare/v0.3.0...HEAD +[0.3.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.2...v0.2.0 +[0.1.2]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.1...v0.1.2 [0.1.1]: https://github.com/TestSprite/testsprite-cli/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/TestSprite/testsprite-cli/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b995b9..1479099 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,21 +20,61 @@ Discussions**, so the issue tracker stays a clean, actionable list. ## Contribution model -- **Small fixes and improvements:** just open a pull request. No issue required. -- **Large or breaking changes** (new commands, changed flags/output, new - dependencies, refactors): **open an issue first** to discuss the design. This - avoids wasted work on something we'd ask you to rework or that's out of scope. +- **Docs and small fixes** (typos, doc corrections, comment-only changes): + just open a pull request. No issue required (linking one is still + appreciated). +- **Features and behavior changes** (new commands or flags, changed output, + new dependencies, refactors) follow **issue-first**: + 1. Find an existing issue, or open a new one describing the change. + 2. Claim it by commenting `/assign` on the issue — the literal slash + command on its own line. The triage bot assigns you automatically; + free-text requests ("can I take this?") are **not** detected. + 3. If the issue is new, wait for triage — we check proposals against + [VISION.md](./VISION.md) and the [standing policies](#standing-scope-policies) + below before any code is written, so you don't invest in something + we'd ask you to rework or decline. + 4. Open your PR with a closing link (e.g. `Closes #123`) in the + description. +- **PR gate:** a bot checks every non-docs community PR for a closing-linked + issue that is **assigned to the PR author**. PRs that don't meet this get + the `needs-issue` label and a failing `PR triage` check, and **are not + reviewed** until it's fixed — file or claim the issue, add the closing + link, then edit the PR description (or push a commit) to re-run the check. +- Suspected **security vulnerabilities** are the exception to "file an + issue": report them privately per [SECURITY.md](./SECURITY.md) instead. - We **do** accept community code contributions — this is an actively maintained open-source CLI, not a read-only distribution mirror. - See [VISION.md](./VISION.md) for what is in and out of scope. +### Standing scope policies + +Pre-decided policies, so proposals don't have to relitigate them: + +- **Runtime dependencies are budgeted.** The CLI ships with a deliberately + tiny runtime dependency set (`commander`, `valibot`, plus `undici` — + approved for `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` support). Any new + runtime dependency needs explicit maintainer sign-off **in the issue, + before the PR**. Utility modules land only together with the consumer + that uses them — standalone libraries are declined. +- **`agent install` targets.** Shipped: `claude`, `antigravity`, `cursor`, + `cline`, `codex`, `kiro`, `windsurf`, `copilot`. Accepted and in progress: + `gemini`. + A proposal for a new target needs (1) the editor's official rules/skill + file mechanism, documented, and (2) the proposer prepared to maintain the + target going forward. +- **Outbound network calls.** The CLI talks only to the configured + TestSprite API endpoint. The one approved exception is an opt-out-able + npm registry version check (at most once per 24h, carrying nothing but + the package name, fully silenced by its env opt-out and in CI/JSON/dry-run + modes or when stderr is not a TTY). Any other outbound call is out of scope. + We aim to give every issue and PR a **first response within 5 business days** (best-effort). If something has gone quiet longer than that, a polite nudge on the thread or in Discord is welcome. ## Prerequisites -- Node 20 or newer (development happens on Node 22). +- Node 20.19+, Node 22.13+, or Node 24+ (development happens on Node 22). ## Build from source @@ -69,11 +109,36 @@ npm run format:check # Prettier (check only) npm run typecheck # tsc --noEmit ``` +## Developing on Windows + +Native Windows (no WSL, no Git Bash required) is a fully supported dev +environment — you only need **git** and **Node ≥ 20**. `npm ci`, `npm run +build`, `npm test`, `npm run lint`, and `npm run typecheck` all run the same +way as on macOS/Linux, and CI runs the unit suite on `windows-latest` as the +reference environment (see [CI gates](#ci-gates-required-for-merge) below) — +if it's green there, it's green on your machine. + +A couple of things worth knowing: + +- Line endings are normalized to LF on checkout via `.gitattributes` + (`core.autocrlf` doesn't need any special local configuration). +- `--out`/bundle-path flags accept native Windows paths (`C:\Users\...`) + including a trailing backslash and a bare drive root (`C:\`). +- A small number of tests that create real filesystem symlinks are skipped on + Windows (creating a symlink there needs Administrator rights or Developer + Mode, which isn't guaranteed on hosted CI) — the safety behavior they cover + is still exercised on the Linux/macOS CI jobs. + +Hit something that doesn't work on Windows? Please file it — see +[Questions & support](#questions--support) above, and feel free to tag it +`good first issue` if it looks like an isolated fix; we'd rather know than +have you route around it silently. + ## CI gates (required for merge) - ESLint + Prettier clean - TypeScript type-check clean -- All unit tests passing +- All unit tests passing on Linux (Node 20 + 22) **and** on Windows (`windows-latest`, Node 22) - Coverage ≥ 80% on lines / statements / functions / branches - Build + smoke test of the CLI binary diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 7e50c36..bf6b1b5 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -15,6 +15,7 @@ The full reference for the TestSprite CLI: install verification, manual setup, e - [Read commands](#read-commands) - [Write commands](#write-commands) - [Run commands](#run-commands) + - [Account & diagnostics](#account--diagnostics) - [Configuration](#configuration) - [Output & scripting](#output--scripting) - [Exit codes](#exit-codes) @@ -35,7 +36,7 @@ Or run it without installing: npx @testsprite/testsprite-cli --version ``` -Requires **Node.js ≥ 20**. +Requires **Node.js 20.19+**, **22.13+**, or **24+**. Confirm the binary works **without** configuring an API key: @@ -112,15 +113,23 @@ testsprite agent install claude # install the skill for Claude Code testsprite agent install codex # install into AGENTS.md for Codex (managed-section) testsprite agent install cursor # .cursor/rules/testsprite-verify.mdc testsprite agent install cline # .clinerules/testsprite-verify.md +testsprite agent install windsurf # .windsurf/rules/testsprite-verify.md testsprite agent install antigravity # .agents/skills/testsprite-verify/SKILL.md -testsprite agent list # list all 5 targets with status + mode + path +testsprite agent install kiro # .kiro/skills/testsprite-verify/SKILL.md +testsprite agent install copilot # .github/instructions/testsprite-verify.instructions.md +testsprite agent list # list all 8 targets with status + mode + path +testsprite agent status # check installed skills against this CLI version ``` -Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental). +Supported targets: `claude` (GA), `codex` (experimental), `cursor` (experimental), `cline` (experimental), `antigravity` (experimental), `kiro` (experimental), `windsurf` (experimental), `copilot` (experimental). + +Omitting `--target` in a non-interactive shell (CI, agent subprocess) defaults to `claude` with an `[info]` note on stderr; in a terminal the CLI prompts (empty answer = `claude`). + +`agent status` checks every installed skill file against the current CLI version and reports one of `ok`, `stale`, `modified`, `unmarked`, `absent`, or `corrupt` per target. It exits `1` when anything needs attention, so `testsprite agent status && …` can gate a CI step; `--dir ` inspects a different project root. The `codex` target uses **managed-section mode** — it writes only a sentinel-delimited section inside your existing `AGENTS.md`, so your project instructions are never clobbered. Re-running without `--force` replaces the section in-place; user content outside the sentinels is always preserved. -Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity) backs up the existing file to `.bak` first. +Re-running with `--force` on **own-file targets** (claude, cursor, cline, antigravity, kiro, windsurf, copilot) backs up the existing file to `.bak` first. ## Command reference @@ -171,7 +180,7 @@ Common flags: #### `testsprite test get ` -Get a single test by id. Test ids look like `test_xxxxxxxx` and come from `test list`. +Get a single test by id. Test ids look like `test_xxxxxxxx` and come from `test list`. Backend tests echo their dependency declarations — `produces` / `consumes` / `category` — when present. ```bash testsprite test get test_xxxxxxxx --output json @@ -201,7 +210,7 @@ Common flags: `--page-size`, `--starting-token`, `--max-items` — same shape as #### `testsprite test result ` -Get the latest result for a test — status, started / finished timestamps, video and failure-analysis URLs, summary counts (`passed / failed / skipped`), and correlation fields (`snapshotId`, `runId`, `codeVersion`). With `--include-analysis`, the response also carries an inline `analysis` block (root-cause hypothesis, recommended fix target, failure kind). +Get the latest result for a test — status, started / finished timestamps, video and failure-analysis URLs, summary counts (`passed / failed / skipped`), and correlation fields (`snapshotId`, `runId`, `codeVersion`). With `--include-analysis`, the response also carries an inline `analysis` block (root-cause hypothesis, recommended fix target, failure kind). Backend tests additionally surface the run's captured stdout (`apiOutput`) and Python traceback (`trace`): full content under `--output json` (and in `result.json` / `failure.json` inside failure bundles); text mode prints a bounded 20-line tail of each with a byte count. ```bash testsprite test result test_xxxxxxxx --output json @@ -217,6 +226,15 @@ testsprite test result test_xxxxxxxx --history --source cli --since 7d --output testsprite test result test_xxxxxxxx --history --dry-run --output json ``` +#### `testsprite test diff ` + +Compare two runs of a test and print what regressed: verdict, `failureKind`, `failedStepIndex`, per-step status flips, and `codeVersion` drift. Exit `0` when the verdicts match, `1` when they differ — so a script can assert "this rerun behaves like the last known-good run" in one call. + +```bash +testsprite test diff run_aaaa run_bbbb --output json +testsprite test diff run_aaaa run_bbbb --dry-run --output json +``` + #### `testsprite test failure get ` The latest-failure agent entry point. Returns one consistent snapshot of the latest failing run as a self-contained bundle: the result, the failed step plus its immediate neighbors with screenshots and DOM snapshots, the test source, a video pointer, a root-cause hypothesis, a recommended fix target, and correlation metadata. For the bundle of a _specific_ run an agent just triggered, prefer `test artifact get ` — it is keyed by `runId` and cannot be raced by another run that lands afterward. @@ -250,11 +268,32 @@ testsprite test failure summary test_xxxxxxxx --dry-run --output json ### Write commands -Require the `write:tests` scope. +Require the `write:tests` scope (project commands require `write:projects`), except `test scaffold` and `test lint`, which are pure-local authoring helpers — no network, no credentials, no scope. + +#### `testsprite test scaffold` + +Emit a schema-correct starter test definition — a frontend plan JSON by default, or a backend Python skeleton with `--type backend`. Pure-local: no network, no credentials. Edit the scaffold, then create the test with `--plan-from` / `--code-file`. + +```bash +testsprite test scaffold > first-test.plan.json +testsprite test scaffold --type backend --out tests/health.py +testsprite test scaffold --out plan.json --force # overwrite an existing file +``` + +#### `testsprite test lint` + +Validate plan/steps files offline with the same validators `test create` runs, collecting **every** problem instead of stopping at the first. No network, no credentials. Exit `0` when all inputs are valid, `5` otherwise. + +```bash +testsprite test lint --plan-from ./checkout.plan.json +testsprite test lint --plan-from-dir ./plans/ # every *.json checked, all errors reported +testsprite test lint --plans ./plans.jsonl # one plan spec per line +testsprite test lint --steps ./refined.plan.json # the shape `test plan put` ingests +``` #### `testsprite test create` -Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. +Create a new test. Backend tests use `--code-file` (agents supply backend code directly); frontend tests use either `--code-file` or `--plan-from`. With `--run --wait`, the CLI chains create → trigger → poll in a single invocation. Backend tests can declare wave-ordering dependencies at create time — `--produces ` / `--needs ` (repeatable) and `--category ` — and amend them later via `test update`. ```bash # Backend test from a code file @@ -280,16 +319,17 @@ testsprite test create-batch --plan-from-dir ./plans/ --dry-run --output json #### `testsprite test update ` -Update test metadata (name, description). +Update test metadata (name, description, priority) — and, for **backend tests**, the dependency declarations: `--produces ` / `--needs ` (repeatable) and `--category `. Updated declarations are echoed back by `test get`. ```bash testsprite test update test_xxxxxxxx --name "Renamed test" --description "Updated" +testsprite test update test_be_xxxx --produces session_token --category setup testsprite test update test_xxxxxxxx --dry-run --output json ``` #### `testsprite test delete ` / `test delete-batch` -Soft-delete one test (or many). `--confirm` is required; absent it, the CLI exits 5 with a local validation error. +Permanently delete one test (or many) — there is **no restore window**. `--confirm` is required; absent it, the CLI exits 5 with a local validation error. ```bash testsprite test delete test_xxxxxxxx --confirm @@ -319,20 +359,75 @@ testsprite test plan put test_xxxxxxxx --steps ./refined.plan.json --dry-run --o #### `testsprite project create` / `project update` -Manage projects from the CLI. Both pre-flight `--target-url` against local addresses for fast feedback. +Manage projects from the CLI. Both pre-flight `--url` against local addresses for fast feedback. Projects have **no description field** — `--description` is rejected client-side with a validation error (descriptions live on tests: `test create --description`). `project update` accepts `--name`, `--url`, `--username`, `--password`, `--password-file`, and `--instruction`. ```bash -testsprite project create --name "Checkout" --target-url https://staging.example.com +testsprite project create --type frontend --name "Checkout" --url https://staging.example.com testsprite project update proj_xxxxxxxx --name "Checkout v2" ``` +#### `testsprite project delete ` + +Permanently delete a project and **everything under it** — its frontend/backend sub-projects, all their tests, and backend fixtures (mirrors the Portal's cascade delete). There is **no restore window**. `--confirm` is required (the CLI never prompts); absent it, the CLI exits 5 with a local validation error. `--dry-run` previews the response shape without a network call. Exit codes: 0 success, 3 auth, 4 not found (or already deleted), 5 validation. + +```bash +testsprite project delete proj_xxxxxxxx --confirm +testsprite project delete proj_xxxxxxxx --dry-run --output json +``` + +#### `testsprite project credential ` + +Set the **static backend credential** injected into every backend test in the project (free tier). Supported types: `public` (no credential), `"Bearer token"`, `"API key"`, `"basic token"`. + +```bash +testsprite project credential proj_xxxxxxxx --type "Bearer token" --credential-file ./token.txt +testsprite project credential proj_xxxxxxxx --type public +testsprite project credential proj_xxxxxxxx --type "API key" --credential sk-live-... --dry-run --output json +``` + +`--credential ` or `--credential-file ` supplies the value (required unless `--type public`). Prefer `--credential-file` in scripts so the secret never lands in shell history. + +#### `testsprite project auto-auth ` + +Configure the **recurring-token (auto-refresh) login** for backend tests (Pro): a fresh token is fetched on each run and injected into every backend test, so long-lived suites survive token expiry. + +```bash +# Password login: POST the login endpoint, extract the token, inject as a Bearer header +testsprite project auto-auth proj_xxxxxxxx \ + --method password --inject bearer \ + --login-url https://api.example.com/login --login-method POST \ + --login-content-type application/json \ + --login-body-template '{"user":"{{username}}","pass":"{{password}}"}' \ + --username ci@example.com --password-file ./pw.txt \ + --token-path '$.data.accessToken' + +# OAuth refresh-token flow +testsprite project auto-auth proj_xxxxxxxx \ + --method refresh_token --inject header --inject-key X-Auth-Token \ + --token-endpoint https://auth.example.com/oauth/token \ + --client-id my-client --client-secret-file ./secret.txt \ + --refresh-token-file ./refresh.txt --scope api.read + +# AWS Cognito refresh +testsprite project auto-auth proj_xxxxxxxx \ + --method aws_cognito_refresh --inject bearer \ + --client-id my-app-client --refresh-token-file ./refresh.txt --region us-east-1 + +# Turn it off (stored config is kept) +testsprite project auto-auth proj_xxxxxxxx --disable +``` + +Required flags: `--method ` and `--inject ` (`--inject-key ` names the header/cookie when not `bearer`). Method-specific flags: password login uses `--login-url/--login-method/--login-content-type/--login-body-template/--username/--password[-file]/--token-path`; OAuth uses `--token-endpoint/--client-id/--client-secret[-file]/--refresh-token[-file]/--scope`; Cognito adds `--region`. File variants (`--password-file`, `--client-secret-file`, `--refresh-token-file`) keep secrets out of shell history. + ### Run commands Require the `run:tests` scope. #### `testsprite test run ` -Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout` (with a `nextAction` pointing at `test wait ` so an agent can resume). +Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", enqueuedAt, codeVersion, targetUrl }` and exits 0. With `--wait`, polls until terminal — exit 0 on `passed`, exit 1 on `failed | blocked | cancelled`, exit 7 on `--timeout`. On a timeout the CLI still prints the partial run object (with `runId`) to stdout **before** exiting 7, plus a `nextAction` pointing at `test wait ` — so a script always has the id to resume with, and stdout is never empty. + +`--all --project ` runs every test in the project in wave order. On the current unified engine that means **all tests, frontend and backend**; on the legacy backend-only engine, frontend tests can't run — they are skipped and enumerated in `skippedFrontend` with a stderr advisory. ```bash # Trigger and return immediately @@ -344,9 +439,19 @@ testsprite test run test_xxxxxxxx --target-url https://staging.example.com \ # Dry-run prints a canned queued response (no network, no credentials) testsprite test run test_xxxxxxxx --dry-run --output json + +# Batch run with JUnit XML for CI (sidecar; --output json unchanged) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test run --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json ``` -`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr at `--verbose`); pass `--idempotency-key ` to control it explicitly. +Batch `--report` flags apply only to `test run --all --wait` (and batch `test rerun --wait`). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. Optional `--report-suite-name ` overrides the default `testsprite:` suite name. + +`--target-url` must be a publicly reachable URL — the CLI pre-flights it against local addresses (`localhost`, `127.x`, `::1`, `0.0.0.0`, `169.254.x`, RFC1918) and the backend resolves it via DNS. For testing against localhost, use the [TestSprite MCP plugin](https://www.testsprite.com/docs), which handles the local tunnel. The CLI auto-mints an idempotency key (printed to stderr under `--output json`, `--verbose`, or `--debug`); pass `--idempotency-key ` to control it explicitly. #### `testsprite test rerun [test-id...]` @@ -365,10 +470,20 @@ testsprite test rerun test_be_xxxx --skip-dependencies --output json # Rerun every test in a project (batch) testsprite test rerun --all --project proj_xxxxxxxx --wait --max-concurrency 4 --output json +# Batch rerun with JUnit XML for CI +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --output json + +# Optional custom suite name (default: testsprite:) +testsprite test rerun --all --project proj_xxxxxxxx --wait \ + --report junit --report-file ./results.xml --report-suite-name my-ci-suite --output json + # Several specific tests testsprite test rerun test_aaaa test_bbbb --wait --output json ``` +Batch `--report` flags apply only to batch `--wait` reruns (`--all` or multiple test ids). `--report junit --report-file ` writes a JUnit XML sidecar after polling completes (atomic write); `--output json` is unchanged. When `--project` is omitted, the CLI infers `projectId` from polled run rows for classname / default suite naming; if inference fails, pass `--project ` explicitly (required under `--dry-run`). + Flags: - `--all` — rerun every test in the resolved project; requires `--project `. @@ -376,20 +491,55 @@ Flags: - `--auto-heal` / `--no-auto-heal` — frontend AI heal-on-drift, **on by default** for FE reruns; opt out with `--no-auto-heal`. Verbatim-replay passes are free; a heal engage costs a small amount of credit. Ignored for backend tests. - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. - `--max-concurrency ` — with `--wait`, cap on in-flight polls during a batch rerun. -- `--idempotency-key ` — auto-minted when omitted. +- `--idempotency-key ` — auto-minted when omitted (the minted key is printed to stderr under `--output json`, `--verbose`, or `--debug`). +- `--report junit --report-file ` — with batch `--wait`, write a JUnit XML sidecar after polling (atomic write). Optional `--report-suite-name ` overrides the default `testsprite:` suite name. Requires `--wait`; not available on single-test reruns. A batch rerun returns `accepted[]` (one `runId` per dispatched test) plus `deferred[]` for any test shed by the per-key run-rate limit; under `--wait`, a non-empty `deferred[]` exits 7 with a `nextAction` you can retry with a fresh idempotency key. -#### `testsprite test wait ` +#### `testsprite test flaky ` + +Detect a **flaky** test by replaying it several times and reporting how often it passes. Each attempt is a rerun with auto-heal **off** (a strict verbatim replay), so healed drift can't disguise a nondeterministic pass/fail — this measures the replay stability of the saved script against the configured URL. Frontend replays are free verbatim script replays; backend tests re-run their dependency closure and may cost credits (a one-line stderr advisory is printed before the run). + +```bash +# Replay 10 times and print a stability score +testsprite test flaky test_xxxxxxxx --runs 10 + +# Fast "is it flaky at all?" — stop at the first non-passing attempt +testsprite test flaky test_xxxxxxxx --runs 10 --until-fail + +# Machine-readable stability report for CI +testsprite test flaky test_xxxxxxxx --runs 10 --output json +``` + +Flags: + +- `--runs ` — number of replays (1–10, default 5). +- `--until-fail` — stop at the first attempt that does not pass. +- `--timeout ` — per-attempt polling deadline (same semantics as `test wait`). + +`--output json` emits `{ testId, runs, passed, failed, stableRatio, verdict, failures: [{ attempt, runId, outcome, failureKind }] }`. Exit codes: **0** when every observed attempt passed (`stable`); **1** when any attempt did not pass (`flaky` or `failing`); **4** when the test has no replayable run (trigger `testsprite test run ` first); **5** on a validation error. + +#### `testsprite test wait ` -Block until a run reaches a terminal status. Same exit-code matrix as `test run --wait`. Used to resume polling after a timed-out `test run --wait`, or when an agent already has a `runId` from a previous invocation. +Block until one **or more** runs reach a terminal status. With a single `run-id` the behavior is unchanged: same exit-code matrix as `test run --wait`. With several ids, the runs are polled concurrently under one shared `--timeout` and the CLI prints a `{ results, summary }` envelope — the worst status wins the exit code — so every re-attach hint the CLI prints can be pasted back as one command. `--max-concurrency ` (1–100, default 10) caps concurrent polls. Used to resume polling after a timed-out `--wait`, or when an agent already holds `runId`s from previous invocations. ```bash testsprite test wait run_01hx3z9p8q4k2y7a --timeout 600 --output json +testsprite test wait run_aaaa run_bbbb run_cccc --timeout 900 --output json testsprite test wait run_01hx3z9p8q4k2y7a --dry-run --output json ``` -Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. +With several ids, a per-member poll error (e.g. one id not found) is recorded as `error:` in that run's row and folded into exit 7, rather than aborting the whole batch. Polling is handled automatically — the CLI uses server-driven long-poll where supported and exponential backoff with jitter otherwise, honoring `Retry-After`. + +#### `testsprite test cancel ` + +Cancel one or more in-flight runs — the counterpart to Ctrl-C, which only **detaches** (the server-side run keeps executing and billing). Cancelling is idempotent: an already-cancelled run reports `alreadyCancelled` as an advisory, not an error; a run that already reached a terminal verdict is a conflict — the verdict is never overwritten, and no credits are refunded. With one id, prints the run card; with several, prints a `{ cancelled, alreadyCancelled, conflicts, notFound }` summary. Exit codes: any unknown id → 4; else any conflict → 6; else 0. + +```bash +testsprite test cancel run_01hx3z9p8q4k2y7a +testsprite test cancel run_aaaa run_bbbb --output json +testsprite test cancel run_01hx3z9p8q4k2y7a --dry-run --output json +``` #### `testsprite test artifact get ` @@ -404,6 +554,30 @@ testsprite test artifact get run_01hx3z9p8q4k2y7a --dry-run --output json Returns 404 (CLI exit 4) when the run passed (`details.reason: "no_failing_run"`), is still in flight (`run_not_ready`), was cancelled (`cancelled_no_artifacts`), or its test was deleted (`no_code`). +### Account & diagnostics + +#### `testsprite usage` (alias: `testsprite credits`) + +Account pre-flight before a large batch: resolves the active key to its identity (`userId`, `keyId`, `env`) and surfaces the credit balance / plan fields when the backend supplies them. Useful right before a `test run --all` fan-out. + +```bash +testsprite usage --output json +testsprite credits +testsprite usage --dry-run --output json +``` + +#### `testsprite doctor` + +One-shot environment diagnostic. Runs a fixed checklist — CLI version, Node.js runtime, active profile, API endpoint, credentials, live connectivity + key validity (`GET /me`), and whether the verify skill is installed in the current project — and prints an OK/WARN/FAIL report. Exits non-zero only when a check **fails** (warnings, e.g. skill not installed, don't fail the process), so it can gate a CI step or an agent preflight: + +```bash +testsprite doctor +testsprite doctor --output json +testsprite doctor && testsprite test run test_xxxxxxxx --wait +``` + +Every check reuses the same helpers the real commands use, so the report reflects exactly what a subsequent command would resolve. + ## Configuration ### Profiles & credentials @@ -426,24 +600,48 @@ These apply to every command: ### Environment variables -| Variable | Purpose | -| ------------------------------- | --------------------------------------------------------------------------------- | -| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | -| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | -| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | -| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| Variable | Purpose | +| ----------------------------------------- | ---------------------------------------------------------------------------------------- | +| `TESTSPRITE_API_KEY` | API key — overrides the credentials file | +| `TESTSPRITE_API_URL` | API endpoint — overrides the credentials file | +| `TESTSPRITE_PROFILE` | Active profile (below `--profile`, above `default`) | +| `TESTSPRITE_REQUEST_TIMEOUT_MS` | Per-request timeout in **milliseconds** (default `120000`, range `1000`–`600000`) | +| `TESTSPRITE_NO_UPDATE_NOTIFIER` | Any non-empty value disables the once-per-24h "new version available" notice | +| `NO_COLOR` | Suppress ANSI escape sequences in ticker output ([no-color.org](https://no-color.org/)) | +| `HTTPS_PROXY` / `HTTP_PROXY` / `NO_PROXY` | Standard proxy support — API traffic is routed through the configured proxy | +| `TESTSPRITE_NO_SKILL_WARNING` | Any non-empty value silences the "verify skill not installed" reminder (CI / manual use) | +| `TESTSPRITE_PORTAL_URL` | Override the Portal origin used for `dashboardUrl` links (non-prod environments) | + +### Update notice + +Interactive runs print a one-line "new version available" notice on stderr when +a newer release exists. To learn this, the CLI contacts the public npm registry +(`registry.npmjs.org`) at most once per 24 hours; the request carries the +package name only — never your API key, project data, or command line. The +check is skipped in CI, when stderr is not a TTY, under `--output json` / +`--dry-run`, and entirely when `TESTSPRITE_NO_UPDATE_NOTIFIER` is set. Any +failure is silent: the notice can never break or delay a command. This is the +only outbound call the CLI makes besides your configured API endpoint. + +Separately, the backend advertises its **minimum supported CLI version** on +every `/api/cli/v1` response. When the running CLI is below that floor, a +one-line upgrade advisory is printed to stderr (same opt-outs as the update +notice; it never changes the exit status). A backend may also reject a +too-old client outright with HTTP 426 — surfaced as `CLIENT_TOO_OLD`, +exit `14`, non-retriable, with upgrade guidance. ### Scopes API-key scopes gate the write and run surfaces: -| Scope | Required by | -| --------------- | -------------------------------------------------------------------- | -| `read:me` | `auth whoami` | -| `read:projects` | `project list / get` | -| `read:tests` | every `test *` read command | -| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | -| `run:tests` | `test run / rerun / wait / artifact get` | +| Scope | Required by | +| ---------------- | -------------------------------------------------------------------- | +| `read:me` | `auth status`, `usage`, `doctor` (connectivity check) | +| `read:projects` | `project list / get` | +| `read:tests` | every `test *` read command | +| `write:tests` | `test create / create-batch / update / delete / code put / plan put` | +| `write:projects` | `project create / update / delete / credential / auto-auth` | +| `run:tests` | `test run / rerun / flaky / wait / cancel / artifact get` | New API keys include the full scope set. If a command returns `AUTH_FORBIDDEN`, the missing scope is named in `details.requiredScope` — regenerate your key from the dashboard to pick up new scopes. @@ -461,20 +659,26 @@ testsprite test wait "$RUN_ID" --timeout 600 --output json || echo "run did not ## Exit codes -| Code | Meaning | -| ---- | --------------------------------------- | -| `0` | Success | -| `1` | Generic failure / non-passed run status | -| `2` | Not yet implemented | -| `3` | Auth error | -| `4` | Not found | -| `5` | Validation error / payload too large | -| `6` | Conflict / precondition failed | -| `7` | Timeout / unsupported | -| `10` | Service unavailable | -| `11` | Rate limited (retriable) | -| `12` | Insufficient credits (non-retriable) | -| `13` | Feature gated (paid plan required) | +| Code | Meaning | +| --------------------- | ------------------------------------------------------------------------------------------------- | +| `0` | Success | +| `1` | Generic failure / non-passed run status | +| `2` | Not yet implemented | +| `3` | Auth error | +| `4` | Not found | +| `5` | Validation error / payload too large | +| `6` | Conflict / precondition failed | +| `7` | Timeout / unsupported | +| `10` | Service unavailable | +| `11` | Rate limited (retriable) | +| `12` | Insufficient credits (non-retriable) | +| `13` | Feature gated (paid plan required) | +| `14` | Client too old — the backend requires a newer CLI (HTTP 426 `CLIENT_TOO_OLD`); upgrade to proceed | +| `129` / `130` / `143` | Interrupted by a signal (SIGHUP / SIGINT / SIGTERM) — `128 + signal number` | + +### Signals & pipes + +During any `--wait`, SIGINT (Ctrl-C), SIGTERM, or SIGHUP triggers a **graceful detach**: the in-flight request aborts immediately, stdout gets the same partial `{ runId, status: "running" }` envelope as the request-timeout path (under `--output json`, stderr carries an `INTERRUPTED` envelope naming the signal), and stderr states the truth — the server-side run keeps executing, and any credit spend continues — with a re-attach hint (`test wait `) and a `test cancel ` pointer. The exit code is `128 + signal` (130 / 143 / 129). A second signal forces an immediate hard exit. Outside a `--wait` (prompts, one-shot commands), signals keep the pre-existing immediate-exit behavior. **Ctrl-C never cancels the server-side run** — `test cancel ` is the explicit stop. A closed stdout pipe (`EPIPE`, e.g. `testsprite test list | head`) exits `0` silently rather than crashing. ## Design principles diff --git a/README.md b/README.md index 2d90012..3bb465e 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ AI ships code in minutes — verifying it hasn't. `testsprite` opens your live a

npm version npm downloads - Node >= 20 + Node 20.19+, 22.13+, or 24+ License Apache 2.0 CI

@@ -54,14 +54,14 @@ If you find `testsprite` useful, a GitHub Star ⭐️ would be greatly appreciat ## Quickstart -Requires **Node.js ≥ 20**. (No global install? `npx @testsprite/testsprite-cli` works too.) +Requires **Node.js 20.19+**, **22.13+**, or **24+**. (No global install? `npx @testsprite/testsprite-cli` works too.) ```bash npm install -g @testsprite/testsprite-cli testsprite setup ``` -`testsprite setup` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `antigravity`, `codex`, etc.) — one command, so your agent is wired to verify its own work. Non-interactive (CI / onboarding scripts): +`testsprite setup` prompts for your [API key](https://www.testsprite.com), verifies it, and installs the verification-loop skill for your coding agent (`claude`, `cursor`, `cline`, `windsurf`, `antigravity`, `codex`, etc.) — one command, so your agent is wired to verify its own work. Non-interactive (CI / onboarding scripts): ```bash TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude @@ -69,6 +69,8 @@ TESTSPRITE_API_KEY=sk-... testsprite setup --from-env --yes --agent claude > **Pointing a coding agent (Claude Code, Cursor, Codex, Cline, …) at TestSprite?** Have it run `testsprite setup` first — that installs the verification skill, so the agent knows how to create, run, and triage tests on its own (instead of guessing from this README). New here? Start with the **[getting-started overview](https://docs.testsprite.com/cli/getting-started/overview)**. +> **Privacy note:** interactive runs check the npm registry at most once per 24 h to offer a "new version available" notice — package name only, never your key or data; `TESTSPRITE_NO_UPDATE_NOTIFIER=1` disables it. The backend also advertises its minimum supported CLI version — a below-floor CLI prints a one-line upgrade advisory on stderr, and a too-old client may be rejected with exit 14 (`CLIENT_TOO_OLD`). Details in [DOCUMENTATION.md → Update notice](./DOCUMENTATION.md#update-notice). + From there, the loop runs on its own — an example session, typed by the coding agent: ```bash @@ -89,28 +91,35 @@ Prefer to configure each step by hand (or learn the surface offline with `--dry- ## Commands -| Group | Command | What it does | -| --------- | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | -| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | -| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | -| | `auth remove` | Remove the active profile from the credentials file | -| **Read** | `project list` / `project get` | List projects / fetch one by id | -| | `test list` / `test get` | List tests under a project / fetch one by id | -| | `test code get` | Print (or write) the generated test source | -| | `test steps` | List the latest run's steps with screenshot / DOM pointers | -| | `test result` | Latest result; `--history` lists a test's prior runs | -| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | -| | `test failure summary` | One-screen triage card (no media download) | -| **Write** | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | -| | `test update` / `test delete` / `test delete-batch` | Edit metadata / soft-delete | -| | `test code put` | Replace generated code (etag-guarded) | -| | `test plan put` | Replace a frontend test's plan-steps | -| | `project create` / `project update` | Manage projects | -| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | -| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | -| | `test wait` | Block on a `runId` until terminal | -| | `test artifact get` | Download the failure bundle for a specific `runId` | -| **Agent** | `agent install` / `agent list` | Add or list coding-agent targets (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity` | +| Group | Command | What it does | +| --------- | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Setup** | `setup` | **Start here** — one command: configure your API key, verify it, and install the agent verification skill | +| | `doctor` | Environment diagnostic — CLI/Node versions, profile, endpoint, credentials, connectivity, agent skill; exits non-zero on failure | +| **Auth** | `auth status` | Resolve the active profile to its user, key, env, and scopes | +| | `auth remove` | Remove the active profile from the credentials file | +| | `usage` (alias `credits`) | Account pre-flight: identity, plus credit balance / plan info when the backend supplies them | +| **Read** | `project list` / `project get` | List projects / fetch one by id | +| | `test list` / `test get` | List tests under a project / fetch one by id | +| | `test code get` | Print (or write) the generated test source | +| | `test steps` | List the latest run's steps with screenshot / DOM pointers | +| | `test result` | Latest result; `--history` lists a test's prior runs | +| | `test failure get` | The agent entry point: one self-contained latest-failure bundle | +| | `test failure summary` | One-screen triage card (no media download) | +| | `test diff` | Compare two runs — verdict, failure kind, per-step status flips, code-version drift | +| **Write** | `test scaffold` / `test lint` | Author plans locally: emit a schema-correct starter, validate plan files offline — no network, no credentials | +| | `test create` / `test create-batch` | Create a test (or bulk-create from a plan file); `--produces` / `--needs` / `--category` wire BE dependency metadata | +| | `test update` / `test delete` / `test delete-batch` | Edit metadata and BE dependency declarations (`--produces` / `--needs` / `--category`) / permanently delete (no restore window; `--confirm` required) | +| | `test code put` | Replace generated code (etag-guarded) | +| | `test plan put` | Replace a frontend test's plan-steps | +| | `project create` / `project update` / `project delete` | Manage projects; `delete` removes a project and everything under it (`--confirm` required, no restore window) | +| | `project credential` / `project auto-auth` | Configure backend-test auth: a static injected credential, or auto-refresh login (Pro) | +| **Run** | `test run` | Trigger a fresh run; `--wait` blocks until terminal; `--all --project ` runs all tests in a project in wave order | +| | `test rerun` | Cheap replay of one/many tests (FE verbatim; BE with deps); `--all --project ` reruns all tests | +| | `test flaky` | Replay a test several times (auto-heal off) and report a stability score | +| | `test wait` | Block on one or more `runId`s until terminal | +| | `test cancel` | Cancel one or more in-flight runs (Ctrl-C during `--wait` only detaches — `cancel` is the real stop) | +| | `test artifact get` | Download the failure bundle for a specific `runId` | +| **Agent** | `agent install` / `agent list` / `agent status` | Add, list, or health-check coding-agent skills (pure-local): `claude`, `codex`, `cursor`, `cline`, `antigravity`, `kiro`, `windsurf`, `copilot` | > The earlier command names — `init`, `auth configure`, `auth whoami`, `auth logout` — still work as hidden, deprecated aliases (each prints a one-line notice pointing at the new name), so existing scripts keep running. `auth configure` now runs the full `setup` (it also installs the skill). @@ -173,7 +182,7 @@ That's the point of all of this: you no longer need the biggest, most expensive ## Contributing -Contributions are welcome — the CLI is plain TypeScript/Node (≥ 20), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: +Contributions are welcome — the CLI is plain TypeScript/Node (20.19+, 22.13+, or 24+), tested with Vitest, built with `tsc`. Getting a dev loop running takes a minute: ```bash git clone https://github.com/TestSprite/testsprite-cli.git diff --git a/docs/cli-v1-agent-install/skill-template.md b/docs/cli-v1-agent-install/skill-template.md index d051a39..764e04e 100644 --- a/docs/cli-v1-agent-install/skill-template.md +++ b/docs/cli-v1-agent-install/skill-template.md @@ -61,13 +61,13 @@ because . Treat this as unverified until that's resolved." Don't claim done. ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials configured? +testsprite auth status # credentials configured? ``` - `--version` fails → the CLI isn't installed. Tell the user to install the TestSprite CLI (see the TestSprite docs) and stop; don't install it for them. -- `auth whoami` fails → no credentials. Tell the user they can run - `testsprite auth configure`, then stop. +- `auth status` fails → no credentials. Tell the user they can run + `testsprite setup`, then stop. ## 2. Find the project diff --git a/package-lock.json b/package-lock.json index 4ea700c..4461016 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.1.2", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@testsprite/testsprite-cli", - "version": "0.1.2", + "version": "0.3.0", "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", + "undici": "^7.16.0", "valibot": "^1.4.1" }, "bin": { @@ -29,7 +30,7 @@ "vitest": "^2.1.4" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" } }, "node_modules/@ampproject/remapping": { @@ -1024,9 +1025,6 @@ "arm" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1041,9 +1039,6 @@ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1058,9 +1053,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1075,9 +1067,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1092,9 +1081,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1109,9 +1095,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1126,9 +1109,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1143,9 +1123,6 @@ "ppc64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1160,9 +1137,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1177,9 +1151,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1194,9 +1165,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1211,9 +1179,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1228,9 +1193,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3855,6 +3817,15 @@ "typescript": ">=4.8.4 <6.1.0" } }, + "node_modules/undici": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", + "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", diff --git a/package.json b/package.json index 2037d26..18336cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testsprite/testsprite-cli", - "version": "0.2.0", + "version": "0.4.0", "description": "Official TestSprite command-line interface", "type": "module", "main": "dist/index.js", @@ -28,7 +28,7 @@ "test:e2e": "npm run build && vitest run -c vitest.e2e.config.ts" }, "engines": { - "node": ">=20" + "node": "^20.19.0 || ^22.13.0 || >=24" }, "publishConfig": { "access": "public" @@ -50,6 +50,7 @@ "license": "Apache-2.0", "dependencies": { "commander": "^12.1.0", + "undici": "^7.16.0", "valibot": "^1.4.1" }, "devDependencies": { diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..7e17420 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,29 @@ +# `scripts/` — what runs where + +Every file here is classified below so nobody has to guess whether it's +safe/expected to run on a laptop, and so a Windows contributor knows exactly +what (if anything) they're missing. Per DEV-356 ("Windows-proof the +toolchain"): **the release/backport shell scripts are CI-only** — nothing in +this directory requires a human to run bash or perl locally. + +| File | Classification | Runs on | Notes | +| ------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `make-public-snapshot.sh` | **CI-ONLY** | the release pipeline's build job (`release-build.yml`) and the nightly divergence sentinel (dry-run mode) | Builds the scrubbed public snapshot. Never ships to the public repo (it's in its own DROP list). Not meant to be run by hand — see [`docs/internal/cli-oss/release-pipeline-ops.md`](../docs/internal/cli-oss/release-pipeline-ops.md) for the operator-facing flow. | +| `backport-public-pr.sh` | **CI-ONLY** (human-runnable for conflict recovery) | the auto-backport workflow, invoked per merged community PR | Also directly runnable by an operator resolving a cherry-pick conflict (`--record-only` after a manual fix) — that's an escape hatch, not the steady-state path. Requires `bash` + `gh` + `jq`; a Windows operator resolving a conflict does it via Git Bash/WSL, or asks another maintainer — this one script is the sole remaining bash dependency in the whole release flow. | +| `generate-version.mjs` | **HUMAN-RUN** (Node) | `npm run prebuild` / `npm run generate:version`, any OS | Pure Node — no shell-out, no bash/perl/BSD-vs-GNU assumptions. | +| `postbuild.mjs` | **HUMAN-RUN** (Node) | `npm run build`, any OS | Pure Node `fs` calls (sets the executable bit on `dist/index.js`; a no-op on Windows, which has no POSIX exec bit). | +| `p0-status-coverage.sql` | **DROPPED-INTERNAL** | ad hoc, by an operator, against Athena | Not part of any npm script or CI workflow. Never ships to the public repo (internal AWS resource references). | + +## The upshot for a Windows contributor / operator + +- **Local dev loop** (`npm ci`, `npm test`, `npm run build`, `npm run lint`, + `npm run typecheck`) needs only **git + Node ≥ 20** — nothing in this + directory runs as part of that loop. See CONTRIBUTING.md's "Developing on + Windows" section. +- **Releasing** does not require running `make-public-snapshot.sh` (or any + bash) on your own machine at all — it's a `workflow_dispatch` you trigger + and approve from a browser. See + [`docs/internal/cli-oss/release-pipeline-ops.md`](../docs/internal/cli-oss/release-pipeline-ops.md). +- The **only** scenario that still touches bash directly is resolving a rare + backport cherry-pick conflict by hand — everything else in the pipeline is + either Node or runs unattended in Actions. diff --git a/skills/testsprite-onboard.skill.md b/skills/testsprite-onboard.skill.md index cc9342c..df04c18 100644 --- a/skills/testsprite-onboard.skill.md +++ b/skills/testsprite-onboard.skill.md @@ -91,6 +91,27 @@ Each file is a COMPLETE plan and must include `projectId` (from step 2), `type: **Backend** — one `.py` file per endpoint, using `requests` with concrete assertions on status code and response body. +**Backend auth — read the injected `__AUTH_HEADERS__`, NEVER hardcode any credential.** This +covers **every** secret the API needs — Bearer/JWT tokens **and** API keys (`sk-…`, +`x-api-key`), basic-auth blobs, cookies. TestSprite prepends a managed credential block +(`__AUTH_CREDENTIAL__` / `__AUTH_TYPE__` / `__AUTH_HEADERS__`) to every backend test from the +project's Authentication settings, and `__AUTH_HEADERS__` already holds the right header(s) for +the configured type (Bearer → `{"Authorization": "Bearer …"}`; API key → `{"X-API-Key": "…"}`; +basic → `{"Authorization": "Basic …"}`). Spread it into your request headers — never paste a +literal `Bearer …` / `sk-…` / key value into the script: + +```python +r = requests.get(f"{TARGET_URL}/orders", headers={**__AUTH_HEADERS__}) +``` + +Configure the credential once on the project (ask the user for the value — never invent it or +reuse a key you happened to see): a static credential with +`testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential `, +or an auto-refreshing login with `testsprite project auto-auth …` so scheduled/repeat +runs keep working after the token expires. A hardcoded token expires within hours and a hardcoded +key can't be rotated centrally — `test create` emits a `[warn]` on an inlined credential; treat it +as a must-fix. + **Assertion rule (this is the whole game for FE):** every `assertion` step must name a **concrete, observable** outcome — an element, text, URL, count, or status. Never write `"verify it works"`, `"check the page loads"`, or other narrative that an AI judge can diff --git a/skills/testsprite-verify.codex.md b/skills/testsprite-verify.codex.md index f2c2456..d3c5373 100644 --- a/skills/testsprite-verify.codex.md +++ b/skills/testsprite-verify.codex.md @@ -10,17 +10,23 @@ docs/config and is about to be reported complete. Run after a feature or fix lands. Skip only for: docs-only edits, pure build/config changes, or when the repo has no TestSprite project linked. +The CLI only tests a reachable deployed URL (it rejects localhost). If the +change is only running locally, hand off to the TestSprite MCP when it's +available — it tunnels your local server; otherwise report the change as +unverified-because-undeployed and stop. If the user explicitly named a tool +(the CLI or the MCP), honor that over this reachability heuristic. + ## Core loop ### 1. Preflight ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials valid? +testsprite auth status # credentials valid? ``` If `--version` fails, tell the user to install the CLI and stop. -If `auth whoami` fails, tell the user to run `testsprite auth configure` and stop. +If `auth status` fails, tell the user to run `testsprite setup` and stop. ### 2. Find the project diff --git a/skills/testsprite-verify.skill.md b/skills/testsprite-verify.skill.md index 650a88b..40d60f2 100644 --- a/skills/testsprite-verify.skill.md +++ b/skills/testsprite-verify.skill.md @@ -21,6 +21,17 @@ Run the loop only once the change is live somewhere reachable (e.g. open the PR, let CI deploy the preview/staging environment) and pass that URL as `--target-url`. Running earlier verifies the previous build, not your change. +This CLI only tests a reachable deployed URL (it rejects localhost). If the +change is only running locally and isn't deployed anywhere reachable yet: + +- if the TestSprite MCP is available in this environment, hand off to it — it + tunnels your local server and tests the running app; +- otherwise report the change as unverified-because-undeployed and stop — don't + run against a stale deployment to manufacture a verdict. + +If the user explicitly named a tool (the CLI or the MCP), honor that over this +reachability heuristic. + ## When to skip The skip list is narrow: @@ -56,13 +67,13 @@ because . Treat this as unverified until that's resolved." Don't claim done. ```bash testsprite --version # CLI installed? -testsprite auth whoami # credentials configured? +testsprite auth status # credentials configured? ``` - `--version` fails → the CLI isn't installed. Tell the user to install the TestSprite CLI (see the TestSprite docs) and stop; don't install it for them. -- `auth whoami` fails → no credentials. Tell the user they can run - `testsprite auth configure`, then stop. +- `auth status` fails → no credentials. Tell the user they can run + `testsprite setup`, then stop. ## 2. Find the project @@ -165,6 +176,39 @@ only the Python **standard library + `requests` + `pytest` + `numpy` + `scipy`** - Get values from the API's responses (and captured variables), not by importing and calling the app's internals. +**Authentication — read the injected credential, NEVER hardcode any credential.** This +applies to **every** secret the API needs — Bearer/JWT tokens **and** API keys, basic-auth +blobs, session cookies. Do not paste a literal `Bearer …`, `sk-…`, `x-api-key` value, or any +other credential into the test. Before your script runs, TestSprite prepends a managed +credential block built from the project's Authentication settings, and `__AUTH_HEADERS__` +already contains the right header(s) for the configured auth type: + +```python +# Auto-injected credentials — do not modify +__AUTH_CREDENTIAL__ = "..." +__AUTH_TYPE__ = "Bearer token" # or "API key" / "basic token" / "public" +__AUTH_HEADERS__ = {"Authorization": "Bearer ..."} # API key → {"X-API-Key": "..."}; basic → {"Authorization": "Basic ..."} +``` + +Spread `__AUTH_HEADERS__` into every authenticated request — it adapts to whatever auth type +the project is configured for, so the same line works for Bearer, API-key, or basic auth: + +```python +r = requests.get(f"{TARGET_URL}/profile", headers={**__AUTH_HEADERS__}) +``` + +Configure the credential **once on the project** (ask the user for the value — never invent +or reuse a key you happened to see), and the block stays correct + refreshable: + +- **Bearer / API key / basic (static):** + `testsprite project credential --type "Bearer token"|"API key"|"basic token" --credential ` +- **Auto-refreshing login (recurring token):** `testsprite project auto-auth …` + +A hardcoded token expires within hours (and a hardcoded key can't be rotated centrally), so +the test breaks on later runs; the managed block is rewritten with a fresh value each run. +`test create` emits a `[warn]` when it detects an inlined credential literal — treat that as +a must-fix, not a nuisance. + **Backend tests that share state declare dependencies at create time.** For a one-off verification, prefer a single self-contained script (log in inside the same file). But when the coverage set splits naturally into producer → consumer @@ -192,10 +236,10 @@ testsprite test create --type backend --project --name "fixture user ordering: trigger the set with `test run --all` (§4) and producers run before consumers, `teardown` last. Chaining `run A --wait && run B --wait` yourself loses the engine's variable passing and conflicts with concurrent runs. -- **Declarations are currently create-only.** `test update` cannot amend them and - `test get` / `test list` don't echo them back, so note what each test - produces/needs as you create it; changing the graph later means delete + - recreate. +- **Declarations are editable.** `test update` amends them (`--produces` / + `--needs` / `--category`, repeatable) and `test get` echoes them back — + still declare the graph at create time when you can, so wave ordering is + right on the first run. **Show the user the drafted plan / code before creating it** — creating writes to their project. One short confirmation; let them edit the tempfile first. diff --git a/src/commands/agent.test.ts b/src/commands/agent.test.ts index 523c5fc..7326aeb 100644 --- a/src/commands/agent.test.ts +++ b/src/commands/agent.test.ts @@ -9,13 +9,21 @@ import { MANAGED_SECTION_END, ONBOARD_CODEX_LINE, SKILLS, + buildSkillMarker, pathFor, renderForTarget, + renderOwnFileWithMarker, TARGETS, type AgentTarget, } from '../lib/agent-targets.js'; -import type { AgentDeps, AgentFs, InstallResult, ListResult } from './agent.js'; -import { AGENTS_MD_CODEX_BUDGET_BYTES, createAgentCommand, runInstall, runList } from './agent.js'; +import type { AgentDeps, AgentFs, InstallResult, ListResult, StatusResult } from './agent.js'; +import { + AGENTS_MD_CODEX_BUDGET_BYTES, + createAgentCommand, + runInstall, + runList, + runStatus, +} from './agent.js'; // --------------------------------------------------------------------------- // In-memory AgentFs backed by a Map @@ -653,29 +661,39 @@ describe('runInstall — multi-target', () => { // --------------------------------------------------------------------------- describe('runInstall — empty target', () => { - it('non-TTY with no target throws exit 5', async () => { - const { fs: agentFs } = makeMemFs(); + it('non-TTY with no target defaults to claude and installs the skill file', async () => { + const { store, fs: agentFs } = makeMemFs(); + const { capture, deps } = makeCapture(); + + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: [], + force: false, + }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); + + const claudeAbs = path.resolve(CWD, TARGETS.claude.path); + expect(store.has(claudeAbs)).toBe(true); + expect(capture.stderr.join('\n')).toContain('defaulting to claude'); + }); + + it('non-TTY default writes the canonical claude content', async () => { + const { store, fs: agentFs } = makeMemFs(); const { deps } = makeCapture(); - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: [], - force: false, - }, - { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, - ); - } catch (err) { - thrown = err; - } + await runInstall( + { profile: 'default', output: 'text', debug: false, dryRun: false, target: [], force: false }, + { cwd: CWD, fs: agentFs, isTTY: false, ...deps }, + ); - expect(thrown).toBeInstanceOf(ApiError); - expect((thrown as ApiError).exitCode).toBe(5); + const { path: relPath, content } = renderForTarget('claude', 'testsprite-verify'); + const abs = path.resolve(CWD, relPath); + expect(store.get(abs)).toBe(content); }); it('TTY with injected prompt returning "claude" installs claude', async () => { @@ -741,6 +759,7 @@ describe('runList', () => { expect(out).toContain('cursor'); expect(out).toContain('cline'); expect(out).toContain('antigravity'); + expect(out).toContain('kiro'); expect(out).toContain('codex'); expect(out).toContain('ga'); expect(out).toContain('experimental'); @@ -749,6 +768,7 @@ describe('runList', () => { expect(out).toContain(TARGETS.cursor.path); expect(out).toContain(TARGETS.cline.path); expect(out).toContain(TARGETS.antigravity.path); + expect(out).toContain(TARGETS.kiro.path); expect(out).toContain(TARGETS.codex.path); }); @@ -759,13 +779,16 @@ describe('runList', () => { const json = JSON.parse(capture.stdout.join('\n')) as ListResult[]; expect(Array.isArray(json)).toBe(true); - // 5 targets × 2 default skills = 10 rows - expect(json).toHaveLength(10); + // 8 targets × 2 default skills = 16 rows + expect(json).toHaveLength(16); const targets = json.map(r => r.target); expect(targets).toContain('claude'); expect(targets).toContain('cursor'); expect(targets).toContain('cline'); + expect(targets).toContain('windsurf'); + expect(targets).toContain('copilot'); expect(targets).toContain('antigravity'); + expect(targets).toContain('kiro'); expect(targets).toContain('codex'); // skill field present on each row const skills = json.map(r => r.skill); @@ -905,11 +928,11 @@ describe('createAgentCommand wiring', () => { }); // --------------------------------------------------------------------------- -// All four own-file targets installed at once +// All own-file targets installed at once // --------------------------------------------------------------------------- -describe('runInstall — all four own-file targets', () => { - it('installs all four own-file targets in one invocation', async () => { +describe('runInstall — all own-file targets', () => { + it('installs every own-file target in one invocation', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -919,7 +942,7 @@ describe('runInstall — all four own-file targets', () => { output: 'text', debug: false, dryRun: false, - target: ['claude', 'cursor', 'cline', 'antigravity'], + target: [...OWN_FILE_TARGETS], skills: ['testsprite-verify'], force: false, }, @@ -937,11 +960,11 @@ describe('runInstall — all four own-file targets', () => { }); // --------------------------------------------------------------------------- -// Dry-run for all four own-file targets +// Dry-run for all seven own-file targets // --------------------------------------------------------------------------- describe('runInstall — dry-run all own-file targets', () => { - it('writes nothing for any of the four own-file targets (default 2 skills = 8 would-write lines)', async () => { + it('writes nothing for any of the seven own-file targets (default 2 skills = 14 would-write lines)', async () => { const { store, fs: agentFs } = makeMemFs(); const { capture, deps } = makeCapture(); @@ -951,7 +974,7 @@ describe('runInstall — dry-run all own-file targets', () => { output: 'text', debug: false, dryRun: true, - target: ['claude', 'cursor', 'cline', 'antigravity'], + target: ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'windsurf', 'copilot'], force: false, }, { cwd: CWD, fs: agentFs, ...deps }, @@ -961,9 +984,9 @@ describe('runInstall — dry-run all own-file targets', () => { const stderrOut = capture.stderr.join('\n'); // Banner appears once expect(stderrOut).toContain('[dry-run] no files written'); - // 4 targets × 2 default skills = 8 would-write lines + // 7 targets × 2 default skills = 14 would-write lines const wouldWriteLines = stderrOut.split('\n').filter(l => l.includes('would write')); - expect(wouldWriteLines.length).toBe(8); + expect(wouldWriteLines.length).toBe(14); }); }); @@ -1085,75 +1108,87 @@ describe('runInstall — default AgentFs (real disk)', () => { expect(readFileSync(abs, 'utf8')).toBe(content); }); - it('refuses to write through a symlinked parent dir (real disk) — exit 5', async () => { - const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-parent-')); - const outside = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-')); - // `.claude` is a real symlink to a directory outside the project root. - symlinkSync(outside, path.join(tmpRoot, '.claude'), 'dir'); - const { deps } = makeCapture(); - - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: ['claude'], - skills: ['testsprite-verify'], - force: false, - dir: tmpRoot, - }, - { ...deps }, - ); - } catch (err) { - thrown = err; - } - - expect(thrown).toBeInstanceOf(CLIError); - expect((thrown as CLIError).exitCode).toBe(5); - // Nothing was created through the symlink, outside --dir. - expect(existsSync(path.join(outside, 'skills'))).toBe(false); - }); - - it('refuses to overwrite a symlinked target file (real disk) with --force — exit 5', async () => { - const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-target-')); - const outsideDir = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-target-')); - const { path: relPath } = renderForTarget('claude', 'testsprite-verify'); - const abs = path.resolve(tmpRoot, relPath); - const nodeFs = await import('node:fs/promises'); - await nodeFs.mkdir(path.dirname(abs), { recursive: true }); - // SKILL.md is a real symlink to a file outside the project root. - const outsideFile = path.join(outsideDir, 'secret.txt'); - await nodeFs.writeFile(outsideFile, 'SECRET', 'utf8'); - symlinkSync(outsideFile, abs, 'file'); - const { deps } = makeCapture(); + // `fs.symlinkSync` needs elevated privileges or Developer Mode on Windows + // (EPERM otherwise) — not guaranteed on hosted CI runners. The underlying + // guard (`inspectTargetPath` fail-closing via `lstat`) is exercised on + // POSIX runners; TODO(DEV-356): revisit if/when a reliable Windows + // symlink-creation path (junctions, or an elevated runner) is available. + it.skipIf(process.platform === 'win32')( + 'refuses to write through a symlinked parent dir (real disk) — exit 5', + async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-parent-')); + const outside = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-')); + // `.claude` is a real symlink to a directory outside the project root. + symlinkSync(outside, path.join(tmpRoot, '.claude'), 'dir'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + skills: ['testsprite-verify'], + force: false, + dir: tmpRoot, + }, + { ...deps }, + ); + } catch (err) { + thrown = err; + } - let thrown: unknown; - try { - await runInstall( - { - profile: 'default', - output: 'text', - debug: false, - dryRun: false, - target: ['claude'], - skills: ['testsprite-verify'], - force: true, - dir: tmpRoot, - }, - { ...deps }, - ); - } catch (err) { - thrown = err; - } + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + // Nothing was created through the symlink, outside --dir. + expect(existsSync(path.join(outside, 'skills'))).toBe(false); + }, + ); + + // Same Windows symlink-privilege caveat as the test above. + it.skipIf(process.platform === 'win32')( + 'refuses to overwrite a symlinked target file (real disk) with --force — exit 5', + async () => { + const tmpRoot = mkdtempSync(path.join(tmpdir(), 'agent-test-symlink-target-')); + const outsideDir = mkdtempSync(path.join(tmpdir(), 'agent-test-outside-target-')); + const { path: relPath } = renderForTarget('claude', 'testsprite-verify'); + const abs = path.resolve(tmpRoot, relPath); + const nodeFs = await import('node:fs/promises'); + await nodeFs.mkdir(path.dirname(abs), { recursive: true }); + // SKILL.md is a real symlink to a file outside the project root. + const outsideFile = path.join(outsideDir, 'secret.txt'); + await nodeFs.writeFile(outsideFile, 'SECRET', 'utf8'); + symlinkSync(outsideFile, abs, 'file'); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude'], + skills: ['testsprite-verify'], + force: true, + dir: tmpRoot, + }, + { ...deps }, + ); + } catch (err) { + thrown = err; + } - expect(thrown).toBeInstanceOf(CLIError); - expect((thrown as CLIError).exitCode).toBe(5); - // The outside file was NOT overwritten (nor clobbered via the .bak path). - expect(readFileSync(outsideFile, 'utf8')).toBe('SECRET'); - }); + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + // The outside file was NOT overwritten (nor clobbered via the .bak path). + expect(readFileSync(outsideFile, 'utf8')).toBe('SECRET'); + }, + ); }); // --------------------------------------------------------------------------- @@ -1311,6 +1346,50 @@ describe('runInstall — symlink safety', () => { expect(writeCalls.length).toBe(0); // never wrote a .bak nor through the link }); + it('dry-run: refuses (exit 5) when the target file is a symlink (parity with real install)', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + // Same planted SKILL.md symlink as the real-install case above: dry-run + // must report the same refusal the real install would, not a success. + seedSymlink(path.resolve(CWD, TARGETS.claude.path)); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false, dryRun: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + + it('dry-run: refuses (exit 5) when a parent path component is a symlink', async () => { + const { fs: agentFs, writeCalls, seedSymlink } = makeMemFs(); + seedSymlink(path.resolve(CWD, '.claude')); + const { deps } = makeCapture(); + + let thrown: unknown; + try { + await runInstall( + { ...BASE_OPTS, target: ['claude'], force: false, dryRun: true }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(5); + expect((thrown as CLIError).message).toContain('symlink'); + expect(writeCalls.length).toBe(0); + }); + it('does not write through a symlinked .bak slot — backs up to a numbered slot', async () => { const { store, fs: agentFs, seedFile, seedSymlink } = makeMemFs(); const abs = path.resolve(CWD, TARGETS.claude.path); @@ -1969,12 +2048,12 @@ describe('[P3 round-2] codex --dry-run: composed-size precision + read-failure s const { capture, deps } = makeCapture(); const agentsAbs = path.resolve(CWD, TARGETS.codex.path); - // Old managed section with a 6 KiB body + 26 KiB of user prose. - // existing (~32.9 KiB) + new section (~4.8 KiB) > 32 KiB → the OLD formula - // would warn; the composed replace result (26 KiB + new section) is - // comfortably under budget → no warn expected. + // Old managed section with a 6 KiB body + 25 KiB of user prose. + // existing (~31.9 KiB) + new section (~6.2 KiB: verify codex body + + // onboard aggregate) > 32 KiB → the OLD formula would warn; the composed + // replace result (25 KiB + new section) is under budget → no warn expected. const oldSection = `${MANAGED_SECTION_BEGIN}\n${'o'.repeat(6 * 1024)}\n${MANAGED_SECTION_END}\n`; - const userProse = `# My own AGENTS.md\n${'u'.repeat(26 * 1024)}\n`; + const userProse = `# My own AGENTS.md\n${'u'.repeat(25 * 1024)}\n`; seedFile(agentsAbs, `${userProse}\n${oldSection}`); await runInstall({ ...BASE_OPTS_DRY, target: ['codex'] }, { cwd: CWD, fs: agentFs, ...deps }); @@ -2383,3 +2462,120 @@ describe('runInstall — SKILLS registry / DEFAULT_SKILLS contract', () => { expect(ONBOARD_CODEX_LINE).toContain('**First-time setup:**'); }); }); + +// --------------------------------------------------------------------------- +// runStatus — `agent status` (issue #123) +// --------------------------------------------------------------------------- + +describe('runStatus — agent status (issue #123)', () => { + const statusOpts = { + profile: 'default' as const, + output: 'json' as const, + debug: false, + dryRun: false, + }; + + /** Run status against the given fs and return the printed rows. */ + async function statusRows(agentFs: AgentFs): Promise<{ rows: StatusResult[]; thrown: unknown }> { + const { capture, deps } = makeCapture(); + let thrown: unknown; + try { + await runStatus(statusOpts, { cwd: CWD, fs: agentFs, ...deps }); + } catch (err) { + thrown = err; + } + return { rows: JSON.parse(capture.stdout.join('')) as StatusResult[], thrown }; + } + + it('nothing installed: every row is absent and the command exits 0', async () => { + const { fs: agentFs } = makeMemFs(); + const { rows, thrown } = await statusRows(agentFs); + expect(thrown).toBeUndefined(); + expect(rows).toHaveLength(Object.keys(TARGETS).length * DEFAULT_SKILLS.length); + expect(rows.every(row => row.state === 'absent')).toBe(true); + }); + + it('fresh installs read ok (own-file and codex managed section), exit 0', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + await runInstall( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + target: ['claude', 'codex'], + skills: [...DEFAULT_SKILLS], + force: false, + }, + { cwd: CWD, fs: agentFs, ...deps }, + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(thrown).toBeUndefined(); + for (const skill of DEFAULT_SKILLS) { + expect(rows.find(r => r.target === 'claude' && r.skill === skill)?.state).toBe('ok'); + expect(rows.find(r => r.target === 'codex' && r.skill === skill)?.state).toBe('ok'); + expect(rows.find(r => r.target === 'cursor' && r.skill === skill)?.state).toBe('absent'); + } + }); + + it('stale: a marker whose hash matches an OLDER body reads stale and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const oldBody = '# TestSprite Verification Loop\n\nold body from a previous CLI release\n'; + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + renderOwnFileWithMarker( + 'claude', + 'testsprite-verify', + buildSkillMarker('testsprite-verify', oldBody), + oldBody, + ), + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'stale', + ); + expect(thrown).toBeInstanceOf(CLIError); + expect((thrown as CLIError).exitCode).toBe(1); + expect((thrown as CLIError).message).toContain('need attention'); + }); + + it('modified: current hash but edited bytes reads modified and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + const canonical = renderForTarget('claude', 'testsprite-verify').content; + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + `${canonical}\n\n`, + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'modified', + ); + expect((thrown as CLIError).exitCode).toBe(1); + }); + + it('unmarked: an artifact without a marker line reads unmarked and exits 1', async () => { + const { fs: agentFs, seedFile } = makeMemFs(); + seedFile( + path.resolve(CWD, pathFor('claude', 'testsprite-verify')), + '# hand-rolled skill file with no marker\n', + ); + + const { rows, thrown } = await statusRows(agentFs); + expect(rows.find(r => r.target === 'claude' && r.skill === 'testsprite-verify')?.state).toBe( + 'unmarked', + ); + expect((thrown as CLIError).exitCode).toBe(1); + }); + + it('rejects an explicit empty --dir (exit 5), matching the resolve-to-cwd hazard', async () => { + const { fs: agentFs } = makeMemFs(); + const { deps } = makeCapture(); + await expect( + runStatus({ ...statusOpts, dir: ' ' }, { cwd: CWD, fs: agentFs, ...deps }), + ).rejects.toMatchObject({ exitCode: 5 }); + }); +}); diff --git a/src/commands/agent.ts b/src/commands/agent.ts index 3347453..7d7e7c8 100644 --- a/src/commands/agent.ts +++ b/src/commands/agent.ts @@ -4,17 +4,23 @@ import { Command } from 'commander'; import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; import { CLIError, localValidationError } from '../lib/errors.js'; import type { OutputMode } from '../lib/output.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptText } from '../lib/prompt.js'; import { type AgentTarget, TARGETS, SKILLS, DEFAULT_SKILLS, + MARKER_SKILL_SEPARATOR, pathFor, loadSkillBodyFor, + bodyHash12, + compactBodyFor, buildCodexAggregate, + buildSkillMarker, + parseSkillMarker, renderForTarget, + renderOwnFileWithMarker, MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, } from '../lib/agent-targets.js'; @@ -150,11 +156,15 @@ async function writeBackup(agentFs: AgentFs, abs: string, existing: string): Pro // --------------------------------------------------------------------------- /** - * Build the section block to inject (sentinels + body + trailing newline). + * Build the section block to inject (sentinels + marker + body + trailing + * newline). The provenance marker line sits just inside the BEGIN sentinel so + * `agent status` can fingerprint the section. The same skill set + CLI version + * + body always produce byte-identical output, so the classifySection + * 'unchanged' fast-path keeps working across re-installs. * Uses \n throughout; the caller handles CRLF normalisation. */ -function buildSection(body: string): string { - return `${MANAGED_SECTION_BEGIN}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; +function buildSection(body: string, markerLine: string): string { + return `${MANAGED_SECTION_BEGIN}\n${markerLine}\n${body.trimEnd()}\n${MANAGED_SECTION_END}\n`; } /** @@ -361,18 +371,19 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (rawTargets.length === 0) { const isTTY = deps.isTTY ?? Boolean(process.stdin.isTTY); if (!isTTY) { - throw localValidationError( - 'target', - `required; pass --target=claude (comma-separated or repeated for several). Supported: ${Object.keys(TARGETS).join(', ')}`, + stderrFn( + '[info] --target not specified; defaulting to claude. Pass --target= to select a different agent.', ); + resolvedTargetStrings = ['claude']; + } else { + const promptFn = deps.prompt ?? ((q: string) => promptText(q)); + const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); + const defaulted = answer || 'claude'; + resolvedTargetStrings = defaulted + .split(',') + .map(s => s.trim()) + .filter(Boolean); } - const promptFn = deps.prompt ?? ((q: string) => promptText(q)); - const answer = (await promptFn('Targets to install (comma-separated) [claude]: ')).trim(); - const defaulted = answer || 'claude'; - resolvedTargetStrings = defaulted - .split(',') - .map(s => s.trim()) - .filter(Boolean); } else { resolvedTargetStrings = rawTargets; } @@ -434,10 +445,32 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr } return b; }; + // Budget-capped own-file targets (e.g. windsurf) render the compact per-skill + // body so the rule file isn't truncated by the agent. Cached separately; must + // match renderForTarget's default selection so written bytes equal the asserted + // render. + const compactBodyCache = new Map(); + const compactBodyForSkill = (skill: string): string => { + let b = compactBodyCache.get(skill); + if (b === undefined) { + b = compactBodyFor(skill); + compactBodyCache.set(skill, b); + } + return b; + }; + const ownFileBodyFor = (t: AgentTarget, skill: string): string => + TARGETS[t].compactBody ? compactBodyForSkill(skill) : bodyForSkill(skill); let codexSectionCache: string | undefined; const getCodexSection = (): string => { if (codexSectionCache === undefined) { - codexSectionCache = buildSection(buildCodexAggregate(skills)); + const aggregate = buildCodexAggregate(skills); + // ONE marker for the whole managed section: it names every aggregated + // skill ('+'-joined) and hashes the canonical aggregate body, so + // `agent status` can attribute and fingerprint the section per skill. + codexSectionCache = buildSection( + aggregate, + buildSkillMarker(skills.join(MARKER_SKILL_SEPARATOR), aggregate), + ); } return codexSectionCache; }; @@ -635,9 +668,20 @@ export async function runInstall(opts: InstallOptions, deps: AgentDeps = {}): Pr if (abs !== root && !abs.startsWith(root + path.sep)) { throw new CLIError(`refusing to write outside --dir: ${relPath}`, 5); } - const content = renderForTarget(t, skill, bodyForSkill(skill)).content; + const content = renderForTarget(t, skill, ownFileBodyFor(t, skill)).content; if (opts.dryRun) { + // Apply the SAME symlink fail-close guard as the real install path + // below (the codex managed-section branch already does this). Without + // it, dry-run reports success for a planted symlink that the real + // install would refuse with exit 5. + const dryRunSt = await inspectTargetPath(agentFs, root, relPath); + if (dryRunSt !== null && !dryRunSt.isFile) { + throw new CLIError( + `${relPath} exists but is not a regular file — remove it and re-run.`, + 5, + ); + } const bytes = Buffer.byteLength(content, 'utf8'); dryRunLines.push({ abs, bytes, note: '' }); results.push({ target: t, path: relPath, action: 'dry-run', skills: [skill] }); @@ -778,6 +822,236 @@ export async function runList(opts: CommonOptions, deps: AgentDeps = {}): Promis }); } +// --------------------------------------------------------------------------- +// runStatus (issue #123: detect silently stale installed skill files) +// --------------------------------------------------------------------------- + +/** + * Health of one installed skill artifact, as reported by `agent status`. + * + * Decision order (first match wins): + * - 'absent' : nothing at the landing path (codex: no managed section, + * including an AGENTS.md that exists without our sentinels). + * - 'corrupt' : codex only. Dangling or duplicated sentinels, the same + * classification `agent install` refuses on; status REPORTS it + * instead of refusing. + * - 'unmarked' : artifact present but carries no testsprite-skill marker + * (installed before markers existed), or the landing path is + * occupied by a non-regular file (never followed). + * - 'stale' : marker present, but its hash differs from the current + * canonical body: a re-install would change the content. Edits + * on top of an OLD install also read stale (older renders + * cannot be reproduced); the remedy is the same re-install. + * - 'modified' : marker hash matches the current body, but the artifact bytes + * differ from the canonical render carrying that same marker + * line: the user edited the artifact after install. + * - 'ok' : marker hash matches and the bytes equal the canonical render + * with the file's own marker line (a version-string-only lag + * with an unchanged body still reads ok). + * + * For the codex managed section, ONE marker names every aggregated skill + * ('+'-joined); skills not named by the marker report 'absent'. + */ +export type SkillArtifactState = 'ok' | 'stale' | 'modified' | 'unmarked' | 'absent' | 'corrupt'; + +export interface StatusResult { + target: AgentTarget; + skill: string; + path: string; + state: SkillArtifactState; +} + +interface StatusOptions extends CommonOptions { + dir?: string; +} + +/** + * Classify one own-file artifact per the {@link SkillArtifactState} contract. + * Comparisons are byte-exact, matching the installer's own skipped/blocked + * comparison for own-file targets. + */ +async function classifyOwnFileState( + agentFs: AgentFs, + abs: string, + target: AgentTarget, + skill: string, + bodyForSkill: (skill: string) => string, +): Promise { + const stat = await agentFs.lstat(abs); + if (stat === null) return 'absent'; + // Occupied by a directory or symlink: not something our installer wrote, and + // never followed (mirrors the installer's fail-closed stance on symlinks). + if (!stat.isFile) return 'unmarked'; + + const existing = await agentFs.readFile(abs); + const marker = parseSkillMarker(existing); + if (marker === null) return 'unmarked'; + + const canonicalBody = bodyForSkill(skill); + if (marker.hash12 !== bodyHash12(canonicalBody)) return 'stale'; + + // Hash matches the current body: pristine iff the file equals the canonical + // render carrying its own marker line, so a marker whose version string lags + // behind an unchanged body still reads ok. + const reRender = renderOwnFileWithMarker(target, skill, marker.line, canonicalBody); + return existing === reRender ? 'ok' : 'modified'; +} + +/** + * Classify the codex managed section per skill. The section is ONE artifact + * carrying ONE marker that names every aggregated skill, so a single + * inspection answers all skill rows; the returned function maps a skill name + * to its state. Comparisons are CRLF-insensitive on the section bytes. + */ +async function classifyManagedSectionStates( + agentFs: AgentFs, + abs: string, +): Promise<(skill: string) => SkillArtifactState> { + const constantState = + (state: SkillArtifactState): ((skill: string) => SkillArtifactState) => + () => + state; + + const stat = await agentFs.lstat(abs); + if (stat === null) return constantState('absent'); + // Occupied by a directory or symlink: never followed (fail-closed). + if (!stat.isFile) return constantState('unmarked'); + + const existing = await agentFs.readFile(abs); + + // Current canonical section for the default skill set. classifySection's + // 'unchanged' answers the common all-defaults-fresh case; its + // corrupt/append classification is reused verbatim for status verdicts. + const defaultAggregate = buildCodexAggregate(DEFAULT_SKILLS); + const defaultSection = buildSection( + defaultAggregate, + buildSkillMarker(DEFAULT_SKILLS.join(MARKER_SKILL_SEPARATOR), defaultAggregate), + ); + const sectionState = classifySection(existing, defaultSection); + + if (sectionState.kind === 'corrupt') return constantState('corrupt'); + // No standalone sentinels anywhere: the managed section is not installed. + if (sectionState.kind === 'append') return constantState('absent'); + if (sectionState.kind === 'unchanged') { + // Byte-identical to today's default install. + return skill => ((DEFAULT_SKILLS as readonly string[]).includes(skill) ? 'ok' : 'absent'); + } + if (sectionState.kind !== 'replace') { + // 'create' is unreachable when the file exists; treat defensively as absent. + return constantState('absent'); + } + + // Sentinels are present but the section differs from today's default + // canonical: slice the live section bytes out of the file and inspect its + // own marker (before/after are exact byte prefix/suffix around the section). + const sectionContent = existing.slice( + sectionState.before.length, + existing.length - sectionState.after.length, + ); + const marker = parseSkillMarker(sectionContent); + if (marker === null) return constantState('unmarked'); + + const installedSkills = marker.skill.split(MARKER_SKILL_SEPARATOR); + const coversSkill = (skill: string): boolean => installedSkills.includes(skill); + + // A marker naming a skill this CLI does not ship cannot be re-rendered; + // report the named skills stale (a re-install refreshes the section). + if (installedSkills.some(name => SKILLS[name] === undefined)) { + return skill => (coversSkill(skill) ? 'stale' : 'absent'); + } + + const canonicalAggregate = buildCodexAggregate(installedSkills); + if (marker.hash12 !== bodyHash12(canonicalAggregate)) { + return skill => (coversSkill(skill) ? 'stale' : 'absent'); + } + + // Hash matches the current aggregate: the section is pristine iff its bytes + // equal a re-render carrying its own marker line (version-string-only lag + // with an unchanged body still reads ok). + const pristine = + sectionContent.replace(/\r\n/g, '\n') === buildSection(canonicalAggregate, marker.line); + return skill => (coversSkill(skill) ? (pristine ? 'ok' : 'modified') : 'absent'); +} + +/** + * `agent status`: one row per (target × default skill), each classified per + * the {@link SkillArtifactState} contract. Exit contract: returns normally + * (exit 0) when every row is 'ok' or 'absent'; throws CLIError exit 1 when any + * row is stale/modified/unmarked/corrupt, so the command can gate CI. + */ +export async function runStatus(opts: StatusOptions, deps: AgentDeps = {}): Promise { + const agentFs = deps.fs ?? defaultAgentFs; + const out = makeOutput(opts.output, deps); + + // An explicit but empty --dir must not silently resolve to cwd + // (path.resolve('') === cwd). + if (opts.dir !== undefined && opts.dir.trim() === '') { + throw localValidationError('dir', 'must not be empty'); + } + const dir = opts.dir !== undefined ? opts.dir.trim() : (deps.cwd ?? process.cwd()); + const root = path.resolve(dir); + + // Canonical own-file bodies, read once per skill (same lazy caching pattern + // as runInstall's bodyForSkill). + const skillBodyCache = new Map(); + const bodyForSkill = (skill: string): string => { + let cachedBody = skillBodyCache.get(skill); + if (cachedBody === undefined) { + cachedBody = loadSkillBodyFor(skill); + skillBodyCache.set(skill, cachedBody); + } + return cachedBody; + }; + + const results: StatusResult[] = []; + for (const [target, spec] of Object.entries(TARGETS) as [ + AgentTarget, + { mode: string; path: string }, + ][]) { + if (spec.mode === 'managed-section') { + const stateFor = await classifyManagedSectionStates(agentFs, path.resolve(root, spec.path)); + for (const skill of DEFAULT_SKILLS) { + results.push({ target, skill, path: spec.path, state: stateFor(skill) }); + } + continue; + } + for (const skill of DEFAULT_SKILLS) { + const relPath = pathFor(target, skill); + results.push({ + target, + skill, + path: relPath, + state: await classifyOwnFileState( + agentFs, + path.resolve(root, relPath), + target, + skill, + bodyForSkill, + ), + }); + } + } + + out.print(results, data => { + const items = data as StatusResult[]; + const header = `${'TARGET'.padEnd(14)} ${'SKILL'.padEnd(20)} ${'STATE'.padEnd(10)} PATH`; + const rows = items.map( + row => `${row.target.padEnd(14)} ${row.skill.padEnd(20)} ${row.state.padEnd(10)} ${row.path}`, + ); + return [header, ...rows].join('\n'); + }); + + const needingAttention = results.filter( + result => result.state !== 'ok' && result.state !== 'absent', + ); + if (needingAttention.length > 0) { + throw new CLIError( + `${needingAttention.length} skill artifact(s) need attention (stale/modified/unmarked/corrupt); re-run \`testsprite agent install\` (add --force for own-file targets) to refresh them.`, + 1, + ); + } +} + // --------------------------------------------------------------------------- // Command factory // --------------------------------------------------------------------------- @@ -788,7 +1062,7 @@ function collect(v: string, prev: string[]): string[] { export function createAgentCommand(deps: AgentDeps = {}): Command { const agent = new Command('agent').description( - 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Codex)', + 'Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, Kiro, Windsurf, Copilot, Codex)', ); agent @@ -798,7 +1072,7 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { ) .option( '--target ', - 'Agent target(s): claude, cursor, cline, antigravity, codex (comma-separated or repeated)', + 'Agent target(s): claude, cursor, cline, antigravity, kiro, windsurf, copilot, codex (comma-separated or repeated)', collect, [], ) @@ -841,6 +1115,17 @@ export function createAgentCommand(deps: AgentDeps = {}): Command { await runList(resolveCommonOptions(command), deps); }); + agent + .command('status') + .description( + 'Check installed TestSprite skill files against this CLI version: ok, stale, modified, unmarked, absent, or corrupt (exits 1 when anything needs attention, so it can gate CI)', + ) + .option('--dir ', 'Project root to inspect (default: cwd)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: { dir?: string }, command: Command) => { + await runStatus({ ...resolveCommonOptions(command), dir: cmdOpts.dir }, deps); + }); + return agent; } @@ -852,7 +1137,7 @@ function resolveCommonOptions(command: Command): CommonOptions { const globals = command.optsWithGlobals() as Partial; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, diff --git a/src/commands/auth.test.ts b/src/commands/auth.test.ts index d65e6cc..0ce0bfb 100644 --- a/src/commands/auth.test.ts +++ b/src/commands/auth.test.ts @@ -110,6 +110,50 @@ describe('runConfigure', () => { ); }); + it('uses requestTimeoutMs for the pre-write key validation ping', async () => { + const { deps } = makeCapture(); + let sawAbort = false; + const fetchImpl = vi.fn( + async (_input: string | URL | Request, init?: RequestInit) => + new Promise((_resolve, reject) => { + const signal = init?.signal; + const timeout = setTimeout(() => { + reject(new Error('requestTimeoutMs was not applied to the validation ping')); + }, 50); + signal?.addEventListener( + 'abort', + () => { + sawAbort = true; + clearTimeout(timeout); + reject(new DOMException('The operation timed out.', 'TimeoutError')); + }, + { once: true }, + ); + }), + ) as unknown as typeof fetch; + + await expect( + runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + requestTimeoutMs: 1, + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl, + }, + ), + ).rejects.toBeInstanceOf(CLIError); + + expect(sawAbort).toBe(true); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + it('throws VALIDATION_ERROR when --from-env is set but key is missing', async () => { const { deps } = makeCapture(); await expect( @@ -120,6 +164,100 @@ describe('runConfigure', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('rejects a malformed endpoint before key validation fetch', async () => { + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: true, + endpointUrl: 'not-a-url', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(capture.stderr.join('\n')).not.toContain('API key rejected'); + }); + + it('rejects a non-http endpoint before key validation fetch', async () => { + const { deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: true, + endpointUrl: 'ftp://example.com', + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + }); + + it('rejects a malformed dry-run endpoint before emitting dry-run output', async () => { + const { capture, deps } = makeCapture(); + const fetchImpl = vi.fn(); + + await expect( + runConfigure( + { + profile: 'default', + output: 'json', + debug: false, + fromEnv: false, + dryRun: true, + endpointUrl: 'not-a-url', + }, + { + ...deps, + env: {}, + credentialsPath, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(readProfile('default', { path: credentialsPath })).toBeUndefined(); + expect(capture.stderr.join('\n')).not.toContain('[dry-run]'); + expect(capture.stdout).toEqual([]); + }); + it('prompts only for the API key (never the endpoint) and defaults to prod', async () => { const { capture, deps } = makeCapture(); // Prompt object exposes ONLY `secret`. If runConfigure tried to prompt for @@ -140,6 +278,35 @@ describe('runConfigure', () => { expect(capture.prelude.join('')).toContain('Configuring profile "default"'); }); + it('routes the interactive prelude to stderr by default, keeping stdout for the result', async () => { + // Regression: the prelude used to default to process.stdout, polluting the + // result stream (and the JSON document under --output json). With no + // injected preludeWrite/stderr, the default must land on stderr, not stdout. + const stdout: string[] = []; + const errChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + try { + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false }, + { + stdout: line => stdout.push(line), + prompt: { secret: vi.fn(async () => 'sk-typed') }, + fetchImpl: meOkFetch, + credentialsPath, + env: {}, + }, + ); + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + } + expect(errChunks.join('')).toContain('Configuring profile "default"'); + expect(stdout.join('\n')).not.toContain('Configuring profile'); + }); + it('interactive path resolves the endpoint from TESTSPRITE_API_URL without prompting', async () => { const { capture, deps } = makeCapture(); const prompt = { secret: vi.fn(async () => 'sk-typed') }; @@ -268,6 +435,46 @@ describe('runConfigure', () => { expect(capture.stderr.join('\n')).toContain('profile NOT updated'); }); + it('key-rejected error preserves the typed ApiError envelope (JSON contract)', async () => { + const { deps } = makeCapture(); + const rejectedFetch: AuthDeps['fetchImpl'] = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'API key is invalid or revoked.', + nextAction: 'Rotate your key.', + requestId: 'req_reject', + details: { reason: 'malformed' }, + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ), + ) as unknown as AuthDeps['fetchImpl']; + + // The thrown error must be an ApiError (with code, nextAction, requestId) + // — not a CLIError wrapper that drops those fields. Under --output json, + // index.ts renders ApiError as the full typed envelope; CLIError would + // render only {"error":"...string..."}, violating the JSON contract. + await expect( + runConfigure( + { profile: 'default', output: 'json', debug: false, fromEnv: true }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-bad' }, + credentialsPath, + fetchImpl: rejectedFetch, + }, + ), + ).rejects.toMatchObject({ + code: 'AUTH_INVALID', + exitCode: 3, + nextAction: 'Rotate your key.', + requestId: 'req_reject', + }); + }); + // The old "run `testsprite agent install`" self-bootstrap tip was removed with // the setup consolidation — runConfigure now runs ONLY as part of `setup`, // which installs the skill itself. These guard that the tip stays gone. @@ -643,6 +850,77 @@ describe('runWhoami', () => { expect(printed).toEqual(sampleMe); }); + it('renders routing: v3 and the gap advisory when v3Enabled is true', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV3 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV3) }, + ); + expect(capture.stdout.join('\n')).toContain('routing: v3'); + expect(capture.stderr.join('\n')).toContain('[advisory]'); + expect(capture.stderr.join('\n')).toContain('test cancel'); + }); + + it('renders routing: v2 and NO advisory when v3Enabled is false', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV2 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: false }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV2) }, + ); + expect(capture.stdout.join('\n')).toContain('routing: v2'); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('omits the routing line when the backend does not return v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meResponse()) }, + ); + expect(capture.stdout.join('\n')).not.toContain('routing:'); + }); + + it('does not emit the advisory in JSON mode even when v3Enabled is true', async () => { + writeProfile('default', { apiKey: 'sk' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const meV3 = new Response(JSON.stringify({ ...sampleMe, v3Enabled: true }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + await runWhoami( + { profile: 'default', output: 'json', debug: false }, + { ...deps, env: {}, credentialsPath, fetchImpl: makeFetch(meV3) }, + ); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + const parsed = JSON.parse(capture.stdout.join('')) as MeResponse; + expect(parsed.v3Enabled).toBe(true); + }); + + it('dry-run: whitespace-only TESTSPRITE_API_URL falls through to prod default endpoint', async () => { + const { capture, deps } = makeCapture(); + await runWhoami( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + { + ...deps, + env: { TESTSPRITE_API_URL: ' ' }, + credentialsPath, + }, + ); + const out = capture.stdout.join('\n'); + expect(out).toContain('endpoint: https://api.testsprite.com'); + }); + it('L1788: text output includes the resolved endpoint URL', async () => { writeProfile( 'default', @@ -1003,3 +1281,90 @@ describe('createAuthCommand surface', () => { expect(err.exitCode).toBe(3); }); }); + +// --------------------------------------------------------------------------- +// runConfigure -- skipIfConfigured +// --------------------------------------------------------------------------- + +describe('runConfigure -- skipIfConfigured', () => { + it('skips the prompt and returns early when credentials already exist', async () => { + const { capture, deps } = makeCapture(); + // Write a saved key first. + writeProfile('default', { apiKey: 'sk-existing' }, { path: credentialsPath }); + const prompt = { secret: vi.fn(async () => 'sk-new') }; + const fetchImpl = vi.fn(); + + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false, skipIfConfigured: true }, + { + ...deps, + credentialsPath, + prompt, + fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'], + }, + ); + + // Prompt must never have fired. + expect(prompt.secret).not.toHaveBeenCalled(); + // No network call -- we never validated or wrote a key. + expect(fetchImpl).not.toHaveBeenCalled(); + // The saved key must be untouched. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-existing'); + // Output indicates already_configured. + expect(capture.stdout.join('\n')).toContain('already configured'); + }); + + it('emits already_configured status in JSON mode', async () => { + const { capture, deps } = makeCapture(); + writeProfile('default', { apiKey: 'sk-saved' }, { path: credentialsPath }); + const fetchImpl = vi.fn(); + + await runConfigure( + { profile: 'default', output: 'json', debug: false, fromEnv: false, skipIfConfigured: true }, + { ...deps, credentialsPath, fetchImpl: fetchImpl as unknown as AuthDeps['fetchImpl'] }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + const parsed = JSON.parse(capture.stdout.join('')); + expect(parsed).toMatchObject({ profile: 'default', status: 'already_configured' }); + }); + + it('proceeds normally when no credentials exist and skipIfConfigured is true', async () => { + const { deps } = makeCapture(); + // No pre-existing profile -- skip has no effect, should fall through to prompt. + const prompt = { secret: vi.fn(async () => 'sk-new') }; + + await runConfigure( + { profile: 'default', output: 'text', debug: false, fromEnv: false, skipIfConfigured: true }, + { ...deps, credentialsPath, prompt, fetchImpl: meOkFetch }, + ); + + expect(prompt.secret).toHaveBeenCalledTimes(1); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-new'); + }); + + it('ignores skipIfConfigured when --from-env is set', async () => { + const { deps } = makeCapture(); + // Pre-existing key -- but fromEnv should override and write a new one. + writeProfile('default', { apiKey: 'sk-old' }, { path: credentialsPath }); + + await runConfigure( + { + profile: 'default', + output: 'text', + debug: false, + fromEnv: true, + skipIfConfigured: true, + }, + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk-from-env' }, + credentialsPath, + fetchImpl: meOkFetch, + }, + ); + + // The env key must overwrite the saved key. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-from-env'); + }); +}); diff --git a/src/commands/auth.ts b/src/commands/auth.ts index de1eda4..7e60c05 100644 --- a/src/commands/auth.ts +++ b/src/commands/auth.ts @@ -1,7 +1,9 @@ import { Command } from 'commander'; import { + assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import type { ErrorCode } from '../lib/errors.js'; @@ -15,11 +17,12 @@ import { readProfile, writeProfile, } from '../lib/credentials.js'; -import { loadConfig } from '../lib/config.js'; +import { loadConfig, normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import type { OutputMode } from '../lib/output.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import { promptSecret } from '../lib/prompt.js'; +import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; export interface MeResponse { userId: string; @@ -35,6 +38,11 @@ export interface MeResponse { email?: string; /** Human-readable display name for the bound account. Absent-safe (dogfood L1866). */ displayName?: string; + /** + * Authoritative per-user V3 routing bit. Absent-safe: older backends omit it, + * so it is only rendered when present. + */ + v3Enabled?: boolean; } export interface AuthDeps { @@ -63,6 +71,17 @@ type CommonOptions = FactoryCommonOptions; interface ConfigureOptions extends CommonOptions { fromEnv: boolean; + /** + * When true and the active profile already has a saved API key, skip the + * interactive key prompt and proceed directly to the skill-install step. + * A CI-safe flag: lets `setup` run idempotently without prompting on + * machines that already have credentials (e.g. re-running setup to + * refresh the agent skill without re-entering the key). + * + * Ignored when an explicit `--api-key` or `--from-env` key source is + * provided -- those paths always overwrite, regardless of existing state. + */ + skipIfConfigured?: boolean; } const DEFAULT_API_URL = 'https://api.testsprite.com'; @@ -73,21 +92,25 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): const env = deps.env ?? process.env; const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const out = makeOutput(opts.output, deps); - const prelude = deps.preludeWrite ?? ((chunk: string) => process.stdout.write(chunk)); + // The "Configuring profile …" prelude is informational, not result data, so + // it defaults to stderr — stdout stays a pure result stream (the configured + // JSON/text), which matters under `--output json` (§8.1 stdout purity). + const prelude = deps.preludeWrite ?? ((chunk: string) => process.stderr.write(chunk)); const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); // Normalize the env endpoint: an empty / whitespace-only TESTSPRITE_API_URL is // treated as unset. Without this, `''` (e.g. `export TESTSPRITE_API_URL=` in a // shell profile) is non-nullish and would short-circuit the `??` chains below to // an empty endpoint instead of falling through to the profile / prod default. - const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); // Dry-run: do not prompt, do not read env, do not write credentials. // Print the canned success shape so an agent sees exactly the JSON it // would get on a real configure (modulo the endpoint string). if (opts.dryRun) { - emitDryRunBanner(stderr); const apiUrl = opts.endpointUrl ?? envApiUrl ?? DEFAULT_API_URL; + assertValidEndpointUrl(apiUrl); + emitDryRunBanner(stderr); stderr(`[dry-run] would write credentials for profile="${opts.profile}" to ${credentialsPath}`); out.print({ profile: opts.profile, apiUrl, status: 'configured' }, data => { const d = data as { profile: string; apiUrl: string }; @@ -113,14 +136,29 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): // api_url doesn't silently validate a new key against the default endpoint. const resolvedFromProfile = existingProfile?.apiUrl; const apiUrl = opts.endpointUrl ?? envApiUrl ?? resolvedFromProfile ?? DEFAULT_API_URL; + assertValidEndpointUrl(apiUrl); if (opts.fromEnv) { apiKey = env.TESTSPRITE_API_KEY?.trim(); if (!apiKey) throw validationError('TESTSPRITE_API_KEY', FROM_ENV_MISSING_KEY); } else { + // --skip-if-configured: when a non-empty API key is already saved for + // this profile, skip the interactive prompt and return early. The + // key is NOT re-validated via GET /me on this path -- the caller + // (runInit) only reaches this when no explicit key source was given, + // and the subsequent whoami call in runInit will surface an expired + // key to the user anyway. + if (opts.skipIfConfigured && existingProfile?.apiKey) { + out.print({ profile: opts.profile, apiUrl, status: 'already_configured' }, data => { + const d = data as { profile: string; apiUrl: string }; + return `Profile "${d.profile}" already configured. Endpoint: ${d.apiUrl}`; + }); + return; + } + const promptApi = deps.prompt ?? { secret: (q: string) => promptSecret(q) }; prelude(`Configuring profile "${opts.profile}".\n`); - // Only the API key is prompted — the endpoint defaults to prod (see above). + // Only the API key is prompted -- the endpoint defaults to prod (see above). apiKey = (await promptApi.secret('TestSprite API key: ')).trim(); if (!apiKey) throw new CLIError('No API key provided.', 5); } @@ -145,6 +183,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): baseUrl: facadeBaseUrl(apiUrl), apiKey, fetchImpl: deps.fetchImpl, + requestTimeoutMs: opts.requestTimeoutMs, }); try { // Tag the validation call with the originating command (when provided) so @@ -158,13 +197,22 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): } catch (err) { const message = err instanceof Error ? err.message : String(err); stderr(`API key rejected by ${apiUrl}: ${message} — profile NOT updated`); - const exitCode = err instanceof ApiError ? err.exitCode : 3; - // Include the resolved endpoint in the thrown message so the user knows - // which host rejected the key. This prevents the "invalid or revoked" - // message from being ambiguous when the key is valid for a different env. + // When the verification call returned a typed API error (AUTH_INVALID, + // AUTH_FORBIDDEN, etc.), re-throw it directly so `index.ts` renders the + // full typed envelope under `--output json` (code, nextAction, requestId, + // details). Previously wrapping it in CLIError discarded those fields and + // emitted a bare `{"error":"...string..."}` — violating the JSON contract. + // Augment the message with the endpoint context so text-mode users still + // see which host rejected the key. + if (err instanceof ApiError) { + err.message = `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`; + throw err; + } + // Non-ApiError (truly unexpected throws like a TypeError from a + // misconfigured fetchImpl). Exit 3 (auth family). throw new CLIError( `API key rejected by ${apiUrl}: ${message} — did you mean to set TESTSPRITE_API_URL?`, - exitCode, + 3, ); } @@ -184,6 +232,7 @@ export async function runConfigure(opts: ConfigureOptions, deps: AuthDeps = {}): export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promise { const out = makeOutput(opts.output, deps); const env = deps.env ?? process.env; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); // Resolve the endpoint URL so it can be surfaced in text output. // Dry-run uses the flag/env/default chain without touching credentials. @@ -191,7 +240,8 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi // displayed URL always matches where requests actually go (dogfood L1788). let resolvedEndpoint: string; if (opts.dryRun) { - resolvedEndpoint = opts.endpointUrl ?? env.TESTSPRITE_API_URL ?? 'https://api.testsprite.com'; + resolvedEndpoint = + opts.endpointUrl ?? normalizeEnvVar(env.TESTSPRITE_API_URL) ?? 'https://api.testsprite.com'; } else { const credentialsPath = deps.credentialsPath ?? defaultCredentialsPath(); const config = loadConfig({ @@ -226,6 +276,8 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi `endpoint: ${resolvedEndpoint}`, `env: ${m.env}`, `scopes: ${m.scopes.join(', ')}`, + // Authoritative routing mode, rendered only when the backend supplies it. + ...(m.v3Enabled !== undefined ? [`routing: ${routingLabel(m.v3Enabled)}`] : []), ]; // C2: warn in text mode when key cannot write/run const missingScopes = (['write:tests', 'run:tests'] as const).filter( @@ -238,6 +290,11 @@ export async function runWhoami(opts: CommonOptions, deps: AuthDeps = {}): Promi } return lines.join('\n'); }); + // When V3 routing is on, warn (text mode only) about the still-open behavior + // gaps. JSON consumers read `v3Enabled` directly; stdout stays pure. + if (opts.output !== 'json' && me.v3Enabled === true) { + emitV3RoutingAdvisory(stderr); + } return me; } @@ -318,7 +375,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, @@ -327,19 +384,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values — the - * factory applies the same clamp so there is no double-clamp risk. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - function makeOutput(mode: OutputMode, deps: AuthDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } diff --git a/src/commands/completion.test.ts b/src/commands/completion.test.ts new file mode 100644 index 0000000..8388a09 --- /dev/null +++ b/src/commands/completion.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest'; +import type { CompletionSpec } from './completion.js'; +import { createCompletionCommand, detectShell, isShell, renderCompletion } from './completion.js'; + +const SPEC: CompletionSpec = { + program: 'testsprite', + commands: ['setup', 'auth', 'test', 'doctor', 'completion', 'help'], + subcommands: { auth: ['status', 'remove'], test: ['run', 'wait'] }, + globalFlags: ['--output', '--profile', '--help'], +}; + +describe('isShell / detectShell', () => { + it('recognizes the three supported shells', () => { + expect(isShell('bash')).toBe(true); + expect(isShell('zsh')).toBe(true); + expect(isShell('fish')).toBe(true); + expect(isShell('powershell')).toBe(false); + }); + + it('detects the shell from a $SHELL path', () => { + expect(detectShell({ SHELL: '/bin/bash' })).toBe('bash'); + expect(detectShell({ SHELL: '/usr/bin/zsh' })).toBe('zsh'); + expect(detectShell({ SHELL: '/usr/local/bin/fish' })).toBe('fish'); + }); + + it('returns undefined for an unknown or missing shell', () => { + expect(detectShell({ SHELL: '/bin/sh' })).toBeUndefined(); + expect(detectShell({})).toBeUndefined(); + }); +}); + +describe('renderCompletion', () => { + it('bash script wires a completion function and lists commands, subcommands, flags', () => { + const script = renderCompletion('bash', SPEC); + expect(script).toContain('complete -F _testsprite_completion testsprite'); + expect(script).toContain('setup'); + expect(script).toContain('auth) COMPREPLY'); + expect(script).toContain('status remove'); + expect(script).toContain('--output'); + }); + + it('zsh script declares #compdef and per-group subcommands', () => { + const script = renderCompletion('zsh', SPEC); + expect(script.startsWith('#compdef testsprite')).toBe(true); + expect(script).toContain('compdef _testsprite testsprite'); + expect(script).toContain('run wait'); + }); + + it('fish script uses complete -c with subcommand conditions and flags', () => { + const script = renderCompletion('fish', SPEC); + expect(script).toContain('complete -c testsprite -f'); + expect(script).toContain('__fish_seen_subcommand_from auth'); + expect(script).toContain('-l output'); + }); +}); + +describe('createCompletionCommand', () => { + function run(args: string[], env: NodeJS.ProcessEnv): Promise { + const out: string[] = []; + const cmd = createCompletionCommand(() => SPEC, { env, stdout: line => out.push(line) }); + return cmd.parseAsync(args, { from: 'user' }).then(() => out); + } + + it('prints the requested shell script from an explicit argument', async () => { + const out = await run(['bash'], {}); + expect(out.join('\n')).toContain('complete -F'); + }); + + it('auto-detects the shell from $SHELL when no argument is given', async () => { + const out = await run([], { SHELL: '/usr/bin/zsh' }); + expect(out.join('\n')).toContain('#compdef testsprite'); + }); + + it('rejects an unsupported shell with VALIDATION_ERROR (exit 5)', async () => { + const cmd = createCompletionCommand(() => SPEC, { env: {}, stdout: () => undefined }); + await expect(cmd.parseAsync(['powershell'], { from: 'user' })).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + }); + }); + + it('errors when the shell cannot be detected and none is given', async () => { + const cmd = createCompletionCommand(() => SPEC, { env: {}, stdout: () => undefined }); + await expect(cmd.parseAsync([], { from: 'user' })).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + }); + }); + + it('is named "completion"', () => { + expect(createCompletionCommand(() => SPEC).name()).toBe('completion'); + }); +}); diff --git a/src/commands/completion.ts b/src/commands/completion.ts new file mode 100644 index 0000000..15cf7a4 --- /dev/null +++ b/src/commands/completion.ts @@ -0,0 +1,164 @@ +/** + * `testsprite completion [bash|zsh|fish]` — emit a shell completion script. + * + * The command names, per-group subcommands, and global flags are NOT hardcoded: + * `index.ts` builds a {@link CompletionSpec} by walking the fully-assembled + * Commander program and passes it in, so the generated script can never drift + * from the real command tree. `renderCompletion` is a pure function of the spec, + * which keeps it unit-testable without a live program. + * + * Usage: + * bash: eval "$(testsprite completion bash)" (add to ~/.bashrc) + * zsh: testsprite completion zsh > ~/.zsh/_testsprite (on your fpath) + * fish: testsprite completion fish | source (add to config.fish) + */ + +import { Command } from 'commander'; +import { localValidationError } from '../lib/errors.js'; + +export const SUPPORTED_SHELLS = ['bash', 'zsh', 'fish'] as const; +export type Shell = (typeof SUPPORTED_SHELLS)[number]; + +export interface CompletionSpec { + /** Binary name, e.g. "testsprite". */ + program: string; + /** Top-level command names. */ + commands: string[]; + /** command name -> its subcommand names (only groups that have subcommands). */ + subcommands: Record; + /** Global long option flags (e.g. "--output"). */ + globalFlags: string[]; +} + +export interface CompletionDeps { + env?: NodeJS.ProcessEnv; + stdout?: (line: string) => void; +} + +export function isShell(value: string): value is Shell { + return (SUPPORTED_SHELLS as readonly string[]).includes(value); +} + +/** Best-effort shell detection from `$SHELL` (e.g. "/bin/zsh" -> "zsh"). */ +export function detectShell(env: NodeJS.ProcessEnv): Shell | undefined { + const shellPath = env.SHELL ?? ''; + const base = shellPath.slice(shellPath.lastIndexOf('/') + 1); + return isShell(base) ? base : undefined; +} + +export function renderCompletion(shell: Shell, spec: CompletionSpec): string { + switch (shell) { + case 'bash': + return renderBash(spec); + case 'zsh': + return renderZsh(spec); + case 'fish': + return renderFish(spec); + } +} + +function renderBash(spec: CompletionSpec): string { + const fn = `_${spec.program}_completion`; + const lines = [ + `# ${spec.program} bash completion. Enable with: eval "$(${spec.program} completion bash)"`, + `${fn}() {`, + ' local cur prev', + ' cur="${COMP_WORDS[COMP_CWORD]}"', + ' prev="${COMP_WORDS[COMP_CWORD-1]}"', + ` local commands="${spec.commands.join(' ')}"`, + ` local global_flags="${spec.globalFlags.join(' ')}"`, + ' case "$prev" in', + ...Object.entries(spec.subcommands).map( + ([group, subs]) => + ` ${group}) COMPREPLY=( $(compgen -W "${subs.join(' ')}" -- "$cur") ); return;;`, + ), + ' esac', + ' if [[ "$cur" == -* ]]; then', + ' COMPREPLY=( $(compgen -W "$global_flags" -- "$cur") ); return', + ' fi', + ' COMPREPLY=( $(compgen -W "$commands" -- "$cur") )', + '}', + `complete -F ${fn} ${spec.program}`, + ]; + return lines.join('\n'); +} + +function renderZsh(spec: CompletionSpec): string { + const fn = `_${spec.program}`; + const lines = [ + `#compdef ${spec.program}`, + `# ${spec.program} zsh completion. Enable with: ${spec.program} completion zsh > "$fpath[1]/_${spec.program}"`, + `${fn}() {`, + ' local -a commands', + ` commands=(${spec.commands.join(' ')})`, + ' if (( CURRENT == 2 )); then', + " _describe 'command' commands", + ' return', + ' fi', + ' case "${words[2]}" in', + ...Object.entries(spec.subcommands).map( + ([group, subs]) => + ` ${group}) local -a subs; subs=(${subs.join(' ')}); _describe 'subcommand' subs;;`, + ), + ' esac', + '}', + `compdef ${fn} ${spec.program}`, + ]; + return lines.join('\n'); +} + +function renderFish(spec: CompletionSpec): string { + const lines = [ + `# ${spec.program} fish completion. Enable with: ${spec.program} completion fish | source`, + `complete -c ${spec.program} -f`, + ...spec.commands.map( + command => `complete -c ${spec.program} -n '__fish_use_subcommand' -a '${command}'`, + ), + ...Object.entries(spec.subcommands).flatMap(([group, subs]) => + subs.map( + sub => `complete -c ${spec.program} -n '__fish_seen_subcommand_from ${group}' -a '${sub}'`, + ), + ), + ...spec.globalFlags.map(flag => `complete -c ${spec.program} -l ${flag.replace(/^--/, '')}`), + ]; + return lines.join('\n'); +} + +export function createCompletionCommand( + getSpec: () => CompletionSpec, + deps: CompletionDeps = {}, +): Command { + return new Command('completion') + .description('Print a shell completion script (bash|zsh|fish)') + .argument( + '[shell]', + 'Shell to generate for (bash|zsh|fish); auto-detected from $SHELL when omitted', + ) + .addHelpText( + 'after', + '\nExamples:\n' + + ' eval "$(testsprite completion bash)" # bash, current session\n' + + ' testsprite completion zsh > ~/.zsh/_testsprite\n' + + ' testsprite completion fish | source # fish, current session', + ) + .action((shellArg: string | undefined, _cmdOpts: unknown) => { + const env = deps.env ?? process.env; + const shell = shellArg ?? detectShell(env); + if (shell === undefined) { + throw localValidationError( + 'shell', + `could not detect the shell from $SHELL; pass one explicitly (${SUPPORTED_SHELLS.join(', ')})`, + [...SUPPORTED_SHELLS], + ); + } + if (!isShell(shell)) { + throw localValidationError( + 'shell', + `unsupported shell "${shell}"; use one of: ${SUPPORTED_SHELLS.join(', ')}`, + [...SUPPORTED_SHELLS], + ); + } + const write = deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)); + write(renderCompletion(shell, getSpec())); + }); +} diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts new file mode 100644 index 0000000..7974947 --- /dev/null +++ b/src/commands/doctor.test.ts @@ -0,0 +1,315 @@ +/** + * Unit tests for `testsprite doctor`. + * + * The command reuses the real resolution helpers (loadConfig, makeHttpClient, + * isVerifySkillInstalled), so these tests inject env/credentials/fetch/fs and + * assert on the rendered report + the exit-on-failure contract. + */ + +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Command } from 'commander'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { ApiError, CLIError } from '../lib/errors.js'; +import { writeProfile } from '../lib/credentials.js'; +import type { DoctorDeps, DoctorReport } from './doctor.js'; +import { createDoctorCommand, runDoctor } from './doctor.js'; + +interface CapturedOutput { + stdout: string[]; + stderr: string[]; +} + +function makeCapture(): { capture: CapturedOutput; deps: Pick } { + const capture: CapturedOutput = { stdout: [], stderr: [] }; + return { + capture, + deps: { + stdout: line => capture.stdout.push(line), + stderr: line => capture.stderr.push(line), + }, + }; +} + +function makeFetch(body: unknown, status = 200): DoctorDeps['fetchImpl'] { + return vi.fn( + async () => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }), + ) as unknown as DoctorDeps['fetchImpl']; +} + +const OK_ME = { userId: 'u-doc', keyId: 'k-doc' }; + +/** Base deps shared by the healthy-path tests: node OK, skill installed, empty env. */ +function healthyDeps(credentialsPath: string, extra: Partial = {}): DoctorDeps { + return { + env: {}, + credentialsPath, + cwd: '/project', + nodeVersion: '22.9.0', + existsSync: () => true, // skill landing file present + fetchImpl: makeFetch(OK_ME), + ...extra, + }; +} + +function makeDoctorProgram(deps: DoctorDeps = {}): Command { + const program = new Command(); + program.exitOverride(); + program.option('--output ', 'output', 'text'); + program.addCommand(createDoctorCommand(deps)); + return program; +} + +let credentialsPath: string; + +beforeEach(() => { + credentialsPath = join(mkdtempSync(join(tmpdir(), 'testsprite-doctor-')), 'credentials'); +}); + +describe('runDoctor — healthy environment', () => { + it('returns an all-passing report and does not throw', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + expect(report.failures).toBe(0); + expect(report.warnings).toBe(0); + const out = capture.stdout.join('\n'); + expect(out).toContain('[OK]'); + expect(out).toContain('All checks passed.'); + expect(out).toContain('reached GET /me'); + }); + + it('adds a Routing check (v3) and the gap advisory when /me reports v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { + ...healthyDeps(credentialsPath, { + fetchImpl: makeFetch({ ...OK_ME, v3Enabled: true }), + }), + ...deps, + }, + ); + expect(report.failures).toBe(0); + expect(report.checks.some(c => c.name === 'Routing' && c.detail.includes('v3'))).toBe(true); + expect(capture.stderr.join('\n')).toContain('[advisory]'); + expect(capture.stderr.join('\n')).toContain('test cancel'); + }); + + it('shows Routing v2 and no advisory when v3Enabled is false', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { + ...healthyDeps(credentialsPath, { + fetchImpl: makeFetch({ ...OK_ME, v3Enabled: false }), + }), + ...deps, + }, + ); + expect(report.checks.some(c => c.name === 'Routing' && c.detail.includes('v2'))).toBe(true); + expect(capture.stderr.join('\n')).not.toContain('[advisory]'); + }); + + it('omits the Routing check when /me does not report v3Enabled', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, // OK_ME has no v3Enabled + ); + expect(report.checks.some(c => c.name === 'Routing')).toBe(false); + expect(report.warnings).toBe(0); + }); + + it('never prints the API key anywhere in the report', async () => { + writeProfile('default', { apiKey: 'sk-super-secret-value' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + const all = capture.stdout.join('\n') + capture.stderr.join('\n'); + expect(all).not.toContain('sk-super-secret-value'); + }); + + it('emits a machine-readable report under --output json without leaking the API key', async () => { + writeProfile('default', { apiKey: 'sk-json-secret-value' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + await runDoctor( + { profile: 'default', output: 'json', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, + ); + const raw = capture.stdout.join(''); + // Security: the JSON serialization path is distinct from the text renderer, + // so assert the key never leaks here either. + expect(raw).not.toContain('sk-json-secret-value'); + const parsed = JSON.parse(raw) as DoctorReport; + expect(parsed.failures).toBe(0); + expect(Array.isArray(parsed.checks)).toBe(true); + expect( + parsed.checks.some(check => check.name === 'Connectivity' && check.status === 'ok'), + ).toBe(true); + }); +}); + +describe('runDoctor — failing checks exit non-zero', () => { + it('missing API key fails Credentials and throws CLIError (exit 1)', async () => { + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath), ...deps }, // no profile written => no key + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + expect(rejection).toMatchObject({ exitCode: 1 }); + const out = capture.stdout.join('\n'); + expect(out).toContain('[FAIL]'); + expect(out).toContain('Credentials'); + }); + + it('invalid endpoint URL fails the API endpoint check', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false, endpointUrl: 'not-a-url' }, + { ...healthyDeps(credentialsPath), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('API endpoint'); + expect(out).toContain('not a valid'); + }); + + it('rejected API key surfaces as a Connectivity failure', async () => { + writeProfile('default', { apiKey: 'sk-bad' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const authError = { + error: { code: 'AUTH_INVALID', message: 'Bad key.', requestId: 'req_x', details: {} }, + }; + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(authError, 401) }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('Connectivity'); + expect(out).toContain('API key rejected (AUTH_INVALID)'); + }); + + it('a non-auth /me error is reported as a Connectivity failure with its code', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const notFound = { + error: { code: 'NOT_FOUND', message: 'nope', requestId: 'req_y', details: {} }, + }; + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { fetchImpl: makeFetch(notFound, 404) }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + expect(capture.stdout.join('\n')).toContain('GET /me failed (NOT_FOUND)'); + }); + + it('an outdated Node runtime fails the Node.js check', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const rejection = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { nodeVersion: '18.0.0' }), ...deps }, + ).catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(CLIError); + const out = capture.stdout.join('\n'); + expect(out).toContain('Node.js'); + expect(out).toContain('below the required Node 20'); + }); +}); + +describe('runDoctor — warnings do not fail', () => { + it('missing verify skill is a warning, not a failure', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false }, + { ...healthyDeps(credentialsPath, { existsSync: () => false }), ...deps }, + ); + expect(report.failures).toBe(0); + expect(report.warnings).toBeGreaterThanOrEqual(1); + const out = capture.stdout.join('\n'); + expect(out).toContain('[WARN]'); + expect(out).toContain('Verify skill'); + }); + + it('--dry-run skips connectivity and never calls fetch, missing key is a warning', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('fetch must not be called under --dry-run'); + }) as unknown as DoctorDeps['fetchImpl']; + const { capture, deps } = makeCapture(); + const report = await runDoctor( + { profile: 'default', output: 'text', debug: false, dryRun: true }, + { + env: {}, + credentialsPath, + cwd: '/project', + nodeVersion: '22.9.0', + existsSync: () => true, + fetchImpl, + ...deps, + }, + ); + expect(report.failures).toBe(0); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(capture.stdout.join('\n')).toContain('skipped under --dry-run'); + }); +}); + +describe('createDoctorCommand wiring', () => { + it('exposes the doctor command name', () => { + expect(createDoctorCommand().name()).toBe('doctor'); + }); + + it('--help describes the diagnostic', () => { + expect(createDoctorCommand().helpInformation()).toContain('Diagnose'); + }); + + it('rejects invalid --output with the shared VALIDATION_ERROR', async () => { + const rejection = await makeDoctorProgram() + .parseAsync(['node', 'ts', '--output', 'yaml', 'doctor']) + .catch((error: unknown) => error); + expect(rejection).toBeInstanceOf(ApiError); + expect(rejection).toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + nextAction: 'Flag `--output` is invalid: must be one of: json, text.', + }); + }); + + it('accepts valid --output modes through command wiring', async () => { + writeProfile('default', { apiKey: 'sk-abc' }, { path: credentialsPath }); + for (const mode of ['text', 'json'] as const) { + const { capture, deps } = makeCapture(); + await makeDoctorProgram({ ...healthyDeps(credentialsPath), ...deps }).parseAsync([ + 'node', + 'ts', + '--output', + mode, + 'doctor', + ]); + const raw = capture.stdout.join(''); + if (mode === 'json') { + expect((JSON.parse(raw) as DoctorReport).failures).toBe(0); + } else { + expect(raw).toContain('All checks passed.'); + } + } + }); +}); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts new file mode 100644 index 0000000..40c1ac8 --- /dev/null +++ b/src/commands/doctor.ts @@ -0,0 +1,315 @@ +/** + * `testsprite doctor` — one-shot environment diagnostic. + * + * Runs a fixed checklist (CLI version, Node.js runtime, active profile, API + * endpoint, credentials, live connectivity + key validity, and whether the + * verify skill is installed in the current project) and prints an OK/WARN/FAIL + * report. Exits non-zero when any check FAILS so it can gate a CI step or an + * agent preflight (`testsprite doctor && testsprite test run ...`). Warnings + * (e.g. skill not installed) do not fail the process. + * + * Every check is reused from the same helpers the real commands use, so the + * report reflects exactly what a subsequent command would resolve: `loadConfig` + * for profile/endpoint/key, `assertValidEndpointUrl` for the endpoint gate, + * `makeHttpClient` + `GET /me` for connectivity, and `isVerifySkillInstalled` + * for the skill check. + */ + +import { Command } from 'commander'; +import { + assertValidEndpointUrl, + makeHttpClient, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { loadConfig } from '../lib/config.js'; +import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; +import type { FetchImpl } from '../lib/http.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; +import { isVerifySkillInstalled } from '../lib/skill-nudge.js'; +import { emitV3RoutingAdvisory, routingLabel } from '../lib/v3-advisory.js'; +import { VERSION } from '../version.js'; +import { MIN_SUPPORTED_NODE_MAJOR, shouldRejectNodeVersion } from '../version-guard.js'; + +export type DoctorStatus = 'ok' | 'warn' | 'fail'; + +export interface DoctorCheck { + /** Short, stable label (also the JSON key-ish name). */ + name: string; + status: DoctorStatus; + /** Human-readable one-line result. Never contains the API key. */ + detail: string; +} + +export interface DoctorReport { + checks: DoctorCheck[]; + failures: number; + warnings: number; +} + +/** Minimal projection of `GET /me` we read for the connectivity detail. */ +interface MeIdentity { + userId?: string; + keyId?: string; + v3Enabled?: boolean; +} + +export interface DoctorDeps { + env?: NodeJS.ProcessEnv; + credentialsPath?: string; + fetchImpl?: FetchImpl; + stdout?: (line: string) => void; + stderr?: (line: string) => void; + /** Project dir for the skill check. Defaults to `process.cwd()`. */ + cwd?: string; + /** Runtime version string (e.g. "22.9.0"). Defaults to `process.versions.node`. */ + nodeVersion?: string; + existsSync?: (p: string) => boolean; + readFileSync?: (p: string) => string; +} + +type CommonOptions = FactoryCommonOptions; + +export async function runDoctor(opts: CommonOptions, deps: DoctorDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const env = deps.env ?? process.env; + const cwd = deps.cwd ?? process.cwd(); + const nodeVersion = deps.nodeVersion ?? process.versions.node; + + const config = loadConfig({ + profile: opts.profile, + endpointUrl: opts.endpointUrl, + env, + credentialsPath: deps.credentialsPath, + }); + const endpointCheck = checkEndpoint(config.apiUrl); + const hasKey = Boolean(config.apiKey); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + const connectivity = await checkConnectivity(opts, deps, { + hasKey, + endpointOk: endpointCheck.status === 'ok', + }); + + const checks: DoctorCheck[] = [ + { name: 'CLI version', status: 'ok', detail: VERSION }, + checkNodeVersion(nodeVersion), + { name: 'Profile', status: 'ok', detail: config.profile }, + endpointCheck, + checkCredentials(hasKey, config.profile, opts.dryRun ?? false), + connectivity.check, + ]; + + // Informational routing line, only when the backend reported it (no new call). + if (connectivity.v3Enabled !== undefined) { + const label = routingLabel(connectivity.v3Enabled); + checks.push({ + name: 'Routing', + status: 'ok', + detail: + connectivity.v3Enabled === true + ? `${label} (V3 execution routing is ON)` + : `${label} (default routing)`, + }); + } + + checks.push(checkSkill(cwd, deps)); + + const failures = checks.filter(check => check.status === 'fail').length; + const warnings = checks.filter(check => check.status === 'warn').length; + const report: DoctorReport = { checks, failures, warnings }; + + out.print(report, () => renderDoctor(report)); + + if (connectivity.v3Enabled === true) { + emitV3RoutingAdvisory(stderr); + } + + if (failures > 0) { + // Non-zero exit so `testsprite doctor && ...` gates a CI step or an agent + // preflight. The full report already printed above; this line is the stderr + // summary index.ts renders before exiting 1. + throw new CLIError(`doctor: ${failures} check(s) failed, ${warnings} warning(s)`, 1); + } + return report; +} + +function checkNodeVersion(nodeVersion: string): DoctorCheck { + // Reuse the CLI's own runtime guard so the verdict matches exactly what the + // entrypoint enforces at startup, rather than a divergent hardcoded check. + // The precise engines floor (20.19+/22.13+/24+) is enforced by npm at install + // time via .npmrc engine-strict. sourceRef: src/version-guard.ts. + const rejected = shouldRejectNodeVersion(nodeVersion); + return { + name: 'Node.js', + status: rejected ? 'fail' : 'ok', + detail: rejected + ? `v${nodeVersion} is below the required Node ${MIN_SUPPORTED_NODE_MAJOR}; upgrade Node.js` + : `v${nodeVersion} (>=${MIN_SUPPORTED_NODE_MAJOR} required)`, + }; +} + +function checkEndpoint(apiUrl: string): DoctorCheck { + try { + assertValidEndpointUrl(apiUrl); + return { name: 'API endpoint', status: 'ok', detail: apiUrl }; + } catch { + return { + name: 'API endpoint', + status: 'fail', + detail: `"${apiUrl}" is not a valid http(s) URL`, + }; + } +} + +function checkCredentials(hasKey: boolean, profile: string, dryRun: boolean): DoctorCheck { + if (hasKey) { + // Never print any part of the key (security). Confirm presence only. + return { + name: 'Credentials', + status: 'ok', + detail: `API key configured (profile "${profile}")`, + }; + } + // Under --dry-run no key is expected, so a missing key is not a failure. + return { + name: 'Credentials', + status: dryRun ? 'warn' : 'fail', + detail: dryRun + ? 'no API key (not needed under --dry-run)' + : 'no API key found; run `testsprite setup` (or set TESTSPRITE_API_KEY)', + }; +} + +function checkSkill(cwd: string, deps: DoctorDeps): DoctorCheck { + const installed = isVerifySkillInstalled(cwd, { + existsSync: deps.existsSync, + readFileSync: deps.readFileSync, + }); + return { + name: 'Verify skill', + status: installed ? 'ok' : 'warn', + detail: installed + ? 'installed in this project' + : 'not installed here; run `testsprite setup` so your agent verifies its changes', + }; +} + +async function checkConnectivity( + opts: CommonOptions, + deps: DoctorDeps, + ctx: { hasKey: boolean; endpointOk: boolean }, +): Promise<{ check: DoctorCheck; v3Enabled?: boolean }> { + const name = 'Connectivity'; + if (opts.dryRun) return { check: { name, status: 'warn', detail: 'skipped under --dry-run' } }; + if (!ctx.hasKey) + return { check: { name, status: 'warn', detail: 'skipped; no API key to test with' } }; + if (!ctx.endpointOk) + return { check: { name, status: 'warn', detail: 'skipped; endpoint URL is invalid' } }; + + try { + const client = makeHttpClient(opts, { + env: deps.env, + credentialsPath: deps.credentialsPath, + fetchImpl: deps.fetchImpl, + stderr: deps.stderr, + }); + const me = await client.get('/me'); + const who = me.userId ? ` (userId ${me.userId})` : ''; + return { + check: { name, status: 'ok', detail: `reached GET /me, API key accepted${who}` }, + v3Enabled: me.v3Enabled, + }; + } catch (error) { + if (error instanceof ApiError) { + if ( + error.code === 'AUTH_REQUIRED' || + error.code === 'AUTH_INVALID' || + error.code === 'AUTH_FORBIDDEN' + ) { + return { check: { name, status: 'fail', detail: `API key rejected (${error.code})` } }; + } + return { check: { name, status: 'fail', detail: `GET /me failed (${error.code})` } }; + } + return { + check: { + name, + status: 'fail', + detail: `GET /me failed (${error instanceof Error ? error.message : String(error)})`, + }, + }; + } +} + +const STATUS_LABEL: Record = { + ok: '[OK] ', + warn: '[WARN]', + fail: '[FAIL]', +}; + +function renderDoctor(report: DoctorReport): string { + const nameWidth = Math.max(...report.checks.map(check => check.name.length)); + const lines: string[] = ['TestSprite doctor', '']; + for (const check of report.checks) { + lines.push(` ${STATUS_LABEL[check.status]} ${check.name.padEnd(nameWidth)} ${check.detail}`); + } + lines.push(''); + lines.push( + report.failures === 0 && report.warnings === 0 + ? 'All checks passed.' + : `${report.failures} failure(s), ${report.warnings} warning(s).`, + ); + return lines.join('\n'); +} + +export function createDoctorCommand(deps: DoctorDeps = {}): Command { + const cmd = new Command('doctor') + .description( + 'Diagnose CLI setup: version, Node, profile, endpoint, credentials, connectivity, skill', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite doctor # run all checks (exit 1 if any fails)\n' + + ' testsprite doctor --output json # machine-readable report\n' + + ' testsprite doctor && testsprite test run # gate a command on a healthy setup', + ) + .action(async (_cmdOpts, command: Command) => { + await runDoctor(resolveCommonOptions(command), deps); + }); + + return cmd; +} + +function resolveCommonOptions(command: Command): CommonOptions { + const globals = command.optsWithGlobals() as Partial & { + requestTimeout?: string; + }; + return { + profile: globals.profile ?? 'default', + output: resolveOutputMode(globals.output), + endpointUrl: globals.endpointUrl, + debug: globals.debug ?? false, + verbose: globals.verbose ?? false, + dryRun: globals.dryRun ?? false, + requestTimeoutMs: parseRequestTimeoutFlag(globals.requestTimeout), + }; +} + +function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const seconds = Number(raw); + if (!Number.isFinite(seconds) || seconds <= 0) { + // Match the other commands: a malformed --request-timeout is a validation + // error, not a silently-ignored default. + throw localValidationError( + 'request-timeout', + `must be a positive number of seconds (got "${raw}")`, + ); + } + return Math.round(seconds * 1000); +} + +function makeOutput(mode: OutputMode, deps: DoctorDeps): Output { + return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); +} diff --git a/src/commands/init.test.ts b/src/commands/init.test.ts index be19095..92e2095 100644 --- a/src/commands/init.test.ts +++ b/src/commands/init.test.ts @@ -9,6 +9,7 @@ import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { ApiError, CLIError } from '../lib/errors.js'; import { resetDryRunBannerForTesting } from '../lib/client-factory.js'; +import { readProfile, writeProfile } from '../lib/credentials.js'; import type { MeResponse } from './auth.js'; import type { AgentFs } from './agent.js'; import type { InitDeps } from './init.js'; @@ -199,8 +200,14 @@ describe('runInit — happy path (interactive)', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('TestSprite initialized.'); expect(stdout).toContain('profile:'); + // Next steps leads with creating a project; no command that fails without --project. expect(stdout).toContain('Next steps:'); - expect(stdout).toContain('testsprite test list'); + expect(stdout).toContain('testsprite project create --type frontend'); + expect(stdout).toContain('testsprite test run --all --project '); + expect(stdout).toContain('the testsprite-onboard skill is installed'); + // No "current project" wording, no bare test list. + expect(stdout).not.toContain('current project'); + expect(stdout).not.toContain('testsprite test list'); }); it('json mode: emits structured InitSummary object', async () => { @@ -231,6 +238,45 @@ describe('runInit — happy path (interactive)', () => { expect(agent.skills).toContain('testsprite-verify'); expect(agent.skills).toContain('testsprite-onboard'); }); + + it('debug mode reports a display-only whoami lookup failure without corrupting JSON stdout', async () => { + const { captured, deps } = makeCapture(); + let callCount = 0; + const fetchMock = vi.fn(async () => { + callCount += 1; + if (callCount === 1) { + return new Response(JSON.stringify(ME), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response( + JSON.stringify({ + error: { + code: 'AUTH_INVALID', + message: 'Invalid API key', + nextAction: 'Provide a valid key.', + requestId: 'r-whoami', + }, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ); + }) as unknown as InitDeps['fetchImpl']; + + await runInit( + makeBaseOpts({ apiKey: 'sk-json-test', debug: true, noAgent: true, output: 'json' }), + { + ...deps, + fetchImpl: fetchMock, + credentialsPath, + isTTY: false, + }, + ); + + const parsed = JSON.parse(captured.stdout.join('\n')) as Record; + expect(parsed.status).toBe('initialized'); + expect(captured.stderr.some(line => line.includes('setup identity lookup failed'))).toBe(true); + }); }); // --------------------------------------------------------------------------- @@ -306,6 +352,15 @@ describe('runInit — --no-agent', () => { const stdout = captured.stdout.join('\n'); expect(stdout).toContain('skipped (--no-agent)'); + // --no-agent points at manual test creation; must not claim the skill is installed. + expect(stdout).toContain('Next steps:'); + expect(stdout).toContain('testsprite project create --type frontend'); + expect(stdout).toContain('testsprite test create --project '); + expect(stdout).toContain('testsprite test run --all --project '); + expect(stdout).not.toContain('skill is installed'); + // No "current project" wording, no bare test list. + expect(stdout).not.toContain('current project'); + expect(stdout).not.toContain('testsprite test list'); }); it('text mode with agent: summary contains skills line with both default skills', async () => { @@ -542,6 +597,36 @@ describe('runInit — codex-review hardening', () => { expect(fetchImpl).toHaveBeenCalled(); }); + it('rejects malformed --endpoint-url before setup key verification', async () => { + const { captured, deps } = makeCapture(); + const fetchImpl = makeOkFetch(); + + await expect( + runInit( + makeBaseOpts({ + fromEnv: true, + endpointUrl: 'not-a-url', + noAgent: true, + output: 'json', + }), + { + ...deps, + env: { TESTSPRITE_API_KEY: 'sk' }, + fetchImpl, + credentialsPath, + isTTY: false, + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'endpoint-url' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(captured.stderr.join('\n')).not.toContain('API key rejected'); + }); + it('whoami banner uses --api-key, not a stale TESTSPRITE_API_KEY in env (E2E 2026-06-09)', async () => { const { captured, deps } = makeCapture(); // Key-aware fetch: only the real key gets a 200 + identity; the stale env key 401s. @@ -961,3 +1046,99 @@ describe('runInit — telemetry attribution (X-CLI-Command)', () => { expect(initTagged).toHaveLength(1); }); }); + +// --------------------------------------------------------------------------- +// runInit -- skipIfConfigured +// --------------------------------------------------------------------------- + +describe('runInit -- skipIfConfigured', () => { + it('skips the API key prompt and reuses saved credentials when the profile exists', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + // Write a saved key before running setup. + writeProfile('default', { apiKey: 'sk-saved' }, { path: credentialsPath }); + // Provide a mock fetch that accepts /me so runWhoami (identity banner) succeeds. + const fetchMock = makeOkFetch(); + const prompt = { secret: vi.fn(async () => 'sk-should-never-be-asked') }; + + await runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'json' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + prompt, + }); + + // The prompt must never have fired. + expect(prompt.secret).not.toHaveBeenCalled(); + // The saved key must be untouched. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-saved'); + // The summary must still be emitted. + const parsed = JSON.parse(captured.stdout.join('')) as { status: string }; + expect(parsed.status).toBe('initialized'); + }); + + it('proceeds to prompt when skipIfConfigured is true but no credentials exist', async () => { + const { captured, deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + // No pre-existing credentials -- skip has no effect. + const fetchMock = makeOkFetch(); + const prompt = { secret: vi.fn(async () => 'sk-fresh') }; + + await runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'text' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: true, + prompt, + }); + + // With no saved key, the prompt should fire. + expect(prompt.secret).toHaveBeenCalledTimes(1); + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-fresh'); + expect(captured.stdout.join('')).toContain('initialized'); + }); + + it('allows non-interactive (isTTY=false) when skipIfConfigured is true and credentials exist', async () => { + const { deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + writeProfile('default', { apiKey: 'sk-ci' }, { path: credentialsPath }); + const fetchMock = makeOkFetch(); + + // Must not throw exit 5 for "non-interactive mode, no key source". + await expect( + runInit(makeBaseOpts({ skipIfConfigured: true, noAgent: true, output: 'json' }), { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + }), + ).resolves.toBeUndefined(); + + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-ci'); + }); + + it('--api-key takes precedence over skipIfConfigured and overwrites the saved key', async () => { + const { deps } = makeCapture(); + const { fs: agentFs } = makeMemFs(); + writeProfile('default', { apiKey: 'sk-old' }, { path: credentialsPath }); + const fetchMock = makeOkFetch(); + + await runInit( + makeBaseOpts({ apiKey: 'sk-new', skipIfConfigured: true, noAgent: true, output: 'text' }), + { + ...deps, + credentialsPath, + fetchImpl: fetchMock, + fs: agentFs, + isTTY: false, + }, + ); + + // Explicit --api-key must overwrite regardless of skipIfConfigured. + expect(readProfile('default', { path: credentialsPath })?.apiKey).toBe('sk-new'); + }); +}); diff --git a/src/commands/init.ts b/src/commands/init.ts index 6a432c2..659945f 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -11,10 +11,14 @@ */ import { Command } from 'commander'; -import type { CommonOptions as FactoryCommonOptions } from '../lib/client-factory.js'; +import { + parseRequestTimeoutFlag, + type CommonOptions as FactoryCommonOptions, +} from '../lib/client-factory.js'; +import { normalizeEnvVar } from '../lib/config.js'; import { emitDeprecationNotice } from '../lib/deprecate.js'; import { CLIError } from '../lib/errors.js'; -import { GLOBAL_OPTS_HINT, Output } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; import type { AuthDeps, MeResponse } from './auth.js'; import { runConfigure, runWhoami } from './auth.js'; import type { AgentDeps, AgentFs, InstallResult } from './agent.js'; @@ -36,7 +40,7 @@ const DEFAULT_API_URL = 'https://api.testsprite.com'; */ function resolveReportedEndpoint(opts: InitOptions, deps: InitDeps): string { const env = deps.env ?? process.env; - const envApiUrl = env.TESTSPRITE_API_URL?.trim() || undefined; + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); let existing: string | undefined; try { existing = readProfile(opts.profile, { path: deps.credentialsPath })?.apiUrl; @@ -86,6 +90,12 @@ interface InitOptions extends CommonOptions { force: boolean; dir?: string; yes: boolean; + /** + * When true and the active profile already has a saved API key, skip the + * interactive key prompt. Forwarded verbatim to runConfigure. Has no + * effect when --api-key or --from-env is also given. + */ + skipIfConfigured?: boolean; /** Set by the command action when both --agent and --no-agent appear in rawArgs. */ rawArgConflict?: boolean; } @@ -192,9 +202,16 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise, --from-env (reads TESTSPRITE_API_KEY), or run interactively.', @@ -204,7 +221,7 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise or --from-env.', @@ -219,7 +236,13 @@ export async function runInit(opts: InitOptions, deps: InitDeps = {}): Promise # re-install or install additional targets', + ' # 2. Generate tests: ask your coding agent (the testsprite-onboard skill is installed),', + ); + lines.push(' # or create one yourself, then run them (use the projectId from step 1):'); + lines.push(' testsprite test run --all --project '); + lines.push(''); + lines.push(' # Manage installed agent skills'); + lines.push(' testsprite agent list'); + lines.push( + ' testsprite agent install --target= # re-install or install additional targets', ); + } else { + lines.push(' # 2. Create a test, then run it (use the projectId from step 1):'); + lines.push(' testsprite test create --project ...'); + lines.push(' testsprite test run --all --project '); + lines.push( + ' # Tip: `testsprite agent install` sets up the onboarding skill for your coding agent', + ); + lines.push(''); + lines.push(' # Manage installed agent skills'); + lines.push(' testsprite agent list'); } return lines.join('\n'); @@ -424,7 +477,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, @@ -433,13 +486,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); -} - const SETUP_DESCRIPTION = 'Set up TestSprite: configure your API key and install the TestSprite agent skills for your coding agent'; @@ -457,6 +503,7 @@ interface SetupCmdOpts { force?: boolean; dir?: string; yes?: boolean; + skipIfConfigured?: boolean; } /** Attach the onboarding flags shared by `setup` and the `init` alias. */ @@ -480,7 +527,11 @@ function addSetupOptions( .option('--no-agent', 'Skip the agent skill install (configure credentials only)') .option('--force', 'Overwrite an existing skill file (a .bak backup is kept)') .option('--dir ', 'Project root for the skill install (default: current directory)') - .option('-y, --yes', 'Non-interactive: accept all defaults, never prompt'); + .option('-y, --yes', 'Non-interactive: accept all defaults, never prompt') + .option( + '--skip-if-configured', + 'Skip the API key prompt when credentials already exist for this profile (CI-safe idempotent re-run)', + ); } /** Build {@link InitOptions} from raw Commander opts + globals. */ @@ -521,6 +572,7 @@ function buildSetupOptions( force: Boolean(cmdOpts.force), dir: cmdOpts.dir, yes: Boolean(cmdOpts.yes), + skipIfConfigured: Boolean(cmdOpts.skipIfConfigured), rawArgConflict, }; } diff --git a/src/commands/project.test.ts b/src/commands/project.test.ts index 057296a..ae98070 100644 --- a/src/commands/project.test.ts +++ b/src/commands/project.test.ts @@ -6,9 +6,13 @@ import { ApiError } from '../lib/errors.js'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; import { type CliProject, + type CliDeleteProjectResponse, type CliUpdateProjectResponse, createProjectCommand, + runAutoAuth, runCreate, + runCredential, + runDelete, runGet, runList, runUpdate, @@ -72,10 +76,10 @@ describe('createProjectCommand', () => { errorSpy.mockRestore(); }); - it('exposes list, get, create and update subcommands', () => { + it('exposes list, get, create, update, delete, credential and auto-auth subcommands', () => { const project = createProjectCommand(); const names = project.commands.map(c => c.name()).sort(); - expect(names).toEqual(['create', 'get', 'list', 'update']); + expect(names).toEqual(['auto-auth', 'create', 'credential', 'delete', 'get', 'list', 'update']); }); it('list exposes the pagination flags from the design contract', () => { @@ -85,6 +89,8 @@ describe('createProjectCommand', () => { expect(flagNames).toContain('--page-size'); expect(flagNames).toContain('--starting-token'); expect(flagNames).toContain('--max-items'); + expect(flagNames).toContain('--columns'); + expect(flagNames).toContain('--no-header'); }); }); @@ -199,6 +205,47 @@ describe('runList', () => { }); }); + it('rejects invalid pagination before requiring credentials', async () => { + const credentialsPath = join(mkdtempSync(join(tmpdir(), 'cli-p2-no-creds-')), 'credentials'); + const fetchImpl = vi.fn(); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, pageSize: 1.5 }, + { credentialsPath, fetchImpl: fetchImpl as unknown as typeof globalThis.fetch }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('rejects invalid dry-run pagination before emitting the dry-run banner', async () => { + const stderr: string[] = []; + const fetchImpl = vi.fn(); + + await expect( + runList( + { profile: 'default', output: 'json', debug: false, dryRun: true, pageSize: 1.5 }, + { + credentialsPath: join(mkdtempSync(join(tmpdir(), 'cli-p2-dryrun-')), 'credentials'), + fetchImpl: fetchImpl as unknown as typeof globalThis.fetch, + stderr: line => stderr.push(line), + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'page-size' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(stderr.join('\n')).not.toContain(DRY_RUN_BANNER); + }); + it('rejects pageSize=101 with VALIDATION_ERROR exit 5 (Fix 7 — upper-bound enforced client-side)', async () => { // Previously silently clamped to 100; now rejected so callers get fast feedback. const { credentialsPath } = makeCreds(); @@ -240,6 +287,72 @@ describe('runList', () => { expect(block).toContain('nextToken: next-please'); }); + it('text output selects/reorders columns and suppresses the header', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + pageSize: 25, + columns: 'name,id', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const block = out.join('\n'); + expect(block).toMatch(/^Checkout\s+project_b3c91efa$/); + expect(block).not.toContain('NAME'); + expect(block).not.toContain('CREATED'); + }); + + it('text output rejects unknown columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runList( + { + profile: 'default', + output: 'text', + debug: false, + pageSize: 25, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json output ignores text-only column flags', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [PROJECT_FIXTURE], nextToken: null }, + })); + + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'json', + debug: false, + pageSize: 25, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(JSON.parse(out.join('\n')).items[0].id).toBe('project_b3c91efa'); + }); + it('text output reads "No projects." when items is empty and nextToken is null', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); @@ -286,6 +399,21 @@ describe('runList', () => { }); }); +describe('DEV-244 — project update no longer accepts the dead --description flag', () => { + it('rejects --description on `project update` as an unknown option', async () => { + const project = createProjectCommand(); + const update = project.commands.find(c => c.name() === 'update')!; + project.exitOverride(); + update.exitOverride(); + + await expect( + project.parseAsync(['update', 'proj_x', '--description', 'should not exist'], { + from: 'user', + }), + ).rejects.toThrow(/unknown option.*--description/i); + }); +}); + describe('createProjectCommand --page-size option parser', () => { it('rejects non-numeric --page-size values via commander', async () => { const project = createProjectCommand(); @@ -569,6 +697,88 @@ describe('runCreate', () => { ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); expect(fetchImpl).not.toHaveBeenCalled(); }); + + it('rejects a whitespace-only --name with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network — validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: ' ', + targetUrl: 'https://example.com', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('rejects a whitespace-only --password with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network - validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Password Guard Project', + targetUrl: 'https://example.com', + password: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('rejects --description with VALIDATION_ERROR (exit 5), no network — projects have no description', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network — validation must fire client-side'); + }); + + await expect( + runCreate( + { + profile: 'default', + output: 'json', + debug: false, + type: 'frontend', + name: 'Desc Project', + targetUrl: 'https://example.com', + description: 'a human description', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ exitCode: 5, code: 'VALIDATION_ERROR' }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); }); // --------------------------------------------------------------------------- @@ -646,6 +856,55 @@ describe('runUpdate', () => { expect(fetchImpl).not.toHaveBeenCalled(); }); + it('rejects a whitespace-only --name with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + name: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it('rejects a whitespace-only --password with VALIDATION_ERROR (exit 5), no network', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not be called'); + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_abc', + password: ' ', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); it('P7 — dry-run returns canned shape without network call', async () => { resetDryRunBannerForTesting(); const { credentialsPath } = makeCreds(); @@ -677,6 +936,33 @@ describe('runUpdate', () => { expect(err).toContain(DRY_RUN_BANNER); }); + it('P7 — dry-run with --password-file does not read the filesystem', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = vi.fn(async () => { + throw new Error('should not hit network'); + }); + const result = await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'proj_dry', + passwordFile: '/tmp/definitely-not-here-testsprite', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => {}, + stderr: () => {}, + }, + ); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(result.id).toBe('proj_dry'); + expect(result.updatedFields).toContain('password'); + }); + it('P7 — renders text mode with updatedFields and updatedAt', async () => { const { credentialsPath } = makeCreds(); const updateResponse: CliUpdateProjectResponse = { @@ -693,7 +979,6 @@ describe('runUpdate', () => { debug: false, projectId: 'proj_text', name: 'New Name', - description: 'New desc', }, { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, ); @@ -758,3 +1043,383 @@ describe('runUpdate', () => { expect(result.updatedFields).toBeUndefined(); }); }); + +describe('runDelete', () => { + it('refuses without --confirm and never hits the network (exit 5)', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: {} }; + }); + await expect( + runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'confirm' }), + }); + expect(called).toBe(0); + }); + + it('DELETEs /projects/{id} with a minted idempotency-key when --confirm is set', async () => { + const { credentialsPath } = makeCreds(); + const deleteResponse: CliDeleteProjectResponse = { + projectId: 'proj_alpha', + deletedAt: '2026-05-16T10:00:00.000Z', + }; + let seenUrl = ''; + let seenMethod = ''; + let seenIdemKey: string | null = null; + const fetchImpl = (async (input: Parameters[0], init: RequestInit = {}) => { + seenUrl = typeof input === 'string' ? input : (input as { url: string }).url; + seenMethod = init.method ?? 'GET'; + seenIdemKey = new Headers(init.headers).get('idempotency-key'); + return new Response(JSON.stringify(deleteResponse), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof fetch; + + const result = await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: true, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(seenMethod).toBe('DELETE'); + expect(seenUrl).toContain('/api/cli/v1/projects/proj_alpha'); + expect(seenIdemKey).toMatch(/^cli-delete-[0-9a-f-]{36}$/); + expect(result.projectId).toBe('proj_alpha'); + expect(result.deletedAt).toBe('2026-05-16T10:00:00.000Z'); + }); + + it('forwards a caller-supplied --idempotency-key verbatim', async () => { + const { credentialsPath } = makeCreds(); + let seenIdemKey: string | null = null; + const fetchImpl = (async (_input: Parameters[0], init: RequestInit = {}) => { + seenIdemKey = new Headers(init.headers).get('idempotency-key'); + return new Response( + JSON.stringify({ projectId: 'proj_alpha', deletedAt: '2026-05-16T10:00:00.000Z' }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + }) as typeof fetch; + + await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'proj_alpha', + confirm: true, + idempotencyKey: 'idem-del-001', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + + expect(seenIdemKey).toBe('idem-del-001'); + }); + + it('--dry-run bypasses --confirm and returns the canned sample without network', async () => { + resetDryRunBannerForTesting(); + const { credentialsPath } = makeCreds(); + const err: string[] = []; + // No fetchImpl → the client-factory dry-run fetch serves the samples.ts value. + const result = await runDelete( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + projectId: 'project_b3c91efa', + confirm: false, + }, + { credentialsPath, stdout: () => {}, stderr: line => err.push(line) }, + ); + + expect(result.projectId).toBe('project_b3c91efa'); + expect(result.deletedAt).toBe('2026-05-16T00:00:00.000Z'); + expect(err).toContain(DRY_RUN_BANNER); + }); + + it('renders text mode with projectId and deletedAt', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { projectId: 'proj_text', deletedAt: '2026-05-16T10:00:00.000Z' }, + })); + const out: string[] = []; + await runDelete( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'proj_text', + confirm: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line), stderr: () => {} }, + ); + const block = out.join('\n'); + expect(block).toContain('projectId proj_text'); + expect(block).toContain('deletedAt 2026-05-16T10:00:00.000Z'); + }); +}); + +describe('runCredential', () => { + interface Captured { + url: string; + method: string; + body: unknown; + headers: Headers; + } + function captureFetch(captured: Captured[], body: unknown) { + return makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : undefined, + headers: new Headers(init.headers as Record), + }); + return { status: 200, body }; + }); + } + + it('PUTs /projects/:id/credential with authType + credential + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured, { + projectId: 'p1', + authType: 'Bearer token', + rewroteCount: 2, + }); + const res = await runCredential( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + authType: 'Bearer token', + credential: 'tok-123', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(res.rewroteCount).toBe(2); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.url).toContain('/projects/p1/credential'); + expect(put.body).toEqual({ authType: 'Bearer token', credential: 'tok-123' }); + expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-cred-[0-9a-f-]{36}$/); + }); + + it('public clears the credential (no credential in body, none required)', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured, { + projectId: 'p1', + authType: 'public', + rewroteCount: 0, + }); + await runCredential( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'public' }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.body).toEqual({ authType: 'public' }); + }); + + it('non-public without --credential → VALIDATION_ERROR (exit 5), no fetch', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runCredential( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', authType: 'API key' }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); + + it('rejects an unknown --type locally (no fetch)', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runCredential( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + authType: 'jwt', + credential: 'x', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('runAutoAuth', () => { + interface Captured { + url: string; + method: string; + body: Record; + headers: Headers; + } + function captureFetch(captured: Captured[]) { + return makeFetch((url, init) => { + captured.push({ + url, + method: init.method ?? 'GET', + body: init.body ? JSON.parse(init.body as string) : {}, + headers: new Headers(init.headers as Record), + }); + return { + status: 200, + body: { projectId: 'p1', enabled: true, method: 'aws_cognito_refresh', inject: 'bearer' }, + }; + }); + } + + it('PUTs /projects/:id/auto-auth with the config body + idempotency-key', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'aws_cognito_refresh', + inject: 'bearer', + region: 'us-east-1', + clientId: 'abc', + refreshToken: 'rt-xyz', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + const put = captured.find(c => c.method === 'PUT')!; + expect(put.url).toContain('/projects/p1/auto-auth'); + expect(put.body).toEqual({ + enabled: true, + method: 'aws_cognito_refresh', + inject: 'bearer', + region: 'us-east-1', + clientId: 'abc', + refreshToken: 'rt-xyz', + }); + expect(put.headers.get('idempotency-key')).toMatch(/^cli-proj-autoauth-[0-9a-f-]{36}$/); + }); + + it('--disable sends enabled:false', async () => { + const { credentialsPath } = makeCreds(); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + disable: true, + method: 'password', + inject: 'bearer', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(captured.find(c => c.method === 'PUT')!.body.enabled).toBe(false); + }); + + it('reads a secret from --refresh-token-file', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-rt-')); + const rtFile = join(dir, 'rt.txt'); + writeFileSync(rtFile, ' rt-from-file\n'); + const captured: Captured[] = []; + const fetchImpl = captureFetch(captured); + await runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'refresh_token', + inject: 'bearer', + tokenEndpoint: 'https://idp.example.com/token', + refreshTokenFile: rtFile, + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ); + expect(captured.find(c => c.method === 'PUT')!.body.refreshToken).toBe('rt-from-file'); + }); + + it('rejects an unknown --method / --inject locally (no fetch)', async () => { + const { credentialsPath } = makeCreds(); + let fetched = false; + const fetchImpl = makeFetch(() => { + fetched = true; + return { body: {} }; + }); + await expect( + runAutoAuth( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'p1', + method: 'magic', + inject: 'bearer', + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetched).toBe(false); + }); +}); + +describe('dogfood 2026-06-30 — whitespace-only --name is rejected (parity with `test create`)', () => { + const noNetwork = () => { + throw new Error('network should not be hit'); + }; + + it('runCreate rejects a whitespace-only --name (exit 5, no network)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runCreate( + { profile: 'default', output: 'json', debug: false, type: 'backend', name: ' ' }, + { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); + + it('runUpdate rejects a whitespace-only --name (exit 5, no network)', async () => { + const { credentialsPath } = makeCreds(); + await expect( + runUpdate( + { profile: 'default', output: 'json', debug: false, projectId: 'p1', name: '\t \n' }, + { credentialsPath, fetchImpl: makeFetch(noNetwork), stdout: () => {}, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); diff --git a/src/commands/project.ts b/src/commands/project.ts index 41abab1..6a0b332 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -4,13 +4,15 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { ApiError } from '../lib/errors.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { assertNotLocal } from '../lib/target-url.js'; +import { renderTextTable, resolveTextColumns, type TextTableColumn } from '../lib/text-table.js'; import { assertIdempotencyKey } from '../lib/validate.js'; import { fetchSinglePage, @@ -43,6 +45,8 @@ interface ListOptions extends CommonOptions { pageSize?: number; startingToken?: string; maxItems?: number; + columns?: string; + noHeader?: boolean; } export async function runList( @@ -50,13 +54,16 @@ export async function runList( deps: ProjectDeps = {}, ): Promise> { const out = makeOutput(opts.output, deps); - const client = makeClient(opts, deps); const paginationFlags: PaginationFlags = validatePaginationFlags({ pageSize: opts.pageSize, startingToken: opts.startingToken, maxItems: opts.maxItems, }); + if (opts.output === 'text') { + resolveTextColumns(opts.columns, PROJECT_LIST_COLUMNS); + } + const client = makeClient(opts, deps); // When the user explicitly passed a page-size flag and did NOT ask // for --max-items, treat that as a "give me one page and the cursor" @@ -83,7 +90,7 @@ export async function runList( out.print(page, data => { const p = data as Page; - return renderProjectListText(p); + return renderProjectListText(p, { columns: opts.columns, noHeader: opts.noHeader }); }); return page; } @@ -109,7 +116,8 @@ export interface CliCreateProjectRequest { type: 'frontend' | 'backend'; name: string; targetUrl?: string; - description?: string; + // `description` is intentionally not part of the wire request — projects have + // no description field. The `--description` flag is rejected client-side. username?: string; password?: string; instruction?: string; @@ -142,11 +150,24 @@ export async function runCreate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. - if (opts.name !== undefined && opts.name.length > 200) { + // Whitespace-only / empty rejection (parity with `test create`'s requireString; + // a truthy `--name " "` otherwise creates a blank-named project on the backend). + if (opts.name === undefined || opts.name.trim().length === 0) { + throw localValidationError('--name is required and must not be empty or whitespace-only'); + } + if (opts.password !== undefined && opts.password.trim().length === 0) { + throw localValidationError('--password must not be empty or whitespace-only'); + } + if (opts.name.length > 200) { throw localValidationError('--name must be at most 200 characters'); } - if (opts.description !== undefined && opts.description.length > 2000) { - throw localValidationError('--description must be at most 2000 characters'); + // `--description` is not supported on projects — no project entity stores a + // description, and the backend rejects it with a 422. Fail fast client-side + // with an actionable message instead of a wasted round trip. + if (opts.description !== undefined) { + throw localValidationError( + '--description is not supported for projects; omit it (test-level descriptions are set on `test create`)', + ); } // P2-7: guard --url against localhost/RFC1918/non-http(s) (same rules as @@ -201,7 +222,6 @@ export async function runCreate( type: opts.type, name: opts.name, ...(opts.targetUrl !== undefined ? { targetUrl: opts.targetUrl } : {}), - ...(opts.description !== undefined ? { description: opts.description } : {}), ...(opts.username !== undefined ? { username: opts.username } : {}), ...(password !== undefined ? { password } : {}), ...(opts.instruction !== undefined ? { instruction: opts.instruction } : {}), @@ -235,7 +255,6 @@ interface UpdateOptions extends CommonOptions { username?: string; password?: string; passwordFile?: string; - description?: string; instruction?: string; idempotencyKey?: string; } @@ -251,36 +270,36 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); // P1-3: client-side length checks matching server limits. - if (opts.name !== undefined && opts.name.length > 200) { - throw localValidationError('--name must be at most 200 characters'); + // Reject a whitespace-only `--name` on update too (parity with create); name + // stays optional here, so only validate when the flag is supplied. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError('--name must not be empty or whitespace-only'); } - if (opts.description !== undefined && opts.description.length > 2000) { - throw localValidationError('--description must be at most 2000 characters'); + if (opts.password !== undefined && opts.password.trim().length === 0) { + throw localValidationError('--password must not be empty or whitespace-only'); } - - // Resolve password - let password = opts.password; - if (password === undefined && opts.passwordFile !== undefined) { - password = readFileSync(opts.passwordFile, 'utf8').trim(); + if (opts.name !== undefined && opts.name.length > 200) { + throw localValidationError('--name must be at most 200 characters'); } - // P2-7: guard --url against localhost/RFC1918/non-http(s). if (opts.targetUrl !== undefined) { assertNotLocal(opts.targetUrl); } - const mutableFields: Record = { - name: opts.name, - targetUrl: opts.targetUrl, - username: opts.username, - password, - description: opts.description, - instruction: opts.instruction, + const passwordSupplied = opts.password !== undefined || opts.passwordFile !== undefined; + const mutableFields: Record = { + name: opts.name !== undefined, + targetUrl: opts.targetUrl !== undefined, + username: opts.username !== undefined, + password: passwordSupplied, + instruction: opts.instruction !== undefined, }; - const presentFields = Object.entries(mutableFields).filter(([, v]) => v !== undefined); - if (presentFields.length === 0) { + const presentFieldNames = Object.entries(mutableFields) + .filter(([, present]) => present) + .map(([field]) => field); + if (presentFieldNames.length === 0) { throw localValidationError( - 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, --description, or --instruction.', + 'At least one mutable flag is required: --name, --url, --username, --password/--password-file, or --instruction.', ); } @@ -296,19 +315,35 @@ export async function runUpdate( } const sample: CliUpdateProjectResponse = { id: opts.projectId, - updatedFields: presentFields.map(([k]) => k), + updatedFields: presentFieldNames, updatedAt: '2026-05-16T00:00:00.000Z', }; out.print(sample, data => renderUpdateText(data as CliUpdateProjectResponse)); return sample; } + // Resolve password only on the real path. Dry-run must not touch the + // filesystem, even when --password-file is present. + let password = opts.password; + if (password === undefined && opts.passwordFile !== undefined) { + password = readFileSync(opts.passwordFile, 'utf8').trim(); + } + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-update-${randomUUID()}`; if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderr(`idempotency-key: ${idempotencyKey}`); } - const body = Object.fromEntries(presentFields) as Record; + const bodyFields: Record = { + name: opts.name, + targetUrl: opts.targetUrl, + username: opts.username, + password, + instruction: opts.instruction, + }; + const body = Object.fromEntries( + Object.entries(bodyFields).filter(([, v]) => v !== undefined), + ) as Record; const client = makeClient(opts, deps); const updated = await client.patch( `/projects/${encodeURIComponent(opts.projectId)}`, @@ -322,6 +357,302 @@ export async function runUpdate( return updated; } +// --------------------------------------------------------------------------- +// project delete +// --------------------------------------------------------------------------- + +export interface CliDeleteProjectResponse { + projectId: string; + deletedAt: string; +} + +interface DeleteOptions extends CommonOptions { + projectId: string; + /** Hard gate — required (unless `--dry-run` is set). No interactive prompts. */ + confirm: boolean; + /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ + idempotencyKey?: string; +} + +/** + * `project delete --confirm` — permanent cascade delete via + * DELETE /projects/{id}. + * + * The server deletes the project together with everything under it — its + * frontend/backend sub-projects, all their tests, and backend fixtures — + * matching the Portal's own delete behavior. There is no restore window. + * + * **`--confirm` is required** (unless `--dry-run`). Without either, the CLI + * exits 5 `VALIDATION_ERROR` with a typed envelope explaining the convention. + * The CLI never prompts interactively (CI-friendly contract). Re-delete on an already-deleted (or missing) project returns 404 from + * the server; the CLI surfaces the envelope as-is (exit 4), no client branching. + */ +export async function runDelete( + opts: DeleteOptions, + deps: ProjectDeps = {}, +): Promise { + assertIdempotencyKey(opts.idempotencyKey); + if (opts.projectId === undefined || opts.projectId.trim().length === 0) { + throw localValidationError(' is required'); + } + + if (!opts.confirm && !opts.dryRun) { + throw ApiError.fromEnvelope({ + error: { + code: 'VALIDATION_ERROR', + message: 'Refusing to delete without --confirm.', + nextAction: + 'This permanently deletes the project and everything under it — its ' + + 'sub-projects, all their tests, and backend fixtures (no restore window). ' + + 'The CLI convention is explicit confirmation for destructive operations. ' + + 'Re-run with --confirm. (--dry-run also works without --confirm.)', + requestId: 'local', + details: { field: 'confirm', reason: 'required for destructive operation' }, + }, + }); + } + + const idempotencyKey = opts.idempotencyKey ?? `cli-delete-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(`idempotency-key: ${idempotencyKey}`); + } + + const client = makeClient(opts, deps); + const out = makeOutput(opts.output, deps); + const response = await client.delete( + `/projects/${encodeURIComponent(opts.projectId)}`, + { + headers: { 'idempotency-key': idempotencyKey }, + }, + ); + + out.print(response, data => renderDeleteText(data as CliDeleteProjectResponse)); + return response; +} + +// --------------------------------------------------------------------------- +// project credential — set the static backend credential +// --------------------------------------------------------------------------- + +const CLI_AUTH_TYPES = ['public', 'Bearer token', 'API key', 'basic token'] as const; + +export interface CliProjectCredentialResponse { + projectId: string; + authType: string; + rewroteCount: number; +} + +interface CredentialOptions extends CommonOptions { + projectId: string; + authType: string; + credential?: string; + credentialFile?: string; + idempotencyKey?: string; +} + +export async function runCredential( + opts: CredentialOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + assertIdempotencyKey(opts.idempotencyKey); + + if (!(CLI_AUTH_TYPES as readonly string[]).includes(opts.authType)) { + throw localValidationError(`--type must be one of: ${CLI_AUTH_TYPES.join(', ')}`); + } + + // Resolve the credential value (flag or file). Required for every type + // except `public` (which clears it). + let credential = opts.credential; + if (credential === undefined && opts.credentialFile !== undefined) { + credential = readFileSync(opts.credentialFile, 'utf8').trim(); + } + if (opts.authType !== 'public' && (credential === undefined || credential === '')) { + throw localValidationError( + '--credential (or --credential-file) is required unless --type is "public"', + ); + } + + const body: Record = { authType: opts.authType }; + if (opts.authType !== 'public' && credential !== undefined) body.credential = credential; + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-cred-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + if (opts.dryRun) { + const sample: CliProjectCredentialResponse = { + projectId: opts.projectId, + authType: opts.authType, + rewroteCount: 0, + }; + out.print(sample, data => renderCredentialText(data as CliProjectCredentialResponse)); + return sample; + } + + const client = makeClient(opts, deps); + const res = await client.put( + `/projects/${encodeURIComponent(opts.projectId)}/credential`, + { body, headers: { 'idempotency-key': idempotencyKey } }, + ); + out.print(res, data => renderCredentialText(data as CliProjectCredentialResponse)); + return res; +} + +function renderCredentialText(r: CliProjectCredentialResponse): string { + return [ + `projectId ${r.projectId}`, + `authType ${r.authType}`, + `rewroteCount ${r.rewroteCount}`, + ].join('\n'); +} + +// --------------------------------------------------------------------------- +// project auto-auth — configure the recurring-token (auto-refresh) login +// --------------------------------------------------------------------------- + +const AUTO_AUTH_METHODS = ['password', 'refresh_token', 'aws_cognito_refresh'] as const; +const AUTO_AUTH_INJECTS = ['bearer', 'header', 'cookie'] as const; + +export interface CliProjectAutoAuthResponse { + projectId: string; + enabled: boolean; + method: string; + inject: string; + /** + * Present when the server's trial refresh failed: `enabled` is then `false` + * and this carries the reason (e.g. a bad refresh token). The config is still + * stored, but auto-auth won't run until the login succeeds. + */ + lastRefreshError?: string; +} + +interface AutoAuthOptions extends CommonOptions { + projectId: string; + disable?: boolean; + method: string; + inject: string; + injectKey?: string; + // password method + loginUrl?: string; + loginMethod?: string; + loginContentType?: string; + loginBodyTemplate?: string; + username?: string; + password?: string; + passwordFile?: string; + tokenPath?: string; + // refresh_token method + tokenEndpoint?: string; + clientId?: string; + clientSecret?: string; + clientSecretFile?: string; + refreshToken?: string; + refreshTokenFile?: string; + scope?: string; + // aws_cognito_refresh method + region?: string; + idempotencyKey?: string; +} + +export async function runAutoAuth( + opts: AutoAuthOptions, + deps: ProjectDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + assertIdempotencyKey(opts.idempotencyKey); + + if (!(AUTO_AUTH_METHODS as readonly string[]).includes(opts.method)) { + throw localValidationError(`--method must be one of: ${AUTO_AUTH_METHODS.join(', ')}`); + } + if (!(AUTO_AUTH_INJECTS as readonly string[]).includes(opts.inject)) { + throw localValidationError(`--inject must be one of: ${AUTO_AUTH_INJECTS.join(', ')}`); + } + + // Resolve secrets from --*-file variants so they stay out of shell history. + const password = + opts.password ?? + (opts.passwordFile !== undefined ? readFileSync(opts.passwordFile, 'utf8').trim() : undefined); + const clientSecret = + opts.clientSecret ?? + (opts.clientSecretFile !== undefined + ? readFileSync(opts.clientSecretFile, 'utf8').trim() + : undefined); + const refreshToken = + opts.refreshToken ?? + (opts.refreshTokenFile !== undefined + ? readFileSync(opts.refreshTokenFile, 'utf8').trim() + : undefined); + + const enabled = opts.disable !== true; + const body: Record = { enabled, method: opts.method, inject: opts.inject }; + const maybe = (k: string, v: string | undefined): void => { + if (v !== undefined) body[k] = v; + }; + maybe('injectKey', opts.injectKey); + maybe('loginUrl', opts.loginUrl); + maybe('loginMethod', opts.loginMethod); + maybe('loginContentType', opts.loginContentType); + maybe('loginBodyTemplate', opts.loginBodyTemplate); + maybe('username', opts.username); + maybe('password', password); + maybe('tokenPath', opts.tokenPath); + maybe('tokenEndpoint', opts.tokenEndpoint); + maybe('clientId', opts.clientId); + maybe('clientSecret', clientSecret); + maybe('refreshToken', refreshToken); + maybe('scope', opts.scope); + maybe('region', opts.region); + + const idempotencyKey = opts.idempotencyKey ?? `cli-proj-autoauth-${randomUUID()}`; + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${idempotencyKey}`); + } + + if (opts.dryRun) { + const sample: CliProjectAutoAuthResponse = { + projectId: opts.projectId, + enabled, + method: opts.method, + inject: opts.inject, + }; + out.print(sample, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); + return sample; + } + + const client = makeClient(opts, deps); + const res = await client.put( + `/projects/${encodeURIComponent(opts.projectId)}/auto-auth`, + { body, headers: { 'idempotency-key': idempotencyKey } }, + ); + out.print(res, data => renderAutoAuthText(data as CliProjectAutoAuthResponse)); + return res; +} + +function renderAutoAuthText(r: CliProjectAutoAuthResponse): string { + const lines = [ + `projectId ${r.projectId}`, + `enabled ${r.enabled}`, + `method ${r.method}`, + `inject ${r.inject}`, + ]; + if (r.lastRefreshError) { + lines.push(`lastRefreshError ${r.lastRefreshError}`); + } + // A disabled result after a write means the trial login failed — call it out + // so the user doesn't assume auto-auth is live. + if (!r.enabled) { + lines.push( + 'note auto-auth was stored but is DISABLED — the trial login failed. Fix the credentials (e.g. a valid refresh token) and re-run.', + ); + } + return lines.join('\n'); +} + export function createProjectCommand(deps: ProjectDeps = {}): Command { const project = new Command('project').description('Manage TestSprite projects'); @@ -338,6 +669,8 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--page-size ', 'service page-size hint (1-100, default 25)') .option('--starting-token ', 'opaque cursor from a previous list response') .option('--max-items ', 'stop after this many items across auto-paged pages') + .option('--columns ', 'select/reorder text table columns (comma-separated keys)') + .option('--no-header', 'suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (cmdOpts: ListFlagOpts, command: Command) => { // Don't parse numeric flags via Commander — its parser throws a @@ -351,6 +684,8 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { pageSize: parseFlag(cmdOpts.pageSize, 'page-size'), startingToken: cmdOpts.startingToken, maxItems: parseFlag(cmdOpts.maxItems, 'max-items'), + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -370,7 +705,10 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--type ', 'project type (required)') .option('--name ', 'project name (required)') .option('--url ', 'target URL (required for frontend)') - .option('--description ', 'optional human description') + .option( + '--description ', + 'not supported — projects have no description (test-level descriptions are set on `test create`)', + ) .option('--username ', 'optional auth username') .option('--password ', 'optional auth password (use --password-file for non-interactive)') .option('--password-file ', 'read password from file instead of inline flag') @@ -415,7 +753,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { .option('--username ', 'new auth username') .option('--password ', 'new auth password') .option('--password-file ', 'read new password from file') - .option('--description ', 'new description') .option('--instruction ', 'new FE plan-gen instruction hint') .option( '--idempotency-key ', @@ -432,7 +769,6 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { username: cmdOpts.username, password: cmdOpts.password, passwordFile: cmdOpts.passwordFile, - description: cmdOpts.description, instruction: cmdOpts.instruction, idempotencyKey: cmdOpts.idempotencyKey, }, @@ -440,6 +776,128 @@ export function createProjectCommand(deps: ProjectDeps = {}): Command { ); }); + project + .command('delete ') + .description( + 'Permanently delete a project and everything under it (sub-projects,\n' + + 'their tests, and backend fixtures). Requires --confirm.\n' + + '\nExit codes:\n' + + ' 0 success\n' + + ' 3 auth error\n' + + ' 4 project not found (or already deleted)\n' + + ' 5 validation error (e.g., missing --confirm)', + ) + .option('--confirm', 'required: explicit confirmation for the destructive operation', false) + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: DeleteFlagOpts, command: Command) => { + await runDelete( + { + ...resolveCommonOptions(command), + projectId, + confirm: cmdOpts.confirm === true, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + project + .command('credential ') + .description( + 'Set the static backend credential injected into every backend test\n' + + '(Bearer token / API key / Basic token / public). Free tier.', + ) + .requiredOption('--type ', 'public | "Bearer token" | "API key" | "basic token"') + .option('--credential ', 'credential value (required unless --type public)') + .option('--credential-file ', 'read the credential value from a file') + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: CredentialFlagOpts, command: Command) => { + await runCredential( + { + ...resolveCommonOptions(command), + projectId, + authType: cmdOpts.type, + credential: cmdOpts.credential, + credentialFile: cmdOpts.credentialFile, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + project + .command('auto-auth ') + .description( + 'Configure the recurring-token (auto-refresh login) for backend tests (Pro).\n' + + 'A fresh token is fetched on each run and injected into every backend test.', + ) + .requiredOption('--method ', 'password | refresh_token | aws_cognito_refresh') + .requiredOption('--inject ', 'bearer | header | cookie') + .option('--disable', 'turn auto-auth off (keeps stored config)') + .option('--inject-key ', 'header/cookie name when --inject is header/cookie') + // password method + .option('--login-url ', 'login endpoint (method=password)') + .option('--login-method ', 'POST | PUT (method=password)') + .option('--login-content-type ', 'application/json | application/x-www-form-urlencoded') + .option('--login-body-template ', 'login body template with {{username}}/{{password}}') + .option('--username ', 'login username (method=password)') + .option('--password ', 'login password (method=password)') + .option('--password-file ', 'read login password from a file') + .option('--token-path ', 'JSONPath to the token in the login response') + // refresh_token method + .option('--token-endpoint ', 'OAuth token endpoint (method=refresh_token)') + .option('--client-id ', 'OAuth client id') + .option('--client-secret ', 'OAuth client secret') + .option('--client-secret-file ', 'read OAuth client secret from a file') + .option('--refresh-token ', 'OAuth/Cognito refresh token') + .option('--refresh-token-file ', 'read the refresh token from a file') + .option('--scope ', 'OAuth scope') + // aws_cognito_refresh method + .option('--region ', "AWS region (method=aws_cognito_refresh, e.g. 'us-east-1')") + .option( + '--idempotency-key ', + 'opaque idempotency token. Defaults to a UUIDv4 minted per invocation.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (projectId: string, cmdOpts: AutoAuthFlagOpts, command: Command) => { + await runAutoAuth( + { + ...resolveCommonOptions(command), + projectId, + disable: cmdOpts.disable, + method: cmdOpts.method, + inject: cmdOpts.inject, + injectKey: cmdOpts.injectKey, + loginUrl: cmdOpts.loginUrl, + loginMethod: cmdOpts.loginMethod, + loginContentType: cmdOpts.loginContentType, + loginBodyTemplate: cmdOpts.loginBodyTemplate, + username: cmdOpts.username, + password: cmdOpts.password, + passwordFile: cmdOpts.passwordFile, + tokenPath: cmdOpts.tokenPath, + tokenEndpoint: cmdOpts.tokenEndpoint, + clientId: cmdOpts.clientId, + clientSecret: cmdOpts.clientSecret, + clientSecretFile: cmdOpts.clientSecretFile, + refreshToken: cmdOpts.refreshToken, + refreshTokenFile: cmdOpts.refreshTokenFile, + scope: cmdOpts.scope, + region: cmdOpts.region, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + return project; } @@ -447,6 +905,8 @@ interface ListFlagOpts { pageSize?: string; startingToken?: string; maxItems?: string; + columns?: string; + header?: boolean; } interface CreateFlagOpts { @@ -467,11 +927,46 @@ interface UpdateFlagOpts { username?: string; password?: string; passwordFile?: string; - description?: string; instruction?: string; idempotencyKey?: string; } +interface DeleteFlagOpts { + confirm?: boolean; + idempotencyKey?: string; +} + +interface CredentialFlagOpts { + type: string; + credential?: string; + credentialFile?: string; + idempotencyKey?: string; +} + +interface AutoAuthFlagOpts { + disable?: boolean; + method: string; + inject: string; + injectKey?: string; + loginUrl?: string; + loginMethod?: string; + loginContentType?: string; + loginBodyTemplate?: string; + username?: string; + password?: string; + passwordFile?: string; + tokenPath?: string; + tokenEndpoint?: string; + clientId?: string; + clientSecret?: string; + clientSecretFile?: string; + refreshToken?: string; + refreshTokenFile?: string; + scope?: string; + region?: string; + idempotencyKey?: string; +} + function parseFlag(raw: string | undefined, flagName: string): number | undefined { if (raw === undefined) return undefined; const n = Number(raw); @@ -494,13 +989,9 @@ function resolveCommonOptions(command: Command): CommonOptions { requestTimeout?: string; }; // P2-8: validate --output before allowing silent fallback to 'text'. - const rawOutput = globals.output; - if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { - throw localValidationError('--output must be one of: json, text'); - } return { profile: globals.profile ?? 'default', - output: (globals.output as OutputMode | undefined) ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, @@ -509,18 +1000,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - function makeClient(opts: CommonOptions, deps: ProjectDeps): HttpClient { return makeHttpClient(opts, { env: deps.env, @@ -534,45 +1013,37 @@ function makeOutput(mode: OutputMode, deps: ProjectDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } -function renderProjectListText(page: Page): string { +const PROJECT_LIST_COLUMNS: ReadonlyArray> = [ + { + header: 'ID', + width: rows => Math.max(2, ...rows.map(project => project.id.length)), + render: project => project.id, + }, + { + header: 'NAME', + width: rows => Math.max(4, ...rows.map(project => project.name.length)), + render: project => project.name, + }, + { header: 'TYPE', width: 8, render: project => project.type }, + { header: 'FROM', width: 6, render: project => project.createdFrom }, + { header: 'CREATED', width: 0, render: project => project.createdAt }, +]; + +function renderProjectListText( + page: Page, + options: { columns?: string; noHeader?: boolean } = {}, +): string { if (page.items.length === 0) { return page.nextToken ? `No projects on this page.\nnextToken: ${page.nextToken}` : 'No projects.'; } - // Compact, AWS-CLI-grade columnar output. Column widths are computed - // per-call so a single absurdly long project name doesn't push the - // whole table off-screen. - const idWidth = Math.max(2, ...page.items.map(p => p.id.length)); - const nameWidth = Math.max(4, ...page.items.map(p => p.name.length)); - const typeWidth = 8; - const fromWidth = 6; - - const header = - pad('ID', idWidth) + - ' ' + - pad('NAME', nameWidth) + - ' ' + - pad('TYPE', typeWidth) + - ' ' + - pad('FROM', fromWidth) + - ' ' + - 'CREATED'; - - const rows = page.items.map( - p => - pad(p.id, idWidth) + - ' ' + - pad(p.name, nameWidth) + - ' ' + - pad(p.type, typeWidth) + - ' ' + - pad(p.createdFrom, fromWidth) + - ' ' + - p.createdAt, - ); - - const lines = [header, ...rows]; + const lines = [ + renderTextTable(page.items, PROJECT_LIST_COLUMNS, { + columns: options.columns, + noHeader: options.noHeader, + }), + ]; if (page.nextToken) lines.push('', `nextToken: ${page.nextToken}`); return lines.join('\n'); } @@ -588,11 +1059,6 @@ function renderProjectText(p: CliProject): string { ].join('\n'); } -function pad(s: string, width: number): string { - if (s.length >= width) return s; - return s + ' '.repeat(width - s.length); -} - function renderUpdateText(r: CliUpdateProjectResponse): string { return [ `id: ${r.id}`, @@ -601,6 +1067,10 @@ function renderUpdateText(r: CliUpdateProjectResponse): string { ].join('\n'); } +function renderDeleteText(r: CliDeleteProjectResponse): string { + return [`projectId ${r.projectId}`, `deletedAt ${r.deletedAt}`].join('\n'); +} + function localValidationError(message: string): ApiError { return ApiError.fromEnvelope({ error: { diff --git a/src/commands/test.artifact.spec.ts b/src/commands/test.artifact.spec.ts index c058079..c0b9283 100644 --- a/src/commands/test.artifact.spec.ts +++ b/src/commands/test.artifact.spec.ts @@ -21,6 +21,7 @@ import { assertOutDirParentExists, createTestArtifactCommand, createTestCommand, + resolveDefaultArtifactDir, runArtifactGet, runFailureGet, } from './test.js'; @@ -322,6 +323,53 @@ describe('runArtifactGet', () => { } }); + it('rejects path-like runId before auth or fetch when default --out is used', async () => { + const fetchImpl = vi.fn(); + + await expect( + runArtifactGet( + { + profile: 'default', + output: 'json', + debug: false, + runId: '../../outside', + failedOnly: false, + }, + { fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it.each([ + '.', + '..', + '. ', + '.. ', + '...', + '.. .', + '. .', + '../outside', + '..\\outside', + 'nested/run', + 'nested\\run', + 'bad\0id', + ])('rejects unsafe default artifact runId segment %j', runId => { + expect(() => resolveDefaultArtifactDir(runId, '/repo')).toThrowError( + expect.objectContaining({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'run-id' }), + }), + ); + }); + + it('keeps the documented default directory for path-safe runIds', () => { + expect(resolveDefaultArtifactDir(SAMPLE_RUN_ID, '/repo')).toBe( + join('/repo', '.testsprite', 'runs', SAMPLE_RUN_ID), + ); + }); + // ---- --failed-only passed through to writeBundle ---- it('passes --failed-only through to writeBundle (steps filtered to failed ± 1)', async () => { diff --git a/src/commands/test.cancel.spec.ts b/src/commands/test.cancel.spec.ts new file mode 100644 index 0000000..72496b0 --- /dev/null +++ b/src/commands/test.cancel.spec.ts @@ -0,0 +1,309 @@ +/** + * Unit tests for `test cancel `. + * + * Covers: single-id happy path (text + json), `alreadyCancelled` advisory, + * multi-id mixed summary + exit precedence, 404, 409, dry-run. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; +import { ApiError } from '../lib/errors.js'; +import type { CancelRunResponse } from '../lib/runs.types.js'; +import { runTestCancel, type CliCancelSummary } from './test.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +type FetchInput = Parameters[0]; + +function makeFetch( + handler: (url: string, init: RequestInit) => { status?: number; body: unknown }, +): typeof globalThis.fetch { + return (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const { status = 200, body } = handler(url, init); + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + }) as typeof globalThis.fetch; +} + +function makeCreds( + apiKey = 'sk-user-test', + apiUrl = 'http://localhost:13503', +): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-cancel-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync(credentialsPath, `[default]\napi_url = ${apiUrl}\napi_key = ${apiKey}\n`, { + mode: 0o600, + }); + return { credentialsPath }; +} + +function makeCancelResponse( + runId: string, + overrides: Partial = {}, +): CancelRunResponse { + return { + runId, + testId: 'test_xyz', + projectId: 'project_1', + userId: 'user_1', + status: 'cancelled', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 2, passedCount: 2, failedCount: 0 }, + alreadyCancelled: false, + ...overrides, + }; +} + +function errorBody(code: string, details: Record = {}) { + const statusMap: Record = { + NOT_FOUND: 404, + CONFLICT: 409, + AUTH_FORBIDDEN: 403, + }; + return { + status: statusMap[code] ?? 400, + body: { + error: { + code, + message: `Error: ${code}`, + nextAction: 'do something', + requestId: 'req_test', + details, + }, + }, + }; +} + +/** Extract the runId embedded in a `POST /runs/{runId}/cancel` URL. */ +function runIdFromCancelUrl(url: string): string | undefined { + const match = /\/runs\/([^/]+)\/cancel/.exec(url); + return match?.[1]; +} + +// --------------------------------------------------------------------------- +// Single id — happy path +// --------------------------------------------------------------------------- + +describe('runTestCancel — single id happy path', () => { + it('CXL-1: fresh cancel of a queued/running run → 200, alreadyCancelled:false, no advisory', async () => { + const { credentialsPath } = makeCreds(); + let seenUrl = ''; + let seenMethod = ''; + const fetchImpl = makeFetch((url, init) => { + seenUrl = url; + seenMethod = init.method ?? 'GET'; + return { body: makeCancelResponse('run_abc') }; + }); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + + expect(seenMethod).toBe('POST'); + expect(runIdFromCancelUrl(seenUrl)).toBe('run_abc'); + expect(result.status).toBe('cancelled'); + expect(result.alreadyCancelled).toBe(false); + expect(stderrLines.some(l => l.includes('already cancelled'))).toBe(false); + }); + + it('renders the run card in text mode with status cancelled', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeCancelResponse('run_abc') })); + const stdoutLines: string[] = []; + await runTestCancel( + { profile: 'default', output: 'text', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: line => stdoutLines.push(line), stderr: () => {} }, + ); + const block = stdoutLines.join('\n'); + expect(block).toContain('run_abc'); + expect(block).toContain('status'); + expect(block).toContain('cancelled'); + }); + + it('CXL-5: alreadyCancelled:true → [advisory] stderr line, still exit 0 (no throw)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: makeCancelResponse('run_abc', { alreadyCancelled: true }), + })); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_abc'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + expect(result.alreadyCancelled).toBe(true); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('already cancelled'))).toBe( + true, + ); + }); + + it('CXL-6: unknown/cross-tenant runId → 404 propagates as ApiError exit 4', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('NOT_FOUND')); + const err = await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_ghost'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(4); + }); + + it('CXL-4: already-terminal run → 409 propagates as ApiError exit 6', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => errorBody('CONFLICT', { status: 'passed' })); + const err = await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: false, runIds: ['run_done'] }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(6); + }); +}); + +// --------------------------------------------------------------------------- +// Multi-id — summary + exit precedence (CXL-11) +// --------------------------------------------------------------------------- + +describe('runTestCancel — multi-id summary + exit precedence (CXL-11)', () => { + it('all cancelled/alreadyCancelled → exit 0, summary buckets correct', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_2') { + return { body: makeCancelResponse(runId, { alreadyCancelled: true }) }; + } + return { body: makeCancelResponse(runId) }; + }); + const result = (await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_1', 'run_2'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + )) as CliCancelSummary; + expect(result.cancelled).toEqual(['run_1']); + expect(result.alreadyCancelled).toEqual(['run_2']); + expect(result.conflicts).toEqual([]); + expect(result.notFound).toEqual([]); + // Stable machine shape (codex finding 2): errors is ALWAYS present, + // an empty array on full success — never an absent key. + expect(result.errors).toEqual([]); + }); + + it('mixed: cancelled + conflict + notFound → notFound wins (exit 4), all buckets populated', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'failed' }); + if (runId === 'run_ghost') return errorBody('NOT_FOUND'); + return { body: makeCancelResponse(runId) }; + }); + const err = await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict', 'run_ghost'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err.exitCode).toBe(4); // notFound outranks conflict + expect(err.message).toContain('run_ghost'); + }); + + it('cancelled + conflict, no notFound → conflict wins (exit 6)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'blocked' }); + return { body: makeCancelResponse(runId) }; + }); + const err = await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict'], + }, + { credentialsPath, fetchImpl, stdout: () => {}, stderr: () => {} }, + ).catch(e => e); + expect(err.exitCode).toBe(6); + expect(err.message).toContain('run_conflict'); + expect(err.message).toContain('blocked'); + }); + + it('JSON summary shape carries the conflicting run status', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(url => { + const runId = runIdFromCancelUrl(url)!; + if (runId === 'run_conflict') return errorBody('CONFLICT', { status: 'passed' }); + return { body: makeCancelResponse(runId) }; + }); + const stdoutLines: string[] = []; + await runTestCancel( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runIds: ['run_ok', 'run_conflict'], + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => {}, + }, + ).catch(() => {}); + const summary = JSON.parse(stdoutLines.join('\n')) as CliCancelSummary; + expect(summary.cancelled).toEqual(['run_ok']); + expect(summary.conflicts).toEqual([{ runId: 'run_conflict', status: 'passed' }]); + }); +}); + +// --------------------------------------------------------------------------- +// Dry-run +// --------------------------------------------------------------------------- + +describe('runTestCancel — dry-run', () => { + it('single id: prints the dry-run banner and a canned cancelled envelope, no real network', async () => { + resetDryRunBannerForTesting(); + const stderrLines: string[] = []; + const result = (await runTestCancel( + { profile: 'default', output: 'json', debug: false, dryRun: true, runIds: ['run_dry'] }, + { stdout: () => {}, stderr: line => stderrLines.push(line) }, + )) as CancelRunResponse; + expect(stderrLines.some(l => l.includes(DRY_RUN_BANNER))).toBe(true); + expect(result.status).toBe('cancelled'); + expect(result.alreadyCancelled).toBe(false); + }); +}); diff --git a/src/commands/test.flaky.spec.ts b/src/commands/test.flaky.spec.ts new file mode 100644 index 0000000..8076f3e --- /dev/null +++ b/src/commands/test.flaky.spec.ts @@ -0,0 +1,431 @@ +/** + * Unit tests for `test flaky` — the repeat-run flaky-test detector. + * + * All HTTP is mocked via `makeFlakyFetch`. The polling loop's sleep is injected + * through `TestDeps.sleep` to avoid real delays. Each rerun POST returns a + * unique runId; each run GET returns a terminal status scripted per attempt, + * so a test can assert stable / flaky / failing verdicts deterministically. + */ + +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { CLIError, ApiError } from '../lib/errors.js'; +import type { FlakyReport } from '../lib/flaky.js'; +import type { FetchImpl } from '../lib/http.js'; +import { runFlaky } from './test.js'; + +type FetchInput = Parameters[0]; +type RunStatus = 'passed' | 'failed' | 'blocked' | 'cancelled'; +type TriggerAuthErrorCode = 'AUTH_REQUIRED' | 'AUTH_INVALID' | 'AUTH_FORBIDDEN'; + +function urlOf(input: FetchInput): string { + return typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; +} + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +/** + * Build a fetch that: + * - GET /tests/{id} → the test record ('frontend' | 'backend') + * - POST /tests/{id}/runs/rerun → a queued rerun with runId run_ (n increments) + * - GET /runs/run_ → a terminal run with statuses[k-1] + * + * `notFoundOnTrigger` makes the rerun POST return 404 (no replayable run). + */ +function makeFlakyFetch(opts: { + statuses: RunStatus[]; + testType?: 'frontend' | 'backend'; + notFoundOnTrigger?: boolean; + triggerAuthError?: TriggerAuthErrorCode; +}): { fetchImpl: FetchImpl; triggerCount: () => number } { + let triggers = 0; + const testType = opts.testType ?? 'frontend'; + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = urlOf(input); + const method = (init.method ?? 'GET').toUpperCase(); + + if (method === 'GET' && /\/tests\/[^/]+$/.test(url.split('?')[0]!)) { + return jsonResponse(200, { + id: 'test_x', + projectId: 'project_abc', + name: 'sample', + type: testType, + createdFrom: 'portal', + status: 'passed', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }); + } + + if (method === 'POST' && url.includes('/runs/rerun')) { + if (opts.triggerAuthError) { + return jsonResponse(opts.triggerAuthError === 'AUTH_FORBIDDEN' ? 403 : 401, { + error: { + code: opts.triggerAuthError, + message: 'auth failed', + nextAction: 'run setup', + requestId: 'req_auth', + details: {}, + }, + }); + } + if (opts.notFoundOnTrigger) { + return jsonResponse(404, { + error: { + code: 'NOT_FOUND', + message: 'no replayable run', + nextAction: 'run it', + requestId: 'req_1', + details: {}, + }, + }); + } + triggers += 1; + return jsonResponse(200, { + runId: `run_${triggers}`, + status: 'queued', + enqueuedAt: '2026-06-03T10:00:00.000Z', + codeVersion: 'v1', + autoHeal: false, + }); + } + + const runMatch = /\/runs\/(run_\d+)/.exec(url); + if (method === 'GET' && runMatch) { + const runId = runMatch[1]!; + const idx = Number(runId.replace('run_', '')) - 1; + const status = opts.statuses[idx] ?? 'passed'; + return jsonResponse(200, { + runId, + testId: 'test_x', + projectId: 'project_abc', + userId: 'user_1', + status, + source: 'cli', + createdAt: '2026-06-03T10:00:00.000Z', + startedAt: '2026-06-03T10:00:01.000Z', + finishedAt: '2026-06-03T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'rerun:prior', + failedStepIndex: status === 'passed' ? null : 2, + failureKind: status === 'passed' ? null : 'assertion', + error: null, + videoUrl: null, + stepSummary: { total: 5, completed: 5, passedCount: 5, failedCount: 0 }, + }); + } + + return jsonResponse(404, { + error: { code: 'NOT_FOUND', message: 'unmatched', requestId: 'x', details: {} }, + }); + }) as FetchImpl; + + return { fetchImpl, triggerCount: () => triggers }; +} + +function makeCreds(): { credentialsPath: string } { + const dir = mkdtempSync(join(tmpdir(), 'cli-flaky-')); + const credentialsPath = join(dir, 'credentials'); + mkdirSync(dir, { recursive: true }); + writeFileSync( + credentialsPath, + `[default]\napi_url = http://localhost:13509\napi_key = sk-user-test\n`, + { + mode: 0o600, + }, + ); + return { credentialsPath }; +} + +const instantSleep = (): Promise => Promise.resolve(); + +function makeDeps(fetchImpl: FetchImpl): { + deps: { + credentialsPath: string; + fetchImpl: FetchImpl; + sleep: () => Promise; + stdout: (l: string) => void; + stderr: (l: string) => void; + }; + stdout: string[]; + stderr: string[]; +} { + const stdout: string[] = []; + const stderr: string[] = []; + const { credentialsPath } = makeCreds(); + return { + deps: { + credentialsPath, + fetchImpl, + sleep: instantSleep, + stdout: (l: string) => stdout.push(l), + stderr: (l: string) => stderr.push(l), + }, + stdout, + stderr, + }; +} + +// --------------------------------------------------------------------------- +// Surface +// --------------------------------------------------------------------------- + +describe('createTestCommand — flaky subcommand exposed', () => { + it('exposes flaky with its flags', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const flaky = test.commands.find(c => c.name() === 'flaky'); + expect(flaky).toBeDefined(); + const flagNames = flaky!.options.map(o => o.long); + expect(flagNames).toContain('--runs'); + expect(flagNames).toContain('--until-fail'); + expect(flagNames).toContain('--timeout'); + }); +}); + +// --------------------------------------------------------------------------- +// Behavior +// --------------------------------------------------------------------------- + +describe('runFlaky', () => { + it('reports STABLE and exits 0 when every attempt passes', async () => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: ['passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + const report = (await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + )) as FlakyReport; + expect(report.verdict).toBe('stable'); + expect(report.runs).toBe(3); + expect(triggerCount()).toBe(3); + }); + + it('reports FLAKY and throws exit 1 on a mix of pass/fail', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'failed', 'passed'] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).exitCode).toBe(1); + expect((err as CLIError).message).toContain('flaky'); + }); + + it('reports FAILING and throws exit 1 when no attempt passes', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['failed', 'failed'] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(CLIError); + expect((err as CLIError).message).toContain('failing'); + }); + + it('--until-fail stops at the first non-passing attempt', async () => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: ['passed', 'failed', 'passed', 'passed', 'passed'], + }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 5, + untilFail: true, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + // Stopped after attempt 2 (the failure) — only 2 triggers fired. + expect(triggerCount()).toBe(2); + expect(err).toBeInstanceOf(CLIError); + }); + + it('prints a backend credit advisory to stderr', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'passed'], testType: 'backend' }); + const { deps, stderr } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 2, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ); + expect(stderr.some(l => l.includes('backend test') && l.includes('credits'))).toBe(true); + }); + + it('emits a machine-readable JSON stability report', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: ['passed', 'failed', 'passed'] }); + const { deps, stdout } = makeDeps(fetchImpl); + await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch(() => undefined); // swallow the exit-1 throw; we only assert stdout + const parsed = JSON.parse(stdout.join('\n')) as FlakyReport; + expect(parsed.testId).toBe('test_x'); + expect(parsed.runs).toBe(3); + expect(parsed.passed).toBe(2); + expect(parsed.verdict).toBe('flaky'); + expect(parsed.failures).toHaveLength(1); + expect(parsed.failures[0]!.failureKind).toBe('assertion'); + }); + + it('throws exit 4 when the test has no replayable run', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [], notFoundOnTrigger: true }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('NOT_FOUND'); + }); + + it.each(['AUTH_REQUIRED', 'AUTH_INVALID', 'AUTH_FORBIDDEN'] as const)( + 'propagates %s during trigger instead of scoring an error attempt', + async code => { + const { fetchImpl, triggerCount } = makeFlakyFetch({ + statuses: [], + triggerAuthError: code, + }); + const { deps, stdout } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'json', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 3, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect(err).toMatchObject({ code, exitCode: 3 }); + expect(stdout).toEqual([]); + expect(triggerCount()).toBe(0); + }, + ); + + it('rejects --runs below the range (0) with a validation error (exit 5)', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 0, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(5); + }); + + it('rejects --runs above the cap (11) with a validation error (exit 5)', async () => { + const { fetchImpl } = makeFlakyFetch({ statuses: [] }); + const { deps } = makeDeps(fetchImpl); + const err = await runFlaky( + { + profile: 'default', + output: 'text', + dryRun: false, + debug: false, + verbose: false, + testId: 'test_x', + runs: 11, + untilFail: false, + timeoutSeconds: 600, + }, + deps, + ).catch((e: unknown) => e); + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).exitCode).toBe(5); + }); +}); diff --git a/src/commands/test.quickwins.spec.ts b/src/commands/test.quickwins.spec.ts index c781300..322000c 100644 --- a/src/commands/test.quickwins.spec.ts +++ b/src/commands/test.quickwins.spec.ts @@ -937,3 +937,113 @@ describe('runDeleteBatch (dogfood L1796)', () => { expect(flagNames).toContain('--status'); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 (codex finding 2) — create-batch --run --wait interrupt partial +// carries the dispatched runIds, not empty placeholders +// --------------------------------------------------------------------------- + +describe('create-batch --run --wait — InterruptError partial names dispatched runIds (DEV-331)', () => { + it('interrupt mid-poll → partial rows carry the runIds recorded at trigger time', async () => { + const creds = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-dev331-cbrun-')); + for (let i = 0; i < 2; i++) { + writeFileSync( + join(dir, `plan_${i}.json`), + JSON.stringify({ ...PLAN_SPEC, name: `Plan ${i}` }), + 'utf8', + ); + } + + const { ShutdownController } = await import('../lib/interrupt.js'); + const { InterruptError } = await import('../lib/errors.js'); + const shutdown = new ShutdownController(); + + // batch create resolves; each trigger POST resolves with a per-test runId; + // every run poll hangs until the composed signal aborts. + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (url.includes('/tests/batch') && init.method === 'POST') { + return new Response(JSON.stringify(batchCreateResp(2)), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + if (init.method === 'POST' && /\/tests\/[^/]+\/runs$/.test(url)) { + const testId = /\/tests\/([^/]+)\/runs$/.exec(url)![1]!; + return new Response( + JSON.stringify({ + runId: `run_${testId}`, + status: 'queued', + enqueuedAt: '2026-07-09T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ); + } + // GET /runs/{id} long-poll: hang until aborted. + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + const pending = runCreateBatch( + { + plans: '', + planFromDir: dir, + run: true, + wait: true, + timeoutSeconds: 600, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + fetchImpl, + stdout: (l: string) => stdoutLines.push(l), + stderr: (l: string) => stderrLines.push(l), + sleep: () => Promise.resolve(), + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 25); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + + // The partial must name the real runIds recorded at trigger time — + // members mid-poll have no settled result, but their runId is known. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + results: Array<{ testId: string; runId: string; status: string }>; + }; + const byTestId = new Map(stdoutJson.results.map(r => [r.testId, r])); + expect(byTestId.get('test_batch_0')?.runId).toBe('run_test_batch_0'); + expect(byTestId.get('test_batch_0')?.status).toBe('running'); + expect(byTestId.get('test_batch_1')?.runId).toBe('run_test_batch_1'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('run_test_batch_0'); + expect(stderrBlock).toContain('run_test_batch_1'); + }); +}); diff --git a/src/commands/test.rerun.spec.ts b/src/commands/test.rerun.spec.ts index 86c8e89..81283d9 100644 --- a/src/commands/test.rerun.spec.ts +++ b/src/commands/test.rerun.spec.ts @@ -280,6 +280,126 @@ describe('runTestRerun — validation', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); }); + it('exit 5 (VALIDATION_ERROR) when explicit test IDs are combined with --all', async () => { + // The --all branch resolves the FULL project test set and overwrites the + // listed ids, so 'rerun test_abc --all' would silently rerun the ENTIRE + // project instead of test_abc — burning rerun/auto-heal credits. The + // guard throws BEFORE any network/dispatch. (Mirrors `test run`'s + // positional+--all guard and delete-batch's ids+--all guard.) + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_abc'], + all: true, + projectId: 'proj_1', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'test-ids' }), + }); + }); + + it('exit 5 (VALIDATION_ERROR) when --status is passed WITHOUT --all', async () => { + // --status is an --all-only narrowing filter. With explicit ids it was + // silently ignored (both tests dispatched, filter dropped) — same + // failure mode as the --filter guard above. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a', 'test_b'], + all: false, + statusFilter: 'failed', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'status' }), + }); + }); + + it('exit 5 (VALIDATION_ERROR) for an INVALID --status value without --all (was silently accepted)', async () => { + // Before the guard, an invalid --status token without --all was never + // even validated: 'rerun test_a --status notastatus' exited 0 while the + // same flag on delete-batch exits 5. + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + statusFilter: 'notastatus', + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + + it('exit 5 (VALIDATION_ERROR) when --skip-terminal is passed WITHOUT --all', async () => { + const creds = makeCreds(); + await expect( + runTestRerun( + { + testIds: ['test_a'], + all: false, + skipTerminal: true, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'skip-terminal' }), + }); + }); + it('exit 5 when --all without --project', async () => { const creds = makeCreds(); try { @@ -1637,6 +1757,74 @@ describe('--idempotency-key passthrough', () => { expect(receivedKey).toBe('my-custom-key-abc'); }); + + it('emits the auto-minted idempotency-key on stderr in JSON output mode (parity with test run)', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + + it('does NOT emit an idempotency-key line in default text mode', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + const stderrLines: string[] = []; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return { body: rerunResp }; + } + return errorBody('NOT_FOUND'); + }); + + await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: false, + timeoutSeconds: 600, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'text', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { ...creds, sleep: instantSleep, fetchImpl, stderr: line => stderrLines.push(line) }, + ); + + expect(stderrLines.some(l => l.includes('idempotency-key:'))).toBe(false); + }); }); // --------------------------------------------------------------------------- @@ -4630,3 +4818,137 @@ describe('rerun --wait — dashboardUrl on terminal output', () => { ); }); }); + +// --------------------------------------------------------------------------- +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty +// --------------------------------------------------------------------------- +describe('[finding-5] batch rerun --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const creds = makeCreds(); + const batchResp: BatchRerunResponse = { + accepted: [ + { testId: 'test_1', runId: 'run_b1', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + { testId: 'test_2', runId: 'run_b2', enqueuedAt: '2026-06-03T10:00:00.000Z' }, + ], + deferred: [], + conflicts: [], + closure: { byProject: [] }, + }; + + const fetchImpl = makeFetch((url) => { + if (url.includes('/tests/batch/rerun')) { + return { status: 202, body: batchResp }; + } + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_rerun'); + } + return errorBody('NOT_FOUND'); + }); + + const stdoutLines: string[] = []; + const err = await runTestRerun( + { + testIds: ['test_1', 'test_2'], + all: false, + wait: true, + timeoutSeconds: 60, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 1, + profile: 'default', + output: 'json', + debug: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch((e) => e); + + expect(err).toMatchObject({ exitCode: 7 }); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map((r) => r.runId).sort()).toEqual(['run_b1', 'run_b2']); + expect(parsed.accepted.every((r) => r.status === 'timeout')).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// TimeoutError on single FE rerun --wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- +describe('[finding-4] single FE rerun --wait: TimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const creds = makeCreds(); + const rerunResp = makeFeRerunResp(); + + let fetchCallCount = 0; + const fetchImpl: typeof globalThis.fetch = async (input, _init) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + fetchCallCount++; + if (url.includes('/tests/test_fe_01/runs/rerun')) { + return new Response(JSON.stringify(rerunResp), { + status: 202, + headers: { 'content-type': 'application/json' }, + }); + } + if (url.includes('/runs/')) { + const runningRun: RunResponse = { + ...makeTerminalRun(rerunResp.runId, 'passed'), + status: 'running', + finishedAt: null, + }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + return new Response(JSON.stringify({ error: { code: 'NOT_FOUND' } }), { status: 404 }); + }; + + const stdoutLines: string[] = []; + + const err = await runTestRerun( + { + testIds: ['test_fe_01'], + all: false, + wait: true, + timeoutSeconds: 0, + autoHeal: false, + autoHealExplicit: false, + skipDependencies: false, + maxConcurrency: 10, + output: 'json', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { + ...creds, + sleep: instantSleep, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: (line) => stdoutLines.push(line), + stderr: () => undefined, + }, + ).catch((e) => e); + + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(parsed.runId).toBe(rerunResp.runId); + expect(parsed.status).toBe('running'); + + void fetchCallCount; + }); +}); diff --git a/src/commands/test.result.history.spec.ts b/src/commands/test.result.history.spec.ts index e3fb963..87df3e0 100644 --- a/src/commands/test.result.history.spec.ts +++ b/src/commands/test.result.history.spec.ts @@ -169,6 +169,24 @@ describe('parseDuration', () => { it('case-insensitive day suffix', () => { expect(parseDuration('7D', NOW)).toBe('2026-05-27T12:00:00.000Z'); }); + + it('overflow hours throws VALIDATION_ERROR instead of crashing', () => { + expect(() => parseDuration('99999999999h', NOW)).toThrow(); + try { + parseDuration('99999999999h', NOW); + } catch (err: unknown) { + expect((err as { code?: string }).code).toBe('VALIDATION_ERROR'); + } + }); + + it('overflow days throws VALIDATION_ERROR instead of crashing', () => { + expect(() => parseDuration('99999999999d', NOW)).toThrow(); + try { + parseDuration('99999999999d', NOW); + } catch (err: unknown) { + expect((err as { code?: string }).code).toBe('VALIDATION_ERROR'); + } + }); }); // --------------------------------------------------------------------------- @@ -207,6 +225,132 @@ describe('runResultHistory — text mode', () => { expect(output).toMatch(/DURATION/); }); + it('selects/reorders columns and suppresses header/separator/detail sub-lines', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_url_001', + targetUrl: 'https://staging.example.com/checkout', + targetUrlSource: 'run', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'status,runid', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output.split('\n')[0]).toMatch(/^passed\s+run_url_001$/); + expect(output).not.toMatch(/^RUN ID/m); + expect(output).not.toMatch(/^-+$/m); + expect(output).not.toContain('targetUrl:'); + }); + + it('no-header suppresses only the history header and separator', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { + body: makeHistoryResp([ + makeHistoryItem({ + runId: 'run_url_001', + targetUrl: 'https://staging.example.com/checkout', + targetUrlSource: 'run', + }), + ]), + }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const output = lines.join('\n'); + expect(output).not.toMatch(/^RUN ID/m); + expect(output).not.toMatch(/^-+$/m); + expect(output).toContain('run_url_001'); + expect(output).toContain('targetUrl: https://staging.example.com/checkout'); + }); + + it('rejects unknown history columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runResultHistory( + { + output: 'text', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json mode ignores text-only history column flags', async () => { + const { credentialsPath } = makeCreds(); + const lines: string[] = []; + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/test_abc/runs')) { + return { body: makeHistoryResp() }; + } + return { status: 404, body: errorEnvelope('NOT_FOUND') }; + }); + + await runResultHistory( + { + output: 'json', + testId: 'test_abc', + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => lines.push(line) }, + ); + + const parsed = JSON.parse(lines.join('')) as { runs: Array<{ runId: string }> }; + expect(parsed.runs[0]?.runId).toBe('run_hist_001'); + }); + it('renders run_hist_001 row with status passed and source cli', async () => { const { credentialsPath } = makeCreds(); const lines: string[] = []; @@ -535,6 +679,32 @@ describe('runResultHistory — pagination', () => { expect(capturedUrl).toContain('pageSize=5'); }); + + it('rejects fractional --page-size before making a request', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => { + throw new Error('should not be called'); + }); + + await expect( + runResultHistory( + { + output: 'json', + testId: 'test_abc', + pageSize: 1.5, + profile: 'default', + dryRun: false, + debug: false, + verbose: false, + }, + { credentialsPath, fetchImpl, stdout: () => {} }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'page-size' }), + }); + }); }); // --------------------------------------------------------------------------- diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 7e2b55e..51b527f 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -10,7 +10,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { Command } from 'commander'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; import type { FetchImpl } from '../lib/http.js'; import type { RunResponse, TriggerRunResponse, BatchRunFreshResponse } from '../lib/runs.types.js'; @@ -1699,6 +1700,88 @@ describe('C2 — backend run renders steps: n/a (backend) in text mode', () => { expect(out).toContain('steps n/a (backend)'); expect(out).not.toContain('0/0'); }); + + it('standalone BE run --wait: text probes /tests/{id} → n/a (backend); JSON never probes (DEV-282)', async () => { + // Standalone `test run ` supplies NO type hint, and the run row is + // terminal on the first poll (BE rows finalize server-side now), so + // `beFallbackUsed` stays false. In TEXT mode the card must still read + // `n/a (backend)`, resolved via a one-time `GET /tests/{id}` probe; in JSON + // mode the output is already correct and must NOT pay for that round-trip. + const { credentialsPath } = makeCreds(); + const passedBeRun: RunResponse = { + runId: 'run_282', + testId: 'be_282', + projectId: 'p1', + userId: 'u1', + status: 'passed', + source: 'cli', + createdAt: '2026-05-15T10:00:00.000Z', + startedAt: '2026-05-15T10:00:01.000Z', + finishedAt: '2026-05-15T10:00:02.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 0, completed: 0, passedCount: 0, failedCount: 0 }, + }; + const makeHandler = (urls: string[]) => (url: string) => { + urls.push(url); + if (url.includes('/tests/be_282/runs')) { + return { + body: { + runId: 'run_282', + status: 'queued', + enqueuedAt: '2026-05-15T10:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }, + }; + } + if (url.includes('/runs/run_282')) return { body: passedBeRun }; + // Bare type probe: GET /tests/be_282 (no /runs, no /result suffix). + if (/\/tests\/be_282$/.test(url)) return { body: { id: 'be_282', type: 'backend' } }; + return { status: 404, body: {} }; + }; + const runOnce = async (output: 'text' | 'json') => { + const urls: string[] = []; + const stdoutLines: string[] = []; + await runTestRun( + { + profile: 'default', + output, + debug: false, + dryRun: false, + testId: 'be_282', + wait: true, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl: makeFetch(makeHandler(urls)), + stdout: line => stdoutLines.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + return { urls, out: stdoutLines.join('\n') }; + }; + + // Text mode: probes the type and renders the honest backend placeholder. + const text = await runOnce('text'); + expect(text.out).toContain('steps n/a (backend)'); + expect(text.out).not.toContain('0/0'); + expect(text.urls.some(u => /\/tests\/be_282$/.test(u))).toBe(true); + + // JSON mode: no extra probe; the wire envelope ships stepSummary verbatim. + const json = await runOnce('json'); + expect(json.urls.some(u => /\/tests\/be_282$/.test(u))).toBe(false); + const parsed = JSON.parse(json.out); + expect(parsed.status).toBe('passed'); + expect(parsed.stepSummary).toEqual({ total: 0, completed: 0, passedCount: 0, failedCount: 0 }); + }); }); // --------------------------------------------------------------------------- @@ -1935,6 +2018,74 @@ describe('runTestRun --wait: Fix 3 — RequestTimeoutError writes partial JSON t }); }); +// --------------------------------------------------------------------------- +// TimeoutError on --wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestRun --wait: TimeoutError writes partial JSON to stdout', () => { + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const { credentialsPath } = makeCreds(); + let dateCallCount = 0; + let fetchCallCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (++dateCallCount > 6 ? base + 2000 : base); + + try { + const fetchImpl: typeof globalThis.fetch = async () => { + ++fetchCallCount; + if (fetchCallCount === 1) { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + const runningRun: RunResponse = { ...makePassedRun(), status: 'running' }; + return new Response(JSON.stringify(runningRun), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }; + + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + verbose: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as FetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + targetUrl: string; + }; + expect(stdoutJson.runId).toBe(TRIGGER_RESP.runId); + expect(stdoutJson.status).toBe('running'); + expect(stdoutJson.targetUrl).toBe(TRIGGER_RESP.targetUrl); + } finally { + Date.now = realDateNow; + } + }); +}); + // --------------------------------------------------------------------------- // Fix 5 — B2(c): --timeout hint fires on default, not on explicit timeout // --------------------------------------------------------------------------- @@ -2487,6 +2638,61 @@ describe('runTestRunAll — batch fresh run', () => { expect(payload.accepted.every(r => r.status === 'passed')).toBe(true); }); + it('run --all --wait: does not start a fresh poll for a queued run after the shared deadline expired', async () => { + const { credentialsPath } = makeCreds(); + const baseNow = new Date('2026-06-09T10:00:00.000Z').getTime(); + let nowMs = baseNow; + const runFetches: string[] = []; + const stdoutLines: string[] = []; + let caughtError: unknown; + const nowSpy = vi.spyOn(Date, 'now').mockImplementation(() => nowMs); + + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'POST') return { body: BATCH_FRESH_RESP }; + + const runId = url.split('/runs/')[1]?.split('?')[0] ?? 'run_unknown'; + runFetches.push(runId); + if (runId === 'run_fresh_01') { + nowMs = baseNow + 2000; + return { body: makePassedRun(runId, 'test_be_01') }; + } + return { body: makePassedRun(runId, 'test_be_02') }; + }); + + try { + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 1, + maxConcurrency: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ); + } catch (err) { + caughtError = err; + } finally { + nowSpy.mockRestore(); + } + + const payload = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ runId: string; status: string }>; + }; + expect(runFetches).toEqual(['run_fresh_01']); + expect(payload.accepted.find(r => r.runId === 'run_fresh_02')?.status).toBe('timeout'); + expect((caughtError as { exitCode?: number } | undefined)?.exitCode).toBe(7); + }); + it('--wait with a failed run → exit 1', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch((url, init) => { @@ -3470,6 +3676,39 @@ describe('dashboardUrl on run completion', () => { ); }); + it('run --all: emits the auto-minted idempotency-key on stderr in JSON output mode (parity with test run)', async () => { + const { credentialsPath } = makeCreds('sk-user-test', PROD_API); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_f_01', enqueuedAt: '2026-06-10T10:00:00.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const stderrLines: string[] = []; + await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: false, + timeoutSeconds: 600, + maxConcurrency: 10, + }, + { + credentialsPath, + fetchImpl: makeFetch(() => ({ body: batchResp })), + stdout: () => undefined, + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ); + expect(stderrLines.some(l => l.startsWith('idempotency-key:'))).toBe(true); + }); + it('run --all --wait (prod endpoint): summary items carry dashboardUrl + stderr Dashboard line', async () => { const { credentialsPath } = makeCreds('sk-user-test', PROD_API); const batchResp: BatchRunFreshResponse = { @@ -3536,3 +3775,136 @@ describe('dashboardUrl on run completion', () => { ).toBe(true); }); }); + +// --------------------------------------------------------------------------- +// Batch --all --wait fan-out: RequestTimeoutError must not leave stdout empty +// --------------------------------------------------------------------------- + +describe('[finding-5] runTestRunAll --wait: RequestTimeoutError during fan-out poll writes JSON stdout + exit 7', () => { + it('stdout contains accepted[] with runIds when member polls throw RequestTimeoutError', async () => { + const { credentialsPath } = makeCreds(); + const batchResp: BatchRunFreshResponse = { + accepted: [ + { testId: 'test_be_01', runId: 'run_fresh_01', enqueuedAt: '2026-06-09T10:00:00.000Z' }, + { testId: 'test_be_02', runId: 'run_fresh_02', enqueuedAt: '2026-06-09T10:00:01.000Z' }, + ], + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + }; + const fetchImpl = makeFetch((url, init) => { + if ((init.method ?? 'GET') === 'POST') return { body: batchResp }; + if (url.includes('/runs/')) { + throw new RequestTimeoutError(120000, 'req_timeout_batch_all'); + } + return errorBody('NOT_FOUND'); + }); + const stdoutLines: string[] = []; + + const err = await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: () => undefined, + sleep: instantSleep, + }, + ).catch(e => e); + + expect(err).toMatchObject({ exitCode: 7 }); + expect(stdoutLines.length).toBeGreaterThan(0); + const parsed = JSON.parse(stdoutLines.join('\n')) as { + accepted: Array<{ testId: string; runId: string; status: string }>; + }; + expect(parsed.accepted).toHaveLength(2); + expect(parsed.accepted.map(r => r.runId).sort()).toEqual(['run_fresh_01', 'run_fresh_02']); + expect(parsed.accepted.every(r => r.status === 'timeout')).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach on SIGINT during test run --wait +// --------------------------------------------------------------------------- + +describe('runTestRun --wait — InterruptError graceful detach (DEV-331)', () => { + it('SIG-1: trigger succeeds, poll interrupted → partial to stdout + honest stderr + exit 130', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + // Trigger POST resolves; the subsequent GET /runs/{id} long-poll hangs + // until the composed signal aborts (real-fetch contract). + const fetchImpl = (async (input: FetchInput, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + if (init.method === 'POST') { + return new Response(JSON.stringify(TRIGGER_RESP), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + } + void url; + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + + const pending = runTestRun( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + testId: 'test_xyz', + wait: true, + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGINT'), 5); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('testsprite test wait run_abc'); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 4d205e0..c8fe5a5 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -27,13 +27,18 @@ import { runCreateBatch, runCreateFromPlan, runDelete, + runDiff, runFailureGet, runFailureSummary, runGet, + runLint, runList, + runOpen, runPlanPut, runResult, + runScaffold, runSteps, + runTestWaitMany, runUpdate, } from './test.js'; @@ -116,18 +121,24 @@ describe('createTestCommand — surface', () => { const names = test.commands.map(c => c.name()).sort(); expect(names).toEqual([ 'artifact', + 'cancel', 'code', 'create', 'create-batch', 'delete', 'delete-batch', + 'diff', 'failure', + 'flaky', 'get', + 'lint', 'list', + 'open', 'plan', 'rerun', 'result', 'run', + 'scaffold', 'steps', 'update', 'wait', @@ -191,6 +202,7 @@ describe('createTestCommand — surface', () => { it('result exposes --include-analysis (M2.1) + M3.4 piece-5 --history flags', () => { // M2.1 piece 3 adds `--include-analysis` to `test result`. // M3.4 piece 5 adds `--history`, `--source`, `--since`, `--page-size`, `--cursor`. + // Issue #165 adds text-table shaping via `--columns` and `--no-header`. // Pinning the surface so a future flag-consolidation sweep keeps every // option intentional. Back-compat: bare `test result ` (no --history) // still calls runResult and returns the M2 CliLatestResult shape. @@ -204,6 +216,8 @@ describe('createTestCommand — surface', () => { '--since', '--page-size', '--cursor', + '--columns', + '--no-header', ]); }); @@ -456,6 +470,14 @@ describe('runList', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); }); + it('rejects --status=junk before reading credentials', async () => { + const test = createTestCommand(); + disableExits(test); + await expect( + test.parseAsync(['list', '--project', 'project_alice', '--status', 'junk'], { from: 'user' }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); + it('rejects --page-size=0 locally with VALIDATION_ERROR (no network call)', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => { @@ -475,6 +497,34 @@ describe('runList', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', details: { field: 'page-size' } }); }); + it('rejects invalid --status before requiring credentials', async () => { + const credentialsPath = join(mkdtempSync(join(tmpdir(), 'cli-list-status-')), 'credentials'); + const fetchImpl = vi.fn(); + + await expect( + runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + status: 'notastatus', + }, + { + credentialsPath, + fetchImpl: fetchImpl as unknown as typeof fetch, + stdout: () => undefined, + }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'status' }, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it('forwards a server-side VALIDATION_ERROR envelope as ApiError exit 5', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ @@ -538,6 +588,74 @@ describe('runList', () => { expect(block).toContain('mcp'); }); + it('text mode selects/reorders columns and suppresses the header', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [FE_TEST, BE_TEST], nextToken: null }, + })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'status,id', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + const lines = out.join('\n').split('\n'); + expect(lines[0]).toMatch(/^failed\s+test_fe$/); + expect(lines[1]).toMatch(/^passed\s+test_be$/); + expect(out.join('\n')).not.toContain('STATUS'); + expect(out.join('\n')).not.toContain('UPDATED'); + }); + + it('text mode rejects unknown columns with VALIDATION_ERROR before auth/network access', async () => { + await expect( + runList( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'bogus', + }, + { stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'columns' }, + }); + }); + + it('json mode ignores text-only column flags', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [FE_TEST], nextToken: null }, + })); + const out: string[] = []; + await runList( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + pageSize: 25, + columns: 'bogus', + noHeader: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + + expect(JSON.parse(out.join('\n')).items[0].id).toBe('test_fe'); + }); + it('text mode reads "No tests." when items is empty and nextToken is null', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: { items: [], nextToken: null } })); @@ -798,6 +916,40 @@ describe('runGet', () => { expect(out.join('\n')).not.toContain('planSteps:'); }); + it('renders produces/consumes/category when the facade ships them', async () => { + const withDeps: CliTest = { + ...FE_TEST, + produces: ['user_id', 'order_id'], + consumes: ['session_token'], + category: 'teardown', + }; + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: withDeps })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).toContain('produces: user_id, order_id'); + expect(block).toContain('consumes: session_token'); + expect(block).toContain('category: teardown'); + }); + + it('omits produces/consumes/category lines when absent', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: FE_TEST })); + const out: string[] = []; + await runGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + expect(block).not.toContain('produces:'); + expect(block).not.toContain('consumes:'); + expect(block).not.toContain('category:'); + }); + it('NOT_FOUND envelope from server propagates as ApiError exit 4', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ @@ -1522,6 +1674,31 @@ describe('runCodeGet', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('--out rejects an existing directory path with VALIDATION_ERROR (exit 5) before any network I/O', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-dir-')); + let fetchCalls = 0; + const fetchImpl = (() => { + fetchCalls += 1; + return Promise.resolve(new Response('{}')); + }) as typeof globalThis.fetch; + + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: dir, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(fetchCalls).toBe(0); + expect(readdirSync(dir)).toEqual([]); + }); + // Regression: a parent dir that doesn't exist used to surface as exit 1 // (TRANSPORT_ERROR) — `createWriteStream` opens lazily and ENOENT fires // mid-write. Synchronous parent stat keeps every `--out` user-input @@ -1591,22 +1768,66 @@ describe('runCodeGet', () => { expect(leftovers).toEqual([]); }); - // Regression: the "no code generated yet" branch writes nothing but - // previously still closed (and thus truncated) the opened file. + it('--out (text mode) rejects empty inline code with VALIDATION_ERROR and leaves no artifact', async () => { + const { credentialsPath } = makeCreds(); + const emptyCode: CliTestCode = { ...TEST_CODE_INLINE, code: '' }; + const fetchImpl = makeFetch(() => ({ body: emptyCode })); + const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-empty-out-')); + const target = join(dir, 'empty.ts'); + await expect( + runCodeGet( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + out: target, + }, + { credentialsPath, fetchImpl }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'out' }), + }); + expect(existsSync(target)).toBe(false); + }); + + // Regression: empty inline code with --out must reject (exit 5) without + // truncating or replacing a pre-existing destination file. it('--out: "no code generated yet" leaves a pre-existing file untouched', async () => { const { credentialsPath } = makeCreds(); const dir = mkdtempSync(join(tmpdir(), 'cli-test-code-out-empty-')); const target = join(dir, 'existing.ts'); writeFileSync(target, 'PRE-EXISTING CONTENT', 'utf8'); const fetchImpl = makeFetch(() => ({ body: { ...TEST_CODE_INLINE, code: '' } })); - await runCodeGet( - { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: target }, - { credentialsPath, fetchImpl, stderr: () => undefined }, - ); + await expect( + runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', out: target }, + { credentialsPath, fetchImpl, stderr: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: expect.objectContaining({ field: 'out' }), + }); expect(readFileSync(target, 'utf-8')).toBe('PRE-EXISTING CONTENT'); const leftovers = readdirSync(dir).filter(f => f !== 'existing.ts'); expect(leftovers).toEqual([]); }); + + it('text mode without --out still hints on stderr when inline code is empty', async () => { + const { credentialsPath } = makeCreds(); + const emptyCode: CliTestCode = { ...TEST_CODE_INLINE, code: '' }; + const fetchImpl = makeFetch(() => ({ body: emptyCode })); + const stderr: string[] = []; + const got = await runCodeGet( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe' }, + { credentialsPath, fetchImpl, stderr: line => stderr.push(line) }, + ); + expect(got.code).toBe(''); + expect(stderr.join('\n')).toContain('no code generated yet'); + }); }); describe('runCodePut', () => { @@ -1661,6 +1882,30 @@ describe('runCodePut', () => { expect(sent.headers.get('content-type')).toBe('application/json'); }); + it('strips a UTF-8 BOM from --code-file before uploading (Windows PowerShell 5.1 default)', async () => { + const { credentialsPath } = makeCreds(); + const dir = mkdtempSync(join(tmpdir(), 'cli-p4-bom-')); + const codeFile = join(dir, 'updated.py'); + writeFileSync(codeFile, '\uFEFF' + 'updated body', 'utf8'); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runCodePut( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + codeFile, + expectedVersion: 'v3', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ); + expect(seenBody).toEqual({ code: 'updated body' }); + }); + it('forwards --language in the body when set', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('print("hi")'); @@ -2198,6 +2443,182 @@ describe('runCodePut', () => { }); }); +describe('runScaffold', () => { + it('frontend scaffold is a valid CliPlanInput and prints as JSON on stdout', async () => { + const out: string[] = []; + const result = await runScaffold( + { profile: 'default', output: 'text', debug: false, scaffoldType: 'frontend', force: false }, + { stdout: line => out.push(line), stderr: () => undefined, env: {} }, + ); + const plan = result as { projectId: string; type: string; planSteps: Array<{ type: string }> }; + expect(plan.type).toBe('frontend'); + // Placeholder project id when TESTSPRITE_PROJECT_ID is unset. + expect(plan.projectId).toContain('testsprite project list'); + expect(plan.planSteps.length).toBeGreaterThanOrEqual(2); + // Every emitted step type must come from the real enum (no drift). + for (const step of plan.planSteps) expect(['action', 'assertion']).toContain(step.type); + // At least one assertion step so the scaffold is a meaningful test. + expect(plan.planSteps.some(step => step.type === 'assertion')).toBe(true); + // stdout body parses back to the same plan (`> plan.json` works). + expect(JSON.parse(out.join('\n'))).toEqual(plan); + }); + + it('pre-fills projectId from TESTSPRITE_PROJECT_ID when set', async () => { + const result = await runScaffold( + { profile: 'default', output: 'json', debug: false, scaffoldType: 'frontend', force: false }, + { + stdout: () => undefined, + stderr: () => undefined, + env: { TESTSPRITE_PROJECT_ID: 'project_env' }, + }, + ); + expect((result as { projectId: string }).projectId).toBe('project_env'); + }); + + it('backend scaffold defines a requests test with a status assertion AND calls it', async () => { + const out: string[] = []; + const result = await runScaffold( + { profile: 'default', output: 'text', debug: false, scaffoldType: 'backend', force: false }, + { stdout: line => out.push(line), stderr: () => undefined, env: {} }, + ); + const code = (result as { code: string }).code; + expect(code).toContain('import requests'); + expect(code).toContain('assert response.status_code == 200'); + // The onboarding rule: the function must be CALLED, not just defined. + expect(code).toContain('\ntest_health_endpoint()'); + expect(out.join('\n')).toContain('import requests'); + }); + + it('--out writes the file and refuses to overwrite without --force', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-')); + const target = join(dir, 'plan.json'); + const opts = { + profile: 'default', + output: 'text', + debug: false, + scaffoldType: 'frontend', + out: target, + force: false, + } as const; + const deps = { stdout: () => undefined, stderr: () => undefined, env: {} }; + await runScaffold({ ...opts }, deps); + const written = JSON.parse(readFileSync(target, 'utf8')) as { type: string }; + expect(written.type).toBe('frontend'); + // Second run without --force must not clobber the (possibly edited) file. + await expect(runScaffold({ ...opts }, deps)).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + // --force overwrites. + await expect(runScaffold({ ...opts, force: true }, deps)).resolves.toBeDefined(); + }); + + it('--out pointing at an existing path (here a directory) rejects without --force', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-scaffold-dir-')); + await expect( + runScaffold( + { + profile: 'default', + output: 'text', + debug: false, + scaffoldType: 'frontend', + out: dir, + force: false, + }, + { stdout: () => undefined, stderr: () => undefined, env: {} }, + ), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + +describe('runOpen', () => { + // The mock endpoint host has no portal mapping; the operator override is the + // supported escape hatch and gives the tests a deterministic base. + beforeEach(() => { + process.env.TESTSPRITE_PORTAL_URL = 'https://portal.example.com'; + }); + afterEach(() => { + delete process.env.TESTSPRITE_PORTAL_URL; + }); + + const TEST_ROW = { + id: 'test_open_me', + projectId: 'project_alice', + projectName: 'Alice', + name: 'Checkout', + type: 'frontend', + createdFrom: 'cli', + status: 'ready', + createdAt: '2026-06-01T10:00:00.000Z', + updatedAt: '2026-06-01T10:00:00.000Z', + }; + + it('prints the dashboard URL and spawns the opener with it', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const out: string[] = []; + const opened: string[] = []; + const result = await runOpen( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_open_me', + noBrowser: false, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + url => opened.push(url), + ); + expect(result.dashboardUrl).toContain('/dashboard/tests/project_alice/test/test_open_me'); + expect(out.join('\n')).toContain(result.dashboardUrl); + expect(opened).toEqual([result.dashboardUrl]); + }); + + it('--no-browser prints the URL but never spawns', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const out: string[] = []; + const opened: string[] = []; + await runOpen( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_open_me', + noBrowser: true, + }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + url => opened.push(url), + ); + expect(opened).toEqual([]); + expect((JSON.parse(out.join('')) as { dashboardUrl: string }).dashboardUrl).toContain( + 'test_open_me', + ); + }); + + it('a broken opener downgrades to a stderr hint, never a failure', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: TEST_ROW })); + const errs: string[] = []; + await expect( + runOpen( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_open_me', + noBrowser: false, + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) }, + () => { + throw new Error('no display'); + }, + ), + ).resolves.toBeDefined(); + expect(errs.join('\n')).toContain('could not launch a browser'); + }); +}); + describe('runSteps', () => { it('JSON mode returns the §6.4 wire shape and forwards pageSize/cursor', async () => { const { credentialsPath } = makeCreds(); @@ -2482,6 +2903,65 @@ describe('runSteps', () => { expect(step2.outcomeContributesToFailure).toBe(true); }); + it('--run-id carries the per-step error text and stepType through to JSON (no silent drop)', async () => { + // Regression lock for the "steps discard RunStepDto.error" gap: the wire + // already returns the failure text in the same response; it must survive + // the CliTestStep mapping instead of forcing an artifact-bundle download. + const { credentialsPath } = makeCreds(); + const runWithStepError = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + steps: [ + RUN_WITH_STEPS.steps[0]!, + { + ...RUN_WITH_STEPS.steps[1]!, + status: 'failed', + error: 'Expected heading "Order confirmed" to be visible, got hidden', + }, + ], + }; + const fetchImpl = makeFetch(() => ({ body: runWithStepError })); + const page = await runSteps( + { profile: 'default', output: 'json', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + const passing = page.items.find(s => s.stepIndex === 1)!; + const failing = page.items.find(s => s.stepIndex === 2)!; + expect(failing.error).toBe('Expected heading "Order confirmed" to be visible, got hidden'); + expect(failing.stepType).toBe('assertion'); + expect(passing.error).toBeNull(); + expect(passing.stepType).toBe('action'); + }); + + it('--run-id text mode prints an indented error: sub-line under the failed row only', async () => { + const { credentialsPath } = makeCreds(); + const runWithStepError = { + ...RUN_WITH_STEPS, + status: 'failed' as const, + failedStepIndex: 2, + steps: [ + RUN_WITH_STEPS.steps[0]!, + { + ...RUN_WITH_STEPS.steps[1]!, + status: 'failed', + error: 'Locator resolved to hidden element\n at assert heading', + }, + ], + }; + const fetchImpl = makeFetch(() => ({ body: runWithStepError })); + const out: string[] = []; + await runSteps( + { profile: 'default', output: 'text', debug: false, testId: 'test_fe', runId: 'run_scoped' }, + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ); + const block = out.join('\n'); + // Newlines in the wire error collapse to one displayable line. + expect(block).toContain('error: Locator resolved to hidden element at assert heading'); + // Exactly one sub-line: the passing step must not grow one. + expect(block.match(/error: /g)).toHaveLength(1); + }); + it('--run-id: rejects a runId that belongs to a different test (exit 4)', async () => { const { credentialsPath } = makeCreds(); // The run-scoped endpoint returns a run whose testId differs from the @@ -2517,139 +2997,497 @@ describe('runSteps', () => { expect(printed.items).toHaveLength(0); }); - it('--run-id: empty steps → text mode emits advisory to stderr pointing at artifact get', async () => { + it('--run-id: empty steps → text mode emits advisory to stderr pointing at artifact get', async () => { + const { credentialsPath } = makeCreds(); + const runNoSteps = { ...RUN_WITH_STEPS, steps: [] }; + const fetchImpl = makeFetch(() => ({ body: runNoSteps })); + const stderrLines: string[] = []; + const out: string[] = []; + const page = await runSteps( + { + profile: 'default', + output: 'text', + debug: false, + testId: 'test_fe', + runId: 'run_scoped', + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => stderrLines.push(line), + }, + ); + expect(page.items).toHaveLength(0); + expect(out).toHaveLength(0); + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('artifact get'))).toBe( + true, + ); + }); + + it('--run-id: 404 propagates as ApiError (exit 4, NOT_FOUND)', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + status: 404, + body: { code: 'NOT_FOUND', message: 'Run not found' }, + })); + await expect( + runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + runId: 'run_unknown', + }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => {} }, + ), + ).rejects.toMatchObject({ code: 'NOT_FOUND' }); + }); + + it('no --run-id: multi-run response emits advisory to stderr when steps span >1 runId', async () => { + const { credentialsPath } = makeCreds(); + const stepA: CliTestStep = { ...STEP_PASSED, stepIndex: 1, runIdIfAvailable: 'run_A' }; + const stepB: CliTestStep = { ...STEP_PASSED, stepIndex: 2, runIdIfAvailable: 'run_B' }; + const fetchImpl = makeFetch(() => ({ + body: { items: [stepA, stepB], nextToken: null }, + })); + const stderrLines: string[] = []; + await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + // Advisory must mention the count and the --run-id flag + expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('--run-id'))).toBe(true); + }); + + it('no --run-id: single-run response produces no advisory', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ + body: { items: [STEP_PASSED, STEP_FAILED], nextToken: null }, + })); + const stderrLines: string[] = []; + await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: line => stderrLines.push(line), + }, + ); + expect(stderrLines.some(l => l.includes('[advisory]'))).toBe(false); + }); + + it('--run-id: pagination flags (--page-size / --starting-token / --max-items) are ignored — run endpoint returns all steps', async () => { + // The run-scoped endpoint returns all steps in a single response. + // Pagination flags are only meaningful on the cumulative /tests/{id}/steps path. + const { credentialsPath } = makeCreds(); + const seenUrls: string[] = []; + const fetchImpl = makeFetch(url => { + seenUrls.push(url); + return { body: RUN_WITH_STEPS }; + }); + const page = await runSteps( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_fe', + runId: 'run_scoped', + pageSize: 5, // ignored for run-scoped path + startingToken: 'tok', // ignored for run-scoped path + maxItems: 1, // ignored for run-scoped path + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + // All 2 steps are returned (maxItems=1 is NOT applied on the run-scoped path) + expect(page.items).toHaveLength(2); + // The run-scoped URL should not carry pagination query params + const runUrl = seenUrls.find(u => u.includes('/runs/run_scoped'))!; + expect(runUrl).toBeDefined(); + expect(runUrl).not.toMatch(/pageSize|cursor|startingToken/); + }); + + it('test steps subcommand exposes --run-id flag', async () => { + const { createTestCommand } = await import('./test.js'); + const test = createTestCommand(); + const steps = test.commands.find(c => c.name() === 'steps')!; + const flagNames = steps.options.map(o => o.long); + expect(flagNames).toContain('--run-id'); + }); +}); + +describe('runDiff', () => { + const baseRun = { + testId: 'test_fe', + projectId: 'project_alice', + userId: 'u1', + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + targetUrl: 'https://example.com', + createdFrom: null, + error: null, + videoUrl: null, + stepSummary: { total: 2, completed: 2, passedCount: 2, failedCount: 0 }, + }; + const makeStep = (index: string, status: string, error: string | null = null) => ({ + stepIndex: index, + type: 'action', + action: `step ${index}`, + status, + description: `Step ${index}`, + error, + screenshotUrl: null, + htmlSnapshotUrl: null, + createdAt: '2026-06-01T10:00:05.000Z', + }); + const RUN_GREEN = { + ...baseRun, + runId: 'run_green', + status: 'passed', + codeVersion: 'v1', + failedStepIndex: null, + failureKind: null, + steps: [makeStep('0001', 'passed'), makeStep('0002', 'passed')], + }; + const RUN_RED = { + ...baseRun, + runId: 'run_red', + status: 'failed', + codeVersion: 'v2', + failedStepIndex: 2, + failureKind: 'assertion', + steps: [makeStep('0001', 'passed'), makeStep('0002', 'failed', 'heading not visible')], + }; + const fetchForRuns = () => + makeFetch(url => ({ body: url.includes('run_green') ? RUN_GREEN : RUN_RED })); + + it('reports the verdict flip, the changed step with its error, and code drift, then exits 1', async () => { + const { credentialsPath } = makeCreds(); + const out: string[] = []; + const rejection = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' }, + { credentialsPath, fetchImpl: fetchForRuns(), stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const printed = JSON.parse(out.join('')) as { + verdictChanged: boolean; + codeVersionChanged: boolean; + crossTest: boolean; + changedSteps: Array<{ stepIndex: number; statusA: string; statusB: string; errorB?: string }>; + }; + expect(printed.verdictChanged).toBe(true); + expect(printed.codeVersionChanged).toBe(true); + expect(printed.crossTest).toBe(false); + expect(printed.changedSteps).toHaveLength(1); + expect(printed.changedSteps[0]).toMatchObject({ + stepIndex: 2, + statusA: 'passed', + statusB: 'failed', + errorB: 'heading not visible', + }); + }); + + it('identical verdicts resolve with exit 0 and no step changes', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: RUN_GREEN })); + const diff = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_green' }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(diff.verdictChanged).toBe(false); + expect(diff.changedSteps).toHaveLength(0); + }); + + it('warns (not fails) when the runs belong to different tests', async () => { + const { credentialsPath } = makeCreds(); + const OTHER = { ...RUN_GREEN, runId: 'run_red', testId: 'test_other' }; + const fetchImpl = makeFetch(url => ({ + body: url.includes('run_green') ? RUN_GREEN : OTHER, + })); + const errs: string[] = []; + const diff = await runDiff( + { profile: 'default', output: 'json', debug: false, runA: 'run_green', runB: 'run_red' }, + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: line => errs.push(line) }, + ); + expect(diff.crossTest).toBe(true); + expect(errs.join('\n')).toContain('different tests'); + }); + + it('--dry-run returns the canned sample fully offline (no credentials, no fetch)', async () => { + // Dry-run must not require credentials or hit the network — it returns a + // canned CliRunDiff so `--dry-run` shows the shape offline. + const diff = await runDiff( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: true, + runA: 'run_aaa', + runB: 'run_bbb', + }, + { stdout: () => undefined, stderr: () => undefined }, + ); + expect(diff.runA.runId).toBe('run_aaa'); + expect(diff.runB.runId).toBe('run_bbb'); + expect(diff.verdictChanged).toBe(true); + expect(diff.changedSteps).toHaveLength(1); + expect(diff.changedSteps[0]).toMatchObject({ + stepIndex: 2, + statusA: 'passed', + statusB: 'failed', + }); + }); +}); + +describe('runLint', () => { + const VALID_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Checkout works', + planSteps: [ + { type: 'action', description: 'Open the cart' }, + { type: 'assertion', description: 'Assert the total is visible' }, + ], + }); + const INVALID_PLAN = JSON.stringify({ + projectId: 'project_alice', + type: 'frontend', + name: 'Broken', + planSteps: [{ type: 'hover', description: 'Bad step type' }], + }); + + it('a directory with valid and invalid plans reports EVERY problem and exits 5', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-')); + writeFileSync(join(dir, 'a-valid.json'), VALID_PLAN, 'utf8'); + writeFileSync(join(dir, 'b-invalid.json'), INVALID_PLAN, 'utf8'); + writeFileSync(join(dir, 'c-notjson.json'), '{oops', 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { + checked: number; + valid: number; + issues: Array<{ file: string; field: string }>; + }; + // All three files were checked (no first-error-fatal bailout). + expect(report.checked).toBe(3); + expect(report.valid).toBe(1); + expect(report.issues.map(issue => issue.file).sort()).toEqual([ + 'b-invalid.json', + 'c-notjson.json', + ]); + }); + + it('an all-valid directory resolves with exit 0 and no network/credentials needed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-ok-')); + writeFileSync(join(dir, 'a.json'), VALID_PLAN, 'utf8'); + const report = await runLint( + { profile: 'default', output: 'json', debug: false, planFromDir: dir }, + { stdout: () => undefined }, + ); + expect(report).toMatchObject({ checked: 1, valid: 1, issues: [] }); + }); + + it('a JSONL file reports each bad line with its PHYSICAL line number (blank lines skipped, not renumbered)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'cli-lint-jsonl-')); + const file = join(dir, 'plans.jsonl'); + // A blank separator line sits between entries: line 1 valid, line 2 blank, + // line 3 bad JSON, line 4 invalid plan. Reported numbers must be 3 and 4. + writeFileSync(file, `${VALID_PLAN}\n\nnot json at all\n${INVALID_PLAN}\n`, 'utf8'); + const out: string[] = []; + const rejection = await runLint( + { profile: 'default', output: 'json', debug: false, plans: file }, + { stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 5 }); + const report = JSON.parse(out.join('')) as { checked: number; issues: Array<{ file: string }> }; + expect(report.checked).toBe(3); + expect(report.issues.some(issue => issue.file.endsWith(':3'))).toBe(true); + expect(report.issues.some(issue => issue.file.endsWith(':4'))).toBe(true); + // The blank line itself is not an entry and never reports. + expect(report.issues.some(issue => issue.file.endsWith(':2'))).toBe(false); + }); + + it('requires exactly one input source', async () => { + await expect( + runLint({ profile: 'default', output: 'json', debug: false }, { stdout: () => undefined }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + }); +}); + +describe('runTestWaitMany', () => { + const terminalRun = (runId: string, status: string) => ({ + runId, + testId: `test_of_${runId}`, + projectId: 'project_alice', + userId: 'u1', + status, + source: 'cli', + createdAt: '2026-06-01T10:00:00.000Z', + startedAt: '2026-06-01T10:00:01.000Z', + finishedAt: '2026-06-01T10:00:30.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }); + + it('polls every run, keeps input order, and exits 1 when one finished non-passed', async () => { const { credentialsPath } = makeCreds(); - const runNoSteps = { ...RUN_WITH_STEPS, steps: [] }; - const fetchImpl = makeFetch(() => ({ body: runNoSteps })); - const stderrLines: string[] = []; + const fetchImpl = makeFetch(url => ({ + body: url.includes('run_bad') + ? terminalRun('run_bad', 'failed') + : terminalRun('run_ok', 'passed'), + })); const out: string[] = []; - const page = await runSteps( + const rejection = await runTestWaitMany( { profile: 'default', - output: 'text', + output: 'json', debug: false, - testId: 'test_fe', - runId: 'run_scoped', - }, - { - credentialsPath, - fetchImpl, - stdout: line => out.push(line), - stderr: line => stderrLines.push(line), + runIds: ['run_ok', 'run_bad'], + timeoutSeconds: 30, + maxConcurrency: 2, }, - ); - expect(page.items).toHaveLength(0); - expect(out).toHaveLength(0); - expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('artifact get'))).toBe( - true, - ); - }); - - it('--run-id: 404 propagates as ApiError (exit 4, NOT_FOUND)', async () => { - const { credentialsPath } = makeCreds(); - const fetchImpl = makeFetch(() => ({ - status: 404, - body: { code: 'NOT_FOUND', message: 'Run not found' }, - })); - await expect( - runSteps( - { - profile: 'default', - output: 'json', - debug: false, - testId: 'test_fe', - runId: 'run_unknown', - }, - { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => {} }, - ), - ).rejects.toMatchObject({ code: 'NOT_FOUND' }); + { credentialsPath, fetchImpl, stdout: line => out.push(line) }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 1 }); + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + summary: { passed: number; failed: number }; + }; + expect(payload.results.map(row => row.runId)).toEqual(['run_ok', 'run_bad']); + expect(payload.summary).toMatchObject({ passed: 1, failed: 1 }); }); - it('no --run-id: multi-run response emits advisory to stderr when steps span >1 runId', async () => { + it('a member whose poll errors is captured as error: and the others survive (exit 7)', async () => { const { credentialsPath } = makeCreds(); - const stepA: CliTestStep = { ...STEP_PASSED, stepIndex: 1, runIdIfAvailable: 'run_A' }; - const stepB: CliTestStep = { ...STEP_PASSED, stepIndex: 2, runIdIfAvailable: 'run_B' }; - const fetchImpl = makeFetch(() => ({ - body: { items: [stepA, stepB], nextToken: null }, - })); - const stderrLines: string[] = []; - await runSteps( + const fetchImpl = makeFetch(url => { + if (url.includes('run_gone')) { + return { + status: 404, + body: { + error: { + code: 'NOT_FOUND', + message: 'no such run', + nextAction: 'check the id', + requestId: 'req_x', + details: {}, + }, + }, + }; + } + return { body: terminalRun('run_ok', 'passed') }; + }); + const out: string[] = []; + const errs: string[] = []; + const rejection = await runTestWaitMany( { profile: 'default', output: 'json', debug: false, - testId: 'test_fe', + runIds: ['run_ok', 'run_gone'], + timeoutSeconds: 30, + maxConcurrency: 2, }, { credentialsPath, fetchImpl, - stdout: () => {}, - stderr: line => stderrLines.push(line), + stdout: line => out.push(line), + stderr: line => errs.push(line), }, - ); - // Advisory must mention the count and the --run-id flag - expect(stderrLines.some(l => l.includes('[advisory]') && l.includes('--run-id'))).toBe(true); + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + const payload = JSON.parse(out.join('')) as { + results: Array<{ runId: string; status: string }>; + }; + // The failing member did not abort the pool: the passed verdict survived. + expect(payload.results[0]).toMatchObject({ runId: 'run_ok', status: 'passed' }); + expect(payload.results[1]!.status).toBe('error:NOT_FOUND'); + // The re-attach hint names the errored member (resumable) but NOT the + // already-terminal passed one. + const hint = errs.find(line => line.includes('Re-attach with:')); + expect(hint).toContain('run_gone'); + expect(hint).not.toContain('run_ok'); }); - it('no --run-id: single-run response produces no advisory', async () => { + it('members dequeued after the shared deadline are not granted extra poll time', async () => { const { credentialsPath } = makeCreds(); - const fetchImpl = makeFetch(() => ({ - body: { items: [STEP_PASSED, STEP_FAILED], nextToken: null }, - })); - const stderrLines: string[] = []; - await runSteps( + let fetches = 0; + const fetchImpl = makeFetch(() => { + fetches += 1; + return { body: terminalRun('run_any', 'passed') }; + }); + // timeoutSeconds 0: the shared deadline is already in the past when the + // pool starts, so every member must resolve to timeout WITHOUT polling + // (previously each dequeued member was granted a fresh 1s minimum). + const rejection = await runTestWaitMany( { profile: 'default', output: 'json', debug: false, - testId: 'test_fe', - }, - { - credentialsPath, - fetchImpl, - stdout: () => {}, - stderr: line => stderrLines.push(line), + runIds: ['run_a', 'run_b', 'run_c'], + timeoutSeconds: 0, + maxConcurrency: 1, }, - ); - expect(stderrLines.some(l => l.includes('[advisory]'))).toBe(false); + { credentialsPath, fetchImpl, stdout: () => undefined, stderr: () => undefined }, + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 7 }); + expect(fetches).toBe(0); }); - it('--run-id: pagination flags (--page-size / --starting-token / --max-items) are ignored — run endpoint returns all steps', async () => { - // The run-scoped endpoint returns all steps in a single response. - // Pagination flags are only meaningful on the cumulative /tests/{id}/steps path. + it('an auth error escalates the exit code to 3', async () => { const { credentialsPath } = makeCreds(); - const seenUrls: string[] = []; - const fetchImpl = makeFetch(url => { - seenUrls.push(url); - return { body: RUN_WITH_STEPS }; - }); - const page = await runSteps( + const fetchImpl = makeFetch(() => ({ + status: 401, + body: { + error: { + code: 'AUTH_INVALID', + message: 'bad key', + nextAction: 'run setup', + requestId: 'req_y', + details: {}, + }, + }, + })); + const rejection = await runTestWaitMany( { profile: 'default', output: 'json', debug: false, - testId: 'test_fe', - runId: 'run_scoped', - pageSize: 5, // ignored for run-scoped path - startingToken: 'tok', // ignored for run-scoped path - maxItems: 1, // ignored for run-scoped path + runIds: ['run_a', 'run_b'], + timeoutSeconds: 30, + maxConcurrency: 2, }, { credentialsPath, fetchImpl, stdout: () => undefined }, - ); - // All 2 steps are returned (maxItems=1 is NOT applied on the run-scoped path) - expect(page.items).toHaveLength(2); - // The run-scoped URL should not carry pagination query params - const runUrl = seenUrls.find(u => u.includes('/runs/run_scoped'))!; - expect(runUrl).toBeDefined(); - expect(runUrl).not.toMatch(/pageSize|cursor|startingToken/); - }); - - it('test steps subcommand exposes --run-id flag', async () => { - const { createTestCommand } = await import('./test.js'); - const test = createTestCommand(); - const steps = test.commands.find(c => c.name() === 'steps')!; - const flagNames = steps.options.map(o => o.long); - expect(flagNames).toContain('--run-id'); + ).catch((error: unknown) => error); + expect(rejection).toMatchObject({ exitCode: 3 }); }); }); @@ -3997,6 +4835,48 @@ describe('runCreate', () => { expect(sent.headers.get('x-api-key')).toBe('sk-user-test'); }); + it('emits backend warnings[] to stderr without polluting stdout JSON', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('BEARER = "eyJhbGciOi.eyJzdWIiOiJ4In0.sig"\n'); + const fetchImpl = makeFetch((url, init) => { + const method = init.method ?? 'GET'; + if (method === 'GET') return { status: 200, body: { items: [] } }; + return { + status: 200, + body: { + ...SAMPLE_RESPONSE, + type: 'backend', + warnings: [ + 'This test appears to hardcode an auth credential — read auth from __AUTH_HEADERS__.', + ], + }, + }; + }); + const out: string[] = []; + const err: string[] = []; + await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_alice', + type: 'backend', + name: 'hardcoded be', + codeFile, + }, + { + credentialsPath, + fetchImpl, + stdout: line => out.push(line), + stderr: line => err.push(line), + }, + ); + // Warning lands on stderr, prefixed `[warn]`. + expect(err.some(l => l.includes('[warn]') && l.includes('__AUTH_HEADERS__'))).toBe(true); + // stdout stays the parseable wire object — no warning noise. + expect(out.join('\n')).not.toContain('[warn]'); + }); + it('respects a caller-supplied --idempotency-key (for safe retries)', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('code body'); @@ -5040,6 +5920,36 @@ describe('runCreate — M4 BE dependency authoring flags', () => { ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); }); + it('[FE guard] --produces rejection is well-formed: bare field, no ----produces, singular verb (dogfood 2026-06-30)', async () => { + const { credentialsPath } = makeCreds(); + const codeFile = writeCodeFile('// fe code'); + const err = (await runCreate( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_fe', + type: 'frontend', + name: 'fe test', + codeFile, + produces: ['some_var'], + }, + { + credentialsPath, + fetchImpl: () => Promise.resolve(new Response('{}')), + stdout: () => undefined, + stderr: () => undefined, + }, + ).catch((e: unknown) => e)) as ApiError; + expect(err).toBeInstanceOf(ApiError); + // details.field is the BARE flag name (was '--produces' → double-dashed subject). + expect(err.details).toMatchObject({ field: 'produces' }); + expect(err.nextAction).toContain('--produces'); + expect(err.nextAction).not.toContain('----'); // regression: '----produces' + expect(err.nextAction).toContain('is a backend-only flag'); // singular for one flag + expect(err.nextAction).not.toContain('backend..'); // regression: double period + }); + it('[FE guard] throws VALIDATION_ERROR exit 5 when --type frontend + --needs', async () => { const { credentialsPath } = makeCreds(); const codeFile = writeCodeFile('// fe code'); @@ -5569,6 +6479,74 @@ describe('runUpdate', () => { expect(seenBody).toEqual({ priority: 'p2' }); }); + it('threads --produces/--needs/--category into the PUT body with wire names', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + produces: ['user_id', 'order_id'], + needs: ['session_token'], + category: 'teardown', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ + produces: ['user_id', 'order_id'], + consumes: ['session_token'], + category: 'teardown', + }); + }); + + it('accepts a dependency-only update (not rejected as a no-op)', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + category: 'teardown', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ category: 'teardown' }); + }); + + it('omits empty dependency arrays from the body', async () => { + const { credentialsPath } = makeCreds(); + let seenBody: unknown; + const fetchImpl = makeFetch((_url, init) => { + seenBody = init.body ? JSON.parse(init.body as string) : undefined; + return { body: SAMPLE_RESPONSE }; + }); + await runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: 'renamed', + produces: [], + needs: [], + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ); + expect(seenBody).toEqual({ name: 'renamed' }); + }); + it('respects a caller-supplied --idempotency-key', async () => { const { credentialsPath } = makeCreds(); let seenKey: string | null = null; @@ -5709,6 +6687,31 @@ describe('runUpdate', () => { expect(called).toBe(0); }); + it('rejects a whitespace-only --name before sending (parity with test create)', async () => { + const { credentialsPath } = makeCreds(); + let called = 0; + const fetchImpl = makeFetch(() => { + called += 1; + return { body: SAMPLE_RESPONSE }; + }); + await expect( + runUpdate( + { + profile: 'default', + output: 'json', + debug: false, + testId: 'test_alpha', + name: ' ', + }, + { credentialsPath, fetchImpl, stdout: () => undefined }, + ), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + details: expect.objectContaining({ field: 'name' }), + }); + expect(called).toBe(0); + }); + it('renders text mode with one line per updated field', async () => { const { credentialsPath } = makeCreds(); const fetchImpl = makeFetch(() => ({ body: SAMPLE_RESPONSE })); diff --git a/src/commands/test.ts b/src/commands/test.ts index 08e4fe2..3949ee4 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -1,11 +1,20 @@ -import { createWriteStream, readFileSync, readdirSync, statSync, type WriteStream } from 'node:fs'; +import { + createWriteStream, + existsSync, + readFileSync, + readdirSync, + statSync, + type WriteStream, +} from 'node:fs'; import { rename, stat, unlink } from 'node:fs/promises'; import { basename, dirname, extname, isAbsolute, join, resolve } from 'node:path'; import { randomUUID } from 'node:crypto'; import { Command } from 'commander'; +import { openInBrowser } from '../lib/browser.js'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { @@ -17,14 +26,25 @@ import { writeBundle, type WriteBundleResult, } from '../lib/bundle.js'; -import { findSample } from '../lib/dry-run/samples.js'; +import { findSample, sampleJUnitReportXml } from '../lib/dry-run/samples.js'; +import { + assertJUnitReportOptions, + buildJUnitReport, + resolveBatchReportProjectId, + writeJUnitReportFile, + type JUnitReportFormat, + parseJUnitReportFormat, + type JUnitTestResult, +} from '../lib/junit-report.js'; import { ApiError, CLIError, + InterruptError, RequestTimeoutError, TransportError, localValidationError, } from '../lib/errors.js'; +import { globalShutdown, type ShutdownHandle } from '../lib/interrupt.js'; import { assertIdempotencyKey, requireArrayLength, @@ -34,7 +54,7 @@ import { import { REQUEST_TIMEOUT_DEFAULT_MS, REQUEST_TIMEOUT_MAX_MS } from '../lib/http.js'; import type { FetchImpl } from '../lib/http.js'; import type { HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import { fetchSinglePage, paginate, @@ -58,13 +78,29 @@ import type { RunSource, BatchRunFreshResponse, BatchRunFreshAccepted, + CancelRunResponse, } from '../lib/runs.types.js'; import { RUN_SOURCES } from '../lib/runs.types.js'; import { assertNotLocal } from '../lib/target-url.js'; +import { + formatTextTableRow, + measureTextColumns, + renderTextTable, + resolveTextColumns, + type TextTableColumn, +} from '../lib/text-table.js'; import { createTicker } from '../lib/ticker.js'; import { RateThrottle } from '../lib/rate-throttle.js'; import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js'; import { loadConfig } from '../lib/config.js'; +import { + flakyExitCode, + renderFlakyText, + summarizeFlaky, + type FlakyAttempt, + type FlakyOutcome, + type FlakyReport, +} from '../lib/flaky.js'; /** * `details` debug block per the CLI OpenAPI `Test` schema @@ -128,6 +164,14 @@ export interface CliTest { * no priority has been set. Text mode surfaces it only when truthy. */ priority?: string | null; + /** + * Backend-only dependency declarations that drive wave ordering. + * Optional on the wire so older facades that don't ship them still + * type-check; text mode surfaces them only when present/non-empty. + */ + produces?: string[] | null; + consumes?: string[] | null; + category?: string | null; } export type CliPublicStatus = @@ -176,6 +220,19 @@ export interface CliTestStep { * that don't emit the field still type-check. */ outcomeContributesToFailure?: boolean | null; + /** + * Per-step failure text, carried from `RunStepDto.error` on the run-scoped + * endpoint (`GET /runs/{id}?includeSteps=true`). Only present on `--run-id` + * responses; the cumulative `/tests/{id}/steps` rows do not carry it, so the + * field stays optional (additive, non-breaking for existing consumers). + */ + error?: string | null; + /** + * Wire step kind from `RunStepDto.type` on the run-scoped endpoint. Same + * availability rules as `error`. Named `stepType` to avoid colliding with + * the free-form `action` label above. + */ + stepType?: 'action' | 'assertion'; } /** @@ -254,6 +311,18 @@ export interface CliLatestResult { * former `{passed,failed,skipped}` count object). */ summary: string; + /** + * Captured stdout (`api_output`) from a backend-test execution. + * Present (possibly null) only for backend tests; omitted for FE/MCP and on + * older backends that omit them. Capped at 50 KB UTF-8 by the server. + */ + apiOutput?: string | null; + /** + * Python traceback from a backend-test execution (ends with the + * `ExceptionType: message` line). Present (possibly null) only for backend + * tests; omitted for FE/MCP and on older backends. + */ + trace?: string | null; /** * §6.5.1 (M2.1 piece 3) — inline failure analysis. Present when the * caller passed `--include-analysis` (`?includeAnalysis=true` on @@ -344,6 +413,14 @@ export interface CliFailureBlock { */ recommendedFixTarget: CliFixTarget | null; evidence: CliEvidence[]; + /** + * Backend-test execution artifacts, surfaced on the failure block + * so triage has stdout + traceback next to the hypothesis (mirrors + * `result.apiOutput` / `result.trace`). Present (possibly null) only for + * backend failures; omitted for FE and on older backends. + */ + apiOutput?: string | null; + trace?: string | null; } /** §6.7 wire shape — one atomic snapshot of the latest failing run. */ @@ -375,6 +452,36 @@ export interface TestDeps { * no-op in tests to avoid real delays. */ sleep?: (ms: number) => Promise; + /** + * Graceful-detach coordinator for the `--wait` paths (DEV-331 piece 1). + * Defaults to the process-wide `globalShutdown`; tests inject their own + * controller and abort it to simulate Ctrl-C deterministically (same + * pattern as `sleep` / `fetchImpl`). + */ + shutdown?: ShutdownHandle; +} + +/** The effective shutdown handle for a command invocation (DEV-331). */ +function shutdownOf(deps: TestDeps): ShutdownHandle { + return deps.shutdown ?? globalShutdown; +} + +/** + * The honest-detach stderr line (DEV-331 D1 — the heart of the ticket): + * a Ctrl-C detaches the local wait only; the server-side run keeps executing + * AND billing. Names both re-attach (`test wait`) and the real cancel + * (`test cancel`, piece 3) so the user always has a path to actually stop it. + */ +function interruptDetachMessage(err: InterruptError, runIds: string[]): string { + const subject = + runIds.length === 1 + ? `Run ${runIds[0]} is still executing on the server and will keep running (and billing) until it finishes.` + : `${runIds.length} runs are still executing on the server and will keep running (and billing) until they finish.`; + return ( + `Interrupted (${err.signal}). ${subject}\n` + + ` Re-attach with: testsprite test wait ${runIds.join(' ')}\n` + + ` Cancel with: testsprite test cancel ${runIds.join(' ')}` + ); } type CommonOptions = FactoryCommonOptions; @@ -393,6 +500,8 @@ interface ListOptions extends CommonOptions { pageSize?: number; startingToken?: string; maxItems?: number; + columns?: string; + noHeader?: boolean; } const TEST_TYPES: ReadonlyArray<'frontend' | 'backend'> = ['frontend', 'backend']; @@ -451,7 +560,16 @@ export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise

= { projectId: opts.projectId, type: opts.type, @@ -491,7 +603,9 @@ export async function runList(opts: ListOptions, deps: TestDeps = {}): Promise

renderTestListText(data as Page)); + out.print(page, data => + renderTestListText(data as Page, { columns: opts.columns, noHeader: opts.noHeader }), + ); return page; } @@ -505,6 +619,12 @@ export interface CliCreateTestResponse { type: 'frontend' | 'backend'; codeVersion: string; createdAt: string; + /** + * Non-fatal advisories from the backend (e.g. the BE auth guardrail + * flagging a hardcoded credential). Rendered on stderr; the create + * still succeeded. + */ + warnings?: string[]; } export const CLI_CREATE_PRIORITIES = ['p0', 'p1', 'p2', 'p3'] as const; @@ -714,14 +834,19 @@ export async function runCreate( // save a round-trip. if (opts.type === 'frontend') { const depFlags: string[] = []; - if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('--produces'); - if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('--needs'); - if (opts.category !== undefined) depFlags.push('--category'); + if (opts.produces !== undefined && opts.produces.length > 0) depFlags.push('produces'); + if (opts.needs !== undefined && opts.needs.length > 0) depFlags.push('needs'); + if (opts.category !== undefined) depFlags.push('category'); if (depFlags.length > 0) { + // Pass the BARE flag name to localValidationError — its kind:'flag' branch + // adds the `--` prefix, so '--produces' would render as '----produces'. + const flagList = depFlags.map(f => `--${f}`); + const verb = depFlags.length === 1 ? 'is a backend-only flag' : 'are backend-only flags'; + // No trailing period: localValidationError appends one after the reason. throw localValidationError( depFlags[0]!, - `${depFlags.join(', ')} are backend-only flags; frontend plans have no wave model. ` + - `Remove ${depFlags.join('/')} or use --type backend.`, + `${flagList.join(', ')} ${verb}; frontend plans have no wave model. ` + + `Remove ${flagList.join('/')} or use --type backend`, ); } } @@ -798,6 +923,11 @@ export async function runCreate( headers: { 'idempotency-key': idempotencyKey }, }); + // Surface backend advisories (e.g. a hardcoded-credential warning for BE + // tests) on stderr so they reach the agent without polluting stdout JSON. + // Emitted before the --run early return so they always show. + emitResponseWarnings(response.warnings, deps); + // --run chain (M3.3 piece-3). Per codex round-1 P1: suppress the // create's own print when chaining; `runTestRun` emits a single // merged envelope `{ ...createResponse, run: }` so @@ -911,7 +1041,7 @@ function readCodeFileGuarded(path: string): string { function readCodeFile(path: string): string { try { - return readFileSync(resolveAbsolute(path), 'utf8'); + return stripBom(readFileSync(resolveAbsolute(path), 'utf8')); } catch (err) { const code = (err as NodeJS.ErrnoException).code; if (code === 'ENOENT') { @@ -960,6 +1090,16 @@ function renderCreateText(response: CliCreateTestResponse): string { ].join('\n'); } +/** + * Emit backend `warnings[]` advisories to stderr (one `[warn]` line each), + * keeping stdout — JSON or text — uncluttered. No-op when absent/empty. + */ +function emitResponseWarnings(warnings: string[] | undefined, deps: TestDeps): void { + if (!warnings || warnings.length === 0) return; + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + for (const w of warnings) stderrFn(`[warn] ${w}`); +} + /** * §6.X / M3.2 piece-6 — response from `PUT /tests/{id}/plan-steps`. * `planStepsHash` is a sha256 over the canonicalized new array so @@ -1210,6 +1350,12 @@ interface UpdateOptions extends CommonOptions { description?: string; /** Optional new priority. Enum-validated CLI-side. */ priority?: CliCreatePriority; + /** Backend-only: variable names this test produces (repeatable --produces). */ + produces?: string[]; + /** Backend-only: variable names this test consumes (repeatable --needs); wire field `consumes`. */ + needs?: string[]; + /** Backend-only: free-text wave category (--category). */ + category?: string; /** Caller-supplied idempotency token; UUIDv4 minted client-side if absent. */ idempotencyKey?: string; } @@ -1242,6 +1388,12 @@ export async function runUpdate( assertIdempotencyKey(opts.idempotencyKey); requireNonEmpty('test-id', opts.testId); // P1-3: client-side length checks matching server limits. + if (opts.name !== undefined && opts.name.trim().length === 0) { + throw localValidationError( + 'name', + 'must be a non-empty string (whitespace-only is not allowed)', + ); + } if (opts.name !== undefined && opts.name.length > 200) { throw localValidationError('name', 'must be at most 200 characters'); } @@ -1260,11 +1412,14 @@ export async function runUpdate( const hasName = opts.name !== undefined; const hasDescription = opts.description !== undefined; const hasPriority = opts.priority !== undefined; - if (!hasName && !hasDescription && !hasPriority) { + const hasProduces = opts.produces !== undefined && opts.produces.length > 0; + const hasNeeds = opts.needs !== undefined && opts.needs.length > 0; + const hasCategory = opts.category !== undefined; + if (!hasName && !hasDescription && !hasPriority && !hasProduces && !hasNeeds && !hasCategory) { throw localValidationError( 'fields', - 'at least one of --name / --description / --priority must be set', - ['name', 'description', 'priority'], + 'at least one of --name / --description / --priority / --produces / --needs / --category must be set', + ['name', 'description', 'priority', 'produces', 'needs', 'category'], ); } @@ -1279,10 +1434,13 @@ export async function runUpdate( // is the intended wire shape — but we build the body deliberately // so the contract is auditable rather than dependent on // JSON.stringify undefined-skipping. - const body: Record = {}; + const body: Record = {}; if (hasName) body.name = opts.name!; if (hasDescription) body.description = opts.description!; if (hasPriority) body.priority = opts.priority!; + if (hasProduces) body.produces = opts.produces!; + if (hasNeeds) body.consumes = opts.needs!; + if (hasCategory) body.category = opts.category!; const client = makeClient(opts, deps); const out = makeOutput(opts.output, deps); @@ -2513,7 +2671,14 @@ async function runBatchRun( * * Returns a CliBatchRunResult. Never throws — errors are captured * into the result's `error` field so one failure doesn't abort siblings. + * (Exception: InterruptError rethrows so the collect point can print the + * partial — DEV-331.) */ + // testId → dispatched runId for members still mid-poll; the interrupt + // partial reads it because a member's runId is local to triggerOne until + // the poll settles (DEV-331, codex finding 2). + const dispatchedRunIds = new Map(); + async function triggerOne(testId: string): Promise { // Mint a fresh idempotency key per run — MUST NOT reuse the create key. const runIdempotencyKey = `cli-batch-run-${randomUUID()}`; @@ -2603,6 +2768,10 @@ async function runBatchRun( triggerResponse = result.body; break; // success — exit the outer retry loop } catch (err) { + // Interrupt must reject the fan-out (the collect point prints the + // partial for every spec), never flatten into a per-member outcome + // that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; // RATE_LIMITED outer retry. Since the HTTP layer no longer retries // RATE_LIMITED (retryOnRateLimit: false above), every 429 reaches here // on the first attempt. @@ -2739,6 +2908,12 @@ async function runBatchRun( } } + // Record the dispatched runId (fresh trigger OR conflict-resume) so the + // interrupt partial at the collect point can name every in-flight run — + // a member's runId is otherwise local until its poll settles (DEV-331, + // codex finding 2). + if (triggerResponse.runId) dispatchedRunIds.set(testId, triggerResponse.runId); + if (!opts.wait) { // No-wait path: return the trigger response as-is. if (opts.output !== 'json') { @@ -2780,11 +2955,14 @@ async function runBatchRun( finalRun = await pollRunUntilTerminal(client, triggerResponse.runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[batch-run][verbose] ${testId}: ${msg}`) : undefined, }); } catch (err) { + // Interrupt rejects the fan-out — see the trigger-stage catch above. + if (err instanceof InterruptError) throw err; if (err instanceof TimeoutError) { if (opts.output !== 'json') { stderrFn( @@ -2850,24 +3028,59 @@ async function runBatchRun( let nextIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && nextIdx < testIds.length) { - const testId = testIds[nextIdx++]!; - inFlight++; - triggerOne(testId) - .then(result => { - batchRunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && nextIdx >= testIds.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && nextIdx < testIds.length) { + const testId = testIds[nextIdx++]!; + inFlight++; + triggerOne(testId) + .then(result => { + batchRunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= testIds.length) resolve(); + }) + .catch(reject); + } + } + startNext(); + if (testIds.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): leave stdout parseable — settled members keep + // their real status, unfinished ones are marked running — then rethrow so + // index.ts exits 128+signum. + if (fanOutErr instanceof InterruptError) { + const settled = new Map(batchRunResults.map(r => [r.testId, r] as const)); + // Members mid-poll have no settled result yet — their runId comes from + // the dispatchedRunIds map recorded at trigger time (codex finding 2). + const partialResults = testIds.map( + (testId): CliBatchRunResult => + settled.get(testId) ?? { + testId, + runId: dispatchedRunIds.get(testId) ?? '', + status: dispatchedRunIds.has(testId) ? 'running' : 'not_dispatched', + codeVersion: '', + }, + ); + out.print({ results: partialResults }, () => + partialResults.map(r => `${r.testId} ${r.runId || '-'} ${r.status}`).join('\n'), + ); + const unfinished = partialResults + .filter(r => r.status === 'running' && r.runId) + .map(r => r.runId); + if (unfinished.length > 0) { + stderrFn(interruptDetachMessage(fanOutErr, unfinished)); + } else { + stderrFn( + `Interrupted (${fanOutErr.signal}). Already-triggered runs keep executing (and billing) server-side; ` + + `check them with: testsprite test list`, + ); } } - startNext(); - if (testIds.length === 0) resolve(); - }); + throw fanOutErr; + } // Sort by testId order (same as input order for stable output). batchRunResults.sort((a, b) => testIds.indexOf(a.testId) - testIds.indexOf(b.testId)); @@ -3292,7 +3505,7 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro return code; } - const fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; + let fileSink = opts.out !== undefined ? openOutputFile(opts.out) : null; const out = fileSink ? makeFileOutput(opts.output, fileSink) : makeOutput(opts.output, deps); const client = makeClient(opts, deps); @@ -3317,9 +3530,20 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro } else if (code.code === '' || code.code === null) { // P2-10: draft test with no code yet — empty body would produce // silent empty stdout. Print a friendly hint to stderr instead so - // the operator knows what happened, and keep exit 0. Nothing was - // written, so the temp file is discarded below without touching - // a pre-existing `--out` file. + // the operator knows what happened, and keep exit 0 when no `--out`. + // + // With `--out`, refuse to leave a zero-byte artifact behind: agents + // and scripts that check file size would otherwise treat exit 0 as + // a successful download. Discard the temp sink without touching a + // pre-existing destination file. + if (fileSink) { + await abortOutputFile(fileSink); + fileSink = null; + throw localValidationError( + 'out', + 'test has no generated code yet — run the test first (refusing to write an empty --out file)', + ); + } const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); stderrFn('(no code generated yet — run the test first)'); } else { @@ -3345,6 +3569,12 @@ export interface CliPutTestCodeResponse { testId: string; codeVersion: string; updatedAt: string; + /** + * Non-fatal advisories (e.g. the BE auth guardrail flagging a hardcoded + * credential in the replaced code). Rendered on stderr; the update still + * succeeded. + */ + warnings?: string[]; } type CodePutLanguage = CliTestCode['language']; @@ -3522,6 +3752,7 @@ export async function runCodePut( }, }, ); + emitResponseWarnings(response.warnings, deps); out.print(response, data => renderCodePutText(data as CliPutTestCodeResponse)); return response; } catch (err) { @@ -3615,7 +3846,510 @@ function mapRunStepToCliTestStep(step: RunStepDto, run: RunResponse): CliTestSte // non-contributors. (Per the CliTestStep contract: null ≠ false.) outcomeContributesToFailure: run.failedStepIndex === null ? null : numericIndex === run.failedStepIndex, + // Carry the per-step failure text and the wire step kind through instead + // of dropping them: the agent asking "why did this step fail?" would + // otherwise have to download the whole artifact bundle to read a string + // this very response already contained. + error: step.error, + stepType: step.type, + }; +} + +export interface DiffOptions extends CommonOptions { + runA: string; + runB: string; +} + +/** One step whose status flipped between the two compared runs. */ +export interface CliDiffStep { + stepIndex: number; + statusA: string; + statusB: string; + /** First divergent failing side's error text, when the wire carried one. */ + errorA?: string | null; + errorB?: string | null; +} + +export interface CliRunDiff { + runA: { + runId: string; + testId: string; + status: string; + failureKind: string | null; + failedStepIndex: number | null; + codeVersion: string | null; + }; + runB: { + runId: string; + testId: string; + status: string; + failureKind: string | null; + failedStepIndex: number | null; + codeVersion: string | null; + }; + verdictChanged: boolean; + failedStepIndexChanged: boolean; + failureKindChanged: boolean; + codeVersionChanged: boolean; + /** True when the two runs belong to DIFFERENT tests (deltas may be meaningless). */ + crossTest: boolean; + changedSteps: CliDiffStep[]; +} + +/** + * `test diff ` (issue #124): isolate what regressed between two + * runs, the first question when CI goes red ("what changed since the last + * green run?"). Pure client-side composition of the existing per-run read + * (`GET /runs/{id}?includeSteps=true`); the endpoint accepts any two run-ids, + * so a cross-test pair is a WARNING, not an error. Exit 0 when the verdicts + * match, exit 1 when they differ, so the command is CI-scriptable. + */ +export async function runDiff(opts: DiffOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const sample: CliRunDiff = { + runA: { + runId: opts.runA, + testId: 'test_dryrun', + status: 'passed', + failureKind: null, + failedStepIndex: null, + codeVersion: 'v1', + }, + runB: { + runId: opts.runB, + testId: 'test_dryrun', + status: 'failed', + failureKind: 'assertion', + failedStepIndex: 2, + codeVersion: 'v1', + }, + verdictChanged: true, + failedStepIndexChanged: true, + failureKindChanged: true, + codeVersionChanged: false, + crossTest: false, + changedSteps: [{ stepIndex: 2, statusA: 'passed', statusB: 'failed' }], + }; + out.print(sample, () => renderRunDiffText(sample)); + return sample; + } + + const client = makeClient(opts, deps); + const [runA, runB] = await Promise.all([ + client.getRun(opts.runA, { includeSteps: true }), + client.getRun(opts.runB, { includeSteps: true }), + ]); + + const crossTest = runA.testId !== runB.testId; + if (crossTest) { + stderrFn( + `⚠ the two runs belong to different tests (${runA.testId} vs ${runB.testId}) — step deltas may be meaningless`, + ); + } + + const stepsByIndex = ( + run: RunResponse, + ): Map => { + const map = new Map(); + for (const step of run.steps ?? []) { + const index = parseInt(step.stepIndex, 10); + if (Number.isInteger(index)) + map.set(index, { status: step.status ?? 'unknown', error: step.error }); + } + return map; + }; + const stepsA = stepsByIndex(runA); + const stepsB = stepsByIndex(runB); + const allIndexes = [...new Set([...stepsA.keys(), ...stepsB.keys()])].sort( + (left, right) => left - right, + ); + const changedSteps: CliDiffStep[] = []; + for (const index of allIndexes) { + const sideA = stepsA.get(index); + const sideB = stepsB.get(index); + const statusA = sideA?.status ?? 'absent'; + const statusB = sideB?.status ?? 'absent'; + if (statusA === statusB) continue; + changedSteps.push({ + stepIndex: index, + statusA, + statusB, + ...(sideA?.error ? { errorA: sideA.error } : {}), + ...(sideB?.error ? { errorB: sideB.error } : {}), + }); + } + + const summarize = (run: RunResponse) => ({ + runId: run.runId, + testId: run.testId, + status: run.status, + failureKind: run.failureKind ?? null, + failedStepIndex: run.failedStepIndex, + codeVersion: run.codeVersion ?? null, + }); + const diff: CliRunDiff = { + runA: summarize(runA), + runB: summarize(runB), + verdictChanged: runA.status !== runB.status, + failedStepIndexChanged: runA.failedStepIndex !== runB.failedStepIndex, + failureKindChanged: (runA.failureKind ?? null) !== (runB.failureKind ?? null), + codeVersionChanged: (runA.codeVersion ?? null) !== (runB.codeVersion ?? null), + crossTest, + changedSteps, + }; + out.print(diff, () => renderRunDiffText(diff)); + + if (diff.verdictChanged) { + // Result already printed; the typed exit makes `test diff` a CI gate. + throw new CLIError( + `verdicts differ: ${runA.runId}=${runA.status} vs ${runB.runId}=${runB.status}`, + 1, + ); + } + return diff; +} + +function renderRunDiffText(diff: CliRunDiff): string { + const lines: string[] = []; + lines.push(`runA: ${diff.runA.runId} ${diff.runA.status} (test ${diff.runA.testId})`); + lines.push(`runB: ${diff.runB.runId} ${diff.runB.status} (test ${diff.runB.testId})`); + lines.push( + `verdict: ${diff.verdictChanged ? `${diff.runA.status} -> ${diff.runB.status}` : `unchanged (${diff.runA.status})`}`, + ); + if (diff.failureKindChanged) + lines.push( + `failureKind: ${diff.runA.failureKind ?? '(none)'} -> ${diff.runB.failureKind ?? '(none)'}`, + ); + if (diff.failedStepIndexChanged) + lines.push( + `failedStepIndex: ${diff.runA.failedStepIndex ?? '(none)'} -> ${diff.runB.failedStepIndex ?? '(none)'}`, + ); + lines.push( + `codeVersion: ${diff.codeVersionChanged ? `${diff.runA.codeVersion ?? '(none)'} -> ${diff.runB.codeVersion ?? '(none)'} (code drift)` : 'unchanged'}`, + ); + if (diff.changedSteps.length === 0) { + lines.push('steps: no per-step status changes'); + } else { + lines.push(`steps changed: ${diff.changedSteps.length}`); + for (const step of diff.changedSteps) { + lines.push(` #${step.stepIndex} ${step.statusA} -> ${step.statusB}`); + if (step.errorB) lines.push(` error(B): ${step.errorB.replace(/\s+/g, ' ').trim()}`); + else if (step.errorA) + lines.push(` error(A): ${step.errorA.replace(/\s+/g, ' ').trim()}`); + } + } + return lines.join('\n'); +} + +export interface LintOptions extends CommonOptions { + planFrom?: string; + planFromDir?: string; + plans?: string; + steps?: string; +} + +export interface CliLintIssue { + file: string; + field: string; + reason: string; +} + +export interface CliLintReport { + checked: number; + valid: number; + issues: CliLintIssue[]; +} + +/** + * `test lint` (issue #98): validate plan/steps files fully OFFLINE with the + * SAME validators the create paths run, but collecting EVERY problem instead + * of dying on the first one, and without any network write. The create-batch + * reader is first-error-fatal and only reachable through a command that POSTs, + * so authoring a 12-plan directory meant one error per paid round-trip. Zero + * network, zero credentials: exit 0 when everything is valid, 5 otherwise, so + * it drops into a pre-commit hook or CI step before `create-batch`. + */ +export async function runLint(opts: LintOptions, deps: TestDeps = {}): Promise { + const out = makeOutput(opts.output, deps); + const sources = [opts.planFrom, opts.planFromDir, opts.plans, opts.steps].filter( + source => source !== undefined, + ); + if (sources.length !== 1) { + throw localValidationError( + 'plan-from', + 'exactly one of --plan-from, --plan-from-dir, --plans, or --steps is required', + ); + } + + const issues: CliLintIssue[] = []; + let checked = 0; + // Run one existing validator, converting its typed throw into a report row + // (same envelopes, so `details.field` pointers like planSteps[2].type + // survive verbatim). + const collect = (file: string, validate: () => void): void => { + checked += 1; + try { + validate(); + } catch (err) { + if (err instanceof ApiError) { + issues.push({ + file, + field: String(err.getDetail('field') ?? '(file)'), + reason: String(err.getDetail('reason') ?? err.nextAction ?? err.message), + }); + } else { + issues.push({ + file, + field: '(file)', + reason: err instanceof Error ? err.message : String(err), + }); + } + } }; + + if (opts.planFrom !== undefined) { + const planFrom = opts.planFrom; + collect(planFrom, () => void readPlanFromGuarded(planFrom)); + } else if (opts.steps !== undefined) { + const steps = opts.steps; + collect(steps, () => void readPlanStepsFileGuarded(steps)); + } else if (opts.planFromDir !== undefined) { + const dir = resolveAbsolute(opts.planFromDir); + let entries: string[]; + try { + entries = readdirSync(dir) + .filter(name => name.endsWith('.json')) + .sort(); + } catch { + throw localValidationError('plan-from-dir', `cannot read directory: ${dir}`); + } + if (entries.length === 0) { + throw localValidationError('plan-from-dir', 'contains no *.json plan files'); + } + for (const entry of entries) { + collect(entry, () => void readPlanFromGuarded(join(dir, entry))); + } + } else if (opts.plans !== undefined) { + // JSONL: validate PER LINE so every bad line reports (the create path's + // reader stays throw-on-first; this is the collecting counterpart). + const absolute = resolveAbsolute(opts.plans); + let content: string; + try { + content = readFileSync(absolute, 'utf8'); + } catch { + throw localValidationError('plans', `cannot read file: ${absolute}`); + } + // Index lines BEFORE dropping blanks so every reported `file:N` points at + // the PHYSICAL line in the file (a blank separator line must not shift all + // subsequent line numbers). + const numberedLines = content + .split('\n') + .map((rawLine, physicalIndex) => ({ line: rawLine.trim(), lineNo: physicalIndex + 1 })) + .filter(entry => entry.line.length > 0); + if (numberedLines.length === 0) throw localValidationError('plans', 'contains no plan lines'); + for (const { line, lineNo } of numberedLines) { + collect(`${opts.plans}:${lineNo}`, () => { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + throw localValidationError( + 'plans', + `line ${lineNo} is not valid JSON`, + undefined, + 'field', + ); + } + assertPlanShape(parsed, { specIndex: lineNo - 1 }); + }); + } + } + + const filesWithIssues = new Set(issues.map(issue => issue.file)).size; + const report: CliLintReport = { checked, valid: checked - filesWithIssues, issues }; + out.print(report, () => + [ + ...issues.map(issue => `${issue.file}: ${issue.field}: ${issue.reason}`), + `${report.valid}/${report.checked} valid, ${issues.length} problem(s)`, + ].join('\n'), + ); + if (issues.length > 0) { + throw new CLIError(`lint: ${issues.length} problem(s) across ${report.checked} file(s)`, 5); + } + return report; +} + +/** Flag options for `test scaffold`. */ +interface ScaffoldFlagOpts { + type?: string; + out?: string; + force?: boolean; +} + +export interface ScaffoldOptions extends CommonOptions { + scaffoldType: 'frontend' | 'backend'; + out?: string; + force: boolean; +} + +/** JSON payload `test scaffold --type backend` prints under --output json. */ +export interface CliBackendScaffold { + type: 'backend'; + language: 'python'; + code: string; +} + +/** + * `test scaffold` — emit a schema-correct starter test definition so a first + * test never starts from hand-copied JSON. Pure-local: no network, no + * credentials, no filesystem reads. The frontend template is a `CliPlanInput` + * (the exact shape `--plan-from` ingests; sourceRef: CliPlanInput / + * PLAN_STEP_TYPES above), so `scaffold | create --plan-from -`-style flows + * validate out of the box. The backend template is the minimal `requests` + * script the onboarding skill mandates: define a test function with a + * concrete status assertion, then CALL it (a defined-but-never-called test + * would pass without asserting anything). + */ +export async function runScaffold( + opts: ScaffoldOptions, + deps: TestDeps = {}, +): Promise { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const env = deps.env ?? process.env; + // Pre-fill the project id from TESTSPRITE_PROJECT_ID when the caller's + // environment carries one; otherwise a clearly-marked placeholder the user + // swaps after running `testsprite project list`. + const projectId = + typeof env.TESTSPRITE_PROJECT_ID === 'string' && env.TESTSPRITE_PROJECT_ID.length > 0 + ? env.TESTSPRITE_PROJECT_ID + : ''; + + let payload: CliPlanInput | CliBackendScaffold; + let body: string; + if (opts.scaffoldType === 'frontend') { + const plan: CliPlanInput = { + projectId, + type: 'frontend', + name: 'My first frontend test', + description: 'Replace with one sentence describing what this test verifies.', + priority: 'p2', + planSteps: [ + { + type: 'action', + description: 'Navigate to /login and sign in with a seeded test account', + }, + { type: 'action', description: 'Open the first product page and click "Add to cart"' }, + { type: 'assertion', description: 'Assert that the cart badge shows 1 item' }, + ], + }; + payload = plan; + body = `${JSON.stringify(plan, null, 2)}\n`; + } else { + const code = [ + 'import requests', + '', + '# Replace with your API base URL (must be reachable from the internet).', + 'BASE_URL = "https://staging.example.com"', + '', + '', + 'def test_health_endpoint() -> None:', + ' response = requests.get(f"{BASE_URL}/health", timeout=30)', + ' assert response.status_code == 200, f"expected 200, got {response.status_code}"', + '', + '', + '# The test function MUST be called: TestSprite executes this file top to', + '# bottom, so a defined-but-never-called function would pass vacuously.', + 'test_health_endpoint()', + '', + ].join('\n'); + payload = { type: 'backend', language: 'python', code }; + body = code; + } + + if (opts.out !== undefined) { + const resolved = isAbsolute(opts.out) ? opts.out : resolve(process.cwd(), opts.out); + // Never clobber silently: scaffolds are starting points the user edits, so + // an accidental re-run must not erase their work. --force opts in. + if (!opts.force && existsSync(resolved)) { + throw localValidationError('out', `already exists: ${resolved}. Pass --force to overwrite`); + } + const sink = openOutputFile(opts.out); // reuses the directory/parent guards + const fileOut = makeFileOutput(opts.output, sink); + await fileOut.writeChunk(body); + await closeOutputFile(sink, true); + stderrFn(`Scaffold written to ${resolved}`); + return payload; + } + + // No --out: the scaffold body IS the stdout payload (`> plan.json` works). + out.print(payload, () => body.trimEnd()); + return payload; +} + +export interface OpenOptions extends CommonOptions { + testId: string; + /** Print the URL only; never spawn a browser (SSH/headless/CI/agents). */ + noBrowser: boolean; +} + +/** + * `test open ` (issue #121): jump from the terminal to the test's + * dashboard page. The CLI already computes this deep-link and prints it as + * text on other commands; this closes the last inch (the `gh browse` / + * `cypress open` hop). The URL is ALWAYS printed to stdout (so `--no-browser` + * and headless use still compose), then the OS browser is spawned unless + * --no-browser. An endpoint with no known portal mapping is a hard error + * rather than a silent no-op. + */ +export async function runOpen( + opts: OpenOptions, + deps: TestDeps = {}, + opener: (url: string) => void = openInBrowser, +): Promise<{ dashboardUrl: string }> { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + // Derive the sample through the SAME resolver as the live path (against + // the canonical prod endpoint and the dry-run project id) so the two can + // never drift; the ?? arm is unreachable for the prod mapping but keeps + // the type total. + const sample = { + dashboardUrl: + resolvePortalUrl('https://api.testsprite.com', 'p_dryrun_2026', opts.testId) ?? + `https://www.testsprite.com/dashboard/tests/p_dryrun_2026/test/${encodeURIComponent(opts.testId)}`, + }; + out.print(sample, data => (data as { dashboardUrl: string }).dashboardUrl); + return sample; + } + + const client = makeClient(opts, deps); + // The deep-link needs the projectId; the test record is the source of truth. + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + const dashboardUrl = resolvePortalUrl(resolveApiUrl(opts, deps), test.projectId, opts.testId); + if (dashboardUrl === undefined) { + throw new CLIError( + `no dashboard mapping for this API endpoint; set TESTSPRITE_PORTAL_URL to your Portal origin`, + 1, + ); + } + out.print({ dashboardUrl }, () => dashboardUrl); + if (!opts.noBrowser) { + try { + opener(dashboardUrl); + } catch { + // The URL is already on stdout; a missing opener (containers, minimal + // hosts) downgrades to "open it yourself" instead of a hard failure. + stderrFn('could not launch a browser; open the URL above manually (or use --no-browser)'); + } + } + return { dashboardUrl }; } export async function runSteps( @@ -3791,12 +4525,20 @@ export function parseDuration(raw: string, now: Date = new Date()): string { const hourMatch = /^(\d+)h$/i.exec(raw); if (hourMatch) { const hours = Number(hourMatch[1]); - return new Date(now.getTime() - hours * 60 * 60 * 1000).toISOString(); + const result = new Date(now.getTime() - hours * 60 * 60 * 1000); + if (!Number.isFinite(result.getTime())) { + throw localValidationError('since', 'duration is too large; maximum is ~1141552511h'); + } + return result.toISOString(); } const dayMatch = /^(\d+)d$/i.exec(raw); if (dayMatch) { const days = Number(dayMatch[1]); - return new Date(now.getTime() - days * 24 * 60 * 60 * 1000).toISOString(); + const result = new Date(now.getTime() - days * 24 * 60 * 60 * 1000); + if (!Number.isFinite(result.getTime())) { + throw localValidationError('since', 'duration is too large; maximum is ~47564688d'); + } + return result.toISOString(); } // Pass-through: ISO timestamp or epoch value — server validates. return raw; @@ -3815,6 +4557,8 @@ interface ResultHistoryOptions extends CommonOptions { pageSize?: number; /** Opaque cursor from a prior page's `nextCursor`. */ cursor?: string; + columns?: string; + noHeader?: boolean; } /** @@ -3836,10 +4580,16 @@ export async function runResultHistory( // configured (codex round-2), matching validatePaginationFlags ordering // in `test list` / `project list`. if (opts.pageSize !== undefined) { - if (!Number.isFinite(opts.pageSize) || opts.pageSize < 1 || opts.pageSize > 100) { + if (!Number.isFinite(opts.pageSize) || !Number.isInteger(opts.pageSize)) { + throw localValidationError('page-size', 'must be an integer between 1 and 100'); + } + if (opts.pageSize < 1 || opts.pageSize > 100) { throw localValidationError('page-size', 'must be between 1 and 100'); } } + if (opts.output === 'text') { + resolveTextColumns(opts.columns, RUN_HISTORY_TABLE_COLUMNS); + } const client = makeClient(opts, deps); const pageSize = opts.pageSize ?? 20; @@ -3852,7 +4602,7 @@ export async function runResultHistory( since: sinceIso, }); - if (opts.output === 'json') { + if (opts.output !== 'text') { out.print({ runs: resp.runs, nextCursor: resp.nextCursor }, data => JSON.stringify(data)); return resp; } @@ -3889,7 +4639,7 @@ export async function runResultHistory( } const lines: string[] = []; - lines.push(renderRunHistoryTable(resp.runs)); + lines.push(renderRunHistoryTable(resp.runs, { columns: opts.columns, noHeader: opts.noHeader })); // Footer: pointer to per-run detail commands. lines.push(''); @@ -3916,13 +4666,18 @@ export async function runResultHistory( return resp; } -const RUN_HISTORY_TABLE_COL_WIDTHS = { - runId: 36, - status: 10, - source: 18, - rerun: 6, - when: 25, -}; +const RUN_HISTORY_TABLE_COLUMNS: ReadonlyArray> = [ + { header: 'RUN ID', width: 36, render: run => run.runId }, + { header: 'STATUS', width: 10, render: run => run.status }, + { header: 'SOURCE', width: 18, render: run => run.source }, + { header: 'RERUN?', width: 6, render: run => (run.isRerun ? 'yes' : 'no') }, + { header: 'WHEN', width: 25, render: run => run.createdAt }, + { + header: 'DURATION', + width: 0, + render: run => formatDurationMs(run.startedAt ?? run.createdAt, run.finishedAt), + }, +]; /** * Max width of the `test steps` DESCRIPTION column in text mode. Long / @@ -3932,6 +4687,14 @@ const RUN_HISTORY_TABLE_COL_WIDTHS = { */ const DESC_COL_MAX = 60; +/** + * Cap, in chars, for the one-line `error:` sub-line under a failed step row + * in `renderStepsText`. Long enough for a full assertion message, short + * enough that a stack-trace blob can't flood the table. Full text is in + * `--output json`. + */ +const ERROR_SUBLINE_MAX = 200; + /** Max chars to show in the TARGETURL sub-line (excess truncated with …). */ const HISTORY_TARGET_URL_MAX = 80; @@ -3950,60 +4713,41 @@ const HISTORY_TARGET_URL_MAX = 80; * run row (truncated to `HISTORY_TARGET_URL_MAX` chars). The table columns * are left intact to avoid width blow-out on terminals. */ -function renderRunHistoryTable(runs: RunHistoryItem[]): string { - const cols = RUN_HISTORY_TABLE_COL_WIDTHS; - const header = [ - padEnd('RUN ID', cols.runId), - padEnd('STATUS', cols.status), - padEnd('SOURCE', cols.source), - padEnd('RERUN?', cols.rerun), - padEnd('WHEN', cols.when), - 'DURATION', - ].join(' '); - const sep = '-'.repeat(header.length); - - const rows = runs.flatMap(r => { - // FE runs never populate `startedAt` today — the RUNNING heartbeat - // that would set it doesn't fire on the legacy/sync execution path - // (dogfood 2026-06-04), so without a fallback DURATION was always - // "—" for every FE run. Fall back to `createdAt` so the column shows - // wall-clock from trigger to finish; on sync dev the queue gap is - // ~0, and `--output json` still exposes raw startedAt/finishedAt for - // consumers that need to exclude queue time. - const duration = formatDurationMs(r.startedAt ?? r.createdAt, r.finishedAt); - const mainRow = [ - padEnd(r.runId, cols.runId), - padEnd(r.status, cols.status), - padEnd(r.source, cols.source), - padEnd(r.isRerun ? 'yes' : 'no', cols.rerun), - padEnd(r.createdAt, cols.when), - duration, - ].join(' '); +function renderRunHistoryTable( + runs: RunHistoryItem[], + options: { columns?: string; noHeader?: boolean } = {}, +): string { + const selectedColumns = resolveTextColumns(options.columns, RUN_HISTORY_TABLE_COLUMNS); + const widths = measureTextColumns(runs, selectedColumns); + const customColumns = options.columns !== undefined && options.columns.trim() !== ''; + const includeDetailLines = !customColumns; + const header = formatTextTableRow( + selectedColumns.map(column => column.header), + widths, + ); + + const rows = runs.flatMap(run => { + const mainRow = formatTextTableRow( + selectedColumns.map(column => column.render(run)), + widths, + ); - // G1b: surface per-run targetUrl as an indented sub-line. - // Render only when truthy (skip null, undefined, empty) and when the - // source is not 'unresolved' (that would mean "backend couldn't resolve - // a URL" — printing "—" is less informative than omitting the line). const lines: string[] = [mainRow]; - if (r.targetUrl && r.targetUrlSource !== 'unresolved') { + if (includeDetailLines && run.targetUrl && run.targetUrlSource !== 'unresolved') { const url = - r.targetUrl.length > HISTORY_TARGET_URL_MAX - ? `${r.targetUrl.slice(0, HISTORY_TARGET_URL_MAX - 1)}…` - : r.targetUrl; + run.targetUrl.length > HISTORY_TARGET_URL_MAX + ? `${run.targetUrl.slice(0, HISTORY_TARGET_URL_MAX - 1)}…` + : run.targetUrl; lines.push(` targetUrl: ${url}`); - } else if (r.targetUrlSource === 'unresolved') { + } else if (includeDetailLines && run.targetUrlSource === 'unresolved') { lines.push(` targetUrl: —`); } return lines; }); - return [header, sep, ...rows].join('\n'); -} - -function padEnd(s: string, width: number): string { - if (s.length >= width) return s; - return s + ' '.repeat(width - s.length); + if (options.noHeader === true) return rows.join('\n'); + return [header, '-'.repeat(header.length), ...rows].join('\n'); } function formatDurationMs(startedAt: string | null, finishedAt: string | null): string { @@ -4281,6 +5025,12 @@ interface RunTestRerunOptions extends CommonOptions { * filters. Client-side only. */ nameFilter?: string; + /** --report junit: write a JUnit XML sidecar after batch --wait completes. */ + report?: JUnitReportFormat; + /** --report-file: destination path for the JUnit XML artifact. */ + reportFile?: string; + /** --report-suite-name: optional override for the JUnit . */ + reportSuiteName?: string; } /** @@ -4546,10 +5296,40 @@ function renderRunResponseText( } /** - * Render a `TriggerRunResponse` (no-wait path) to human-readable text. + * Best-effort resolve whether a finished run's test is a backend test, for the + * text run-card's step line + failure hint only (DEV-282). Returns true with no + * network call when already known (create-chain `--type`, or the BE wait + * fallback fired). Otherwise, in TEXT mode only, issues one `GET /tests/{id}`; + * any error → false (render the numeric step summary as before). JSON mode + * never probes — its envelope carries `stepSummary` verbatim and has no + * "n/a (backend)" concept, so it is already correct. + * + * This closes the standalone-path gap: `test run ` / `test wait ` + * never supply a type hint, and now that BE run rows finalize server-side + * (backend-v2.0 #551/#555) the wait fallback rarely fires — so a backend run + * card would otherwise show a misleading `steps 0/0 (passed=0, failed=0)`. */ -function renderTriggerRunText(r: TriggerRunResponse): string { - // P2-9: omit null-valued fields to avoid printing literal "null". +async function resolveRunCardIsBackend( + client: ResultReadClient, + testId: string | undefined, + output: string, + alreadyKnown: boolean, +): Promise { + if (alreadyKnown) return true; + if (output === 'json' || !testId) return false; + try { + const test = await client.get(`/tests/${encodeURIComponent(testId)}`); + return test.type === 'backend'; + } catch { + return false; // best-effort — fall back to the numeric step summary. + } +} + +/** + * Render a `TriggerRunResponse` (no-wait path) to human-readable text. + */ +function renderTriggerRunText(r: TriggerRunResponse): string { + // P2-9: omit null-valued fields to avoid printing literal "null". const lines: string[] = [ `runId ${r.runId}`, `status ${r.status}`, @@ -4704,7 +5484,8 @@ export async function runTestRun( stderrFn( `[advisory] Run already in flight (runId: ${currentRunId}, ` + `target: ${inFlightRun.targetUrl}). ` + - `Attaching to that run's --wait poll instead of creating a new one.`, + `Attaching to that run's --wait poll instead of creating a new one. ` + + `To stop it instead: testsprite test cancel ${currentRunId}`, ); triggerResponse = { runId: currentRunId, @@ -4730,12 +5511,16 @@ export async function runTestRun( // Auto-resume but emit a stronger advisory so the caller is aware // they are attaching to the project default. + // SIG-9 (DEV-331 final): the in-flight run is NOT auto-cancelled — + // name the real `test cancel` command instead of the old (false) + // "cancel with Ctrl-C" claim. stderrFn( `[advisory] Run already in flight (runId: ${currentRunId}` + (inFlightTargetUrl ? `, target: ${inFlightTargetUrl}` : '') + `). Auto-resuming wait on in-flight run. ` + - `If you needed a specific target URL, cancel with Ctrl-C and ` + - `re-trigger with --target-url.`, + `If you needed a specific target URL, cancel it with ` + + `testsprite test cancel ${currentRunId}, or re-trigger with ` + + `--target-url after it finishes.`, ); triggerResponse = { runId: currentRunId, @@ -4804,6 +5589,7 @@ export async function runTestRun( finalRun = await pollRunUntilTerminal(client, triggerResponse.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -4817,11 +5603,32 @@ export async function runTestRun( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${triggerResponse.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, timeoutPartial, opts.createContext, data => { + const p = data as typeof timeoutPartial; + const lines = [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + ]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); + return lines.join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${triggerResponse.runId}.`, - nextAction: `Resume polling: testsprite test wait ${triggerResponse.runId}`, + nextAction: `Resume polling: testsprite test wait ${triggerResponse.runId}, or cancel it: testsprite test cancel ${triggerResponse.runId}`, requestId: 'local', details: { runId: triggerResponse.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -4848,14 +5655,39 @@ export async function runTestRun( const lines = [`runId ${p.runId}`, `status ${p.status} (request timed out)`]; if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); return lines.join('\n'); }); stderrFn( `Run ${triggerResponse.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${triggerResponse.runId}`, + `Re-attach with: testsprite test wait ${triggerResponse.runId}, or cancel with: testsprite test cancel ${triggerResponse.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1): same partial- + // envelope shape as the timeout paths so stdout stays parseable, plus the + // honest "keeps running and billing" stderr line. Rethrow → index.ts + // renders the INTERRUPTED envelope and exits 128+signum. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${triggerResponse.runId} — interrupted (${err.signal})`); + const partial = { + runId: triggerResponse.runId, + status: 'running' as const, + enqueuedAt: triggerResponse.enqueuedAt, + codeVersion: triggerResponse.codeVersion, + targetUrl: triggerResponse.targetUrl || null, + }; + printRunOrChain(out, partial, opts.createContext, data => { + const p = data as typeof partial; + const lines = [`runId ${p.runId}`, `status ${p.status} (interrupted)`]; + if (p.targetUrl) lines.push(`targetUrl ${p.targetUrl}`); + lines.push(`hint Re-attach with: testsprite test wait ${p.runId}`); + lines.push(`hint Cancel with: testsprite test cancel ${p.runId}`); + return lines.join('\n'); + }); + stderrFn(interruptDetachMessage(err, [triggerResponse.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -4866,16 +5698,21 @@ export async function runTestRun( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, ); + // BE detection: type hint (create-chain) OR beFallbackUsed (slow runs); on + // the standalone `test run ` path neither is set, so probe the test type + // once (text mode only, best-effort) — DEV-282. + const isBackend = await resolveRunCardIsBackend( + client, + opts.testId, + opts.output, + beFallbackUsed || opts.type === 'backend', + ); + printRunOrChain( out, withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), opts.createContext, - data => - renderRunResponseText(data as RunResponse, { - // BE detection: type hint (create-chain) OR beFallbackUsed (slow runs). - // This ensures fast BE runs terminal on first poll still render n/a. - isBackend: beFallbackUsed || opts.type === 'backend', - }), + data => renderRunResponseText(data as RunResponse, { isBackend }), ); // Surface the trigger requestId under --verbose/--debug or JSON mode so @@ -4885,10 +5722,10 @@ export async function runTestRun( stderrFn(`requestId: ${triggerRequestId}`); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { - // BE runs (resolved via the testId fallback) have no run-scoped artifact - // bundle — their failure bundle is addressed by testId, not runId. + // BE runs have no run-scoped artifact bundle — their failure bundle is + // addressed by testId, not runId. stderrFn( - beFallbackUsed + isBackend ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${finalRun.testId}' to download the bundle.` : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); @@ -4904,6 +5741,412 @@ export async function runTestRun( return finalRun; } +/** One row of the `test wait ` multi-run payload. */ +export interface CliMultiWaitResult { + runId: string; + /** Terminal run status, or 'timeout', or 'error:' when the poll failed. */ + status: string; + /** Test the run belongs to, when the poll observed it. */ + testId?: string; +} + +export interface RunTestWaitManyOptions extends CommonOptions { + runIds: string[]; + timeoutSeconds: number; + maxConcurrency: number; +} + +/** + * `test wait ` with two or more ids: attach to N already-dispatched + * runs in ONE invocation. This closes the loop the CLI itself opens: every + * batch/closure timeout prints one `testsprite test wait ` hint PER + * member, which previously meant N sequential blocking invocations. The runs + * are polled concurrently under a bounded pool with ONE shared deadline + * (`--timeout` bounds the whole invocation, not each member), each member's + * poll is total (a transient error on one run never discards the others), and + * the exit code is the worst status across members: auth errors escalate to + * exit 3, any timeout or poll error exits 7, any non-passed terminal exits 1. + * Distinct from a run journal (issue #80): no persistence, just N known ids. + */ +export async function runTestWaitMany( + opts: RunTestWaitManyOptions, + deps: TestDeps = {}, +): Promise<{ results: CliMultiWaitResult[]; summary: Record }> { + const out = makeOutput(opts.output, deps); + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + const results: CliMultiWaitResult[] = opts.runIds.map(runId => ({ + runId, + status: 'passed', + })); + const payload = { + results, + summary: { total: results.length, passed: results.length, failed: 0, timedOut: 0, errors: 0 }, + }; + out.print(payload, () => results.map(r => `${r.runId} ${r.status}`).join('\n')); + return payload; + } + + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + + // One shared deadline across every member (the whole point of the shared + // pool: `--timeout 600` means the invocation ends within ~600s, not + // 600s x ceil(N/concurrency)). + const deadlineMs = Date.now() + opts.timeoutSeconds * 1000; + + type WaitOutcome = + | { kind: 'result'; run: RunResponse } + | { kind: 'timeout' } + | { kind: 'error'; code: string; exitCode: number }; + + const pollOne = async (runId: string): Promise => { + // A member dequeued AFTER the shared deadline has passed must not be + // granted a fresh minimum poll window (with --max-concurrency 1 that + // would extend the invocation by ~1s per queued run past --timeout). + const remainingSeconds = Math.ceil((deadlineMs - Date.now()) / 1000); + if (remainingSeconds <= 0) return { kind: 'timeout' }; + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: run => run.testId, + resolveNotBefore: run => run.createdAt, + onResolved: () => undefined, + }); + try { + const run = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: remainingSeconds, + sleep: deps.sleep, + shutdown: shutdownOf(deps), + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + onTick: (run, elapsedMs) => { + const elapsed = Math.round(elapsedMs / 1000); + ticker.update(`Run ${run.runId} — ${run.status} (elapsed=${elapsed}s)`); + }, + resolveAlternate, + }); + return { kind: 'result', run }; + } catch (err) { + if (err instanceof TimeoutError) return { kind: 'timeout' }; + if (err instanceof RequestTimeoutError) throw err; + // Interrupt must reject the fan-out (handled at the collect point), not + // be flattened into a per-member 'error' outcome that would swallow the + // 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; + if (err instanceof ApiError) return { kind: 'error', code: err.code, exitCode: err.exitCode }; + return { kind: 'error', code: 'TRANSPORT', exitCode: 10 }; + } + }; + + const outcomes = new Map(); + let inFlight = 0; + let nextIdx = 0; + try { + await new Promise((resolve, reject) => { + const startNext = (): void => { + while (inFlight < opts.maxConcurrency && nextIdx < opts.runIds.length) { + const runId = opts.runIds[nextIdx++]!; + inFlight++; + pollOne(runId) + .then(outcome => { + outcomes.set(runId, outcome); + inFlight--; + startNext(); + if (inFlight === 0 && nextIdx >= opts.runIds.length) resolve(); + }) + // pollOne is total except for RequestTimeoutError (handled below). + .catch(reject); + } + }; + startNext(); + if (opts.runIds.length === 0) resolve(); + }); + } catch (fanOutErr) { + if (fanOutErr instanceof RequestTimeoutError || fanOutErr instanceof InterruptError) { + // Same contract as the batch pollers: leave stdout parseable before + // exiting. Members that already settled keep their real status; only + // the still-unfinished ids are marked running and named in the hint + // (re-attaching to an already-terminal run would be a wasted command). + ticker.finalize( + fanOutErr instanceof InterruptError + ? `Multi-run wait — interrupted (${fanOutErr.signal})` + : 'Multi-run wait — request timed out', + ); + const partial = { + results: opts.runIds.map((runId): CliMultiWaitResult => { + const outcome = outcomes.get(runId); + if (outcome === undefined) return { runId, status: 'running' }; + if (outcome.kind === 'timeout') return { runId, status: 'timeout' }; + if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; + return { runId, status: outcome.run.status, testId: outcome.run.testId }; + }), + summary: { total: opts.runIds.length }, + }; + out.print(partial, () => partial.results.map(r => `${r.runId} ${r.status}`).join('\n')); + const unfinished = partial.results + .filter(r => r.status === 'running' || r.status === 'timeout') + .map(r => r.runId); + if (unfinished.length > 0) { + if (fanOutErr instanceof InterruptError) { + stderrFn(interruptDetachMessage(fanOutErr, unfinished)); + } else { + stderrFn( + `Re-attach with: testsprite test wait ${unfinished.join(' ')}, or cancel with: testsprite test cancel ${unfinished.join(' ')}`, + ); + } + } + } + throw fanOutErr; + } + ticker.finalize(); + + const results: CliMultiWaitResult[] = opts.runIds.map(runId => { + const outcome = outcomes.get(runId); + if (outcome === undefined || outcome.kind === 'timeout') return { runId, status: 'timeout' }; + if (outcome.kind === 'error') return { runId, status: `error:${outcome.code}` }; + return { runId, status: outcome.run.status, testId: outcome.run.testId }; + }); + const passed = results.filter(r => r.status === 'passed').length; + const timedOut = results.filter(r => r.status === 'timeout').length; + const errors = results.filter(r => r.status.startsWith('error:')).length; + const failed = results.length - passed - timedOut - errors; + const payload = { + results, + summary: { total: results.length, passed, failed, timedOut, errors }, + }; + out.print(payload, () => + [ + ...results.map(r => `${r.runId} ${r.status}`), + '', + `${passed}/${results.length} passed, ${failed} failed/blocked, ${timedOut} timed out, ${errors} poll errors`, + ].join('\n'), + ); + + // Every member that did not reach a terminal verdict is re-attachable: + // timeouts (still running server-side) and poll errors (e.g. a transient + // transport failure) both belong in the hint; terminal runs do not. + const unfinishedIds = results + .filter(r => r.status === 'timeout' || r.status.startsWith('error:')) + .map(r => r.runId); + if (unfinishedIds.length > 0) { + stderrFn( + `Re-attach with: testsprite test wait ${unfinishedIds.join(' ')}, or cancel with: testsprite test cancel ${unfinishedIds.join(' ')}`, + ); + } + + // Worst-status exit: auth escalates (a rejected key fails every member the + // same way), then timeout/poll-error (7, resumable), then plain failure (1). + const authError = [...outcomes.values()].find( + o => + o.kind === 'error' && + (o.code === 'AUTH_REQUIRED' || o.code === 'AUTH_INVALID' || o.code === 'AUTH_FORBIDDEN'), + ); + if (authError !== undefined && authError.kind === 'error') { + throw new CLIError( + `Multi-run wait: authentication failed (${authError.code})`, + authError.exitCode, + ); + } + if (timedOut > 0 || errors > 0) { + throw new CLIError( + `Multi-run wait: ${timedOut} timed out, ${errors} poll error(s) out of ${results.length} runs`, + 7, + ); + } + if (failed > 0) { + throw new CLIError(`Multi-run wait: ${failed} run(s) finished non-passed`, 1); + } + return payload; +} + +// --------------------------------------------------------------------------- +// DEV-331 piece 3 — `test cancel ` +// --------------------------------------------------------------------------- + +export interface RunTestCancelOptions extends CommonOptions { + runIds: string[]; +} + +/** One run's outcome in a multi-id `test cancel` summary. */ +export interface CliCancelResultRow { + runId: string; + status: 'cancelled' | 'alreadyCancelled' | 'conflict' | 'notFound' | 'error'; + /** Terminal status the run was already in, for a `conflict` row. */ + conflictStatus?: string; + error?: string; +} + +/** JSON payload for a multi-id `test cancel` — per piece-3 CXL-11. */ +export interface CliCancelSummary { + cancelled: string[]; + alreadyCancelled: string[]; + conflicts: Array<{ runId: string; status: string }>; + notFound: string[]; + /** + * Runs that errored for a reason other than 404/409 (e.g. auth, transport). + * Always present — an empty array on full success — so machine consumers + * can rely on a stable shape (DEV-331 codex finding 2). + */ + errors: Array<{ runId: string; message: string }>; +} + +/** + * `test cancel ` — DEV-331 piece 3. + * + * User-initiated cancel of one or more queued/running runs via + * `POST /api/cli/v1/runs/{runId}/cancel`. Naturally idempotent (D10): a + * repeat cancel of the same run is a 200 `alreadyCancelled` success, not an + * error. Dispatches serially (per-id volume is tiny — no batch endpoint, + * D8 "Batch-level cancel endpoint... YAGNI"). + * + * Single id: renders the returned run card (`status cancelled`); an + * `alreadyCancelled` response adds an `[advisory]` line. Exit code mirrors + * the server response directly — 0 on success (fresh or already-cancelled), + * 4 on 404 (unknown/cross-tenant), 6 on 409 (already terminal). + * + * Multi-id: prints a `{cancelled, alreadyCancelled, conflicts, notFound}` + * summary. Exit precedence (CXL-11): any `notFound` → 4 (outranks conflict — + * it signals a caller bug, wrong id/tenant); else any `conflicts` → 6; else 0. + */ +export async function runTestCancel( + opts: RunTestCancelOptions, + deps: TestDeps = {}, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + const client = makeClient(opts, deps); + + if (opts.dryRun) { + emitDryRunBanner(stderrFn); + } + + if (opts.runIds.length === 1) { + const runId = opts.runIds[0]!; + const result = await client.cancelRun(runId); + out.print(result, data => { + const r = data as CancelRunResponse; + return renderRunResponseText(r); + }); + if (result.alreadyCancelled) { + stderrFn(`[advisory] run ${runId} was already cancelled`); + } + return result; + } + + const rows: CliCancelResultRow[] = []; + for (const runId of opts.runIds) { + try { + const result = await client.cancelRun(runId); + rows.push({ + runId, + status: result.alreadyCancelled ? 'alreadyCancelled' : 'cancelled', + }); + } catch (err) { + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + rows.push({ runId, status: 'notFound' }); + continue; + } + if (err instanceof ApiError && err.code === 'CONFLICT') { + const conflictStatus = + err.getDetail('status', (v): v is string => typeof v === 'string') ?? 'unknown'; + rows.push({ runId, status: 'conflict', conflictStatus }); + continue; + } + const message = err instanceof Error ? err.message : String(err); + rows.push({ runId, status: 'error', error: message }); + } + } + + const errorRows = rows.filter(r => r.status === 'error'); + const summary: CliCancelSummary = { + cancelled: rows.filter(r => r.status === 'cancelled').map(r => r.runId), + alreadyCancelled: rows.filter(r => r.status === 'alreadyCancelled').map(r => r.runId), + conflicts: rows + .filter(r => r.status === 'conflict') + .map(r => ({ runId: r.runId, status: r.conflictStatus ?? 'unknown' })), + notFound: rows.filter(r => r.status === 'notFound').map(r => r.runId), + errors: errorRows.map(r => ({ runId: r.runId, message: r.error ?? 'unknown error' })), + }; + + out.print(summary, data => renderCancelSummaryText(data as CliCancelSummary)); + + const parts = [ + `${summary.cancelled.length} cancelled`, + `${summary.alreadyCancelled.length} already cancelled`, + ]; + if (summary.conflicts.length > 0) parts.push(`${summary.conflicts.length} conflict`); + if (summary.notFound.length > 0) parts.push(`${summary.notFound.length} not found`); + if (errorRows.length > 0) parts.push(`${errorRows.length} error`); + stderrFn(`Cancel summary: ${parts.join(', ')}.`); + + // Exit precedence (CXL-11): notFound outranks conflict — a caller-side bug + // (wrong id / wrong tenant) is more actionable to surface than "it already + // finished". A bare transport/auth error on any member also fails loudly + // rather than being silently absorbed into a 0 exit. + if (summary.notFound.length > 0) { + throw new CLIError( + `${summary.notFound.length} run id${summary.notFound.length !== 1 ? 's' : ''} not found: ${summary.notFound.join(' ')}`, + 4, + ); + } + if (errorRows.length > 0) { + throw new CLIError( + `${errorRows.length} cancel request${errorRows.length !== 1 ? 's' : ''} failed: ${errorRows.map(r => r.runId).join(' ')}`, + 1, + ); + } + if (summary.conflicts.length > 0) { + throw new CLIError( + `${summary.conflicts.length} run${summary.conflicts.length !== 1 ? 's' : ''} already terminal: ${summary.conflicts.map(c => `${c.runId} (${c.status})`).join(', ')}`, + 6, + ); + } + return summary; +} + +function renderCancelSummaryText(summary: CliCancelSummary): string { + const lines: string[] = []; + for (const runId of summary.cancelled) lines.push(`${runId} cancelled`); + for (const runId of summary.alreadyCancelled) lines.push(`${runId} alreadyCancelled`); + for (const c of summary.conflicts) lines.push(`${c.runId} conflict (${c.status})`); + for (const runId of summary.notFound) lines.push(`${runId} notFound`); + for (const e of summary.errors ?? []) lines.push(`${e.runId} error (${e.message})`); + return lines.join('\n'); +} + +export function createTestCancelCommand(deps: TestDeps): Command { + const cancel = new Command('cancel'); + cancel + .argument('', 'one or more run ids to cancel') + .description( + 'Cancel one or more queued/running runs.\n' + + '\nCtrl-C during --wait only detaches — it does NOT cancel the server-side\n' + + 'run. This is the real stop button. No refund is issued for the credits\n' + + 'already charged at trigger time (D3); an in-flight Lambda finishes on its\n' + + 'own and its result is discarded once cancelled.\n' + + '\nExit codes:\n' + + ' 0 cancelled (fresh or already-cancelled — naturally idempotent)\n' + + ' 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict)\n' + + ' 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict\n' + + '\nMulti-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (runIds: string[], _cmdOpts: unknown, command: Command) => { + await runTestCancel( + { + ...resolveCommonOptions(command), + runIds, + }, + deps, + ); + }); + return cancel; +} + /** * `test wait ` — M3.3 piece-3. * @@ -4964,6 +6207,7 @@ export async function runTestWait( finalRun = await pollRunUntilTerminal(client, opts.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -4977,11 +6221,24 @@ export async function runTestWait( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${opts.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { runId: opts.runId, status: 'running' as const }; + out.print(timeoutPartial, data => { + const p = data as typeof timeoutPartial; + return [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', // exit 7 per errors.md message: `Timed out after ${opts.timeoutSeconds}s waiting for run ${opts.runId}.`, - nextAction: `Resume polling: testsprite test wait ${opts.runId}`, + nextAction: `Resume polling: testsprite test wait ${opts.runId}, or cancel it: testsprite test cancel ${opts.runId}`, requestId: 'local', details: { runId: opts.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -4999,14 +6256,31 @@ export async function runTestWait( `runId ${p.runId}`, `status ${p.status} (request timed out)`, `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, ].join('\n'); }); stderrFn( `Run ${opts.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${opts.runId}`, + `Re-attach with: testsprite test wait ${opts.runId}, or cancel with: testsprite test cancel ${opts.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1) — see runTestRun. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${opts.runId} — interrupted (${err.signal})`); + const partial = { runId: opts.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (interrupted)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); + stderrFn(interruptDetachMessage(err, [opts.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -5017,15 +6291,24 @@ export async function runTestWait( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps elapsed=${elapsed}s)`, ); + // `test wait` has no type hint; probe the test type once (text mode only, + // best-effort) so a backend run's card reads `n/a (backend)` — DEV-282. + const isBackend = await resolveRunCardIsBackend( + client, + finalRun.testId, + opts.output, + beFallbackUsed, + ); + out.print(withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), data => - renderRunResponseText(data as RunResponse, { isBackend: beFallbackUsed }), + renderRunResponseText(data as RunResponse, { isBackend }), ); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { - // BE runs (resolved via the testId fallback) have no run-scoped artifact - // bundle — their failure bundle is addressed by testId, not runId. + // BE runs have no run-scoped artifact bundle — their failure bundle is + // addressed by testId, not runId. stderrFn( - beFallbackUsed + isBackend ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${finalRun.testId}' to download the bundle.` : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); @@ -5044,7 +6327,7 @@ export async function runTestWait( // --------------------------------------------------------------------------- interface RunTestRunAllOptions extends CommonOptions { - /** projectId to run all BE tests in. */ + /** projectId to run all tests in. */ projectId: string; /** --filter : only run tests whose name contains this substring (case-insensitive). */ nameFilter?: string; @@ -5056,6 +6339,32 @@ interface RunTestRunAllOptions extends CommonOptions { maxConcurrency: number; /** Caller-supplied idempotency token; auto-minted if absent. */ idempotencyKey?: string; + /** --report junit: write a JUnit XML sidecar after batch --wait completes. */ + report?: JUnitReportFormat; + /** --report-file: destination path for the JUnit XML artifact. */ + reportFile?: string; + /** --report-suite-name: optional override for the JUnit . */ + reportSuiteName?: string; +} + +async function writeBatchJUnitReportIfRequested( + opts: { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + projectId?: string; + }, + results: readonly JUnitTestResult[], +): Promise { + if (opts.report !== 'junit' || opts.reportFile === undefined) return; + const projectId = resolveBatchReportProjectId(opts, results); + const suiteName = opts.reportSuiteName ?? `testsprite:${projectId}`; + const xml = buildJUnitReport({ + suiteName, + classname: projectId, + results, + }); + await writeJUnitReportFile(opts.reportFile, xml); } /** @@ -5064,6 +6373,8 @@ interface RunTestRunAllOptions extends CommonOptions { interface CliBatchRunFreshResult { testId: string; runId: string | undefined; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; status: string; error?: { code: string; message: string; exitCode: number }; /** CLIENT-synthesized Portal deep link (projectId from opts, testId per item). */ @@ -5090,6 +6401,13 @@ export async function runTestRunAll( ) { throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); } + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: true, + }); const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const out = makeOutput(opts.output, deps); @@ -5113,6 +6431,12 @@ export async function runTestRunAll( idempotencyKey, ...(opts.wait ? { thenPoll: '/api/cli/v1/runs/?waitSeconds=25' } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(opts.projectId, opts.reportSuiteName), + ); + } out.print(batchRunSample ?? envelope); return undefined; } @@ -5136,14 +6460,11 @@ export async function runTestRunAll( }; const idempotencyKey = opts.idempotencyKey ?? `cli-batch-run-fresh-${randomUUID()}`; - if (opts.idempotencyKey === undefined && opts.debug) { + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderrFn(`idempotency-key: ${idempotencyKey}`); } - if (opts.idempotencyKey === undefined && opts.verbose) { - stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); - } - // Resolve testIds: fetch all BE tests in the project, apply --filter. + // Resolve testIds: fetch all tests in the project, apply --filter. let testIds: string[] | undefined; if (opts.nameFilter !== undefined && opts.nameFilter !== '') { // We need to resolve the full test set to apply the name filter. @@ -5181,7 +6502,8 @@ export async function runTestRunAll( `Resolved ${testIds.length} test${testIds.length !== 1 ? 's' : ''} in project ${opts.projectId} for batch run.`, ); } - // When no --filter, omit testIds → server runs ALL BE tests in the project. + // When no --filter, omit testIds → server runs ALL tests in the project + // (BE tests on the legacy V2 wave engine; FE + BE on the V3 unified engine). const batchResp = await client.triggerBatchRunFresh( { @@ -5421,7 +6743,20 @@ export async function runTestRunAll( async function pollFreshAccepted(entry: BatchRunFreshAccepted): Promise { const runId = entry.runId; - const remainingSeconds = Math.max(1, Math.ceil((batchDeadlineMs - Date.now()) / 1000)); + const remainingMs = batchDeadlineMs - Date.now(); + if (remainingMs <= 0) { + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: `Timed out after ${opts.timeoutSeconds}s`, + exitCode: 7, + }, + }; + } + const remainingSeconds = Math.ceil(remainingMs / 1000); const resolveAlternate = makeBackendWaitFallback({ client, resolveTestId: () => entry.testId, @@ -5432,6 +6767,7 @@ export async function runTestRunAll( const finalRun = await pollRunUntilTerminal(client, runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -5442,7 +6778,12 @@ export async function runTestRunAll( }, resolveAlternate, }); - return { testId: entry.testId, runId, status: finalRun.status }; + return { + testId: entry.testId, + runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -5456,6 +6797,26 @@ export async function runTestRunAll( }, }; } + // Interrupt must reject the fan-out (the collect point below prints the + // partial for every dispatched run), never flatten into a per-member + // outcome that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7) so the fan-out completes and stdout carries every runId. + // Without this, RequestTimeoutError rejects the fan-out before out.print(), + // leaving JSON consumers with empty stdout (mirrors create-batch --run). + return { + testId: entry.testId, + runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: err.message, + exitCode: err.exitCode, + }, + }; + } if (err instanceof ApiError) { // Preserve the real exit code + envelope (AUTH_INVALID=3, NOT_FOUND=4, // RATE_LIMITED=11, …) instead of flattening every member failure to 1 @@ -5475,24 +6836,45 @@ export async function runTestRunAll( let pollIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && pollIdx < pollable.length) { - const entry = pollable[pollIdx++]!; - inFlight++; - pollFreshAccepted(entry) - .then(result => { - freshRunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && pollIdx >= pollable.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && pollIdx < pollable.length) { + const entry = pollable[pollIdx++]!; + inFlight++; + pollFreshAccepted(entry) + .then(result => { + freshRunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && pollIdx >= pollable.length) resolve(); + }) + .catch(reject); + } } + startNext(); + if (pollable.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): stdout stays parseable — settled members + // keep their real status, unfinished ones are marked running — and the + // honest stderr line names every runId still executing (and billing). + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Batch run — interrupted (${fanOutErr.signal})`); + const settled = new Map(freshRunResults.map(r => [r.runId, r] as const)); + const partialResults = pollable.map( + (e): CliBatchRunFreshResult => + settled.get(e.runId) ?? { testId: e.testId, runId: e.runId, status: 'running' }, + ); + out.print( + { accepted: partialResults, conflicts, deferred, skippedFrontend, skippedIntegration }, + () => partialResults.map(r => `${r.runId} ${r.status}`).join('\n'), + ); + const unfinished = pollable.filter(e => !settled.has(e.runId)).map(e => e.runId); + if (unfinished.length > 0) stderrFn(interruptDetachMessage(fanOutErr, unfinished)); } - startNext(); - if (pollable.length === 0) resolve(); - }); + throw fanOutErr; + } ticker.finalize(); @@ -5524,6 +6906,7 @@ export async function runTestRunAll( total: pollable.length, }, }; + await writeBatchJUnitReportIfRequested(opts, freshRunResults); out.print(jsonPayload); // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7), @@ -5545,7 +6928,10 @@ export async function runTestRunAll( error: { code: 'UNSUPPORTED', message: `${timedOut} run${timedOut !== 1 ? 's' : ''} timed out.`, - nextAction: timedOutRunIds.map(rid => `Resume: testsprite test wait ${rid}`).join('\n'), + nextAction: [ + ...timedOutRunIds.map(rid => `Resume: testsprite test wait ${rid}`), + ...timedOutRunIds.map(rid => `Cancel: testsprite test cancel ${rid}`), + ].join('\n'), requestId: 'local', details: { timedOutRunIds, timeoutSeconds: opts.timeoutSeconds }, }, @@ -5586,6 +6972,8 @@ export async function runTestRunAll( interface CliRerunResult { testId: string; runId: string; + /** Observed on polled runs; used for JUnit report naming when --project omitted. */ + projectId?: string; /** Terminal status, or 'timeout' for per-run deadline exceeded. */ status: string; /** Set when the test is a closure member (not the user's named test). */ @@ -5628,6 +7016,18 @@ export async function runTestRerun( 'provide at least one , or use --all to rerun all tests in the project', ); } + // Explicit ids + --all is ambiguous: the --all branch resolves the FULL + // project test set and overwrites the listed ids, so the user's narrowing + // intent would be silently replaced by a whole-project batch rerun — + // burning rerun/auto-heal credits. Reject early. (Mirrors `test run`'s + // positional+--all guard and delete-batch's ids+--all data-loss guard.) + if (opts.all && opts.testIds.length > 0) { + throw localValidationError( + 'test-ids', + 'pass either explicit test IDs or --all, not both — --all reruns every test in the ' + + 'project and would ignore the listed IDs. Drop the IDs, or drop --all.', + ); + } if (opts.all && !opts.projectId) { throw localValidationError( 'project', @@ -5646,6 +7046,24 @@ export async function runTestRerun( 'Remove --filter, or add --all --project .', ); } + // --status and --skip-terminal are --all-only narrowing filters with the + // same silent-ignore failure mode as --filter above: without --all the + // explicit ids get reran unfiltered (and an invalid --status value is + // never even validated). Reject both, mirroring the --filter guard. + if (opts.statusFilter !== undefined && !opts.all) { + throw localValidationError( + 'status', + '--status only applies with --all (it narrows which project tests get reran). ' + + 'Remove --status, or add --all --project .', + ); + } + if (opts.skipTerminal && !opts.all) { + throw localValidationError( + 'skip-terminal', + '--skip-terminal only applies with --all (it narrows which project tests get reran). ' + + 'Remove --skip-terminal, or add --all --project .', + ); + } if ( !Number.isInteger(opts.maxConcurrency) || opts.maxConcurrency < 1 || @@ -5655,6 +7073,13 @@ export async function runTestRerun( } const isSingle = !opts.all && opts.testIds.length === 1; + assertJUnitReportOptions({ + report: opts.report, + reportFile: opts.reportFile, + reportSuiteName: opts.reportSuiteName, + wait: opts.wait, + batchPath: !isSingle, + }); // ------------------------------------------------------------------------- // Pre-flight: auto-heal + Free-tier hint (best-effort, non-blocking) @@ -5694,6 +7119,13 @@ export async function runTestRerun( idempotencyKey, ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), }; + if (opts.report === 'junit' && opts.reportFile !== undefined) { + const projectKey = resolveBatchReportProjectId(opts, []); + await writeJUnitReportFile( + opts.reportFile, + sampleJUnitReportXml(projectKey, opts.reportSuiteName), + ); + } out.print(findSample('POST', '/api/cli/v1/tests/batch/rerun')?.body() ?? envelope); } void client; @@ -5704,12 +7136,9 @@ export async function runTestRerun( // slow rerun trigger / long-poll under load isn't cut at the 120s default. const client = makeClient({ ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs(opts) }, deps); const idempotencyKey = opts.idempotencyKey ?? `cli-rerun-${randomUUID()}`; - if (opts.idempotencyKey === undefined && opts.debug) { + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { stderrFn(`idempotency-key: ${idempotencyKey}`); } - if (opts.idempotencyKey === undefined && opts.verbose) { - stderrFn(`[verbose] auto-minted idempotency-key: ${idempotencyKey}`); - } // ------------------------------------------------------------------------- // Single rerun path @@ -5890,6 +7319,7 @@ export async function runTestRerun( return await pollRunUntilTerminal(client, member.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -5985,8 +7415,35 @@ export async function runTestRerun( const reattachHints = closureMembers .map(m => `testsprite test wait ${m.runId}`) .join('\n'); + const cancelHints = closureMembers + .map(m => `testsprite test cancel ${m.runId}`) + .join('\n'); stderrFn( - `Closure members are still in progress (request timed out). Re-attach with:\n${reattachHints}`, + `Closure members are still in progress (request timed out). Re-attach with:\n${reattachHints}\n` + + `Or cancel with:\n${cancelHints}`, + ); + throw fanOutErr; + } + // Graceful detach (DEV-331): same partial shape as the timeout path — + // SIG-6 requires the partial to list ALL dispatched runIds. + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Closure fan-out — interrupted (${fanOutErr.signal})`); + const dispatchedRunIds = closureMembers.map(m => ({ + runId: m.runId, + testId: m.testId, + role: m.role, + status: 'running' as const, + })); + out.print({ runId: namedRunId, status: 'running', closure: dispatchedRunIds }, () => + dispatchedRunIds + .map(m => `${m.role.padEnd(9)} ${m.testId} (runId: ${m.runId}) — running`) + .join('\n'), + ); + stderrFn( + interruptDetachMessage( + fanOutErr, + closureMembers.map(m => m.runId), + ), ); throw fanOutErr; } @@ -6028,7 +7485,7 @@ export async function runTestRerun( error: { code: 'UNSUPPORTED', message: `Timed out after ${opts.timeoutSeconds}s waiting for rerun ${namedRunId}.`, - nextAction: `Resume polling: testsprite test wait ${namedRunId}`, + nextAction: `Resume polling: testsprite test wait ${namedRunId}, or cancel it: testsprite test cancel ${namedRunId}`, requestId: 'local', details: { runId: namedRunId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -6048,7 +7505,10 @@ export async function runTestRerun( // as a whole was never observed to reach terminal. const timedOutMembers = closureFailures.filter(f => f.status === 'timeout'); if (timedOutMembers.length > 0) { - const resumeHints = timedOutMembers.map(f => `testsprite test wait ${f.runId}`).join('\n'); + const timedOutIds = timedOutMembers.map(f => f.runId); + const resumeHints = + timedOutIds.map(runId => `testsprite test wait ${runId}`).join('\n') + + `\nCancel instead: testsprite test cancel ${timedOutIds.join(' ')}`; throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', @@ -6086,6 +7546,7 @@ export async function runTestRerun( finalRun = await pollRunUntilTerminal(client, rerunResp.runId, { timeoutSeconds: opts.timeoutSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -6099,11 +7560,23 @@ export async function runTestRerun( } catch (err) { if (err instanceof TimeoutError) { ticker.finalize(`Run ${rerunResp.runId} — timed out after ${opts.timeoutSeconds}s`); + // Mirror the RequestTimeoutError path: emit a partial run to stdout so + // JSON consumers and AI agents can grab the runId and chain into + // `testsprite test wait ` without parsing the stderr error envelope. + const timeoutPartial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(timeoutPartial, data => { + const p = data as typeof timeoutPartial; + return [ + `runId ${p.runId}`, + `status ${p.status} (timed out after ${opts.timeoutSeconds}s)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + ].join('\n'); + }); throw ApiError.fromEnvelope({ error: { code: 'UNSUPPORTED', message: `Timed out after ${opts.timeoutSeconds}s waiting for rerun ${rerunResp.runId}.`, - nextAction: `Resume polling: testsprite test wait ${rerunResp.runId}`, + nextAction: `Resume polling: testsprite test wait ${rerunResp.runId}, or cancel it: testsprite test cancel ${rerunResp.runId}`, requestId: 'local', details: { runId: rerunResp.runId, timeoutSeconds: opts.timeoutSeconds }, }, @@ -6120,14 +7593,31 @@ export async function runTestRerun( `runId ${p.runId}`, `status ${p.status} (request timed out)`, `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, ].join('\n'); }); stderrFn( `Run ${rerunResp.runId} is still in progress (request timed out). ` + - `Re-attach with: testsprite test wait ${rerunResp.runId}`, + `Re-attach with: testsprite test wait ${rerunResp.runId}, or cancel with: testsprite test cancel ${rerunResp.runId}`, ); throw err; } + // Graceful detach on SIGINT/SIGTERM (DEV-331 piece 1) — see runTestRun. + if (err instanceof InterruptError) { + ticker.finalize(`Run ${rerunResp.runId} — interrupted (${err.signal})`); + const partial = { runId: rerunResp.runId, status: 'running' as const }; + out.print(partial, data => { + const p = data as typeof partial; + return [ + `runId ${p.runId}`, + `status ${p.status} (interrupted)`, + `hint Re-attach with: testsprite test wait ${p.runId}`, + `hint Cancel with: testsprite test cancel ${p.runId}`, + ].join('\n'); + }); + stderrFn(interruptDetachMessage(err, [rerunResp.runId])); + throw err; + } ticker.finalize(); throw err; } @@ -6139,13 +7629,21 @@ export async function runTestRerun( `Run ${finalRun.runId} — ${finalRun.status} (${s.completed}/${s.total} steps replay)`, ); + // Probe the test type once (text mode only, best-effort) so a backend + // rerun's card reads `n/a (backend)` even when the fallback never fired + // (BE run rows finalize server-side now) — DEV-282. + const isBackend = await resolveRunCardIsBackend(client, testId, opts.output, beFallbackUsed); + out.print(withRunDashboardUrl(finalRun, resolveApiUrl(opts, deps)), data => - renderRunResponseText(data as RunResponse, { isBackend: beFallbackUsed }), + renderRunResponseText(data as RunResponse, { isBackend }), ); if (finalRun.status === 'failed' || finalRun.status === 'blocked') { + // BE reruns have no run-scoped artifact bundle — address by testId. stderrFn( - `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, + isBackend + ? `Run finished with status: ${finalRun.status}. Backend failure artifacts are addressed by testId — use 'testsprite test failure get ${testId}' to download the bundle.` + : `Run finished with status: ${finalRun.status}. Use 'testsprite test artifact get ${finalRun.runId}' to download the failure bundle.`, ); } @@ -6263,7 +7761,15 @@ export async function runTestRerun( chunkResponses = []; for (let idx = 0; idx < chunks.length; idx++) { const chunk = chunks[idx]!; - const chunkKey = chunks.length === 1 ? idempotencyKey : `${idempotencyKey}:chunk${idx}`; + // Bound the per-chunk idempotency key to <=256 chars (mirrors the retry + // path). A long base key plus the `:chunkN` suffix could otherwise exceed + // the server cap and be rejected or truncated inconsistently. + const chunkSuffix = chunks.length === 1 ? '' : `:chunk${idx}`; + const chunkBase = + chunkSuffix.length > 0 && idempotencyKey.length + chunkSuffix.length > 256 + ? idempotencyKey.slice(0, 256 - chunkSuffix.length) + : idempotencyKey; + const chunkKey = `${chunkBase}${chunkSuffix}`; const chunkResp = await client.triggerBatchRerun( { source: 'cli', @@ -6584,6 +8090,7 @@ export async function runTestRerun( const finalRun = await pollRunUntilTerminal(client, entry.runId, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, + shutdown: shutdownOf(deps), onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, onTick: (run, elapsedMs) => { const elapsed = Math.round(elapsedMs / 1000); @@ -6594,7 +8101,12 @@ export async function runTestRerun( }, resolveAlternate, }); - return { testId: entry.testId, runId: entry.runId, status: finalRun.status }; + return { + testId: entry.testId, + runId: entry.runId, + projectId: finalRun.projectId, + status: finalRun.status, + }; } catch (err) { if (err instanceof TimeoutError) { return { @@ -6608,12 +8120,35 @@ export async function runTestRerun( }, }; } + // Interrupt must reject the fan-out (the collect point below prints the + // partial for every dispatched run), never flatten into a per-member + // outcome that would swallow the 128+signum exit (DEV-331). + if (err instanceof InterruptError) throw err; + if (err instanceof RequestTimeoutError) { + // Client-side per-request timeout during polling — classify as timeout + // (exit 7) so the fan-out completes and stdout carries every runId. + // Without this, RequestTimeoutError rejects the fan-out before out.print(), + // leaving JSON consumers with empty stdout (mirrors create-batch --run). + return { + testId: entry.testId, + runId: entry.runId, + status: 'timeout', + error: { + code: 'UNSUPPORTED', + message: err.message, + exitCode: err.exitCode, + }, + }; + } if (err instanceof ApiError) { + // Preserve the real exit code (AUTH_INVALID=3, RATE_LIMITED=11, …) so the + // batch exit-code aggregator can escalate auth failures correctly. Mirroring + // the identical fix already applied to runTestRunAll's pollFreshAccepted. return { testId: entry.testId, runId: entry.runId, status: 'error', - error: { code: err.code, message: err.message, exitCode: 1 }, + error: { code: err.code, message: err.message, exitCode: err.exitCode }, }; } throw err; @@ -6624,24 +8159,44 @@ export async function runTestRerun( let acceptedIdx = 0; let inFlight = 0; - await new Promise((resolve, reject) => { - function startNext(): void { - while (inFlight < concurrencyLimit && acceptedIdx < accepted.length) { - const entry = accepted[acceptedIdx++]!; - inFlight++; - pollAccepted(entry) - .then(result => { - rerunResults.push(result); - inFlight--; - startNext(); - if (inFlight === 0 && acceptedIdx >= accepted.length) resolve(); - }) - .catch(reject); + try { + await new Promise((resolve, reject) => { + function startNext(): void { + while (inFlight < concurrencyLimit && acceptedIdx < accepted.length) { + const entry = accepted[acceptedIdx++]!; + inFlight++; + pollAccepted(entry) + .then(result => { + rerunResults.push(result); + inFlight--; + startNext(); + if (inFlight === 0 && acceptedIdx >= accepted.length) resolve(); + }) + .catch(reject); + } } + startNext(); + if (accepted.length === 0) resolve(); + }); + } catch (fanOutErr) { + // Graceful detach (DEV-331): stdout stays parseable — settled members + // keep their real status, unfinished ones are marked running — and the + // honest stderr line names every runId still executing (and billing). + if (fanOutErr instanceof InterruptError) { + ticker.finalize(`Batch rerun — interrupted (${fanOutErr.signal})`); + const settled = new Map(rerunResults.map(r => [r.runId, r] as const)); + const partialResults = accepted.map( + (e): CliRerunResult => + settled.get(e.runId) ?? { testId: e.testId, runId: e.runId, status: 'running' }, + ); + out.print({ accepted: partialResults, deferred, conflicts, notFound }, () => + partialResults.map(r => `${r.runId} ${r.status}`).join('\n'), + ); + const unfinished = accepted.filter(e => !settled.has(e.runId)).map(e => e.runId); + if (unfinished.length > 0) stderrFn(interruptDetachMessage(fanOutErr, unfinished)); } - startNext(); - if (accepted.length === 0) resolve(); - }); + throw fanOutErr; + } ticker.finalize(); @@ -6682,6 +8237,7 @@ export async function runTestRerun( total: accepted.length, }, }; + await writeBatchJUnitReportIfRequested(opts, rerunResults); out.print(jsonPayload); // Determine exit code: timeout (deferred or any timeout) → 7; any fail → 1; all pass → 0 @@ -6704,6 +8260,7 @@ export async function runTestRerun( // `test wait` accepts exactly one run id — emit one command per // timed-out run so the hint is always valid. ...(timedOut > 0 ? stillRunning.map(rid => `Resume: testsprite test wait ${rid}`) : []), + ...(timedOut > 0 ? stillRunning.map(rid => `Cancel: testsprite test cancel ${rid}`) : []), ] .filter(Boolean) .join('\n'), @@ -6714,6 +8271,17 @@ export async function runTestRerun( } if (failed > 0) { + // Auth failure on any member is a batch-wide condition — the credential is + // bad, not the test. Propagate exit 3 so the operator fixes auth rather than + // chasing a "rerun failed" (exit 1). Mirrors the identical logic already + // applied to runTestRunAll lines 5462-5468. + const authErr = rerunResults.find(r => r.error?.exitCode === 3); + if (authErr) { + throw new CLIError( + `${failed} rerun${failed !== 1 ? 's' : ''} failed — auth error (${authErr.error?.code}): ${authErr.error?.message}`, + 3, + ); + } throw new CLIError(`${failed} rerun${failed !== 1 ? 's' : ''} failed.`, 1); } @@ -6748,6 +8316,25 @@ export interface ArtifactGetResult { bundle?: WriteBundleResult; } +export function resolveDefaultArtifactDir(runId: string, cwd: string = process.cwd()): string { + requireNonEmpty('run-id', runId); + const windowsNormalizedSegment = runId.replace(/[ .]+$/u, ''); + if ( + windowsNormalizedSegment === '' || + windowsNormalizedSegment === '.' || + windowsNormalizedSegment === '..' || + runId.includes('/') || + runId.includes('\\') || + runId.includes('\0') + ) { + throw localValidationError( + 'run-id', + 'must be a single path-safe segment for the default output directory; pass --out

to choose a custom path', + ); + } + return join(cwd, '.testsprite', 'runs', runId); +} + /** * Validate that the parent directory of `resolvedDir` exists and is a * directory. Surfaces `VALIDATION_ERROR` (exit 5) — matches the convention @@ -6797,14 +8384,11 @@ export async function runArtifactGet( deps: TestDeps = {}, ): Promise { const out = makeOutput(opts.output, deps); - const client = makeClient(opts, deps); const { runId } = opts; // Resolve output dir: explicit --out or the default .testsprite/runs// const resolvedDir = - opts.out !== undefined - ? resolveBundleDir(opts.out) - : join(process.cwd(), '.testsprite', 'runs', runId); + opts.out !== undefined ? resolveBundleDir(opts.out) : resolveDefaultArtifactDir(runId); // --dry-run: no network, no disk write. // The client (makeClient) is already wired with createDryRunFetch() when @@ -6850,6 +8434,8 @@ export async function runArtifactGet( await assertOutDirParentExists(resolvedDir); } + const client = makeClient(opts, deps); + // Fetch the run-scoped failure bundle. const { body: context, requestId: fetchRequestId } = await client.getWithMeta( `/runs/${encodeURIComponent(runId)}/failure`, @@ -6974,6 +8560,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { 'alias for --starting-token; accepted for parity with `test result --history`', ) .option('--max-items ', 'stop after this many items across auto-paged pages') + .option('--columns ', 'select/reorder text table columns (comma-separated keys)') + .option('--no-header', 'suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (cmdOpts: ListFlagOpts, command: Command) => { // Same parser strategy as `project list`: skip Commander's number @@ -6994,6 +8582,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { pageSize: parseNumericFlag(cmdOpts.pageSize, 'page-size'), startingToken: cmdOpts.startingToken ?? cmdOpts.cursor, maxItems: parseNumericFlag(cmdOpts.maxItems, 'max-items'), + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -7172,14 +8762,60 @@ export function createTestCommand(deps: TestDeps = {}): Command { await runCreateBatch( { ...resolveCommonOptions(command), - plans: cmdOpts.plans, - planFromDir: cmdOpts.planFromDir, - run: cmdOpts.run === true, - maxConcurrency: parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency'), - wait: cmdOpts.wait === true, - timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), - targetUrl: cmdOpts.targetUrl, - idempotencyKey: cmdOpts.idempotencyKey, + plans: cmdOpts.plans, + planFromDir: cmdOpts.planFromDir, + run: cmdOpts.run === true, + maxConcurrency: parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency'), + wait: cmdOpts.wait === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + targetUrl: cmdOpts.targetUrl, + idempotencyKey: cmdOpts.idempotencyKey, + }, + deps, + ); + }); + + test + .command('scaffold') + .description( + 'Emit a schema-correct starter test definition (frontend plan JSON by default, or a backend Python skeleton). Pure-local: no network, no credentials.', + ) + .option('--type ', 'frontend|backend (default: frontend)') + .option('--out ', 'write the scaffold to a file instead of stdout') + .option('--force', 'overwrite an existing --out file', false) + .addHelpText( + 'after', + '\nExamples:\n' + + ' testsprite test scaffold > first-test.plan.json\n' + + ' testsprite test scaffold --type backend --out tests/health.py\n' + + ' testsprite test scaffold --out plan.json # then edit, and create with --plan-from plan.json', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (cmdOpts: ScaffoldFlagOpts, command: Command) => { + await runScaffold( + { + ...resolveCommonOptions(command), + scaffoldType: parseEnumFlag(cmdOpts.type, 'type', TEST_TYPES) ?? 'frontend', + out: cmdOpts.out, + force: cmdOpts.force === true, + }, + deps, + ); + }); + + test + .command('open ') + .description( + 'Open the test in the TestSprite dashboard: prints the deep-link URL, then spawns your default browser unless --no-browser.', + ) + .option('--no-browser', 'print the URL only (SSH, headless, CI, agents)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, cmdOpts: { browser?: boolean }, command: Command) => { + await runOpen( + { + ...resolveCommonOptions(command), + testId, + noBrowser: cmdOpts.browser === false, }, deps, ); @@ -7212,6 +8848,47 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); }); + test + .command('diff ') + .description( + 'Compare two runs and print what regressed: verdict, failureKind, failedStepIndex, per-step status flips, codeVersion drift. Exit 0 when verdicts match, 1 when they differ.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (runA: string, runB: string, _cmdOpts: unknown, command: Command) => { + await runDiff({ ...resolveCommonOptions(command), runA, runB }, deps); + }); + + test + .command('lint') + .description( + 'Validate plan/steps files offline with the same validators `create` runs, collecting EVERY problem. No network, no credentials. Exit 0 when all valid, 5 otherwise.', + ) + .option('--plan-from ', 'single plan JSON file') + .option( + '--plan-from-dir ', + 'directory of *.json plan files (each checked, all errors reported)', + ) + .option('--plans ', 'JSONL file with one plan spec per line (each line checked)') + .option('--steps ', 'plan-steps JSON file (the shape `test plan put` ingests)') + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + cmdOpts: { planFrom?: string; planFromDir?: string; plans?: string; steps?: string }, + command: Command, + ) => { + await runLint( + { + ...resolveCommonOptions(command), + planFrom: cmdOpts.planFrom, + planFromDir: cmdOpts.planFromDir, + plans: cmdOpts.plans, + steps: cmdOpts.steps, + }, + deps, + ); + }, + ); + test .command('result ') .description( @@ -7235,6 +8912,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option('--page-size ', 'with --history: number of runs per page (1–100, default 20)') .option('--cursor ', 'with --history: opaque cursor from a prior page') + .option('--columns ', 'with --history: select/reorder text table columns') + .option('--no-header', 'with --history: suppress the text table header row') .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testId: string, cmdOpts: ResultFlagOpts, command: Command) => { if (cmdOpts.history) { @@ -7250,6 +8929,8 @@ export function createTestCommand(deps: TestDeps = {}): Command { ? parseNumericFlag(cmdOpts.pageSize, 'page-size') : undefined, cursor: cmdOpts.cursor, + columns: cmdOpts.columns, + noHeader: cmdOpts.header === false, }, deps, ); @@ -7272,6 +8953,22 @@ export function createTestCommand(deps: TestDeps = {}): Command { .option('--name ', 'new human-readable test name') .option('--description ', 'new human description (≤ 2000 chars)') .option('--priority ', 'new priority — one of: p0, p1, p2, p3') + .option( + '--produces ', + 'BE only: variable name this test captures (repeatable). Drives dependency-aware wave ordering.', + (val: string, prev: string[]) => [...(prev ?? []), val], + [] as string[], + ) + .option( + '--needs ', + 'BE only: variable name this test consumes (repeatable). Declares an upstream producer dependency.', + (val: string, prev: string[]) => [...(prev ?? []), val], + [] as string[], + ) + .option( + '--category ', + "BE only: test category. Use 'teardown' or 'cleanup' to mark a final-wave cleanup test.", + ) .option( '--idempotency-key ', 'opaque idempotency token (1-256 ASCII chars). Defaults to a UUIDv4 minted per invocation; pin one yourself for safe retries.', @@ -7287,6 +8984,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { priority: parseEnumFlag(cmdOpts.priority, 'priority', CLI_CREATE_PRIORITIES) as | CliCreatePriority | undefined, + produces: cmdOpts.produces, + needs: cmdOpts.needs, + category: cmdOpts.category, idempotencyKey: cmdOpts.idempotencyKey, }, deps, @@ -7363,7 +9063,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { .command('run [test-id]') .description( 'Trigger a test run. With --wait, polls until terminal status.\n' + - 'Use --all --project for a wave-ordered batch run of all BE tests (M4).\n' + + 'Use --all --project for a wave-ordered batch run of all tests in a project (M4).\n' + '\nExit codes:\n' + ' 0 passed (or queued without --wait)\n' + ' 1 failed / blocked / cancelled\n' + @@ -7371,10 +9071,13 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' 4 test not found\n' + ' 5 validation error (e.g., bad --target-url, or positional + --all both set)\n' + ' 6 conflict (already running — see nextAction for the active runId)\n' + - ' 7 timeout — resume with: testsprite test wait \n' + + ' 7 timeout — resume with: testsprite test wait , ' + + 'or stop it with: testsprite test cancel \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + ' 11 rate limited — honor Retry-After\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C during --wait detaches only (the run keeps executing and billing);\n' + + 'stop it for real with: testsprite test cancel ', ) .option( '--target-url ', @@ -7391,7 +9094,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--all', - 'run all BE tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', + 'run all tests in the project (wave-ordered fresh run; requires --project). Mutually exclusive with .', false, ) .option( @@ -7406,13 +9109,26 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--max-concurrency ', `with --all --wait, max in-flight polls at once (1-100, default: ${DEFAULT_BATCH_RUN_CONCURRENCY})`, ) + .option( + '--report ', + 'with --all --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nDependency-aware fresh run (M4):\n' + - ' testsprite test run --all --project run all BE tests in wave order\n' + + ' testsprite test run --all --project run all project tests in wave order\n' + ' testsprite test run --all --project --filter name-glob subset\n' + + ' testsprite test run --all --project --wait --report junit --report-file ./results.xml\n' + '\nBE tests can declare --produces/--needs at create time to drive wave ordering\n' + - '(see `testsprite test create --help` for details).', + '(see `testsprite test create --help` for details).\n' + + '\nFrontend tests: the current unified engine runs FE tests too (they are billed\n' + + 'like any run). On the legacy backend-only engine FE tests cannot run — they are\n' + + "reported under skippedFrontend with an advisory; run those with 'test run '.", ) .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testIdArg: string | undefined, cmdOpts: RunFlagOpts, command: Command) => { @@ -7428,7 +9144,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { if (testIdArg === undefined && !isAll) { throw localValidationError( 'test-id', - 'provide a , or use --all --project to run all BE tests in a project', + 'provide a , or use --all --project to run all tests in a project', ); } // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). @@ -7440,6 +9156,14 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all --project .', ); } + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isAll, + }); if (isAll) { // --all path: wave-ordered fresh batch run. @@ -7449,14 +9173,16 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--all requires a project id — pass --project ', ); } - // --target-url has no effect on the --all batch path: it is BE-only - // (FE tests are skipped server-side) and a BE test's base URL is baked - // into its code. Silently dropping it could run the suite against an - // unintended environment in the caller's mind — reject loudly instead. + // --target-url has no effect on the --all batch path: a BE test's base + // URL is baked into its code, and the unified engine resolves each + // project's configured environment server-side (per-run URL overrides + // are not applied to batch FE runs either). Silently dropping it could + // run the suite against an unintended environment in the caller's mind + // — reject loudly instead. if (cmdOpts.targetUrl !== undefined && cmdOpts.targetUrl !== '') { throw localValidationError( 'target-url', - '--target-url has no effect with --all (the batch path is the BE-only wave engine; a BE test’s URL is baked into its code). Remove --target-url.', + '--target-url has no effect with --all (the batch path does not apply a per-run URL override — BE test URLs are baked into their code and the unified engine resolves the project environment server-side). Remove --target-url.', ); } await runTestRunAll( @@ -7470,6 +9196,9 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); @@ -7493,26 +9222,55 @@ export function createTestCommand(deps: TestDeps = {}): Command { }); test - .command('wait ') + .command('wait ') .description( - 'Wait for a run to reach a terminal status.\n' + + 'Wait for one or more runs to reach a terminal status.\n' + + '\nWith several run-ids the runs are polled concurrently under one shared\n' + + '--timeout and a {results, summary} envelope is printed (worst status wins\n' + + 'the exit code), so every re-attach hint the CLI prints can be pasted as\n' + + 'ONE command.\n' + '\nExit codes:\n' + ' 0 passed\n' + ' 1 failed / blocked / cancelled\n' + ' 3 auth error\n' + - ' 4 run not found\n' + - ' 7 timeout — resume with: testsprite test wait \n' + + ' 4 run not found (single run-id; with several ids a per-member poll error\n' + + ' is recorded as error: in its row and folded into exit 7)\n' + + ' 7 timeout or per-member poll error — resume with: testsprite test wait \n' + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C detaches only (the run keeps executing and billing); stop it for\n' + + 'real with: testsprite test cancel ', ) .option('--timeout ', `max seconds to wait (1–3600, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`) + .option( + '--max-concurrency ', + 'with several run-ids, max concurrent polls (1-100, default: 10)', + ) .addHelpText('after', GLOBAL_OPTS_HINT) - .action(async (runId: string, cmdOpts: WaitFlagOpts, command: Command) => { - await runTestWait( + .action(async (runIds: string[], cmdOpts: WaitFlagOpts, command: Command) => { + // One id keeps the historical single-run path byte-identical (same + // output shape, same exit codes); two or more fan out. + if (runIds.length === 1) { + await runTestWait( + { + ...resolveCommonOptions(command), + runId: runIds[0]!, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + return; + } + const maxConcurrency = parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? 10; + if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 100) { + throw localValidationError('max-concurrency', 'must be an integer between 1 and 100'); + } + await runTestWaitMany( { ...resolveCommonOptions(command), - runId, + runIds, timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + maxConcurrency, }, deps, ); @@ -7533,9 +9291,12 @@ export function createTestCommand(deps: TestDeps = {}): Command { ' 4 test not found\n' + ' 5 validation error\n' + ' 6 conflict (already running — see nextAction for the active runId)\n' + - ' 7 timeout or deferred — resume with: testsprite test wait \n' + + ' 7 timeout or deferred — resume with: testsprite test wait , ' + + 'or stop it with: testsprite test cancel \n' + ' 11 rate limited — honor Retry-After\n' + - '\nOn failure/blocked/cancelled, run: testsprite test artifact get ', + '\nOn failure/blocked/cancelled, run: testsprite test artifact get \n' + + '\nCtrl-C during --wait detaches only (the run keeps executing and billing);\n' + + 'stop it for real with: testsprite test cancel ', ) .option('--all', 'rerun all tests in the resolved project (requires --project)', false) .option( @@ -7577,6 +9338,15 @@ export function createTestCommand(deps: TestDeps = {}): Command { '--idempotency-key ', 'opaque key for safe retries (1–256 chars). Printed to stderr at --verbose if auto-generated.', ) + .option( + '--report ', + 'with batch --wait: write a JUnit XML sidecar report after polling (accepted: junit)', + ) + .option('--report-file ', 'output path for --report (atomic write)') + .option( + '--report-suite-name ', + 'optional JUnit override (default: testsprite:)', + ) .addHelpText( 'after', '\nNotes:\n' + @@ -7602,10 +9372,20 @@ export function createTestCommand(deps: TestDeps = {}): Command { // `--no-auto-heal`. There is no explicit `--auto-heal` flag, so // autoHealExplicit is always false in this design — the default-on value // is never a deliberate user choice to opt in. + const testIds = testIdsArg ?? []; + const isBatch = cmdOpts.all === true || testIds.length !== 1; + const report = parseJUnitReportFormat(cmdOpts.report); + assertJUnitReportOptions({ + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, + wait: cmdOpts.wait === true, + batchPath: isBatch, + }); await runTestRerun( { ...resolveCommonOptions(command), - testIds: testIdsArg ?? [], + testIds, all: cmdOpts.all === true, projectId: cmdOpts.project, skipTerminal: cmdOpts.skipTerminal === true, @@ -7620,19 +9400,272 @@ export function createTestCommand(deps: TestDeps = {}): Command { parseNumericFlag(cmdOpts.maxConcurrency, 'max-concurrency') ?? DEFAULT_BATCH_RUN_CONCURRENCY, idempotencyKey: cmdOpts.idempotencyKey, + report, + reportFile: cmdOpts.reportFile, + reportSuiteName: cmdOpts.reportSuiteName, }, deps, ); }); + // ------------------------------------------------------------------------- + // `test flaky` — repeat-run flaky-test detector + // ------------------------------------------------------------------------- + + test + .command('flaky ') + .description( + 'Repeatedly replay a test to measure stability and surface flakiness.\n' + + 'Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail.\n' + + '\nExit codes:\n' + + ' 0 stable (every attempt passed)\n' + + ' 1 flaky or failing (at least one attempt did not pass)\n' + + ' 3 auth error\n' + + ' 4 test not found (no replayable run — trigger `testsprite test run ` first)\n' + + ' 5 validation error', + ) + .option( + '--runs ', + `number of replays to run (1-${MAX_FLAKY_RUNS}, default ${DEFAULT_FLAKY_RUNS})`, + ) + .option( + '--until-fail', + 'stop at the first non-passing attempt (fast "is it flaky at all?" check)', + false, + ) + .option( + '--timeout ', + `per-attempt max seconds to wait (1-${MAX_RUN_TIMEOUT_SECONDS}, default ${DEFAULT_RUN_TIMEOUT_SECONDS})`, + ) + .addHelpText( + 'after', + '\nNotes:\n' + + ' • Frontend replays are free verbatim script replays (no credit); backend replays\n' + + ' re-run the dependency closure and may cost credits — a one-line advisory is printed.\n' + + ' • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass;\n' + + ' this measures replay stability of the saved script against the configured URL.\n' + + ' • `--output json` emits a machine-readable stability report for CI gating.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + testIdArg: string, + cmdOpts: { runs?: string; untilFail?: boolean; timeout?: string }, + command: Command, + ) => { + await runFlaky( + { + ...resolveCommonOptions(command), + testId: testIdArg, + runs: parseNumericFlag(cmdOpts.runs, 'runs') ?? DEFAULT_FLAKY_RUNS, + untilFail: cmdOpts.untilFail === true, + timeoutSeconds: parseTimeoutFlag(cmdOpts.timeout, 'timeout'), + }, + deps, + ); + }, + ); + test.addCommand(createTestCodeCommand(deps)); test.addCommand(createTestPlanCommand(deps)); test.addCommand(createTestFailureCommand(deps)); test.addCommand(createTestArtifactCommand(deps)); + test.addCommand(createTestCancelCommand(deps)); return test; } +// --------------------------------------------------------------------------- +// `test flaky` — repeat-run flaky-test detector +// --------------------------------------------------------------------------- + +/** Upper bound on `--runs` so a repeat-runner can't amplify free FE replays. */ +const MAX_FLAKY_RUNS = 10; +/** Default replay count when `--runs` is omitted. */ +const DEFAULT_FLAKY_RUNS = 5; + +function isFlakyFatalTriggerError(err: unknown): boolean { + if (!(err instanceof ApiError)) return false; + switch (err.code) { + case 'AUTH_REQUIRED': + case 'AUTH_INVALID': + case 'AUTH_FORBIDDEN': + return true; + default: + return false; + } +} + +interface RunTestFlakyOptions extends CommonOptions { + testId: string; + /** Number of replays to run (1..MAX_FLAKY_RUNS). */ + runs: number; + /** Stop at the first non-passing attempt. */ + untilFail: boolean; + /** Per-attempt polling deadline in seconds. */ + timeoutSeconds: number; +} + +/** + * `test flaky ` — replay a test N times and report a stability score. + * + * Each attempt is a `POST /tests/{id}/runs/rerun` with auto-heal OFF (a strict + * verbatim replay) followed by `pollRunUntilTerminal`. Frontend replays are + * free verbatim script replays; backend replays re-run the dependency closure + * (a one-line credit advisory is printed). The pure scoring lives in + * `lib/flaky.ts`; this function is the I/O orchestrator. + * + * Exit code: 0 when every observed attempt passed (stable), else 1 — so CI can + * gate a merge on flakiness. + */ +export async function runFlaky( + opts: RunTestFlakyOptions, + deps: TestDeps = {}, +): Promise { + const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + const out = makeOutput(opts.output, deps); + + if (typeof opts.testId !== 'string' || opts.testId.length === 0) { + throw localValidationError('test-id', 'is required'); + } + if (!Number.isInteger(opts.runs) || opts.runs < 1 || opts.runs > MAX_FLAKY_RUNS) { + throw localValidationError('runs', `must be an integer between 1 and ${MAX_FLAKY_RUNS}`); + } + + if (opts.dryRun) { + out.print({ + dryRun: true, + command: 'test flaky', + testId: opts.testId, + runs: opts.runs, + untilFail: opts.untilFail, + method: 'POST', + path: `/api/cli/v1/tests/${opts.testId}/runs/rerun`, + note: `Would replay the test up to ${opts.runs}x with auto-heal OFF and report a stability score.`, + }); + return undefined; + } + + // Under the implicit wait, raise the per-request timeout to cover --timeout + // so a slow trigger / long-poll under load isn't cut at the 120s default. + const client = makeClient( + { ...opts, requestTimeoutMs: resolveWaitRequestTimeoutMs({ ...opts, wait: true }) }, + deps, + ); + + // Best-effort test-type detection for the credit advisory. A probe failure + // never blocks the run — we just skip the advisory. + let isBackend = false; + try { + const test = await client.get(`/tests/${encodeURIComponent(opts.testId)}`); + isBackend = test.type === 'backend'; + } catch { + // best-effort — proceed without the advisory. + } + if (isBackend) { + stderrFn( + `[advisory] ${opts.testId} is a backend test — each replay re-runs its dependency closure ` + + `and may cost credits. Frontend replays are free verbatim script replays; backend replays are not.`, + ); + } + + const ticker = createTicker(stderrFn, opts.output === 'json' ? false : undefined); + const attempts: FlakyAttempt[] = []; + + for (let i = 1; i <= opts.runs; i++) { + const idempotencyKey = `cli-flaky-${randomUUID()}`; + + let rerunResp: RerunResponse; + try { + // auto-heal is intentionally OFF: flaky detection needs a strict verbatim + // replay so healed drift cannot mask a nondeterministic pass/fail. + rerunResp = await client.triggerRerun(opts.testId, { source: 'cli' }, { idempotencyKey }); + } catch (err) { + // A missing replayable run is fatal for the whole command (mirror rerun): + // there is nothing to repeat, so point the user at a fresh `test run`. + if (err instanceof ApiError && err.code === 'NOT_FOUND') { + throw ApiError.fromEnvelope({ + error: { + code: 'NOT_FOUND', + message: `Test ${opts.testId} has no replayable run (unknown/cross-tenant id, or it has never completed a clean run).`, + nextAction: `Trigger a fresh run first: testsprite test run ${opts.testId}`, + requestId: err.requestId ?? 'local', + details: { testId: opts.testId, reason: 'no_replayable_run' }, + }, + }); + } + if (isFlakyFatalTriggerError(err)) { + throw err; + } + // Any other trigger error is recorded as an errored attempt so a single + // transient blip doesn't abort a long stability probe. + const code = err instanceof ApiError ? err.code : 'ERROR'; + attempts.push({ attempt: i, runId: null, outcome: 'error', failureKind: code }); + ticker.update(`Attempt ${i}/${opts.runs} — error (${code})`); + if (opts.untilFail) break; + continue; + } + + const runId = rerunResp.runId; + // Backend run rows never finalize server-side; resolve the verdict from the + // testId-scoped result on non-terminal ticks (same fallback as `test rerun`). + const resolveAlternate = makeBackendWaitFallback({ + client, + resolveTestId: () => opts.testId, + resolveNotBefore: () => rerunResp.enqueuedAt, + }); + + let outcome: FlakyOutcome; + let failureKind: string | null = null; + try { + const finalRun = await pollRunUntilTerminal(client, runId, { + timeoutSeconds: opts.timeoutSeconds, + sleep: deps.sleep, + shutdown: shutdownOf(deps), + onTransition: opts.verbose ? (msg: string) => stderrFn(`[verbose] ${msg}`) : undefined, + resolveAlternate, + }); + outcome = finalRun.status as FlakyOutcome; + failureKind = finalRun.failureKind; + } catch (err) { + // Graceful detach (DEV-331): clean up the ticker line, name the run + // still executing server-side, and let index.ts exit 128+signum. + if (err instanceof InterruptError) { + ticker.finalize(`Attempt ${i}/${opts.runs} — interrupted (${err.signal})`); + stderrFn(interruptDetachMessage(err, [runId])); + throw err; + } + // A per-attempt deadline (poll TimeoutError) or a client-side request + // timeout both count as a non-passing "timeout" outcome for this attempt. + if (err instanceof TimeoutError || err instanceof RequestTimeoutError) { + outcome = 'timeout'; + } else { + throw err; + } + } + + attempts.push({ attempt: i, runId, outcome, failureKind }); + const passedSoFar = attempts.filter(a => a.outcome === 'passed').length; + ticker.update(`Attempt ${i}/${opts.runs} — ${outcome} (${passedSoFar} passed so far)`); + + if (opts.untilFail && outcome !== 'passed') break; + } + + ticker.finalize(); + + const report = summarizeFlaky(opts.testId, attempts); + out.print(report, data => renderFlakyText(data as FlakyReport)); + + const exitCode = flakyExitCode(report); + if (exitCode !== 0) { + throw new CLIError( + `Test ${opts.testId} is ${report.verdict} — ${report.passed}/${report.runs} attempts passed`, + exitCode, + ); + } + return report; +} + interface RunFlagOpts { targetUrl?: string; wait?: boolean; @@ -7643,10 +9676,14 @@ interface RunFlagOpts { project?: string; filter?: string; maxConcurrency?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface WaitFlagOpts { timeout?: string; + maxConcurrency?: string; } interface RerunFlagOpts { @@ -7661,12 +9698,18 @@ interface RerunFlagOpts { skipDependencies?: boolean; maxConcurrency?: string; idempotencyKey?: string; + report?: string; + reportFile?: string; + reportSuiteName?: string; } interface UpdateFlagOpts { name?: string; description?: string; priority?: string; + produces?: string[]; + needs?: string[]; + category?: string; idempotencyKey?: string; } @@ -7694,6 +9737,8 @@ interface ResultFlagOpts { pageSize?: string; /** Opaque pagination cursor from a prior page's nextCursor. */ cursor?: string; + columns?: string; + header?: boolean; } interface CreateFlagOpts { @@ -7741,6 +9786,8 @@ interface ListFlagOpts { */ cursor?: string; maxItems?: string; + columns?: string; + header?: boolean; } interface StepsFlagOpts { @@ -7806,13 +9853,9 @@ function resolveCommonOptions(command: Command): CommonOptions { // P2-8: validate --output before allowing silent fallback to 'text'. // An invalid value (e.g. `--output yaml`) must exit 5 with a clear error // rather than silently treating the request as text mode. - const rawOutput = globals.output; - if (rawOutput !== undefined && rawOutput !== 'json' && rawOutput !== 'text') { - throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); - } return { profile: globals.profile ?? 'default', - output: (globals.output as OutputMode | undefined) ?? 'text', + output: resolveOutputMode(globals.output), dryRun: globals.dryRun ?? false, endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, @@ -7821,18 +9864,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -/** - * Parse the `--request-timeout ` flag value into milliseconds. - * Returns `undefined` when the flag was not supplied (factory falls back to - * the env var / default). Silently clamps out-of-range values. - */ -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); // seconds → milliseconds -} - /** D4: headroom added on top of `--timeout` when deriving the per-request window under `--wait`. */ const WAIT_REQUEST_TIMEOUT_CUSHION_MS = 5_000; @@ -7867,6 +9898,7 @@ function makeClient(opts: CommonOptions, deps: TestDeps): HttpClient { credentialsPath: deps.credentialsPath, fetchImpl: deps.fetchImpl, stderr: deps.stderr, + shutdownSignal: shutdownOf(deps).signal, }); } @@ -7933,6 +9965,17 @@ function openOutputFile(rawPath: string): FileSink { if (!parentStat.isDirectory()) { throw localValidationError('out', `parent path is not a directory: ${parent}`); } + let targetStat; + try { + targetStat = statSync(resolved); + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'ENOENT') { + throw localValidationError('out', `cannot stat output path: ${resolved}`); + } + } + if (targetStat?.isDirectory()) { + throw localValidationError('out', `must point to a file, not a directory: ${resolved}`); + } const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); const sink: FileSink = { stream, path: resolved, tmpPath, error: null }; @@ -7998,6 +10041,19 @@ async function closeOutputFile(sink: FileSink, commit: boolean): Promise { await rename(sink.tmpPath, sink.path); } +/** Tear down an opened `--out` sink without leaving a zero-byte artifact. */ +async function abortOutputFile(sink: FileSink): Promise { + await new Promise(resolve => { + if (sink.stream.destroyed) { + resolve(); + return; + } + sink.stream.once('close', () => resolve()); + sink.stream.destroy(); + }); + await unlink(sink.tmpPath).catch(() => undefined); +} + /** A presigned `code` body is any `https://` URL — never anything else. */ export function isPresignedCodeUrl(code: string): boolean { return code.startsWith('https://'); @@ -8079,45 +10135,36 @@ async function streamPresignedBody(url: string, out: Output, deps: TestDeps): Pr } } -function renderTestListText(page: Page): string { +const TEST_LIST_COLUMNS: ReadonlyArray> = [ + { + header: 'ID', + width: rows => Math.max(2, ...rows.map(test => test.id.length)), + render: test => test.id, + }, + { + header: 'NAME', + width: rows => Math.max(4, ...rows.map(test => test.name.length)), + render: test => test.name, + }, + { header: 'TYPE', width: 8, render: test => test.type }, + { header: 'FROM', width: 6, render: test => test.createdFrom }, + { header: 'STATUS', width: 9, render: test => test.status }, + { header: 'UPDATED', width: 0, render: test => test.updatedAt }, +]; + +function renderTestListText( + page: Page, + options: { columns?: string; noHeader?: boolean } = {}, +): string { if (page.items.length === 0) { return page.nextToken ? `No tests on this page.\nnextToken: ${page.nextToken}` : 'No tests.'; } - const idWidth = Math.max(2, ...page.items.map(t => t.id.length)); - const nameWidth = Math.max(4, ...page.items.map(t => t.name.length)); - const typeWidth = 8; - const fromWidth = 6; - const statusWidth = 9; - - const header = - pad('ID', idWidth) + - ' ' + - pad('NAME', nameWidth) + - ' ' + - pad('TYPE', typeWidth) + - ' ' + - pad('FROM', fromWidth) + - ' ' + - pad('STATUS', statusWidth) + - ' ' + - 'UPDATED'; - - const rows = page.items.map( - t => - pad(t.id, idWidth) + - ' ' + - pad(t.name, nameWidth) + - ' ' + - pad(t.type, typeWidth) + - ' ' + - pad(t.createdFrom, fromWidth) + - ' ' + - pad(t.status, statusWidth) + - ' ' + - t.updatedAt, - ); - - const lines = [header, ...rows]; + const lines = [ + renderTextTable(page.items, TEST_LIST_COLUMNS, { + columns: options.columns, + noHeader: options.noHeader, + }), + ]; if (page.nextToken) lines.push('', `nextToken: ${page.nextToken}`); return lines.join('\n'); } @@ -8150,6 +10197,16 @@ function renderTestText(t: CliTest): string { if (typeof t.planStepCount === 'number') { lines.push(`planSteps: ${t.planStepCount}`); } + // Surface backend dependency declarations when present. + if (Array.isArray(t.produces) && t.produces.length > 0) { + lines.push(`produces: ${t.produces.join(', ')}`); + } + if (Array.isArray(t.consumes) && t.consumes.length > 0) { + lines.push(`consumes: ${t.consumes.join(', ')}`); + } + if (t.category) { + lines.push(`category: ${t.category}`); + } lines.push(`createdAt: ${t.createdAt}`, `updatedAt: ${t.updatedAt}`); return lines.join('\n'); } @@ -8202,9 +10259,9 @@ function renderStepsText(page: Page): string { ' ' + 'UPDATED'; - const rows = page.items.map(s => { + const rows = page.items.flatMap(s => { const marker = s.outcomeContributesToFailure === true ? '* ' : ' '; - return [ + const row = [ marker, pad(String(s.stepIndex), indexWidth), pad(s.action, actionWidth), @@ -8212,6 +10269,19 @@ function renderStepsText(page: Page): string { pad(descOf(s), descWidth), s.updatedAt, ].join(' '); + // Run-scoped rows carry the per-step failure text; surface it as an + // indented sub-line under failed rows (mirrors the history table's + // `targetUrl:` sub-line). Collapsed to one line and capped so a huge + // stack blob can't wreck the table; full text ships in --output json. + if (s.status === 'failed' && typeof s.error === 'string' && s.error.length > 0) { + const oneLine = s.error.replace(/\s+/g, ' ').trim(); + const shown = + oneLine.length > ERROR_SUBLINE_MAX + ? `${oneLine.slice(0, ERROR_SUBLINE_MAX - 1)}…` + : oneLine; + return [row, ` error: ${shown}`]; + } + return [row]; }); const lines: string[] = [header, ...rows, '']; @@ -8270,6 +10340,14 @@ function renderFailureContextText(ctx: CliFailureContext): string { lines.push(`evidence: ${ctx.failure.evidence.length} items (${breakdown})`); } if (ctx.result.videoUrl !== null) lines.push(`videoUrl: ${ctx.result.videoUrl}`); + // backend stdout + traceback (prefer the failure block, falling + // back to the embedded result; identical values). Bounded tail; full content + // in --output json / the written failure.json. + appendBackendArtifactLines( + lines, + ctx.failure.apiOutput ?? ctx.result.apiOutput, + ctx.failure.trace ?? ctx.result.trace, + ); return lines.join('\n'); } @@ -8391,6 +10469,9 @@ function renderResultText(r: CliLatestResult): string { lines.push(`summary: ${r.summary}`); if (r.videoUrl !== null) lines.push(`videoUrl: ${r.videoUrl}`); if (r.failureAnalysisUrl !== null) lines.push(`failureAnalysisUrl: ${r.failureAnalysisUrl}`); + // backend stdout + traceback (null/absent for FE, passed, and + // older backends). Full content always available via `--output json`. + appendBackendArtifactLines(lines, r.apiOutput, r.trace); if (r.analysis !== undefined) { // §6.5.1 (M2.1 piece 3) — render the inline analysis block under // the result summary. Only fires when the caller passed @@ -8414,6 +10495,44 @@ function renderResultText(r: CliLatestResult): string { * level null. Non-null wrappers render `kind=...` plus an optional * reference and indented rationale. */ +/** + * Render backend-test stdout + traceback in text mode. Shared by + * `renderResultText` and `renderFailureContextText`. Both are bounded to a + * tail (last {@link BACKEND_ARTIFACT_TAIL_LINES} lines) so a large stdout can't + * flood the terminal; the full, untruncated content is always in the + * `--output json` envelope. No-op when both are null/absent (FE / passed / + * older backends), so non-backend output stays byte-identical. + */ +const BACKEND_ARTIFACT_TAIL_LINES = 20; + +function appendArtifactTail( + lines: string[], + label: string, + value: string | null | undefined, +): void { + if (value == null || value === '') return; + const allLines = value.replace(/\n+$/, '').split('\n'); + const dropped = Math.max(0, allLines.length - BACKEND_ARTIFACT_TAIL_LINES); + const tail = dropped > 0 ? allLines.slice(-BACKEND_ARTIFACT_TAIL_LINES) : allLines; + const bytes = Buffer.byteLength(value, 'utf8'); + lines.push(''); + lines.push( + `${label} (${bytes} bytes${dropped > 0 ? `, showing last ${tail.length} lines` : ''}):`, + ); + for (const l of tail) lines.push(` ${l}`); + if (dropped > 0) + lines.push(` … ${dropped} earlier line(s) omitted — full content in --output json`); +} + +function appendBackendArtifactLines( + lines: string[], + apiOutput: string | null | undefined, + trace: string | null | undefined, +): void { + appendArtifactTail(lines, 'stdout', apiOutput); + appendArtifactTail(lines, 'trace', trace); +} + function appendFixTargetLines(lines: string[], fix: CliFixTarget | null, label: string): void { if (fix === null) { lines.push(`${label}— (analysis pipeline did not propose one)`); @@ -8643,7 +10762,11 @@ export function createTestArtifactCommand(deps: TestDeps): Command { 'Parent must exist. The bundle dir itself is created if absent.', ].join(' '), ) - .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .option( + '--failed-only', + 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + + 'so this is usually a no-op; use `test steps ` for the full run trail.', + ) .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (runId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { @@ -8673,7 +10796,11 @@ function createTestFailureCommand(deps: TestDeps): Command { '--out ', 'Directory to write the §7 disk layout into (default: print wire envelope to stdout)', ) - .option('--failed-only', 'Keep only the failed step plus its immediate neighbors (±1)') + .option( + '--failed-only', + 'Trim to the failed step ±1. The bundle is already failure-focused server-side, ' + + 'so this is usually a no-op; use `test steps ` for the full run trail.', + ) .addHelpText('after', GLOBAL_OPTS_HINT) .action( async (testId: string, cmdOpts: { out?: string; failedOnly?: boolean }, command: Command) => { diff --git a/src/commands/test.wait.spec.ts b/src/commands/test.wait.spec.ts index 68a4c99..4830e50 100644 --- a/src/commands/test.wait.spec.ts +++ b/src/commands/test.wait.spec.ts @@ -10,7 +10,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DRY_RUN_BANNER, resetDryRunBannerForTesting } from '../lib/client-factory.js'; -import { ApiError, RequestTimeoutError } from '../lib/errors.js'; +import { ApiError, InterruptError, RequestTimeoutError } from '../lib/errors.js'; +import { ShutdownController } from '../lib/interrupt.js'; import type { RunResponse } from '../lib/runs.types.js'; import { runTestWait } from './test.js'; @@ -1021,6 +1022,69 @@ describe('runTestWait: Fix 3 — RequestTimeoutError writes partial JSON to stdo }); }); +// --------------------------------------------------------------------------- +// TimeoutError on test wait: partial stdout + exit 7 +// --------------------------------------------------------------------------- + +describe('runTestWait: TimeoutError writes partial JSON to stdout', () => { + let logSpy: ReturnType; + let errorSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('exit 7 AND stdout contains {runId, status:"running"} when --timeout polling deadline is exceeded', async () => { + const { credentialsPath } = makeCreds(); + const fetchImpl = makeFetch(() => ({ body: makeRun('running') })); + + let callCount = 0; + const base = Date.now(); + const realDateNow = Date.now; + Date.now = () => (callCount++ > 4 ? base + 2000 : base); + + try { + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + await expect( + runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 1, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + }, + ), + ).rejects.toMatchObject({ exitCode: 7 }); + + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { + runId: string; + status: string; + }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + } finally { + Date.now = realDateNow; + } + }); +}); + // --------------------------------------------------------------------------- // FIX 4 — D5-UX: text mode shows the `error` string for failed/blocked runs // --------------------------------------------------------------------------- @@ -1242,3 +1306,122 @@ describe('runTestWait — dashboardUrl on terminal output', () => { expect(printed.dashboardUrl).toBeUndefined(); }); }); + +// --------------------------------------------------------------------------- +// DEV-331 piece 1 — graceful detach on SIGINT/SIGTERM during test wait +// --------------------------------------------------------------------------- + +describe('runTestWait — InterruptError graceful detach (DEV-331)', () => { + function hangingFetch(): typeof globalThis.fetch { + return (async (_input: FetchInput, init: RequestInit = {}) => { + return new Promise((_resolve, reject) => { + const signal = init.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + reject(reason instanceof Error ? reason : new Error('aborted')); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as typeof globalThis.fetch; + } + + it('SIG-2/SIG-4: emits partial JSON to stdout, honest billing line to stderr, rethrows exit 130', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdoutLines: string[] = []; + const stderrLines: string[] = []; + + const pending = runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: hangingFetch(), + stdout: line => stdoutLines.push(line), + stderr: line => stderrLines.push(line), + sleep: instantSleep, + shutdown, + }, + ); + // Simulate Ctrl-C while the long-poll fetch is in flight. + setTimeout(() => shutdown.interrupt('SIGINT'), 5); + + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(130); + expect((err as InterruptError).signal).toBe('SIGINT'); + + // Stdout stays parseable and carries the runId for re-attach. + const stdoutJson = JSON.parse(stdoutLines.join('\n')) as { runId: string; status: string }; + expect(stdoutJson.runId).toBe('run_abc'); + expect(stdoutJson.status).toBe('running'); + + // The honest line: run keeps executing AND billing; re-attach hint. + const stderrBlock = stderrLines.join('\n'); + expect(stderrBlock).toContain('Interrupted (SIGINT)'); + expect(stderrBlock).toContain('billing'); + expect(stderrBlock).toContain('testsprite test wait run_abc'); + }); + + it('SIG-3: SIGTERM maps to exit 143', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const pending = runTestWait( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 600, + }, + { + credentialsPath, + fetchImpl: hangingFetch(), + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + shutdown, + }, + ); + setTimeout(() => shutdown.interrupt('SIGTERM'), 5); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).exitCode).toBe(143); + }); + + it('arms the shutdown scope during the wait and disarms after a terminal result', async () => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const fetchImpl = makeFetch(() => ({ body: makeRun('passed') })); + await runTestWait( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + runId: 'run_abc', + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: () => {}, + stderr: () => {}, + sleep: instantSleep, + shutdown, + }, + ); + expect(shutdown.isArmed).toBe(false); // disarmed after the poll completes + }); +}); diff --git a/src/commands/usage.ts b/src/commands/usage.ts index d8a0f21..1980d26 100644 --- a/src/commands/usage.ts +++ b/src/commands/usage.ts @@ -17,12 +17,13 @@ import { Command } from 'commander'; import { emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, type CommonOptions as FactoryCommonOptions, } from '../lib/client-factory.js'; import { loadConfig } from '../lib/config.js'; import { resolvePortalBase } from '../lib/facade.js'; import type { FetchImpl } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, type OutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; /** * Usage/balance response from `/me` (when the backend supplies it) or a future @@ -153,8 +154,12 @@ function renderUsage(u: UsageResponse, portalBase?: string): string { } if (u.creditsPerRun !== undefined) { lines.push(`cost per frontend run: ${u.creditsPerRun} credit(s)`); + // Backend runs DO consume credits (confirmed by design 2026-06-30 / DEV-289). + // The API exposes no backend-specific per-run cost field, and it differs from + // the frontend rate, so state that it bills without asserting a possibly-wrong + // number — check your balance before/after, or see the billing page. lines.push( - `cost per backend run: 0 credit(s) (backend tests bill at code-generation, not at run time)`, + `cost per backend run: also consumes credits (exact amount not reported by the API)`, ); } @@ -199,7 +204,12 @@ export function createUsageCommand(deps: UsageDeps = {}): Command { '\nExamples:\n' + ' testsprite usage # show balance + plan\n' + ' testsprite usage --output json # machine-readable balance\n' + + ' testsprite usage --debug # trace HTTP method/path, request id, latency\n' + ' testsprite credits # alias for usage\n' + + '\nExit codes:\n' + + ' 0 success (or --dry-run)\n' + + ' 3 auth error — run `testsprite setup` to configure credentials\n' + + ' 10 transport/network failure (UNAVAILABLE) — retry the command\n' + '\nNote: credit balance requires a backend update to /me. Until shipped,\n' + " check your portal's Billing page (/dashboard/settings/billing) for your balance.", ) @@ -216,7 +226,7 @@ function resolveCommonOptions(command: Command): CommonOptions { }; return { profile: globals.profile ?? 'default', - output: globals.output ?? 'text', + output: resolveOutputMode(globals.output), endpointUrl: globals.endpointUrl, debug: globals.debug ?? false, verbose: globals.verbose ?? false, @@ -225,13 +235,6 @@ function resolveCommonOptions(command: Command): CommonOptions { }; } -function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { - if (raw === undefined) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n <= 0) return undefined; - return Math.round(n * 1000); -} - function makeOutput(mode: OutputMode, deps: UsageDeps): Output { return new Output(mode, { stdout: deps.stdout, stderr: deps.stderr }); } diff --git a/src/index.ts b/src/index.ts index 7842ec9..31d3399 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,10 @@ #!/usr/bin/env node + import { Command, CommanderError } from 'commander'; import { createAgentCommand } from './commands/agent.js'; import { createAuthCommand } from './commands/auth.js'; +import { createCompletionCommand, type CompletionSpec } from './commands/completion.js'; +import { createDoctorCommand } from './commands/doctor.js'; import { createDeprecatedInitCommand, createSetupCommand, @@ -10,11 +13,24 @@ import { import { createProjectCommand } from './commands/project.js'; import { createTestCommand } from './commands/test.js'; import { createUsageCommand } from './commands/usage.js'; -import { ApiError, CLIError, RequestTimeoutError } from './lib/errors.js'; +import { ApiError, CLIError, InterruptError, RequestTimeoutError } from './lib/errors.js'; +import { installBrokenPipeGuard, installSignalHandlers } from './lib/interrupt.js'; import { Output, isOutputMode } from './lib/output.js'; +import { maybeInstallProxyAgent } from './lib/proxy.js'; import { renderCommanderError, rephraseUnknownOption } from './lib/render-error.js'; import { maybeEmitSkillNudge } from './lib/skill-nudge.js'; +import { maybeNotifyUpdate } from './lib/update-check.js'; import { VERSION } from './version.js'; +import { shouldRejectNodeVersion } from './version-guard.js'; + +// Guard: exit early with a clear message on unsupported Node.js versions, +// rather than failing later with a cryptic ESM/runtime error. +if (shouldRejectNodeVersion(process.versions.node)) { + process.stderr.write( + `Error: testsprite requires Node.js >= 20 (found ${process.versions.node}).\nInstall the latest LTS from https://nodejs.org\n`, + ); + process.exit(1); +} const program = new Command(); @@ -37,7 +53,7 @@ program .option('--debug', 'Print HTTP method/path, request id, latency, retry decisions to stderr') .option( '--dry-run', - 'Skip the network, credentials, and filesystem; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key.', + 'Skip the network and credentials; emit a canned sample matching the OpenAPI contract. Useful for learning the CLI surface without an API key. Note: file inputs you pass (--plan-from/--plans/--steps) are still read and validated locally; only --code-file uses a placeholder.', ) .option( '--request-timeout ', @@ -76,6 +92,29 @@ program.addCommand(createProjectCommand({})); program.addCommand(createTestCommand()); program.addCommand(createAgentCommand({})); program.addCommand(createUsageCommand()); +program.addCommand(createDoctorCommand()); +program.addCommand(createCompletionCommand(() => buildCompletionSpec())); + +// Derive the shell-completion spec from the fully-assembled command tree at call +// time (not module-load), so `testsprite completion` can never drift from the +// real commands, subcommands, and global flags. +function buildCompletionSpec(): CompletionSpec { + const subcommands: Record = {}; + for (const command of program.commands) { + const subs = command.commands.map(sub => sub.name()).filter(name => name !== 'help'); + if (subs.length > 0) subcommands[command.name()] = subs; + } + const flags = program.options + .map(option => option.long) + .filter((long): long is string => typeof long === 'string'); + if (!flags.includes('--help')) flags.push('--help'); + return { + program: 'testsprite', + commands: [...new Set([...program.commands.map(command => command.name()), 'help'])], + subcommands, + globalFlags: flags, + }; +} // Buffer Commander error messages instead of writing immediately. The catch // block re-emits in the correct format (JSON or text) once the requested @@ -128,16 +167,41 @@ program.hook('preAction', (_thisCommand, actionCommand) => { profile?: string; dryRun?: boolean; }; + const commandPath = commandPathOf(actionCommand); maybeEmitSkillNudge({ - commandPath: commandPathOf(actionCommand), + commandPath, output: isOutputMode(globals.output) ? globals.output : 'text', dryRun: globals.dryRun ?? false, profile: globals.profile ?? 'default', cwd: process.cwd(), env: process.env, }); + + // Best-effort update notice (see lib/update-check.ts): self-gates on the + // opt-out env, CI, TTY, and a 24h cache; the wiring adds the flag-level + // gates the lib cannot see. Skipped for `completion` (its stdout is eval'd + // by shells), under --output json, and under --dry-run. Deliberately not + // awaited: an advisory must never delay the real command. + if (globals.output !== 'json' && globals.dryRun !== true && commandPath !== 'completion') { + void maybeNotifyUpdate(); + } }); +// Clean process lifecycle (DEV-331 piece 1, errors.md §8.1): during a `--wait` +// poll the scope is armed — the first SIGINT/SIGTERM/SIGHUP aborts gracefully +// and the wait path prints an honest partial (the run KEEPS executing and +// billing server-side) + re-attach hint before exiting 128+signum; a second +// signal hard-exits. Outside an armed scope: a clear one-line message + +// immediate exit (instead of Node's silent abrupt kill). Plus an EPIPE guard +// so piping to a reader that closes early (`| head`) exits cleanly instead of +// dumping a raw `write EPIPE` stack. +installSignalHandlers(); +installBrokenPipeGuard(); + +// Corporate/CI proxies: honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY (Node's fetch +// ignores them by default). No-op when no proxy variable is set. +maybeInstallProxyAgent(); + try { await program.parseAsync(process.argv); } catch (err) { @@ -171,10 +235,44 @@ try { process.stderr.write(` granted: ${(granted as string[]).join(', ')}\n`); } } + // Surface the version gap on CLIENT_TOO_OLD so the user sees exactly what + // moved without parsing the message string. (No "latest" line — the npm + // update-notice is the single source of truth for the newest release.) + if (err.code === 'CLIENT_TOO_OLD') { + const your = err.getDetail('yourVersion'); + const min = err.getDetail('minVersion'); + if (typeof your === 'string' && typeof min === 'string') { + process.stderr.write(` your version: ${your}, minimum supported: ${min}\n`); + } + } } process.exit(err.exitCode); } const output = new Output(mode); + if (err instanceof InterruptError) { + // Graceful detach (DEV-331 piece 1, errors.md §8.1): the wait-path catch + // block already printed the honest partial + re-attach hint. Exit with + // the conventional 128+signum code; `INTERRUPTED` is deliberately outside + // the error catalog. Note: Ctrl-C does NOT cancel the server-side run. + if (mode === 'json') { + const envelope = { + error: { + code: 'INTERRUPTED', + message: err.message, + nextAction: + 'The server-side run (if any) keeps executing and billing. ' + + 'Re-attach with: testsprite test wait , or stop it with: testsprite test cancel ' + + '(runId is in the partial JSON on stdout).', + requestId: 'local', + details: { signal: err.signal }, + }, + }; + process.stderr.write(`${JSON.stringify(envelope, null, 2)}\n`); + } else { + process.stderr.write(`Error: ${err.message}\n`); + } + process.exit(err.exitCode); + } if (err instanceof RequestTimeoutError) { // Structured rendering for per-request timeouts: JSON mode emits a // machine-readable envelope; text mode emits the message with a hint. diff --git a/src/lib/agent-targets.test.ts b/src/lib/agent-targets.test.ts index b049711..73feee3 100644 --- a/src/lib/agent-targets.test.ts +++ b/src/lib/agent-targets.test.ts @@ -1,5 +1,7 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { describe, expect, it } from 'vitest'; +import { VERSION } from '../version.js'; import { DEFAULT_SKILLS, MANAGED_SECTION_BEGIN, @@ -9,13 +11,17 @@ import { SKILL_NAME, SKILLS, TARGETS, + bodyHash12, buildCodexAggregate, + buildSkillMarker, codexContentFor, loadCodexSkillBody, loadSkillBody, loadSkillBodyFor, + parseSkillMarker, pathFor, renderForTarget, + renderOwnFileWithMarker, } from './agent-targets.js'; // --------------------------------------------------------------------------- @@ -28,7 +34,9 @@ import { * The description value is a single line (no folded/literal block scalars). */ function parseFrontmatterDescription(content: string): string | undefined { - const lines = content.split('\n'); + // Tolerate CRLF so a Windows checkout (autocrlf) doesn't leave a trailing + // \r on the description and break the byte-identical comparisons. + const lines = content.split(/\r?\n/); let inFrontmatter = false; for (const line of lines) { if (line.trim() === '---') { @@ -74,19 +82,31 @@ testsprite test artifact get --out ./out/ // --------------------------------------------------------------------------- describe('TARGETS', () => { - it('has all five required keys', () => { + it('has all eight required keys', () => { const keys = Object.keys(TARGETS).sort(); - expect(keys).toEqual(['antigravity', 'claude', 'cline', 'codex', 'cursor']); + expect(keys).toEqual([ + 'antigravity', + 'claude', + 'cline', + 'codex', + 'copilot', + 'cursor', + 'kiro', + 'windsurf', + ]); }); it('claude is GA', () => { expect(TARGETS.claude.status).toBe('ga'); }); - it('cursor, cline, antigravity, and codex are experimental', () => { + it('cursor, cline, windsurf, copilot, antigravity, kiro, and codex are experimental', () => { expect(TARGETS.cursor.status).toBe('experimental'); expect(TARGETS.cline.status).toBe('experimental'); + expect(TARGETS.windsurf.status).toBe('experimental'); + expect(TARGETS.copilot.status).toBe('experimental'); expect(TARGETS.antigravity.status).toBe('experimental'); + expect(TARGETS.kiro.status).toBe('experimental'); expect(TARGETS.codex.status).toBe('experimental'); }); @@ -102,6 +122,9 @@ describe('TARGETS', () => { expect(TARGETS.antigravity.mode).toBe('own-file'); expect(TARGETS.cursor.mode).toBe('own-file'); expect(TARGETS.cline.mode).toBe('own-file'); + expect(TARGETS.kiro.mode).toBe('own-file'); + expect(TARGETS.windsurf.mode).toBe('own-file'); + expect(TARGETS.copilot.mode).toBe('own-file'); }); it('codex target has mode managed-section', () => { @@ -200,6 +223,22 @@ describe('renderForTarget("antigravity")', () => { }); }); +describe('renderForTarget("kiro")', () => { + const result = renderForTarget('kiro', 'testsprite-verify', STUB_BODY); + + it('returns the correct path', () => { + expect(result.path).toBe('.kiro/skills/testsprite-verify/SKILL.md'); + }); + + it('frontmatter contains name: testsprite-verify', () => { + expect(result.content).toContain('name: testsprite-verify'); + }); + + it('frontmatter contains description:', () => { + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); +}); + describe('renderForTarget("claude") vs renderForTarget("antigravity")', () => { it('produce the same frontmatter lines (name + description)', () => { const claude = renderForTarget('claude', 'testsprite-verify', STUB_BODY); @@ -269,16 +308,97 @@ describe('renderForTarget("cline")', () => { }); }); +describe('renderForTarget("windsurf")', () => { + const result = renderForTarget('windsurf', 'testsprite-verify', STUB_BODY); + + it('returns the .windsurf/rules path', () => { + expect(result.path).toBe('.windsurf/rules/testsprite-verify.md'); + }); + + it('uses the Cascade frontmatter (trigger: model_decision + description)', () => { + expect(result.content.startsWith('---\n')).toBe(true); + expect(result.content).toContain('trigger: model_decision'); + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + }); + + it('does NOT carry the Claude/Cursor frontmatter keys', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); // claude key + expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key + }); +}); + +describe('windsurf renders within the rules-file budget', () => { + // Regression: a `.windsurf/rules/*.md` file caps at ~12 K characters and + // Cascade silently truncates beyond that. The full verify body (~22 KB) would + // be cut in half, so windsurf renders the COMPACT body for verify (its trimmed + // codex asset) and the full body for onboard (which already fits). Uses the + // REAL bodies (no stub) so the size reflects what a user receives. + for (const skill of DEFAULT_SKILLS) { + it(`${skill} fits under 12 000 characters`, () => { + const r = renderForTarget('windsurf', skill); + expect(r.content.length).toBeLessThan(12_000); + }); + } + + it('verify uses the compact body (smaller than the full claude render)', () => { + const windsurf = renderForTarget('windsurf', 'testsprite-verify'); + const claude = renderForTarget('claude', 'testsprite-verify'); + expect(windsurf.content.length).toBeLessThan(claude.content.length); + // The full-body-only intro line is absent from the compact body... + expect(claude.content).toContain('The verification loop that flies'); + expect(windsurf.content).not.toContain('The verification loop that flies'); + // ...but the load-bearing command survives. + expect(windsurf.content).toContain('testsprite test run'); + }); +}); + +describe('renderForTarget("copilot")', () => { + const result = renderForTarget('copilot', 'testsprite-verify', STUB_BODY); + + it('returns the .github/instructions path', () => { + expect(result.path).toBe('.github/instructions/testsprite-verify.instructions.md'); + }); + + it('uses the Copilot frontmatter (applyTo + description)', () => { + expect(result.content.startsWith('---\n')).toBe(true); + expect(result.content).toContain(`description: ${SKILL_DESCRIPTION}`); + expect(result.content).toContain("applyTo: '**'"); + }); + + it('does NOT carry the Claude/Cursor/Windsurf frontmatter keys', () => { + const match = /^---\n([\s\S]*?)\n---/.exec(result.content); + const fm = match?.[1] ?? ''; + expect(fm).not.toContain('name:'); // claude key + expect(fm).not.toContain('alwaysApply:'); // cursor .mdc key + expect(fm).not.toContain('trigger:'); // windsurf Cascade key + }); + + it('renders the compact verify body (applyTo:** is always-on, so keep it small)', () => { + // Uses the REAL bodies (no stub): copilot always-injects, so like windsurf it + // ships the trimmed verify body while keeping the load-bearing command. + const copilot = renderForTarget('copilot', 'testsprite-verify'); + const claude = renderForTarget('claude', 'testsprite-verify'); + expect(copilot.content.length).toBeLessThan(claude.content.length); + expect(copilot.content).not.toContain('The verification loop that flies'); + expect(copilot.content).toContain('testsprite test run'); + }); +}); + // --------------------------------------------------------------------------- // Content integrity — load-bearing command strings must survive any body trim // --------------------------------------------------------------------------- describe('content integrity — own-file targets', () => { - const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity'> = [ + // Full-body own-file targets. Compact-body targets (windsurf, copilot) are + // excluded — they render the trimmed verify body; see their dedicated tests. + const ownFileTargets: Array<'claude' | 'cursor' | 'cline' | 'antigravity' | 'kiro'> = [ 'claude', 'cursor', 'cline', 'antigravity', + 'kiro', ]; // Use the real body for these checks, since we're guarding against trimming. @@ -329,6 +449,27 @@ describe('loadCodexSkillBody', () => { }); }); +// --------------------------------------------------------------------------- +// content integrity — current auth command names +// --------------------------------------------------------------------------- + +describe('content integrity — testsprite-verify auth commands', () => { + it('uses current auth/setup commands in every canonical verify skill asset', () => { + const assets = [ + ['docs template', templateRaw], + ['own-file skill body', loadSkillBodyFor('testsprite-verify')], + ['codex skill body', codexContentFor('testsprite-verify')], + ] as const; + + for (const [name, content] of assets) { + expect(content, name).toContain('testsprite auth status'); + expect(content, name).toContain('testsprite setup'); + expect(content, name).not.toContain('auth whoami'); + expect(content, name).not.toContain('auth configure'); + } + }); +}); + // --------------------------------------------------------------------------- // MANAGED_SECTION sentinels // --------------------------------------------------------------------------- @@ -680,3 +821,110 @@ describe('renderForTarget for testsprite-onboard', () => { expect(() => renderForTarget('claude', 'testsprite-unknown')).toThrow('unknown skill'); }); }); + +// --------------------------------------------------------------------------- +// Install marker (issue #123): format, parsing, and render placement +// --------------------------------------------------------------------------- + +describe('buildSkillMarker / parseSkillMarker / bodyHash12', () => { + it('marker line is an HTML comment carrying name, vVERSION, and a 12-hex hash', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + expect(marker).toBe( + ``, + ); + expect(marker).toMatch( + /^$/, + ); + }); + + it('bodyHash12 equals the first 12 hex chars of the body SHA-256', () => { + const fullHex = createHash('sha256').update(STUB_BODY, 'utf8').digest('hex'); + expect(bodyHash12(STUB_BODY)).toBe(fullHex.slice(0, 12)); + }); + + it('parseSkillMarker round-trips a built marker embedded in surrounding content', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + const parsed = parseSkillMarker(`# heading\n${marker}\nbody text\n`); + expect(parsed).not.toBeNull(); + expect(parsed?.skill).toBe('testsprite-verify'); + expect(parsed?.version).toBe(VERSION); + expect(parsed?.hash12).toBe(bodyHash12(STUB_BODY)); + expect(parsed?.line).toBe(marker); + }); + + it('parseSkillMarker strips a trailing CR so CRLF checkouts parse identically', () => { + const marker = buildSkillMarker('testsprite-verify', STUB_BODY); + const parsed = parseSkillMarker(`${marker}\r\nrest\r\n`); + expect(parsed?.line).toBe(marker); + }); + + it('parseSkillMarker returns null when no marker line is present', () => { + expect(parseSkillMarker('# Just a heading\n\nProse without any marker.\n')).toBeNull(); + }); + + it('parseSkillMarker ignores the managed-section sentinels (also HTML comments)', () => { + expect(parseSkillMarker(`${MANAGED_SECTION_BEGIN}\nbody\n${MANAGED_SECTION_END}\n`)).toBeNull(); + }); +}); + +describe('render marker placement (own-file targets)', () => { + it('claude render carries the marker on the line right after the closing frontmatter fence', () => { + const { content } = renderForTarget('claude', 'testsprite-verify', STUB_BODY); + const closingFence = '\n---\n'; + const fenceEnd = content.indexOf(closingFence) + closingFence.length; + expect(content.slice(fenceEnd).startsWith(''; + const withForeign = renderOwnFileWithMarker( + 'claude', + 'testsprite-verify', + foreignMarker, + STUB_BODY, + ); + expect(withForeign).toContain(foreignMarker); + // Marker line aside, the bytes match the canonical render exactly. + const canonical = renderForTarget('claude', 'testsprite-verify', STUB_BODY).content; + const currentMarker = buildSkillMarker('testsprite-verify', STUB_BODY); + expect(withForeign.replace(foreignMarker, currentMarker)).toBe(canonical); + }); + + it('renderOwnFileWithMarker rejects the managed-section target', () => { + expect(() => renderOwnFileWithMarker('codex', 'testsprite-verify', 'marker', 'body')).toThrow( + 'own-file', + ); + }); + + it('renderOwnFileWithMarker throws on an unknown skill', () => { + expect(() => renderOwnFileWithMarker('claude', 'testsprite-unknown', 'marker')).toThrow( + 'unknown skill', + ); + }); +}); diff --git a/src/lib/agent-targets.ts b/src/lib/agent-targets.ts index 2547daf..7e6f94d 100644 --- a/src/lib/agent-targets.ts +++ b/src/lib/agent-targets.ts @@ -1,6 +1,16 @@ +import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; +import { VERSION } from '../version.js'; -export type AgentTarget = 'claude' | 'cursor' | 'cline' | 'antigravity' | 'codex'; +export type AgentTarget = + | 'claude' + | 'cursor' + | 'cline' + | 'antigravity' + | 'codex' + | 'kiro' + | 'windsurf' + | 'copilot'; export interface TargetSpec { status: 'ga' | 'experimental'; @@ -12,11 +22,19 @@ export interface TargetSpec { */ path: string; /** - * 'own-file': the CLI owns the whole file (claude/cursor/cline/antigravity). + * 'own-file': the CLI owns the whole file (claude/cursor/cline/antigravity/windsurf). * 'managed-section': the CLI writes only a sentinel-delimited section inside * a potentially user-authored file (codex target, AGENTS.md). */ mode: 'own-file' | 'managed-section'; + /** + * When true, render the budget-friendly body (see {@link compactBodyFor}) + * instead of the full own-file skill body. Used for own-file targets whose + * rule files are size-capped — currently `windsurf` (`.windsurf/rules/*.md` + * files cap at ~12 K characters and Cascade silently truncates beyond that, + * which would cut the full ~22 KB verify skill in half). + */ + compactBody?: boolean; /** * Wrap a skill body in this target's frontmatter/header. Takes the skill's * `name`+`description` (own-file targets emit them as frontmatter) and the body. @@ -120,6 +138,31 @@ function wrapMdc(_name: string, description: string, body: string): string { return `---\ndescription: ${description}\nalwaysApply: false\n---\n\n${body}\n`; } +/** + * Windsurf (Cascade) reads workspace rules from `.windsurf/rules/*.md` with YAML + * frontmatter. `trigger: model_decision` is the Cascade equivalent of the Cursor + * `.mdc` `alwaysApply: false` mode: only the `description` is surfaced up front, + * and Cascade pulls in the full rule body when the description shows it is + * relevant — exactly the on-demand activation these skills want. (The other + * triggers are `always_on`, `manual`, and `glob`.) + */ +function wrapWindsurf(_name: string, description: string, body: string): string { + return `---\ntrigger: model_decision\ndescription: ${description}\n---\n\n${body}\n`; +} + +/** + * GitHub Copilot reads path-specific custom instructions from + * `.github/instructions/*.instructions.md` (VS Code / Visual Studio / GitHub + * Copilot Chat). Each file carries YAML frontmatter with `applyTo` — a glob that + * scopes when the instructions attach. `applyTo: '**'` attaches the guidance to + * every request in the repo, which is what a persistent verification skill wants + * (there is no on-demand "model decides" mode for Copilot instruction files, so + * always-apply is the correct idiom). `description` is surfaced in Copilot's UI. + */ +function wrapCopilot(_name: string, description: string, body: string): string { + return `---\ndescription: ${description}\napplyTo: '**'\n---\n\n${body}\n`; +} + // --------------------------------------------------------------------------- // Landing paths // --------------------------------------------------------------------------- @@ -140,6 +183,12 @@ export function pathFor(target: AgentTarget, skill: string): string { return `.cursor/rules/${skill}.mdc`; case 'cline': return `.clinerules/${skill}.md`; + case 'kiro': + return `.kiro/skills/${skill}/SKILL.md`; + case 'windsurf': + return `.windsurf/rules/${skill}.md`; + case 'copilot': + return `.github/instructions/${skill}.instructions.md`; case 'codex': return 'AGENTS.md'; } @@ -170,6 +219,35 @@ export const TARGETS: Record = { mode: 'own-file', wrap: (_name, _description, body) => body, }, + kiro: { + status: 'experimental', + path: pathFor('kiro', SKILL_NAME), + mode: 'own-file', + // kiro reads SKILL.md files with name/description frontmatter, same as + // claude/antigravity, so it shares the wrapSkill wrapper. + wrap: wrapSkill, + }, + windsurf: { + status: 'experimental', + path: pathFor('windsurf', SKILL_NAME), + mode: 'own-file', + // Windsurf rules files are budget-capped (~12 K chars per `.windsurf/rules/*.md`), + // so render the compact body per skill (see compactBodyFor). + compactBody: true, + wrap: wrapWindsurf, + }, + copilot: { + status: 'experimental', + path: pathFor('copilot', SKILL_NAME), + mode: 'own-file', + // GitHub Copilot path-specific instructions: frontmatter carries `applyTo`. + // `applyTo: '**'` means the file is ALWAYS injected into Copilot requests + // (there is no on-demand "model decides" mode like Cursor/Windsurf), so + // render the compact body to keep the always-on context cost small — the + // same reasoning that drives windsurf's compact render. + compactBody: true, + wrap: wrapCopilot, + }, /** * codex target — managed-section mode. * @@ -201,6 +279,85 @@ export const MANAGED_SECTION_BEGIN = ''; export const MANAGED_SECTION_END = ''; +// --------------------------------------------------------------------------- +// Install marker (stale-skill detection, issue #123) +// --------------------------------------------------------------------------- + +/** + * Hex characters of the canonical body's SHA-256 kept in the install marker. + * 12 hex chars (48 bits) is ample for drift DETECTION (equality against bodies + * this CLI ships); the marker is provenance metadata, not a security boundary. + */ +const MARKER_HASH_HEX_LENGTH = 12; + +/** + * When one marker covers several skills (the codex managed section aggregates + * every installed skill), their names are joined with this separator in the + * marker's skill field. Skill names never contain '+' (see {@link SKILLS} keys). + */ +export const MARKER_SKILL_SEPARATOR = '+'; + +/** + * Marker line shape: ``. + * An HTML comment is inert in every target format (SKILL.md, .mdc, .clinerules + * markdown, AGENTS.md). Built via `new RegExp` so the hash length stays bound + * to {@link MARKER_HASH_HEX_LENGTH}. + */ +const SKILL_MARKER_LINE_RE = new RegExp( + `^$`, +); + +/** + * First {@link MARKER_HASH_HEX_LENGTH} hex chars of the SHA-256 of a canonical + * skill body. The hash covers the CANONICAL BODY ONLY (pre-wrap, pre-marker), + * so writing the marker into the rendered artifact never changes the hash the + * marker itself carries. + */ +export function bodyHash12(canonicalBody: string): string { + return createHash('sha256') + .update(canonicalBody, 'utf8') + .digest('hex') + .slice(0, MARKER_HASH_HEX_LENGTH); +} + +/** + * Build the provenance marker line for a skill (or a + * {@link MARKER_SKILL_SEPARATOR}-joined skill set) and its canonical body. + * `agent status` compares this fingerprint against the bodies the running CLI + * ships to detect silently stale installs. + */ +export function buildSkillMarker(skillName: string, canonicalBody: string): string { + return ``; +} + +/** A marker line parsed back into its fields. */ +export interface ParsedSkillMarker { + /** Skill name, or several names joined with {@link MARKER_SKILL_SEPARATOR}. */ + skill: string; + /** CLI version that wrote the artifact. */ + version: string; + /** First 12 hex chars of the canonical body's SHA-256 at install time. */ + hash12: string; + /** The exact marker line (trailing CR/whitespace stripped) as found. */ + line: string; +} + +/** + * Find the first testsprite-skill marker line in `content`, or null when the + * content carries none (a pre-marker install). Lines are matched whole with + * trailing CR/whitespace stripped, so CRLF checkouts parse identically. + */ +export function parseSkillMarker(content: string): ParsedSkillMarker | null { + for (const rawLine of content.split('\n')) { + const line = rawLine.trimEnd(); + const matched = SKILL_MARKER_LINE_RE.exec(line); + if (matched) { + return { skill: matched[1]!, version: matched[2]!, hash12: matched[3]!, line }; + } + } + return null; +} + type ReadFn = (url: URL) => string; const defaultRead: ReadFn = (url: URL) => readFileSync(url, 'utf8'); @@ -227,6 +384,24 @@ export function loadSkillBodyFor(skill: string, read: ReadFn = defaultRead): str return readSkillAsset(spec.bodyFile, read); } +/** + * Budget-friendly body for an own-file target whose rule files are size-capped + * (e.g. windsurf). For a skill that ships a trimmed codex asset (`codex.kind === + * 'full'`, e.g. `testsprite-verify` — full body ~22 KB, codex ~5 KB) we render + * that compact asset so the wrapped file stays under the cap. For skills whose + * codex contribution is only a one-liner (`'line'`/`'none'`, e.g. + * `testsprite-onboard`), the one-liner is useless as a standalone rule and the + * full own-file body (~6.5 KB) already fits the budget — so the full body is + * used. + */ +export function compactBodyFor(skill: string, read: ReadFn = defaultRead): string { + const spec = SKILLS[skill]; + if (!spec) throw new Error(`unknown skill: ${skill}`); + return spec.codex.kind === 'full' + ? readSkillAsset(spec.codex.file, read) + : loadSkillBodyFor(skill, read); +} + /** * Resolve a skill's codex (AGENTS.md) contribution as a Markdown string. * 'full' → read the `*.codex.md` asset; 'line' → the inline one-liner; 'none' → ''. @@ -275,15 +450,67 @@ export function loadCodexSkillBody(read: ReadFn = defaultRead): string { // renderForTarget // --------------------------------------------------------------------------- +/** + * Place the marker line inside a wrapped own-file render. + * + * - Wraps that emit YAML frontmatter (claude/antigravity/cursor): the marker + * lands on the line right after the closing `---` fence, before the body. + * - Wrapless targets (cline, body verbatim): the marker is appended as the + * LAST line instead. Cline surfaces the file's first heading as the rule + * title, so a leading comment would displace the body's H1. + */ +function injectMarkerLine(wrapped: string, markerLine: string): string { + if (wrapped.startsWith('---\n')) { + // The name/description frontmatter values are single-line, so the first + // `\n---\n` after the opening fence is always the closing fence. + const closingFence = '\n---\n'; + const fenceIdx = wrapped.indexOf(closingFence); + if (fenceIdx !== -1) { + const insertAt = fenceIdx + closingFence.length; + return `${wrapped.slice(0, insertAt)}${markerLine}\n${wrapped.slice(insertAt)}`; + } + } + const separator = wrapped.endsWith('\n') ? '' : '\n'; + return `${wrapped}${separator}${markerLine}\n`; +} + +/** + * Exact own-file bytes for a skill on a target, carrying the GIVEN marker line. + * `agent status` uses this to re-render the current canonical body with a + * file's own (possibly older-versioned) marker: when only the marker's version + * string lags but the body is unchanged, the artifact still compares pristine. + */ +export function renderOwnFileWithMarker( + target: AgentTarget, + skill: string, + markerLine: string, + body?: string, +): string { + const spec = TARGETS[target]; + if (spec.mode !== 'own-file') { + throw new Error(`renderOwnFileWithMarker: ${target} is not an own-file target`); + } + const skillSpec = SKILLS[skill]; + if (!skillSpec) throw new Error(`unknown skill: ${skill}`); + const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); + return injectMarkerLine( + spec.wrap(skillSpec.name, skillSpec.description, resolvedBody), + markerLine, + ); +} + /** * The exact bytes to write for one skill on one target. * * - own-file targets: `body` defaults to the skill's own-file asset, wrapped in - * the target's frontmatter/header. + * the target's frontmatter/header, and carrying a provenance marker line so + * `agent status` can tell fresh, stale, and hand-edited installs apart. * - codex (managed-section): returns the skill's codex contribution unwrapped - * (plain Markdown, no frontmatter). The real install does NOT call this for - * codex — it aggregates all skills via {@link buildCodexAggregate} — but it is - * kept single-skill here for tests and parity. Pass an explicit `body` to override. + * and marker-free (plain Markdown, no frontmatter). The real install does NOT + * call this for codex: it aggregates all skills via + * {@link buildCodexAggregate} and writes ONE marker just inside the BEGIN + * sentinel. It is kept single-skill here for tests and parity. Pass an + * explicit `body` to override. */ export function renderForTarget( t: AgentTarget, @@ -298,6 +525,10 @@ export function renderForTarget( const resolvedBody = body !== undefined ? body : codexContentFor(skill); return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; } - const resolvedBody = body !== undefined ? body : loadSkillBodyFor(skill); - return { path, content: spec.wrap(skillSpec.name, skillSpec.description, resolvedBody) }; + const resolvedBody = + body !== undefined ? body : spec.compactBody ? compactBodyFor(skill) : loadSkillBodyFor(skill); + return { + path, + content: renderOwnFileWithMarker(t, skill, buildSkillMarker(skill, resolvedBody), resolvedBody), + }; } diff --git a/src/lib/browser.test.ts b/src/lib/browser.test.ts new file mode 100644 index 0000000..3d6717b --- /dev/null +++ b/src/lib/browser.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +// Mock the real spawner so the default `exec` path (detached spawn + unref) +// is exercised without launching a real process. +const spawnMock = vi.fn(); +vi.mock('node:child_process', () => ({ + spawn: (command: string, args: readonly string[], opts: unknown) => + spawnMock(command, args, opts), +})); + +import { openInBrowser } from './browser.js'; +import { ApiError } from './errors.js'; + +describe('openInBrowser', () => { + const url = 'https://portal.example.com/tests/t_123'; + + it('uses `open ` on darwin', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'darwin', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'open', args: [url] }]); + }); + + it('uses rundll32 FileProtocolHandler on win32', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'win32', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'rundll32', args: ['url.dll,FileProtocolHandler', url] }]); + }); + + it('uses xdg-open on other platforms', () => { + const calls: Array<{ command: string; args: readonly string[] }> = []; + openInBrowser(url, { + platform: 'linux', + exec: (command, args) => calls.push({ command, args }), + }); + expect(calls).toEqual([{ command: 'xdg-open', args: [url] }]); + }); + + it('refuses a non-http(s) URL with exit 5 before spawning', () => { + const exec = vi.fn(); + let error: unknown; + try { + openInBrowser('file:///etc/passwd', { platform: 'linux', exec }); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiError); + expect((error as ApiError).exitCode).toBe(5); + expect(exec).not.toHaveBeenCalled(); + }); + + it('throws on a malformed URL', () => { + const exec = vi.fn(); + expect(() => openInBrowser('not a url', { exec })).toThrow(); + expect(exec).not.toHaveBeenCalled(); + }); + + describe('default spawner', () => { + beforeEach(() => { + spawnMock.mockReset(); + }); + + it('spawns detached, ignores stdio, and unrefs the child', () => { + const unref = vi.fn(); + spawnMock.mockReturnValue({ unref, on: vi.fn() }); + openInBrowser(url, { platform: 'darwin' }); + expect(spawnMock).toHaveBeenCalledWith('open', [url], { detached: true, stdio: 'ignore' }); + expect(unref).toHaveBeenCalledTimes(1); + }); + + it("handles the child's async 'error' (missing binary) with a stderr hint, not a crash", () => { + // spawn() reports ENOENT asynchronously on the child; an unhandled + // 'error' event would crash the CLI. The default spawner must register + // a listener that degrades to the manual-open hint. + const listeners = new Map void>(); + spawnMock.mockReturnValue({ + unref: vi.fn(), + on: (event: string, listener: (err: Error) => void) => { + listeners.set(event, listener); + }, + }); + const stderrSpy = vi.spyOn(process.stderr, 'write').mockReturnValue(true); + try { + openInBrowser(url, { platform: 'linux' }); + const onError = listeners.get('error'); + expect(onError).toBeDefined(); + // Firing the listener must not throw and must print the hint. + expect(() => onError!(new Error('spawn xdg-open ENOENT'))).not.toThrow(); + expect(String(stderrSpy.mock.calls.at(-1)?.[0])).toContain('could not launch a browser'); + } finally { + stderrSpy.mockRestore(); + } + }); + }); +}); diff --git a/src/lib/browser.ts b/src/lib/browser.ts new file mode 100644 index 0000000..ccd2b8a --- /dev/null +++ b/src/lib/browser.ts @@ -0,0 +1,63 @@ +/** + * Cross-platform "open this URL in the default browser" helper for + * `test open` (issue #121). Spawns the platform opener with an argv array + * (never a shell string) so a URL can never be shell-injected, and refuses + * anything that is not http(s) before any process is spawned. + * + * Platform openers: + * darwin open + * win32 rundll32 url.dll,FileProtocolHandler (avoids `cmd /c start`, + * whose re-parsing would mangle `&` and other metachars in the URL) + * other xdg-open + * + * The child is detached and unref'd so the CLI exits immediately; failures + * are the caller's to surface (it already printed the URL as the fallback). + */ +import { spawn } from 'node:child_process'; +import { localValidationError } from './errors.js'; + +export interface OpenInBrowserDeps { + platform?: NodeJS.Platform; + /** Process spawner taking an argv array. Defaults to a detached spawn. */ + exec?: (command: string, args: readonly string[]) => void; +} + +export function openInBrowser(url: string, deps: OpenInBrowserDeps = {}): void { + const parsed = new URL(url); + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { + // User-input error, not an internal failure: classify as VALIDATION_ERROR + // so it maps to exit 5 (like every other bad-argument path), not exit 1. + throw localValidationError( + 'url', + `must be an http(s) URL (got ${parsed.protocol})`, + undefined, + 'field', + ); + } + const platform = deps.platform ?? process.platform; + const exec = + deps.exec ?? + ((command: string, args: readonly string[]) => { + const child = spawn(command, [...args], { detached: true, stdio: 'ignore' }); + // spawn() reports a missing binary (ENOENT) ASYNCHRONOUSLY on the child, + // so the caller's try/catch cannot see it — and an unhandled 'error' + // event would crash the whole CLI. The URL is already on stdout, so + // degrade to the same manual-open hint the sync failure path prints. + child.on('error', () => { + process.stderr.write( + 'could not launch a browser; open the URL above manually (or use --no-browser)\n', + ); + }); + child.unref(); + }); + + if (platform === 'darwin') { + exec('open', [url]); + return; + } + if (platform === 'win32') { + exec('rundll32', ['url.dll,FileProtocolHandler', url]); + return; + } + exec('xdg-open', [url]); +} diff --git a/src/lib/bundle.commit.test.ts b/src/lib/bundle.commit.test.ts new file mode 100644 index 0000000..83cbce8 --- /dev/null +++ b/src/lib/bundle.commit.test.ts @@ -0,0 +1,124 @@ +import type * as NodeFsPromises from 'node:fs/promises'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const renameMock = vi.hoisted(() => vi.fn()); +const rmMock = vi.hoisted(() => vi.fn()); + +vi.mock('node:fs/promises', async importOriginal => { + const actual = (await importOriginal()) as typeof NodeFsPromises; + return { + ...actual, + rename: renameMock, + rm: rmMock, + }; +}); + +const { commitBundle } = await import('./bundle.js'); + +describe('commitBundle', () => { + let realRename: typeof NodeFsPromises.rename; + let realRm: typeof NodeFsPromises.rm; + + beforeEach(async () => { + const actual = (await vi.importActual('node:fs/promises')) as typeof NodeFsPromises; + realRename = actual.rename; + realRm = actual.rm; + renameMock.mockImplementation(realRename); + rmMock.mockImplementation(realRm); + }); + + afterEach(() => { + renameMock.mockReset(); + rmMock.mockReset(); + }); + + async function withTempParent(run: (parent: string) => Promise): Promise { + const parent = mkdtempSync(join(tmpdir(), 'bundle-commit-parent-')); + try { + await run(parent); + } finally { + await realRm(parent, { recursive: true, force: true }).catch(() => undefined); + } + } + + function seedBundleDirs(parent: string): { dir: string; tmpDir: string; files: string[] } { + const dir = join(parent, 'bundle'); + const tmpDir = join(dir, '.tmp'); + + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'notes.txt'), 'foreign notes\n', 'utf8'); + mkdirSync(join(dir, 'steps'), { recursive: true }); + writeFileSync(join(dir, 'meta.json'), '{"snapshotId":"snap_old"}\n', 'utf8'); + writeFileSync(join(dir, 'steps', '01-evidence.json'), '{"step":1}\n', 'utf8'); + + mkdirSync(join(tmpDir, 'steps'), { recursive: true }); + writeFileSync(join(tmpDir, 'meta.json'), '{"snapshotId":"snap_new"}\n', 'utf8'); + writeFileSync(join(tmpDir, 'result.json'), '{}\n', 'utf8'); + writeFileSync(join(tmpDir, 'steps', '01-evidence.json'), '{"step":9}\n', 'utf8'); + + return { dir, tmpDir, files: ['result.json', 'meta.json', 'steps/01-evidence.json'] }; + } + + it('rolls back to the prior complete bundle when a staged rename fails', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + renameMock.mockImplementation(async (oldPath, newPath) => { + const dest = String(newPath); + if (dest.endsWith('result.json') && !dest.includes('.aside.')) { + throw Object.assign(new Error('simulated install failure'), { code: 'EACCES' }); + } + return realRename(oldPath, newPath); + }); + + await expect(commitBundle(tmpDir, dir, files)).rejects.toThrow('simulated install failure'); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_old"}\n'); + expect(readFileSync(join(dir, 'steps', '01-evidence.json'), 'utf8')).toBe('{"step":1}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + const leftovers = readdirSync(parent).filter(name => name.includes('.aside.')); + expect(leftovers).toEqual([]); + }); + }); + + it('preserves foreign files while installing the new bundle on success', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + await expect(commitBundle(tmpDir, dir, files)).resolves.toBeUndefined(); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_new"}\n'); + expect(readFileSync(join(dir, 'steps', '01-evidence.json'), 'utf8')).toBe('{"step":9}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + expect(existsSync(join(dir, 'result.json'))).toBe(true); + }); + }); + + it('keeps the new bundle when post-commit aside cleanup fails', async () => { + await withTempParent(async parent => { + const { dir, tmpDir, files } = seedBundleDirs(parent); + + rmMock.mockImplementation(async (path, options) => { + if (String(path).includes('.aside.')) { + throw Object.assign(new Error('simulated aside cleanup failure'), { code: 'EACCES' }); + } + return realRm(path, options); + }); + + await expect(commitBundle(tmpDir, dir, files)).resolves.toBeUndefined(); + + expect(readFileSync(join(dir, 'meta.json'), 'utf8')).toBe('{"snapshotId":"snap_new"}\n'); + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('foreign notes\n'); + }); + }); +}); diff --git a/src/lib/bundle.test.ts b/src/lib/bundle.test.ts index 5b746da..d954381 100644 --- a/src/lib/bundle.test.ts +++ b/src/lib/bundle.test.ts @@ -8,9 +8,9 @@ * the full http+fetch path is wired against MSW). */ -import { existsSync, mkdtempSync } from 'node:fs'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { isAbsolute, join, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; import { applyFailedOnly, @@ -18,6 +18,7 @@ import { assertNoEscape, BUNDLE_SCHEMA_VERSION, buildMeta, + isBundleOwnedEntry, pickCodeExtension, resolveBundleDir, STREAM_URL_MAX_RETRIES, @@ -592,14 +593,33 @@ describe('resolveBundleDir', () => { it('resolves a relative path against cwd', () => { const out = resolveBundleDir('./tmp/x'); - expect(out.endsWith('/tmp/x')).toBe(true); - expect(out.startsWith('/')).toBe(true); + expect(out).toBe(resolve(process.cwd(), 'tmp', 'x')); + expect(isAbsolute(out)).toBe(true); }); it('strips a trailing slash', () => { const out = resolveBundleDir('/tmp/x/'); expect(out).toBe('/tmp/x'); }); + + // `C:\...` is only recognized as absolute by node:path when the process + // itself is running on win32 (path.isAbsolute/resolve are platform-native, + // not path.win32.* explicitly) — these two assertions are only meaningful + // under an actual Windows runtime, hence gated rather than run everywhere. + it.runIf(process.platform === 'win32')( + 'strips a trailing backslash (native Windows path)', + () => { + const out = resolveBundleDir('C:\\Users\\me\\bundle\\'); + expect(out).toBe('C:\\Users\\me\\bundle'); + }, + ); + + it.runIf(process.platform === 'win32')( + 'preserves a bare Windows drive root instead of truncating it to a drive-relative path', + () => { + expect(resolveBundleDir('C:\\')).toBe('C:\\'); + }, + ); }); describe('streamUrlToFile retry', () => { @@ -640,17 +660,24 @@ describe('streamUrlToFile retry', () => { calls++; throw new Error('ENETUNREACH dns lookup failed'); }; - await expect( - streamUrlToFile( - 'https://example.com/x', + let caught: unknown; + + try { + await streamUrlToFile( + 'https://example.com/x?X-Amz-Signature=secret-token', '/tmp/will-not-be-written', fetchImpl as typeof globalThis.fetch, { sleep: noSleep }, - ), - ).rejects.toMatchObject({ + ); + } catch (err) { + caught = err; + } + + expect(caught).toMatchObject({ name: 'TransportError', message: expect.stringContaining('ENETUNREACH'), }); + expect(caught).not.toMatchObject({ message: expect.stringContaining('secret-token') }); expect(calls).toBe(STREAM_URL_MAX_RETRIES); }); @@ -660,17 +687,46 @@ describe('streamUrlToFile retry', () => { calls++; return new Response('Forbidden', { status: 403 }); }; - await expect( - streamUrlToFile( - 'https://example.com/x', + let caught: unknown; + const presignedUrl = 'https://example.com/x?X-Amz-Signature=secret-token#download'; + + try { + await streamUrlToFile( + presignedUrl, '/tmp/will-not-be-written', fetchImpl as typeof globalThis.fetch, { sleep: noSleep }, - ), - ).rejects.toMatchObject({ code: 'UNAVAILABLE' }); + ); + } catch (err) { + caught = err; + } + + expect(caught).toMatchObject({ + code: 'UNAVAILABLE', + details: { status: 403, artifactUrl: 'https://example.com/x' }, + }); + const details = (caught as { details?: Record }).details; + expect(details).not.toHaveProperty('url'); + expect(JSON.stringify(details)).not.toContain('secret-token'); expect(calls).toBe(1); }); + it('disables automatic redirects so unsafe redirect targets cannot bypass URL validation', async () => { + const dir = mkdtempSync(join(tmpdir(), 'stream-test-')); + const dest = join(dir, 'out.bin'); + const redirects: Array = []; + const fetchImpl = async (_url: Parameters[0], init?: RequestInit) => { + redirects.push(init?.redirect); + return new Response('hello', { status: 200 }); + }; + + await streamUrlToFile('https://example.com/x', dest, fetchImpl as typeof globalThis.fetch, { + sleep: noSleep, + }); + + expect(redirects).toEqual(['error']); + }); + it('sleeps between retries', async () => { const sleepDelays: number[] = []; const fetchImpl = async () => { @@ -694,6 +750,37 @@ describe('streamUrlToFile retry', () => { }); }); +describe('isBundleOwnedEntry', () => { + it('owns the fixed bundle file set', () => { + for (const entry of [ + 'result.json', + 'failure.json', + 'video.mp4', + 'meta.json', + 'steps', + '.tmp', + '.partial', + ]) { + expect(isBundleOwnedEntry(entry)).toBe(true); + } + }); + + it('owns code. for any single-token extension', () => { + expect(isBundleOwnedEntry('code.ts')).toBe(true); + expect(isBundleOwnedEntry('code.js')).toBe(true); + expect(isBundleOwnedEntry('code.py')).toBe(true); + }); + + it('does not own foreign entries', () => { + expect(isBundleOwnedEntry('notes.txt')).toBe(false); + expect(isBundleOwnedEntry('src')).toBe(false); + expect(isBundleOwnedEntry('.git')).toBe(false); + expect(isBundleOwnedEntry('code.tar.gz')).toBe(false); + expect(isBundleOwnedEntry('mycode.ts')).toBe(false); + expect(isBundleOwnedEntry('code.')).toBe(false); + }); +}); + describe('step artifact path validation', () => { // A fetchImpl that fails the test if called — proves validation rejects // before any write happens. @@ -807,6 +894,49 @@ describe('step artifact path validation', () => { expect(existsSync(join(res.dir, 'meta.json'))).toBe(true); }); + describe('commit sweep ownership (data-loss guard)', () => { + it('preserves pre-existing foreign files and directories in the --out dir', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'notes.txt'), 'important notes\n', 'utf8'); + mkdirSync(join(dir, 'src')); + writeFileSync(join(dir, 'src', 'app.js'), "console.log('app')\n", 'utf8'); + + const res = await writeBundle(stepCtx(3), { + dir, + failedOnly: false, + fetchImpl: throwIfFetched, + }); + + // The bundle landed… + expect(existsSync(join(res.dir, 'meta.json'))).toBe(true); + expect(existsSync(join(res.dir, 'result.json'))).toBe(true); + // …and the user's unrelated files survived the commit sweep. + expect(readFileSync(join(dir, 'notes.txt'), 'utf8')).toBe('important notes\n'); + expect(readFileSync(join(dir, 'src', 'app.js'), 'utf8')).toBe("console.log('app')\n"); + }); + + it('still sweeps a stale bundle-owned video.mp4 the new bundle does not write', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'video.mp4'), 'stale-bytes', 'utf8'); + + // stepCtx has videoUrl: null → the fresh bundle ships no video. + await writeBundle(stepCtx(3), { dir, failedOnly: false, fetchImpl: throwIfFetched }); + + expect(existsSync(join(dir, 'video.mp4'))).toBe(false); + }); + + it('sweeps a stale code file with a different extension than the new bundle writes', async () => { + const dir = mkdtempSync(join(tmpdir(), 'bundle-test-')); + writeFileSync(join(dir, 'code.py'), '# stale python code\n', 'utf8'); + + // baseCtx.code.language is 'typescript' → the fresh bundle writes code.ts. + await writeBundle(stepCtx(3), { dir, failedOnly: false, fetchImpl: throwIfFetched }); + + expect(existsSync(join(dir, 'code.ts'))).toBe(true); + expect(existsSync(join(dir, 'code.py'))).toBe(false); + }); + }); + describe('assertNoEscape', () => { it('returns the resolved path for an in-bounds segment', () => { const base = mkdtempSync(join(tmpdir(), 'bundle-test-')); diff --git a/src/lib/bundle.ts b/src/lib/bundle.ts index 7795115..0685fd7 100644 --- a/src/lib/bundle.ts +++ b/src/lib/bundle.ts @@ -33,12 +33,13 @@ * when the agent re-runs. M3 may add resume. */ +import { randomUUID } from 'node:crypto'; import { mkdir, mkdtemp, readdir, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'; import type { Writable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import type { ReadableStream as NodeReadableStream } from 'node:stream/web'; import { createWriteStream } from 'node:fs'; -import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'; import type { CliFailureContext, CliTestStep } from '../commands/test.js'; import { ApiError, TransportError, localValidationError } from './errors.js'; import { requireEnum } from './validate.js'; @@ -306,6 +307,23 @@ export function applyFailedOnly(ctx: CliFailureContext): CliFailureContext { }; } +/** + * Strip trailing path separators, tolerating both `/` and `\` since a + * native Windows `--out` value (typed or pasted from Explorer) commonly + * ends in a backslash. Preserves a bare drive root (`C:\`) — stripping + * its separator would turn it into `C:`, which Windows resolves as + * "current directory on drive C", not the drive root. + */ +function stripTrailingSeparators(rawPath: string): string { + if (rawPath.length <= 1) return rawPath; + let end = rawPath.length; + while (end > 1 && (rawPath[end - 1] === '/' || rawPath[end - 1] === '\\')) { + if (end === 3 && rawPath[1] === ':' && /[A-Za-z]/.test(rawPath[0]!)) break; + end--; + } + return rawPath.slice(0, end); +} + /** * Resolve the user-supplied `--out` path into an absolute directory. * Empty strings are rejected with `VALIDATION_ERROR` for consistency @@ -325,7 +343,7 @@ export function resolveBundleDir(rawPath: string): string { }, }); } - const trimmed = rawPath.endsWith('/') ? rawPath.slice(0, -1) : rawPath; + const trimmed = stripTrailingSeparators(rawPath); return isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed); } @@ -520,83 +538,124 @@ async function freshTmpDir(dir: string): Promise { } /** - * Rename `/` → `/` for every file in `files`. - * - * Critical ordering for atomicity (the §3 "agent-safe" contract): + * Whether a top-level directory entry belongs to the bundle format — + * i.e. something a prior `writeBundle` could have produced and this + * commit is therefore allowed to clean up. `code.` is matched by + * pattern (not the current run's extension) so a stale `code.py` is + * still swept when the new bundle writes `code.ts`. Everything else in + * the directory is the user's and must never be deleted (`--out` can + * point at a pre-existing, populated directory). + */ +export function isBundleOwnedEntry(entry: string): boolean { + if ( + entry === 'result.json' || + entry === 'failure.json' || + entry === 'video.mp4' || + entry === 'meta.json' || + entry === 'steps' || + entry === '.tmp' || + entry === '.partial' + ) { + return true; + } + return /^code\.[A-Za-z0-9]+$/.test(entry); +} + +/** + * Atomically install the complete bundle staged in `tmpDir` into `dir`. * - * 1. **Remove the OLD `meta.json` first.** The bundle's completion - * signal is `meta.json`'s presence; an agent reading `` while - * we're mutating it must see "no meta → bundle absent or - * mid-write" rather than "meta points at a snapshot that's already - * been partially overwritten." Removing the old meta is what - * makes the rest of the swap safe to do in place. - * 2. Wipe stale top-level files (e.g. an old `video.mp4` when the new - * bundle has no video). Without this, a fresh bundle could ship - * with a stale video lingering at the top level. - * 3. Replace `/steps/` wholesale. - * 4. Rename top-level files into place. - * 5. **Rename `meta.json` LAST.** Its visible presence is the atomic - * completion signal; until step 5 lands, agents see "incomplete." + * Compatible with the #162 data-loss guard: only `isBundleOwnedEntry` + * names are moved aside or replaced — foreign files in `--out` survive. * - * The window between (1) and (5) is bounded by a handful of `rename` - * syscalls — small enough that a SIGKILL there is rare, and any agent - * caught reading the dir during it sees no meta and refuses to consume - * (per §7.3). That's what we want. + * Re-commit safety: bundle-owned entries are renamed to a sibling + * aside directory before the new artifacts land. The prior `meta.json` + * is moved aside first (§3/§7.3 — no meta ⇒ refuse to consume) so a + * concurrent reader never sees meta pointing at steps already gone. + * The new `meta.json` is installed last. On failure, aside entries are + * restored so a failed re-commit never leaves the directory unusable. */ -async function commitBundle( +export async function commitBundle( tmpDir: string, dir: string, files: ReadonlyArray, ): Promise { - // (1) Remove the prior bundle's completion signal FIRST. - await unlink(join(dir, 'meta.json')).catch(() => undefined); - - // (2) Sweep stale top-level files that the new bundle won't write. - // If the prior run wrote `video.mp4` and the new run has no video, - // an in-place rename leaves the old video lingering. Enumerate - // current top-level entries and remove anything that isn't being - // freshly renamed in. - const topLevel = files.filter(f => !f.startsWith('steps/')); - const newTopLevelSet = new Set(topLevel); - newTopLevelSet.add('meta.json'); // about to land last, do not delete - const existing = await readdir(dir).catch(() => [] as string[]); - for (const entry of existing) { - // Preserve the writer's own scratch dir + the .partial marker - // (we'll re-evaluate .partial at the end of commit). Anything else - // not-listed in the new bundle is stale. - if (entry === '.tmp' || entry === '.partial') continue; - if (newTopLevelSet.has(entry)) continue; - if (entry === 'steps') continue; // handled below - await rm(join(dir, entry), { recursive: true, force: true }); - } + const parent = dirname(dir); + const base = basename(dir); + const asideDir = join(parent, `.${base}.aside.${randomUUID()}`); + const asideLog: Array<{ asidePath: string; restorePath: string }> = []; + + const asideIfPresent = async (entry: string): Promise => { + const restorePath = join(dir, entry); + if (!(await pathExists(restorePath))) return; + const asidePath = join(asideDir, entry); + await mkdir(dirname(asidePath), { recursive: true }); + await rename(restorePath, asidePath); + asideLog.push({ asidePath, restorePath }); + }; - // (3) Replace `/steps/` with `/steps/`. - const stepsTmp = join(tmpDir, 'steps'); - const stepsDir = join(dir, 'steps'); - await rm(stepsDir, { recursive: true, force: true }); - if (await dirExists(stepsTmp)) { - await rename(stepsTmp, stepsDir); - } + const rollback = async (): Promise => { + for (const { asidePath, restorePath } of [...asideLog].reverse()) { + await rm(restorePath, { recursive: true, force: true }).catch(() => undefined); + await rename(asidePath, restorePath).catch(() => undefined); + } + await rm(asideDir, { recursive: true, force: true }).catch(() => undefined); + }; - // (4) Top-level files (result/failure/code/video). meta.json renames - // LAST; track it separately. - const metaIdx = topLevel.indexOf('meta.json'); - const beforeMeta = metaIdx >= 0 ? topLevel.filter((_, i) => i !== metaIdx) : topLevel; - for (const file of beforeMeta) { - await rename(join(tmpDir, file), join(dir, file)); - } + try { + const topLevel = files.filter(f => !f.startsWith('steps/')); + const newTopLevelSet = new Set(topLevel); + newTopLevelSet.add('meta.json'); + + // meta.json first — its presence is the completion signal; do not + // move steps (or anything else) aside while a stale meta is still visible. + await asideIfPresent('meta.json'); + + const existing = await readdir(dir).catch(() => [] as string[]); + for (const entry of existing) { + if (entry === '.tmp' || entry === 'meta.json') continue; + if (!isBundleOwnedEntry(entry)) continue; + + const isStale = entry !== 'steps' && entry !== '.partial' && !newTopLevelSet.has(entry); + const willReplace = entry === 'steps' || newTopLevelSet.has(entry); + if (isStale || willReplace) { + await asideIfPresent(entry); + } + } - // (5) meta.json LAST → atomic completion signal. - if (metaIdx >= 0) { - await rename(join(tmpDir, 'meta.json'), join(dir, 'meta.json')); - } + const stepsTmp = join(tmpDir, 'steps'); + const stepsDir = join(dir, 'steps'); + if (await dirExists(stepsTmp)) { + await rename(stepsTmp, stepsDir); + } - // .partial from a prior aborted run is now stale. Remove it so an - // agent inspecting the dir sees only the fresh bundle. - await unlink(join(dir, '.partial')).catch(() => undefined); + const metaIdx = topLevel.indexOf('meta.json'); + const beforeMeta = metaIdx >= 0 ? topLevel.filter((_, i) => i !== metaIdx) : topLevel; + for (const file of beforeMeta) { + await rename(join(tmpDir, file), join(dir, file)); + } - // Clean up the now-empty tmp dir. - await rm(tmpDir, { recursive: true, force: true }); + if (metaIdx >= 0) { + await rename(join(tmpDir, 'meta.json'), join(dir, 'meta.json')); + } + + await unlink(join(dir, '.partial')).catch(() => undefined); + await rm(tmpDir, { recursive: true, force: true }); + // Best-effort aside cleanup — failures must not roll back a committed bundle. + await rm(asideDir, { recursive: true, force: true }).catch(() => undefined); + } catch (err) { + await rollback(); + await rm(tmpDir, { recursive: true, force: true }).catch(() => undefined); + throw err; + } +} + +async function pathExists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } } async function dirExists(path: string): Promise { @@ -746,17 +805,18 @@ export async function streamUrlToFile( deps?: { sleep?: (ms: number) => Promise }, ): Promise { const sleepFn = deps?.sleep ?? ((ms: number) => new Promise(r => setTimeout(r, ms))); + const artifactUrl = redactArtifactUrlForDetails(url); for (let attempt = 1; attempt <= STREAM_URL_MAX_RETRIES; attempt++) { let response: Response; try { - response = await fetchImpl(url); + response = await fetchImpl(url, { redirect: 'error' }); } catch (err) { const message = err instanceof Error ? err.message : String(err); if (attempt < STREAM_URL_MAX_RETRIES) { await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + throw new TransportError(`Failed to download presigned URL ${artifactUrl}: ${message}`); } if (!response.ok) { // Non-2xx: the URL itself is bad (expired, unauthorized, not found). @@ -768,7 +828,7 @@ export async function streamUrlToFile( nextAction: 'Re-run `testsprite test failure get`. Presigned URLs in the bundle expire after 15 minutes.', requestId: 'local', - details: { status: response.status, url }, + details: { status: response.status, artifactUrl }, }, }); } @@ -788,7 +848,7 @@ export async function streamUrlToFile( await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed to download presigned URL ${url}: ${message}`); + throw new TransportError(`Failed to download presigned URL ${artifactUrl}: ${message}`); } } await mkdir(dirname(filePath), { recursive: true }); @@ -810,11 +870,20 @@ export async function streamUrlToFile( await sleepFn(STREAM_URL_RETRY_DELAY_MS); continue; } - throw new TransportError(`Failed mid-download of ${url}: ${message}`); + throw new TransportError(`Failed mid-download of ${artifactUrl}: ${message}`); } } } +function redactArtifactUrlForDetails(url: string): string { + try { + const parsed = new URL(url); + return `${parsed.origin}${parsed.pathname}`; + } catch { + return ''; + } +} + function isPresignedUrl(value: string): boolean { return value.startsWith('https://'); } diff --git a/src/lib/client-factory.test.ts b/src/lib/client-factory.test.ts index f9fdc44..d5135cc 100644 --- a/src/lib/client-factory.test.ts +++ b/src/lib/client-factory.test.ts @@ -1,10 +1,12 @@ -import { afterEach, describe, expect, it } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { DRY_RUN_API_KEY, DRY_RUN_BANNER, + assertValidApiKeyHeaderValue, assertValidEndpointUrl, emitDryRunBanner, makeHttpClient, + parseRequestTimeoutFlag, resetDryRunBannerForTesting, resolveRequestTimeoutMs, } from './client-factory.js'; @@ -213,6 +215,45 @@ describe('resolveRequestTimeoutMs', () => { }); }); +// --------------------------------------------------------------------------- +// parseRequestTimeoutFlag — strict flag parsing (seconds → ms) +// --------------------------------------------------------------------------- + +describe('parseRequestTimeoutFlag', () => { + it('returns undefined when the flag is omitted (factory falls back to env/default)', () => { + expect(parseRequestTimeoutFlag(undefined)).toBeUndefined(); + }); + + it('converts a positive number of seconds to milliseconds', () => { + expect(parseRequestTimeoutFlag('30')).toBe(30_000); + expect(parseRequestTimeoutFlag('1')).toBe(1_000); + expect(parseRequestTimeoutFlag('2.5')).toBe(2_500); + }); + + it('does NOT reject positive out-of-range values — resolveRequestTimeoutMs clamps them', () => { + // 700s is above the 600s cap, but parsing succeeds; the clamp lives in + // resolveRequestTimeoutMs so a large script-supplied value still works. + expect(parseRequestTimeoutFlag('700')).toBe(700_000); + }); + + it.each(['abc', '30s', '0', '-5', 'NaN', 'Infinity', ''])( + 'throws a VALIDATION_ERROR (exit 5) on the invalid flag value %j', + bad => { + let caught: unknown; + try { + parseRequestTimeoutFlag(bad); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('request-timeout'); + }, + ); +}); + // --------------------------------------------------------------------------- // makeHttpClient — requestTimeoutMs propagation // --------------------------------------------------------------------------- @@ -291,6 +332,77 @@ describe('makeHttpClient — real path (regression)', () => { // assertValidEndpointUrl — endpoint syntax guard (NOT an SSRF guard) // --------------------------------------------------------------------------- +describe('makeHttpClient - API key validation', () => { + it.each([ + ['newline', 'sk-user-abc\ndef'], + ['carriage return', 'sk-user-abc\rdef'], + ['smart dash', 'sk-user-abc\u2013def'], + ['smart quote', 'sk-user-\u201cabc\u201d'], + ['emoji', 'sk-user-abc\u{1f600}'], + ])('rejects a malformed configured API key with %s before fetch/retry', (_label, apiKey) => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: apiKey } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }); + + it('treats a whitespace-only TESTSPRITE_API_KEY env var as unset (AUTH_REQUIRED)', () => { + const fetchImpl = vi.fn(); + let caught: unknown; + try { + makeHttpClient( + { profile: 'default', output: 'json', debug: false, dryRun: false }, + { + env: { TESTSPRITE_API_KEY: ' ' } as NodeJS.ProcessEnv, + credentialsPath: NO_CREDS_PATH, + fetchImpl, + }, + ); + } catch (err) { + caught = err; + } + expect(fetchImpl).not.toHaveBeenCalled(); + expect(caught).toBeInstanceOf(ApiError); + expect((caught as ApiError).code).toBe('AUTH_REQUIRED'); + }); +}); + +describe('assertValidApiKeyHeaderValue', () => { + it('accepts a normal ASCII API key value', () => { + expect(() => assertValidApiKeyHeaderValue('sk-user-abc-def_123')).not.toThrow(); + }); + + it('throws a VALIDATION_ERROR (exit 5) for a key that cannot be sent as x-api-key', () => { + let caught: unknown; + try { + assertValidApiKeyHeaderValue('sk-user-abc\u2013def'); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('api-key'); + }); +}); + describe('assertValidEndpointUrl', () => { it('accepts http(s) URLs, including private / localhost hosts (self-hosted, dev, mock)', () => { for (const url of [ diff --git a/src/lib/client-factory.ts b/src/lib/client-factory.ts index fa1f8dc..96d9dc0 100644 --- a/src/lib/client-factory.ts +++ b/src/lib/client-factory.ts @@ -24,8 +24,11 @@ import { REQUEST_TIMEOUT_MAX_MS, REQUEST_TIMEOUT_MIN_MS, } from './http.js'; +import { globalShutdown } from './interrupt.js'; import type { OutputMode } from './output.js'; import { createDryRunFetch } from './dry-run/fetch.js'; +import { noteServerVersion } from './version-notice.js'; +import { VERSION } from '../version.js'; export interface CommonOptions { profile: string; @@ -68,6 +71,12 @@ export interface ClientFactoryDeps { credentialsPath?: string; fetchImpl?: FetchImpl; stderr?: (line: string) => void; + /** + * Shutdown signal composed into every outgoing fetch (DEV-331 piece 1). + * Defaults to `globalShutdown.signal` so an armed SIGINT/SIGTERM aborts an + * in-flight request; tests inject their own controller's signal. + */ + shutdownSignal?: AbortSignal; } /** @@ -180,6 +189,59 @@ export function assertValidEndpointUrl(rawUrl: string): void { } } +export function assertValidApiKeyHeaderValue(apiKey: string): void { + const reason = + 'must be a non-empty HTTP header value; paste the raw key without smart punctuation, emoji, or line breaks'; + + if (apiKey.trim().length === 0) { + throw localValidationError('api-key', reason, undefined, 'field'); + } + + for (let i = 0; i < apiKey.length; i += 1) { + const code = apiKey.charCodeAt(i); + if (code < 0x20 || code === 0x7f || code > 0xff) { + throw localValidationError('api-key', reason, undefined, 'field'); + } + } +} + +/** + * Parse the `--request-timeout ` flag value into milliseconds. + * + * Returns `undefined` when the flag was omitted (the factory then falls back to + * the `TESTSPRITE_REQUEST_TIMEOUT_MS` env var, else the 120s default). + * + * A supplied-but-invalid value (non-numeric, zero, or negative) throws a typed + * VALIDATION_ERROR (exit 5) rather than being silently dropped. An explicit + * `--request-timeout 30s` typo previously resolved to `undefined` and the + * command ran with the default 120s deadline — the operator believed they had + * set a timeout but had not, with no signal. Failing loudly here is consistent + * with every other validated flag (`--page-size`, `--output`, `--type`). + * + * Out-of-range but positive values are intentionally NOT rejected — they flow + * through to {@link resolveRequestTimeoutMs}, which clamps to + * `[REQUEST_TIMEOUT_MIN_MS, REQUEST_TIMEOUT_MAX_MS]`. The env-var path stays + * lenient by design (a stray global env var should not hard-fail every + * command); only the explicit per-invocation flag is strict. + * + * This single definition replaces five byte-identical copies that previously + * lived in `auth`, `project`, `usage`, `init`, and `test` — drift between them + * would have silently changed timeout behaviour depending on the command. + */ +export function parseRequestTimeoutFlag(raw: string | undefined): number | undefined { + if (raw === undefined) return undefined; + const n = Number(raw); + if (!Number.isFinite(n) || n <= 0) { + // Surface the offending value in the message (same as assertValidEndpointUrl) + // so the operator sees exactly what they typed. + throw localValidationError( + 'request-timeout', + `"${raw}" is not valid — must be a positive number of seconds`, + ); + } + return Math.round(n * 1000); // seconds → milliseconds +} + export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {}): HttpClient { const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); const env = deps.env ?? process.env; @@ -199,6 +261,7 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} onDebug: opts.debug ? (event: DebugEvent) => stderr(formatDryRunDebug(event)) : undefined, onTransition: opts.verbose ? (msg: string) => stderr(`[verbose] ${msg}`) : undefined, requestTimeoutMs, + shutdownSignal: deps.shutdownSignal ?? globalShutdown.signal, }); } @@ -214,13 +277,27 @@ export function makeHttpClient(opts: CommonOptions, deps: ClientFactoryDeps = {} // VALIDATION_ERROR rather than an opaque URL throw or a retried "fetch failed". assertValidEndpointUrl(config.apiUrl); if (!config.apiKey) throw ApiError.authRequired(); + assertValidApiKeyHeaderValue(config.apiKey); return new HttpClient({ baseUrl: facadeBaseUrl(config.apiUrl), apiKey: config.apiKey, fetchImpl: deps.fetchImpl, onDebug: opts.debug ? (event: DebugEvent) => stderr(formatDebug(event)) : undefined, onTransition: opts.verbose ? (msg: string) => stderr(`[verbose] ${msg}`) : undefined, + // Warn once if the backend advertises a minimum supported version above + // this binary's. Gating (opt-out env, TTY, output mode, dry-run) lives in + // noteServerVersion; the client just forwards the observed headers. + onServerVersion: info => + noteServerVersion(info, { + currentVersion: VERSION, + env, + isTTY: process.stderr.isTTY === true, + outputMode: opts.output, + dryRun: opts.dryRun, + stderr, + }), requestTimeoutMs, + shutdownSignal: deps.shutdownSignal ?? globalShutdown.signal, }); } diff --git a/src/lib/config.test.ts b/src/lib/config.test.ts index 463ddcc..2a58fc8 100644 --- a/src/lib/config.test.ts +++ b/src/lib/config.test.ts @@ -55,6 +55,23 @@ describe('loadConfig', () => { ).toBe('option-profile'); }); + it('treats empty TESTSPRITE_PROFILE as unset (falls back to default)', () => { + expect(loadConfig({ env: { TESTSPRITE_PROFILE: '' }, credentialsPath }).profile).toBe( + 'default', + ); + }); + + it('treats whitespace-only TESTSPRITE_PROFILE as unset (falls back to default)', () => { + const config = loadConfig({ env: { TESTSPRITE_PROFILE: ' ' }, credentialsPath }); + expect(config.profile).toBe('default'); + }); + + it('reads credentials from the default profile when TESTSPRITE_PROFILE is blank', () => { + writeProfile('default', { apiKey: 'sk-default' }, { path: credentialsPath }); + const config = loadConfig({ env: { TESTSPRITE_PROFILE: ' ' }, credentialsPath }); + expect(config.apiKey).toBe('sk-default'); + }); + it('option.endpointUrl overrides everything', () => { writeProfile('default', { apiUrl: 'https://file' }, { path: credentialsPath }); const config = loadConfig({ @@ -81,6 +98,28 @@ describe('loadConfig', () => { const config = loadConfig({ profile: 'dev', env: {}, credentialsPath }); expect(config.apiKey).toBe('sk-dev'); }); + + it('treats empty / whitespace TESTSPRITE_API_URL as unset (falls through to profile)', () => { + writeProfile( + 'default', + { apiKey: 'sk-file', apiUrl: 'https://api.example.com:8443' }, + { path: credentialsPath }, + ); + const config = loadConfig({ + env: { TESTSPRITE_API_URL: ' ' }, + credentialsPath, + }); + expect(config.apiUrl).toBe('https://api.example.com:8443'); + }); + + it('treats empty / whitespace TESTSPRITE_API_KEY as unset (falls through to profile)', () => { + writeProfile('default', { apiKey: 'sk-file' }, { path: credentialsPath }); + const config = loadConfig({ + env: { TESTSPRITE_API_KEY: '' }, + credentialsPath, + }); + expect(config.apiKey).toBe('sk-file'); + }); }); describe('defaultConfigPath', () => { diff --git a/src/lib/config.ts b/src/lib/config.ts index 363c394..a69f2e4 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -17,6 +17,11 @@ export interface LoadConfigOptions { const DEFAULT_API_URL = 'https://api.testsprite.com'; +/** Treat empty / whitespace-only env values as unset for `??` resolution chains. */ +export function normalizeEnvVar(value: string | undefined): string | undefined { + return value?.trim() || undefined; +} + export function defaultConfigPath(): string { return join(homedir(), '.testsprite', 'config'); } @@ -34,13 +39,23 @@ export function defaultConfigPath(): string { */ export function loadConfig(options: LoadConfigOptions = {}): Config { const env = options.env ?? process.env; - const profile = options.profile ?? env.TESTSPRITE_PROFILE ?? DEFAULT_PROFILE; + const profile = options.profile ?? normalizeEnvVar(env.TESTSPRITE_PROFILE) ?? DEFAULT_PROFILE; const credentialsPath = options.credentialsPath ?? defaultCredentialsPath(); const fileEntry = readProfile(profile, { path: credentialsPath }); + // Empty / whitespace-only env vars are treated as unset so they do not + // short-circuit the `??` chain (e.g. `export TESTSPRITE_API_URL=` or + // `export TESTSPRITE_PROFILE=` in a shell profile). For the profile this + // also avoids a confusing VALIDATION_ERROR: an empty name fails the INI + // section-name guard, so without normalization a blank env var would break + // every command instead of falling back to the default profile. Matches the + // normalization in auth configure and init/setup. + const envApiUrl = normalizeEnvVar(env.TESTSPRITE_API_URL); + const envApiKey = normalizeEnvVar(env.TESTSPRITE_API_KEY); + return { - apiUrl: options.endpointUrl ?? env.TESTSPRITE_API_URL ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, - apiKey: env.TESTSPRITE_API_KEY ?? fileEntry?.apiKey, + apiUrl: options.endpointUrl ?? envApiUrl ?? fileEntry?.apiUrl ?? DEFAULT_API_URL, + apiKey: envApiKey ?? fileEntry?.apiKey, profile, }; } diff --git a/src/lib/credentials.test.ts b/src/lib/credentials.test.ts index ac038f3..194c1ff 100644 --- a/src/lib/credentials.test.ts +++ b/src/lib/credentials.test.ts @@ -1,7 +1,7 @@ import { mkdtempSync, statSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; -import { tmpdir } from 'node:os'; +import { homedir, tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DEFAULT_PROFILE, assertValidProfileName, @@ -88,6 +88,38 @@ describe('serializeCredentials', () => { expect(text).toContain('api_key = sk'); expect(text).not.toContain('api_url'); }); + + it('strips newline characters from values to prevent INI injection', () => { + // A malicious apiUrl with embedded newlines could inject new key-value + // pairs or section headers into the credentials file. The serializer + // must strip \n and \r so the written file has exactly one value per + // field and no injected content parsed as separate keys/sections. + const malicious = 'https://evil.com\napi_key = sk-HIJACKED\n[admin]\napi_key = sk-admin'; + const text = serializeCredentials({ default: { apiKey: 'sk-real', apiUrl: malicious } }); + // The output must NOT contain a standalone [admin] section header + // (it would be on its own line if injection succeeded) + const lines = text.split('\n'); + // Only one section header exists: [default] + const sectionHeaders = lines.filter(l => /^\[.+\]$/.test(l.trim())); + expect(sectionHeaders).toEqual(['[default]']); + // Only one api_key line exists (the real one, not an injected duplicate) + const apiKeyLines = lines.filter(l => l.trim().startsWith('api_key')); + expect(apiKeyLines).toHaveLength(1); + expect(apiKeyLines[0]).toContain('sk-real'); + // Round-trip: reading back must return only the real key, not the injected one + const parsed = parseCredentials(text); + expect(parsed['default']?.apiKey).toBe('sk-real'); + expect(parsed['admin']).toBeUndefined(); + }); + + it('strips \\r\\n (CRLF) injection from values', () => { + const text = serializeCredentials({ default: { apiUrl: 'https://x.com\r\napi_key = pwned' } }); + const parsed = parseCredentials(text); + // The injected api_key must NOT be parsed as a real key + expect(parsed['default']?.apiKey).toBeUndefined(); + // The api_url value is on one line (newlines stripped) + expect(parsed['default']?.apiUrl).toContain('https://x.com'); + }); }); describe('readCredentialsFile / readProfile', () => { @@ -107,8 +139,11 @@ describe('writeProfile', () => { it('creates the file with mode 0600 and writes the profile', () => { writeProfile(DEFAULT_PROFILE, { apiKey: 'sk-new' }, { path: credentialsPath }); expect(existsSync(credentialsPath)).toBe(true); - const mode = statSync(credentialsPath).mode & 0o777; - expect(mode).toBe(0o600); + // POSIX file modes don't exist on Windows (stat reports 0666). + if (process.platform !== 'win32') { + const mode = statSync(credentialsPath).mode & 0o777; + expect(mode).toBe(0o600); + } expect(readProfile(DEFAULT_PROFILE, { path: credentialsPath })).toEqual({ apiKey: 'sk-new' }); }); @@ -156,7 +191,47 @@ describe('ensureRestrictiveMode', () => { expect(() => ensureRestrictiveMode(credentialsPath)).not.toThrow(); }); - it('downgrades over-permissive modes', () => { + it('tightens the Windows ACL with icacls instead of POSIX chmod', () => { + mkdirSync(tmpRoot, { recursive: true }); + writeFileSync(credentialsPath, 'data', { mode: 0o666 }); + const spawn = vi.fn(() => ({ status: 0, signal: null, output: [], pid: 123 })) as never; + + ensureRestrictiveMode(credentialsPath, { + platform: 'win32', + env: { USERNAME: 'alice' } as NodeJS.ProcessEnv, + spawnSync: spawn, + }); + + expect(spawn).toHaveBeenCalledWith( + 'icacls', + [credentialsPath, '/inheritance:r', '/grant:r', 'alice:F'], + { + shell: false, + stdio: 'ignore', + windowsHide: true, + }, + ); + }); + + it('warns on Windows when credentials ACL tightening cannot run', () => { + mkdirSync(tmpRoot, { recursive: true }); + writeFileSync(credentialsPath, 'data'); + const warnings: string[] = []; + const spawn = vi.fn(() => ({ status: 0, signal: null, output: [], pid: 123 })) as never; + + ensureRestrictiveMode(credentialsPath, { + platform: 'win32', + env: {} as NodeJS.ProcessEnv, + spawnSync: spawn, + warn: line => warnings.push(line), + }); + + expect(spawn).not.toHaveBeenCalled(); + expect(warnings.join('\n')).toContain('credentials file permissions were not tightened'); + }); + + // POSIX-only premise: Windows has no 0644/0600 distinction to downgrade. + it.skipIf(process.platform === 'win32')('downgrades over-permissive modes', () => { mkdirSync(tmpRoot, { recursive: true }); writeFileSync(credentialsPath, 'data', { mode: 0o644 }); ensureRestrictiveMode(credentialsPath); @@ -167,7 +242,7 @@ describe('ensureRestrictiveMode', () => { describe('defaultCredentialsPath', () => { it('points at ~/.testsprite/credentials', () => { - expect(defaultCredentialsPath().endsWith('/.testsprite/credentials')).toBe(true); + expect(defaultCredentialsPath()).toBe(join(homedir(), '.testsprite', 'credentials')); }); }); diff --git a/src/lib/credentials.ts b/src/lib/credentials.ts index c885c66..6c20b1f 100644 --- a/src/lib/credentials.ts +++ b/src/lib/credentials.ts @@ -7,6 +7,7 @@ import { statSync, writeFileSync, } from 'node:fs'; +import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { localValidationError } from './errors.js'; @@ -61,6 +62,17 @@ export interface CredentialsOptions { path?: string; } +interface RestrictiveModeOptions { + platform?: NodeJS.Platform; + env?: NodeJS.ProcessEnv; + spawnSync?: ( + command: string, + args: readonly string[], + options: { shell: false; stdio: 'ignore'; windowsHide: true }, + ) => SpawnSyncReturns; + warn?: (line: string) => void; +} + const FILE_KEY_TO_FIELD: Record = { api_key: 'apiKey', api_url: 'apiUrl', @@ -116,7 +128,15 @@ export function serializeCredentials(file: CredentialsFile): string { for (const field of fields) { const value = entry[field]; if (value === undefined || value === '') continue; - lines.push(`${FIELD_TO_FILE_KEY[field]} = ${value}`); + // Guard against INI injection: a value containing newline characters + // would be serialized across multiple lines, allowing an attacker to + // inject arbitrary key-value pairs (or new section headers) into the + // credentials file. A valid API key or URL never contains \n or \r. + // Strip them so a compromised env var or MITM'd backend response + // cannot override the stored api_key on subsequent reads. + const sanitized = value.replace(/[\r\n]/g, ''); + if (sanitized === '') continue; + lines.push(`${FIELD_TO_FILE_KEY[field]} = ${sanitized}`); } lines.push(''); } @@ -164,12 +184,62 @@ export function deleteProfile(profile: string, options: CredentialsOptions = {}) return true; } -export function ensureRestrictiveMode(path: string): void { +/** + * Enforce restrictive access on the credentials file after atomic writes. + * POSIX hosts use chmod(0600); Windows hosts use ACL tightening via icacls. + */ +export function ensureRestrictiveMode(path: string, options: RestrictiveModeOptions = {}): void { if (!existsSync(path)) return; + if ((options.platform ?? process.platform) === 'win32') { + ensureWindowsRestrictiveAcl(path, options); + return; + } const overpermissive = (statSync(path).mode & 0o077) !== 0; if (overpermissive) chmodSync(path, 0o600); } +/** + * Restrict a Windows credentials file to the current user using icacls. + * The command is invoked with an args array so credential paths are never shell-interpreted. + */ +function ensureWindowsRestrictiveAcl(path: string, options: RestrictiveModeOptions): void { + const username = (options.env ?? process.env).USERNAME?.trim(); + if (!username) { + warnWindowsAcl( + 'could not determine the Windows username; credentials file permissions were not tightened', + options, + ); + return; + } + + const run = options.spawnSync ?? spawnSync; + const result = run('icacls', [path, '/inheritance:r', '/grant:r', `${username}:F`], { + shell: false, + stdio: 'ignore', + windowsHide: true, + }); + + if (result.error) { + warnWindowsAcl( + `icacls failed while tightening credentials file permissions: ${result.error.message}`, + options, + ); + return; + } + if (result.status !== 0) { + warnWindowsAcl( + `icacls exited with status ${result.status ?? 'unknown'}; credentials file permissions may be too broad`, + options, + ); + } +} + +/** Emit an explicit warning when Windows ACL tightening cannot be completed. */ +function warnWindowsAcl(message: string, options: RestrictiveModeOptions): void { + const warn = options.warn ?? ((line: string) => process.stderr.write(`${line}\n`)); + warn(`[warning] ${message}`); +} + function resolvePath(options: CredentialsOptions): string { return options.path ?? defaultCredentialsPath(); } diff --git a/src/lib/dry-run/samples.test.ts b/src/lib/dry-run/samples.test.ts index e1bf8ff..078c653 100644 --- a/src/lib/dry-run/samples.test.ts +++ b/src/lib/dry-run/samples.test.ts @@ -1,5 +1,23 @@ import { describe, expect, it } from 'vitest'; -import { DRY_RUN_SAMPLE_ENTRIES, findSample } from './samples.js'; +import { DRY_RUN_SAMPLE_ENTRIES, findSample, sampleJUnitReportXml } from './samples.js'; + +describe('sampleJUnitReportXml', () => { + it('returns well-formed JUnit XML with canned batch ids', () => { + const xml = sampleJUnitReportXml('proj_dry'); + expect(xml).toContain(''); + expect(xml).toContain(' { + const xml = sampleJUnitReportXml('proj_dry', 'custom-ci-suite'); + expect(xml).toContain(' { it('resolves /me', () => { @@ -315,6 +333,23 @@ describe('findSample', () => { updatedAt: expect.any(String), }); break; + case 'cancelRun': + // DEV-331 piece 3 — POST /runs/{runId}/cancel → CancelRunResponse + // (RunResponse shape + alreadyCancelled). + expect(body).toMatchObject({ + runId: expect.any(String), + testId: expect.any(String), + status: 'cancelled', + alreadyCancelled: false, + }); + break; + case 'deleteProject': + // DELETE /projects/{id} → CliDeleteProjectResponse shape. + expect(body).toMatchObject({ + projectId: expect.any(String), + deletedAt: expect.any(String), + }); + break; default: throw new Error(`Unexpected operationId in samples: ${e.operationId}`); } @@ -405,6 +440,22 @@ describe('findSample', () => { expect(body.stepSummary.failedCount).toBe(0); }); + // DEV-331 piece 3: POST /runs/{runId}/cancel must resolve to `cancelRun`, + // never fall through to the GET-only `getRun` entry despite sharing the + // `/runs/{runId}` path prefix — findSample filters by method first. + it('POST /runs/{runId}/cancel resolves cancelRun (not getRun)', () => { + const e = findSample('POST', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz/cancel'); + expect(e?.operationId).toBe('cancelRun'); + const body = e?.body() as { status: string; alreadyCancelled: boolean; runId: string }; + expect(body.status).toBe('cancelled'); + expect(body.alreadyCancelled).toBe(false); + }); + + it('GET /runs/{runId}/cancel has no sample (cancel is POST-only)', () => { + const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/runs/run_xyz/cancel'); + expect(e).toBeUndefined(); + }); + it('getTest sample carries priority field (G1a)', () => { const e = findSample('GET', 'https://api.testsprite.com/api/cli/v1/tests/test_abc'); expect(e?.operationId).toBe('getTest'); diff --git a/src/lib/dry-run/samples.ts b/src/lib/dry-run/samples.ts index 28fa1bb..f6f5ec2 100644 --- a/src/lib/dry-run/samples.ts +++ b/src/lib/dry-run/samples.ts @@ -16,7 +16,11 @@ * If the CLI OpenAPI spec changes, both this file AND * `test/mock-backend/fixtures.ts` must be updated in the same PR. */ -import type { CliProject, CliUpdateProjectResponse } from '../../commands/project.js'; +import type { + CliProject, + CliUpdateProjectResponse, + CliDeleteProjectResponse, +} from '../../commands/project.js'; import type { CliBulkDeleteSummary, CliFailureContext, @@ -27,6 +31,7 @@ import type { CliTestStep, } from '../../commands/test.js'; import type { MeResponse } from '../../commands/auth.js'; +import { buildJUnitReport } from '../junit-report.js'; import type { Page } from '../pagination.js'; import type { TriggerRunResponse, @@ -35,6 +40,7 @@ import type { BatchRerunResponse, BatchRunFreshResponse, ListRunsResponse, + CancelRunResponse, } from '../runs.types.js'; const SAMPLE_USER_ID = '11111111-1111-4111-8111-111111111111'; @@ -67,11 +73,43 @@ const SAMPLE_REQUEST_ID = 'req_dry-run'; export const SAMPLE_DRY_RUN_REQUEST_ID = SAMPLE_REQUEST_ID; +/** + * Canned JUnit XML for batch `--wait --report junit --dry-run`. Mirrors the + * fresh batch-run sample ids so agents can learn the sidecar shape offline. + */ +export function sampleJUnitReportXml( + projectId: string = SAMPLE_PROJECT_ID, + reportSuiteName?: string, +): string { + return buildJUnitReport({ + suiteName: reportSuiteName ?? `testsprite:${projectId}`, + classname: projectId, + results: [ + { + testId: SAMPLE_TEST_ID_FRESH_1, + runId: SAMPLE_BATCH_FRESH_RUN_ID_1, + status: 'passed', + }, + { + testId: SAMPLE_TEST_ID_FRESH_2, + runId: SAMPLE_BATCH_FRESH_RUN_ID_2, + status: 'failed', + error: { + code: 'ASSERTION', + message: 'Expected checkout heading to be visible', + exitCode: 1, + }, + }, + ], + }); +} + const me: MeResponse = { userId: SAMPLE_USER_ID, keyId: SAMPLE_KEY_ID, scopes: ['read:projects', 'read:tests', 'write:tests', 'run:tests'], env: 'development', + v3Enabled: true, }; const projects: CliProject[] = [ @@ -365,6 +403,11 @@ const ENTRIES: DryRunSampleEntry[] = [ updatedFields: ['name'], updatedAt: '2026-05-16T00:00:00.000Z', } satisfies CliUpdateProjectResponse), + // DELETE /projects/{id} (cascade delete project + tests + fixtures). + entry('deleteProject', 'DELETE', '/projects/{projectId}', { + projectId: SAMPLE_PROJECT_ID, + deletedAt: '2026-05-16T00:00:00.000Z', + } satisfies CliDeleteProjectResponse), entry('listTests', 'GET', '/tests', pageOf(tests)), entry('getTestCode', 'GET', '/tests/{testId}/code', testCode), entry('listTestSteps', 'GET', '/tests/{testId}/steps', pageOf(testSteps)), @@ -715,6 +758,36 @@ const ENTRIES: DryRunSampleEntry[] = [ }, ], } satisfies RunResponse), + // DEV-331 piece 3 — POST /runs/{runId}/cancel. Method-guarded in + // `findSample` (POST vs `getRun`'s GET), so this can't be shadowed by the + // broader `/runs/{runId}` pattern above despite sharing its path prefix. + // `alreadyCancelled: false` — a fresh cancel is the more instructive shape + // for a dry-run learner than the idempotent no-op. + entry('cancelRun', 'POST', '/runs/{runId}/cancel', { + runId: SAMPLE_RUN_ID, + testId: SAMPLE_TEST_ID_PASSED, + projectId: SAMPLE_PROJECT_ID, + userId: SAMPLE_USER_ID, + status: 'cancelled', + source: 'cli', + createdAt: '2026-05-15T19:32:00.000Z', + startedAt: '2026-05-15T19:32:05.000Z', + finishedAt: '2026-05-15T19:33:12.000Z', + codeVersion: 'v1', + targetUrl: SAMPLE_TARGET_URL, + createdFrom: null, + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { + total: 8, + completed: 3, + passedCount: 3, + failedCount: 0, + }, + alreadyCancelled: false, + } satisfies CancelRunResponse), ]; function entry( diff --git a/src/lib/errors.test.ts b/src/lib/errors.test.ts index aa3926b..5c5ba66 100644 --- a/src/lib/errors.test.ts +++ b/src/lib/errors.test.ts @@ -59,6 +59,8 @@ describe('exitCodeFor', () => { ['UNAVAILABLE', 10], ['RATE_LIMITED', 11], ['INSUFFICIENT_CREDITS', 12], + ['FEATURE_GATED', 13], + ['CLIENT_TOO_OLD', 14], ['INTERNAL', 1], ] as const)('%s → exit %d', (code, expected) => { expect(exitCodeFor(code)).toBe(expected); @@ -138,6 +140,7 @@ describe('ApiError.fromEnvelope status fallback', () => { [403, 'AUTH_FORBIDDEN' as const], [404, 'NOT_FOUND' as const], [409, 'CONFLICT' as const], + [426, 'CLIENT_TOO_OLD' as const], [429, 'RATE_LIMITED' as const], [501, 'UNSUPPORTED' as const], [503, 'UNAVAILABLE' as const], @@ -157,6 +160,25 @@ describe('ApiError.fromEnvelope status fallback', () => { expect(err.exitCode).toBe(10); }); + it('recognizes a well-formed CLIENT_TOO_OLD (426) envelope and echoes the version details', () => { + const err = ApiError.fromEnvelope( + { + error: { + code: 'CLIENT_TOO_OLD', + message: 'Your CLI is older than the minimum supported version.', + nextAction: 'Upgrade with npm i -g @testsprite/testsprite-cli@latest.', + requestId: 'req_old', + details: { minVersion: '1.0.0', yourVersion: '0.9.0' }, + }, + }, + 426, + ); + // The code is in the union, so it is trusted directly — NOT remapped. + expect(err.code).toBe('CLIENT_TOO_OLD'); + expect(err.exitCode).toBe(14); + expect(err.details).toEqual({ minVersion: '1.0.0', yourVersion: '0.9.0' }); + }); + // Track A dogfood: NestJS raw 404 (route not registered) has the // shape `{ message, error: "Not Found", statusCode }`. Previously the // parser saw `obj.error = "Not Found"` (a string) and fell through to diff --git a/src/lib/errors.ts b/src/lib/errors.ts index f64d8e2..40b6c05 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -45,6 +45,12 @@ export const ERROR_CODES = [ // so the CLI can drop the client-side detection heuristic. 'FEATURE_GATED', 'UNSUPPORTED', + // The running CLI is older than the backend's minimum supported version. + // Backend emits this as HTTP 426 when version enforcement is enabled. Exit + // 14, non-retriable — a too-old client cannot self-heal by retrying; the + // user must upgrade. `nextAction` carries the upgrade instruction and + // `details` echoes { minVersion, latestVersion, yourVersion }. + 'CLIENT_TOO_OLD', 'INTERNAL', 'UNAVAILABLE', ] as const; @@ -77,6 +83,8 @@ export function exitCodeFor(code: ErrorCode): number { return 6; case 'UNSUPPORTED': return 7; + case 'CLIENT_TOO_OLD': + return 14; case 'UNAVAILABLE': return 10; case 'RATE_LIMITED': @@ -100,6 +108,41 @@ export class CLIError extends Error { } } +/** + * Termination signals with graceful handling, mapped to their conventional + * `128 + signum` exit code. sourceRef: POSIX signal numbers (SIGHUP=1, + * SIGINT=2, SIGTERM=15). Lives here (not interrupt.ts) so `InterruptError` + * needs no import cycle; `interrupt.ts` re-exports for back-compat. + */ +export const TERMINATION_EXIT_CODES = { + SIGINT: 130, // 128 + 2 + SIGTERM: 143, // 128 + 15 + SIGHUP: 129, // 128 + 1 +} as const; + +export type TerminationSignal = keyof typeof TERMINATION_EXIT_CODES; + +/** + * User-initiated interrupt (SIGINT/SIGTERM/SIGHUP) observed while a + * graceful-detach scope (the `--wait` polling window) was armed — see + * `interrupt.ts::ShutdownController`. The `--wait` catch blocks render the + * honest partial envelope + re-attach hint, then rethrow to `index.ts`. + * + * Deliberately NOT in `ERROR_CODES` — on a signal the CLI + * exits 130/143 without consulting the error catalog. The JSON-mode stderr + * envelope uses the out-of-catalog code `"INTERRUPTED"`. Same client-local + * synthetic category as `RequestTimeoutError`. + */ +export class InterruptError extends CLIError { + readonly signal: TerminationSignal; + + constructor(signal: TerminationSignal) { + super(`Interrupted by ${signal}.`, TERMINATION_EXIT_CODES[signal]); + this.name = 'InterruptError'; + this.signal = signal; + } +} + export class NotImplementedError extends CLIError { constructor(commandPath: string) { super(`Command not yet implemented: ${commandPath}`, 2); @@ -314,6 +357,8 @@ function codeFromHttpStatus(status: number | undefined): ErrorCode { return 'PRECONDITION_FAILED'; case 413: return 'PAYLOAD_TOO_LARGE'; + case 426: + return 'CLIENT_TOO_OLD'; case 429: return 'RATE_LIMITED'; case 501: diff --git a/src/lib/failing-fe-resolver.spec.ts b/src/lib/failing-fe-resolver.spec.ts index d6524e7..1fe05a9 100644 --- a/src/lib/failing-fe-resolver.spec.ts +++ b/src/lib/failing-fe-resolver.spec.ts @@ -43,6 +43,40 @@ function makeItem( return { id, type, status, updatedAt }; } +/** + * URL-routing fetch stub for the preferredId direct-probe path: + * `GET /tests/{id}` (no query string) is answered from `preferred`; + * `GET /tests?...` list calls are answered from `pages` in order. + * Call counts are exposed so tests can assert which endpoints were hit. + */ +function makeRoutedFetch(opts: { + preferred?: { status: number; body: unknown }; + preferredThrows?: boolean; + pages: Array<{ items: TestListItem[]; nextToken: string | null }>; +}) { + let listCalls = 0; + let preferredCalls = 0; + const impl = (async (url: string | URL | Request) => { + const u = String(url); + if (/\/tests\/[^/?]+$/.test(u)) { + preferredCalls++; + if (opts.preferredThrows) throw new Error('ECONNRESET (direct probe)'); + const p = opts.preferred ?? { status: 404, body: { error: 'not found' } }; + return new Response(JSON.stringify(p.body), { + status: p.status, + headers: { 'Content-Type': 'application/json' }, + }); + } + const page = opts.pages[listCalls] ?? { items: [], nextToken: null }; + listCalls++; + return new Response(JSON.stringify(page), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }) as typeof fetch; + return { impl, counts: () => ({ listCalls, preferredCalls }) }; +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -180,4 +214,163 @@ describe('resolveFailingFrontendTestId', () => { expect(result.testId).toBe('test_abc'); expect(result.reason).toContain('2026-05-27T09:00:00Z'); }); + + // DEV-393: dedicated pinned fixture must win over a fresher incidental + // failure — live-reproduced 2026-07-16 (a passingFrontendTestId fresh-run + // flip immediately outranked a dedicated "always red" fixture by + // updatedAt). See ResolverOptions.preferredId doc comment. + describe('preferredId (DEV-393 pinned-fixture preference)', () => { + test('prefers the pinned id over a fresher candidate when the direct probe is non-ok', async () => { + // Probe answers HTTP 500 (not ok, no throw) → falls through to the list + // scan, where the in-candidates preference must still pick the pinned id. + const routed = makeRoutedFetch({ + preferred: { status: 500, body: { error: 'internal' } }, + pages: [ + { + items: [ + makeItem('test_pinned', '2026-05-01T00:00:00Z'), + makeItem('test_incidental', '2026-05-20T00:00:00Z'), + ], + nextToken: null, + }, + ], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('Preferred pinned'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('falls back to freshest when the pinned id is not in the failed set', async () => { + // Probe returns 200 with a malformed body (no type/status fields) → + // falls through; pinned is absent from the list → freshest wins. + const routed = makeRoutedFetch({ + preferred: { status: 200, body: {} }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned_but_now_passing', + }); + expect(result.testId).toBe('test_incidental'); + }); + + test('behaves exactly as before when preferredId is not supplied', async () => { + const older = makeItem('test_old', '2026-05-01T00:00:00Z'); + const newer = makeItem('test_new', '2026-05-20T00:00:00Z'); + const fetchImpl = makeFetchReturning([{ items: [older, newer], nextToken: null }]); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: fetchImpl as typeof fetch, + }); + expect(result.testId).toBe('test_new'); + }); + + // Codex F3: the direct probe makes the preference immune to the maxPages + // list-scan cap — a pinned fixture beyond the last scanned page must + // still win, without any list call at all on the happy path. + test('direct probe wins without any list call when the pinned test is failed', async () => { + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { + ...makeItem('test_pinned', '2026-05-01T00:00:00Z'), + projectId: BASE_OPTS.projectId, + }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + maxPages: 1, + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('direct GET'); + expect(routed.counts()).toEqual({ listCalls: 0, preferredCalls: 1 }); + }); + + test('direct probe on a now-passing pinned test falls through to freshest', async () => { + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { + ...makeItem('test_pinned', '2026-05-25T00:00:00Z', 'frontend', 'passed'), + projectId: BASE_OPTS.projectId, + }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe error falls through to the list scan, where the pinned id still wins', async () => { + const routed = makeRoutedFetch({ + preferredThrows: true, + pages: [ + { + items: [ + makeItem('test_pinned', '2026-05-01T00:00:00Z'), + makeItem('test_incidental', '2026-05-20T00:00:00Z'), + ], + nextToken: null, + }, + ], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_pinned'); + expect(result.reason).toContain('Preferred pinned'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe on a pinned test from a DIFFERENT project falls through (codex round 2)', async () => { + // The pin points at a genuinely failed FE test — but in another + // project. The old list scan (projectId-filtered server-side) would + // never have returned it, so the direct probe must not let it win. + const routed = makeRoutedFetch({ + preferred: { + status: 200, + body: { ...makeItem('test_pinned', '2026-05-01T00:00:00Z'), projectId: 'proj_OTHER' }, + }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + + test('direct probe 404 falls through and freshest wins when pinned is absent everywhere', async () => { + const routed = makeRoutedFetch({ + preferred: { status: 404, body: { error: 'not found' } }, + pages: [{ items: [makeItem('test_incidental', '2026-05-20T00:00:00Z')], nextToken: null }], + }); + const result = await resolveFailingFrontendTestId({ + ...BASE_OPTS, + fetchImpl: routed.impl, + preferredId: 'test_pinned_deleted', + }); + expect(result.testId).toBe('test_incidental'); + expect(routed.counts()).toEqual({ listCalls: 1, preferredCalls: 1 }); + }); + }); }); diff --git a/src/lib/failing-fe-resolver.ts b/src/lib/failing-fe-resolver.ts index 14df49f..f8d6b5a 100644 --- a/src/lib/failing-fe-resolver.ts +++ b/src/lib/failing-fe-resolver.ts @@ -50,6 +50,21 @@ export interface ResolverOptions { * Each page uses pageSize=50, so 5 pages = 250 tests scanned. */ maxPages?: number; + /** + * DEV-393: the operator's pinned `failingFrontendTestId` (the static value + * from fixtures.local.json). "Freshest failed" alone is fragile in a shared + * project: any OTHER FE test that fails more recently than a dedicated, + * permanently-red fixture silently steals the slot (live-reproduced + * 2026-07-16 — a `passingFrontendTestId` fresh-run flip immediately + * outranked the dedicated fixture on `updatedAt`). When set, the pinned id + * is probed directly first (one GET /tests/{id} — immune to the maxPages + * list-scan cap); if it is currently `status:failed` it wins outright, + * regardless of timestamp. If the probe errors, the list scan below still + * prefers the pinned id when it appears among the candidates. Falls through + * to the existing freshest-wins behavior when unset or no longer failing — + * fully backward compatible. + */ + preferredId?: string; } export interface ResolverResult { @@ -67,9 +82,46 @@ export interface ResolverResult { * when no Failed test exists — never throws. */ export async function resolveFailingFrontendTestId(opts: ResolverOptions): Promise { - const { baseUrl, apiKey, projectId, maxPages = 5 } = opts; + const { baseUrl, apiKey, projectId, maxPages = 5, preferredId } = opts; const fetchImpl = opts.fetchImpl ?? fetch; + // DEV-393 (codex F3): probe the pinned fixture directly first, so the + // preference cannot be defeated by the maxPages list-scan cap (a pinned + // fixture beyond page `maxPages` would otherwise silently lose the slot to + // an incidental fresher failure). Any probe error or non-failed status + // falls through to the list scan; the in-candidates preference there + // remains as a second chance after a transient probe failure. + if (preferredId) { + try { + const resp = await fetchImpl(`${baseUrl}/tests/${encodeURIComponent(preferredId)}`, { + headers: { + 'x-api-key': apiKey, + 'x-request-id': `dev-e2e-resolver-preferred-${Date.now()}`, + accept: 'application/json', + }, + signal: AbortSignal.timeout(10_000), + }); + if (resp.ok) { + const test = (await resp.json()) as Partial & { projectId?: string }; + // The probe must honor the resolver's project scope (codex round 2): + // the list path filters by projectId server-side, so a pin that + // points at a failed FE test in a DIFFERENT project must not win + // here either. A response without a matching projectId (absent or + // mismatched) falls through to the project-scoped list scan. + if (test.type === 'frontend' && test.status === 'failed' && test.projectId === projectId) { + return { + testId: preferredId, + reason: + `Preferred pinned failingFrontendTestId (${preferredId}) confirmed status:failed ` + + `via direct GET (updatedAt=${test.updatedAt ?? 'unknown'}); list scan skipped`, + }; + } + } + } catch { + // Transient probe failure — fall through to the list scan below. + } + } + const candidates: TestListItem[] = []; let cursor: string | undefined; let pagesFetched = 0; @@ -134,6 +186,19 @@ export async function resolveFailingFrontendTestId(opts: ResolverOptions): Promi }; } + // DEV-393: a pinned, dedicated "always red" fixture wins over freshest + // whenever it is still in the failed set — see the doc comment on + // ResolverOptions.preferredId for why "freshest" alone is not enough. + if (preferredId) { + const pinned = candidates.find(c => c.id === preferredId); + if (pinned) { + return { + testId: pinned.id, + reason: `Preferred pinned failingFrontendTestId (${pinned.id}) is still status:failed (updatedAt=${pinned.updatedAt}); took priority over freshest-updatedAt candidate`, + }; + } + } + // Pick the most recently updated candidate — freshest failure is the // most likely to have a valid failure bundle attached. candidates.sort((a, b) => { diff --git a/src/lib/flaky.test.ts b/src/lib/flaky.test.ts new file mode 100644 index 0000000..a6acd9c --- /dev/null +++ b/src/lib/flaky.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest'; +import { flakyExitCode, renderFlakyText, summarizeFlaky, type FlakyAttempt } from './flaky.js'; + +function pass(attempt: number): FlakyAttempt { + return { attempt, runId: `run_${attempt}`, outcome: 'passed' }; +} +function fail(attempt: number, failureKind = 'assertion'): FlakyAttempt { + return { attempt, runId: `run_${attempt}`, outcome: 'failed', failureKind }; +} + +describe('summarizeFlaky', () => { + it('reports STABLE when every attempt passed', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2), pass(3)]); + expect(report.verdict).toBe('stable'); + expect(report.runs).toBe(3); + expect(report.passed).toBe(3); + expect(report.failed).toBe(0); + expect(report.stableRatio).toBe(1); + expect(report.failures).toEqual([]); + expect(flakyExitCode(report)).toBe(0); + }); + + it('reports FLAKY on a mix of pass and fail', () => { + const report = summarizeFlaky('test_x', [pass(1), fail(2), pass(3)]); + expect(report.verdict).toBe('flaky'); + expect(report.passed).toBe(2); + expect(report.failed).toBe(1); + expect(report.stableRatio).toBe(0.6667); + expect(report.failures).toEqual([ + { attempt: 2, runId: 'run_2', outcome: 'failed', failureKind: 'assertion' }, + ]); + expect(flakyExitCode(report)).toBe(1); + }); + + it('reports FAILING when no attempt passed', () => { + const report = summarizeFlaky('test_x', [fail(1), fail(2)]); + expect(report.verdict).toBe('failing'); + expect(report.passed).toBe(0); + expect(report.stableRatio).toBe(0); + expect(flakyExitCode(report)).toBe(1); + }); + + it('treats an empty attempt list as FAILING with a 0 ratio', () => { + const report = summarizeFlaky('test_x', []); + expect(report.verdict).toBe('failing'); + expect(report.runs).toBe(0); + expect(report.stableRatio).toBe(0); + expect(flakyExitCode(report)).toBe(1); + }); + + it('counts timeout and error outcomes as non-passing', () => { + const attempts: FlakyAttempt[] = [ + pass(1), + { attempt: 2, runId: 'run_2', outcome: 'timeout' }, + { attempt: 3, runId: null, outcome: 'error', failureKind: 'UNAVAILABLE' }, + ]; + const report = summarizeFlaky('test_x', attempts); + expect(report.verdict).toBe('flaky'); + expect(report.passed).toBe(1); + expect(report.failed).toBe(2); + expect(report.failures.map(f => f.outcome)).toEqual(['timeout', 'error']); + // error attempt with no runId is preserved as null in the report + expect(report.failures[1]).toEqual({ + attempt: 3, + runId: null, + outcome: 'error', + failureKind: 'UNAVAILABLE', + }); + }); + + it('rounds stableRatio to 4 decimal places', () => { + const report = summarizeFlaky('test_x', [pass(1), pass(2), fail(3)]); + expect(report.stableRatio).toBe(0.6667); + }); + + it('reflects a short-circuited (--until-fail) run — fewer runs than requested', () => { + // Only two attempts were observed before the probe stopped at the failure. + const report = summarizeFlaky('test_x', [pass(1), fail(2)]); + expect(report.runs).toBe(2); + expect(report.verdict).toBe('flaky'); + }); +}); + +describe('renderFlakyText', () => { + it('summarizes a stable run on one line with no failure list', () => { + const text = renderFlakyText(summarizeFlaky('test_login', [pass(1), pass(2)])); + expect(text).toBe('Ran test_login 2x — 2 passed, 0 failed → STABLE (100% stable)'); + }); + + it('lists failing attempts with runId and failureKind', () => { + const text = renderFlakyText( + summarizeFlaky('test_login', [pass(1), fail(2, 'network_timeout')]), + ); + expect(text).toContain('→ FLAKY (50% stable)'); + expect(text).toContain('failed attempts:'); + expect(text).toContain('#2 run_2 failed failureKind=network_timeout'); + }); + + it('shows (no runId) when an errored attempt never got a runId', () => { + const text = renderFlakyText( + summarizeFlaky('test_login', [{ attempt: 1, runId: null, outcome: 'error' }]), + ); + expect(text).toContain('#1 (no runId) error'); + }); +}); diff --git a/src/lib/flaky.ts b/src/lib/flaky.ts new file mode 100644 index 0000000..402eaf3 --- /dev/null +++ b/src/lib/flaky.ts @@ -0,0 +1,133 @@ +/** + * Pure aggregation + rendering for `test flaky` — the repeat-run flaky-test + * detector. This module performs no I/O: the orchestrator in + * `commands/test.ts` feeds it the per-attempt outcomes, and it returns a + * machine-readable stability report plus the text rendering and exit code. + * + * Keeping the scoring logic pure makes it trivially unit-testable in isolation + * (deterministic, no network / credentials), matching the repo's mock-based + * test convention. + */ + +/** + * Outcome of a single flaky-detector attempt. The first four mirror the + * terminal `RunStatus` values; `timeout` and `error` are orchestration + * outcomes (per-attempt deadline exceeded, or a non-fatal trigger/transport + * error that was recorded rather than aborting the whole probe). + */ +export type FlakyOutcome = 'passed' | 'failed' | 'blocked' | 'cancelled' | 'timeout' | 'error'; + +/** Overall stability verdict across all observed attempts. */ +export type FlakyVerdict = 'stable' | 'flaky' | 'failing'; + +/** One recorded attempt in a flaky run. */ +export interface FlakyAttempt { + /** 1-based attempt index. */ + attempt: number; + /** The runId of this attempt, or `null` when the trigger never returned one. */ + runId: string | null; + outcome: FlakyOutcome; + /** + * Server `failureKind` for non-passing runs (or the error code for `error` + * outcomes). `null` / omitted when the attempt passed or the kind is unknown. + */ + failureKind?: string | null; +} + +/** A single non-passing attempt as surfaced in the report. */ +export interface FlakyFailure { + attempt: number; + runId: string | null; + outcome: FlakyOutcome; + failureKind: string | null; +} + +/** + * Machine-readable stability report. This is also the exact `--output json` + * shape, so dashboards / agents / CI can consume it directly. + */ +export interface FlakyReport { + testId: string; + /** + * Attempts actually observed. May be fewer than requested when + * `--until-fail` short-circuits on the first non-passing attempt. + */ + runs: number; + passed: number; + failed: number; + /** `passed / runs`, rounded to 4 decimal places. `0` when `runs === 0`. */ + stableRatio: number; + verdict: FlakyVerdict; + /** Non-passing attempts, in attempt order. */ + failures: FlakyFailure[]; +} + +/** + * Aggregate per-attempt outcomes into a stability report. + * + * Verdict rules: + * - every attempt passed → `stable` + * - no attempt passed → `failing` + * - a mix of pass and non-pass → `flaky` + * An empty attempt list (no runs observed) is reported as `failing` with a + * `0` ratio — there is no evidence the test is stable. + */ +export function summarizeFlaky(testId: string, attempts: FlakyAttempt[]): FlakyReport { + const runs = attempts.length; + const passed = attempts.filter(a => a.outcome === 'passed').length; + const failed = runs - passed; + const stableRatio = runs === 0 ? 0 : Math.round((passed / runs) * 10000) / 10000; + const verdict: FlakyVerdict = + runs > 0 && passed === runs ? 'stable' : passed === 0 ? 'failing' : 'flaky'; + const failures: FlakyFailure[] = attempts + .filter(a => a.outcome !== 'passed') + .map(a => ({ + attempt: a.attempt, + runId: a.runId, + outcome: a.outcome, + failureKind: a.failureKind ?? null, + })); + return { testId, runs, passed, failed, stableRatio, verdict, failures }; +} + +/** + * Exit code for the command: `0` only when the verdict is `stable` (every + * observed attempt passed). Anything else is non-zero so CI can gate a merge + * on flakiness (`testsprite test flaky --runs 5 || exit 1`). + */ +export function flakyExitCode(report: FlakyReport): number { + return report.verdict === 'stable' ? 0 : 1; +} + +/** Human-readable label for a verdict. */ +function verdictLabel(verdict: FlakyVerdict): string { + switch (verdict) { + case 'stable': + return 'STABLE'; + case 'flaky': + return 'FLAKY'; + case 'failing': + return 'FAILING'; + } +} + +/** + * Render a report to human-readable text. JSON-mode callers ship the report + * verbatim via `out.print`; this is the text-mode rendering. + */ +export function renderFlakyText(report: FlakyReport): string { + const pct = Math.round(report.stableRatio * 100); + const lines: string[] = [ + `Ran ${report.testId} ${report.runs}x — ${report.passed} passed, ${report.failed} failed → ` + + `${verdictLabel(report.verdict)} (${pct}% stable)`, + ]; + if (report.failures.length > 0) { + lines.push(' failed attempts:'); + for (const f of report.failures) { + const kind = f.failureKind ? ` failureKind=${f.failureKind}` : ''; + const rid = f.runId ?? '(no runId)'; + lines.push(` #${f.attempt} ${rid} ${f.outcome}${kind}`); + } + } + return lines.join('\n'); +} diff --git a/src/lib/http.test.ts b/src/lib/http.test.ts index 3387eac..e8f4c35 100644 --- a/src/lib/http.test.ts +++ b/src/lib/http.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; -import { ApiError, RequestTimeoutError, TransportError } from './errors.js'; +import { ApiError, InterruptError, RequestTimeoutError, TransportError } from './errors.js'; import type { DebugEvent } from './http.js'; import { HttpClient, REQUEST_TIMEOUT_DEFAULT_MS, buildUrl, parseRetryAfter } from './http.js'; import { VERSION } from '../version.js'; @@ -29,7 +29,11 @@ function errorEnvelopeResponse(status: number, code: string, init: ResponseInit function makeClient( fetchImpl: typeof fetch, - options: { apiKey?: string | null; onDebug?: (e: DebugEvent) => void } = {}, + options: { + apiKey?: string | null; + onDebug?: (e: DebugEvent) => void; + onServerVersion?: (info: { minVersion?: string }) => void; + } = {}, ): HttpClient { const apiKey = 'apiKey' in options ? (options.apiKey ?? undefined) : 'sk-test'; return new HttpClient({ @@ -39,9 +43,73 @@ function makeClient( sleep: () => Promise.resolve(), random: () => 0, onDebug: options.onDebug, + onServerVersion: options.onServerVersion, }); } +describe('CLIENT_TOO_OLD (426)', () => { + it('is not retried — fails fast with the typed error', async () => { + const fetchImpl = vi.fn().mockResolvedValue(errorEnvelopeResponse(426, 'CLIENT_TOO_OLD')); + const client = makeClient(fetchImpl as unknown as typeof fetch); + + const err = await client.get('/me').catch((e: unknown) => e); + + expect(err).toBeInstanceOf(ApiError); + expect((err as ApiError).code).toBe('CLIENT_TOO_OLD'); + expect((err as ApiError).exitCode).toBe(14); + expect(fetchImpl).toHaveBeenCalledTimes(1); // no retry + }); +}); + +describe('onServerVersion hook', () => { + it('fires with the parsed floor header on a 2xx response', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue( + jsonResponse( + { ok: true }, + { + headers: { + 'content-type': 'application/json', + 'x-testsprite-cli-min-version': '1.0.0', + }, + }, + ), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/me'); + + expect(onServerVersion).toHaveBeenCalledWith({ minVersion: '1.0.0' }); + }); + + it('fires on a non-2xx response too (the floor header rides on every response)', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue( + errorEnvelopeResponse(404, 'NOT_FOUND', { + headers: { + 'content-type': 'application/json', + 'x-testsprite-cli-min-version': '1.0.0', + }, + }), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/tests/missing').catch(() => undefined); + + expect(onServerVersion).toHaveBeenCalledWith({ minVersion: '1.0.0' }); + }); + + it('does not fire when the floor header is absent', async () => { + const onServerVersion = vi.fn(); + const fetchImpl = vi.fn().mockResolvedValue(jsonResponse({ ok: true })); + const client = makeClient(fetchImpl as unknown as typeof fetch, { onServerVersion }); + + await client.get('/me'); + + expect(onServerVersion).not.toHaveBeenCalled(); + }); +}); + describe('buildUrl', () => { it('handles trailing slashes and absolute paths', () => { expect(buildUrl('https://api.example.com/api/cli/v1', '/me')).toBe( @@ -140,6 +208,53 @@ describe('HttpClient happy path', () => { }); }); +describe('HttpClient — 200 response with a non-JSON body', () => { + function htmlResponse(status = 200): Response { + return new Response('Login', { + status, + headers: { 'content-type': 'text/html' }, + }); + } + + it('throws a typed INTERNAL ApiError (not a raw SyntaxError) with the requestId and details', async () => { + const fetchImpl = vi.fn(async () => htmlResponse()); + const client = makeClient(fetchImpl as unknown as typeof fetch); + let caught: unknown; + try { + await client.get('/me'); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('INTERNAL'); + expect(apiErr.exitCode).toBe(1); + expect(apiErr.requestId).toMatch(/^cli_/); + expect(apiErr.message).toContain('non-JSON response'); + expect(apiErr.nextAction).toContain('TESTSPRITE_API_URL'); + expect(apiErr.details).toMatchObject({ httpStatus: 200, contentType: 'text/html' }); + expect(String(apiErr.details.parseError)).toContain('JSON'); + }); + + it('does not retry a malformed 200 body (single fetch call)', async () => { + const fetchImpl = vi.fn(async () => htmlResponse()); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.get('/me')).rejects.toBeInstanceOf(ApiError); + // A non-JSON success body is a hard config error, not a transient transport + // failure — it must not burn the retry budget. + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('handles an empty 200 body the same way', async () => { + const fetchImpl = vi.fn( + async () => + new Response('', { status: 200, headers: { 'content-type': 'application/json' } }), + ); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.get('/me')).rejects.toMatchObject({ code: 'INTERNAL', exitCode: 1 }); + }); +}); + describe('HttpClient error mapping', () => { it('does not retry AUTH_INVALID and exits 3', async () => { const fetchImpl = vi.fn(async () => errorEnvelopeResponse(401, 'AUTH_INVALID')); @@ -213,6 +328,31 @@ describe('HttpClient error mapping', () => { expect(fetchImpl).toHaveBeenCalledTimes(4); }); + it('does not retry transport errors for keyless writes', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('ECONNRESET'); + }); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.post('/projects', { body: { name: 'Checkout' } })).rejects.toBeInstanceOf( + TransportError, + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('retries transport errors for writes with an idempotency key', async () => { + const fetchImpl = vi.fn(async () => { + throw new Error('ECONNRESET'); + }); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect( + client.post('/projects', { + body: { name: 'Checkout' }, + headers: { 'Idempotency-Key': 'op_123' }, + }), + ).rejects.toBeInstanceOf(TransportError); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + it('does not retry AbortError', async () => { const fetchImpl = vi.fn(async () => { const err = new Error('aborted'); @@ -284,6 +424,27 @@ describe('HttpClient transport-edge statuses', () => { }, ); + it('does not retry bare transport-edge responses for keyless writes', async () => { + const fetchImpl = vi.fn(async () => new Response('proxy gateway html', { status: 502 })); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect(client.post('/projects', { body: { name: 'Checkout' } })).rejects.toBeInstanceOf( + TransportError, + ); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('retries bare transport-edge responses for writes with an idempotency key', async () => { + const fetchImpl = vi.fn(async () => new Response('proxy gateway html', { status: 502 })); + const client = makeClient(fetchImpl as unknown as typeof fetch); + await expect( + client.post('/projects', { + body: { name: 'Checkout' }, + headers: { 'idempotency-key': 'op_123' }, + }), + ).rejects.toBeInstanceOf(TransportError); + expect(fetchImpl).toHaveBeenCalledTimes(4); + }); + it('502 carrying our envelope still maps to its catalog code', async () => { const body = { error: { @@ -402,6 +563,52 @@ describe('HttpClient per-request timeout', () => { expect(callCount).toBe(1); }); + it('clears the per-attempt timeout after a successful response body is read', async () => { + let capturedSignal: AbortSignal | undefined; + const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { + capturedSignal = init?.signal ?? undefined; + return jsonResponse({ ok: true }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => Promise.resolve(), + random: () => 0, + requestTimeoutMs: 25, + }); + + await expect(client.get('/me')).resolves.toEqual({ ok: true }); + expect(capturedSignal?.aborted).toBe(false); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(capturedSignal?.aborted).toBe(false); + }); + + it('clears a failed attempt timeout before retry backoff sleeps', async () => { + const attemptSignals: AbortSignal[] = []; + let calls = 0; + const fetchImpl = vi.fn(async (_input: unknown, init?: RequestInit) => { + if (init?.signal) attemptSignals.push(init.signal); + calls += 1; + if (calls === 1) return errorEnvelopeResponse(500, 'INTERNAL'); + return jsonResponse({ ok: true }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => new Promise(resolve => setTimeout(resolve, 50)), + random: () => 0, + requestTimeoutMs: 25, + }); + + await expect(client.get('/me')).resolves.toEqual({ ok: true }); + expect(attemptSignals).toHaveLength(2); + expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(attemptSignals.every(signal => signal.aborted === false)).toBe(true); + }); + it('caller-supplied AbortSignal still propagates as AbortError (not RequestTimeoutError)', async () => { const controller = new AbortController(); // Abort immediately @@ -428,6 +635,33 @@ describe('HttpClient per-request timeout', () => { expect((err as Error).name).toBe('AbortError'); }); + it('preserves RequestTimeoutError when the caller aborts after the request timeout wins', async () => { + const controller = new AbortController(); + const fetchImpl = vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => { + const reason = init.signal?.reason; + controller.abort(new Error('caller aborted after timeout')); + const err = new Error(reason?.message ?? 'timed out'); + err.name = reason?.name ?? 'TimeoutError'; + reject(err); + }); + }); + }); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: fetchImpl as unknown as typeof fetch, + sleep: () => Promise.resolve(), + random: () => 0, + requestTimeoutMs: 1, + }); + const err = await client.get('/me', { signal: controller.signal }).catch(e => e); + expect(err).toBeInstanceOf(RequestTimeoutError); + expect((err as RequestTimeoutError).exitCode).toBe(7); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + it('defaults to REQUEST_TIMEOUT_DEFAULT_MS when no requestTimeoutMs is supplied', () => { // Verify the default is wired without actually waiting 120s. // We test via the exported constant rather than exercising the stall. @@ -521,3 +755,177 @@ describe('HttpClient per-request timeout', () => { expect(err).toBeInstanceOf(RequestTimeoutError); }); }); + +describe('HttpClient shutdown signal (DEV-331 graceful detach)', () => { + /** Stalled fetch that rejects with the effective signal's reason on abort. */ + function stalledFetch(callCounter?: { count: number }): typeof fetch { + return vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + if (callCounter) callCounter.count += 1; + return new Promise((_resolve, reject) => { + const signal = init?.signal; + const rejectWithReason = (): void => { + const reason: unknown = signal?.reason; + if (reason instanceof Error) { + reject(reason); + return; + } + const err = new Error('aborted'); + err.name = 'AbortError'; + reject(err); + }; + if (signal?.aborted) { + rejectWithReason(); + return; + } + signal?.addEventListener('abort', rejectWithReason, { once: true }); + }); + }) as unknown as typeof fetch; + } + + function makeShutdownClient( + shutdownSignal: AbortSignal, + callCounter?: { count: number }, + ): HttpClient { + return new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl: stalledFetch(callCounter), + sleep: () => Promise.resolve(), + random: () => 0, + shutdownSignal, + }); + } + + it('aborts an in-flight fetch with the InterruptError reason (no retry, no re-wrap)', async () => { + const counter = { count: 0 }; + const shutdown = new AbortController(); + const client = makeShutdownClient(shutdown.signal, counter); + const pending = client.get('/runs/run_1'); + queueMicrotask(() => shutdown.abort(new InterruptError('SIGINT'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + expect((err as InterruptError).exitCode).toBe(130); + expect(counter.count).toBe(1); // never retried + }); + + it('classifies an anonymous AbortError as the interrupt when the shutdown signal fired', async () => { + // Some runtimes reject with a bare AbortError instead of the abort reason; + // rethrowIfAbort must still classify shutdown-first (never TransportError, + // never RequestTimeoutError). + const shutdown = new AbortController(); + const fetchImpl = vi.fn(async (_input: unknown, init?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener( + 'abort', + () => { + const err = new Error('aborted'); + err.name = 'AbortError'; + reject(err); + }, + { once: true }, + ); + }); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => Promise.resolve(), + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/runs/run_1'); + queueMicrotask(() => shutdown.abort(new InterruptError('SIGTERM'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + }); + + it('skips dispatch entirely when the shutdown signal is already aborted', async () => { + const counter = { count: 0 }; + const shutdown = new AbortController(); + shutdown.abort(new InterruptError('SIGINT')); + const client = makeShutdownClient(shutdown.signal, counter); + const err = await client.get('/me').catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(counter.count).toBe(0); + }); + + it('passes an InterruptError from a caller-composed signal through untouched (poll path, no shutdownSignal configured)', async () => { + // The polling loop composes the shutdown signal into its per-iteration + // caller signal; the fetch then rejects with the InterruptError reason + // even though the client itself has no shutdownSignal. + const counter = { count: 0 }; + const fetchImpl = stalledFetch(counter); + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => Promise.resolve(), + random: () => 0, + }); + const caller = new AbortController(); + const pending = client.get('/runs/run_1', { signal: caller.signal }); + queueMicrotask(() => caller.abort(new InterruptError('SIGINT'))); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(counter.count).toBe(1); // no transport retry + }); +}); + +describe('HttpClient retry-delay sleep bails on shutdown (DEV-331 codex finding 1)', () => { + it('a shutdown during a transport-retry sleep rejects with InterruptError immediately', async () => { + // First fetch throws a transport error → the client schedules a retry + // sleep. The injected sleep NEVER resolves, so only the shutdown race can + // end it — without the bail, the interrupt would wait out the delay + // (e.g. a Retry-After: 60) before surfacing. + const shutdown = new AbortController(); + let calls = 0; + const fetchImpl = vi.fn(async () => { + calls += 1; + throw new Error('socket hang up'); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => new Promise(() => {}), // never resolves + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/me'); + // Let the first attempt fail and the retry sleep begin, then interrupt. + await new Promise(resolve => setTimeout(resolve, 10)); + shutdown.abort(new InterruptError('SIGINT')); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect(calls).toBe(1); // the retry never dispatched + }); + + it('a shutdown during a RATE_LIMITED Retry-After sleep rejects with InterruptError immediately', async () => { + const shutdown = new AbortController(); + let calls = 0; + const fetchImpl = vi.fn(async () => { + calls += 1; + return errorEnvelopeResponse(429, 'RATE_LIMITED', { + headers: { 'content-type': 'application/json', 'retry-after': '60' }, + }); + }) as unknown as typeof fetch; + const client = new HttpClient({ + baseUrl: 'https://api.example.com/api/cli/v1', + apiKey: 'sk-test', + fetchImpl, + sleep: () => new Promise(() => {}), // never resolves + random: () => 0, + shutdownSignal: shutdown.signal, + }); + const pending = client.get('/me'); + await new Promise(resolve => setTimeout(resolve, 10)); + shutdown.abort(new InterruptError('SIGTERM')); + const err = await pending.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + expect(calls).toBe(1); + }); +}); diff --git a/src/lib/http.ts b/src/lib/http.ts index 83ca0f1..21369de 100644 --- a/src/lib/http.ts +++ b/src/lib/http.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; import type { ErrorCode } from './errors.js'; -import { ApiError, RequestTimeoutError, TransportError } from './errors.js'; +import { ApiError, InterruptError, RequestTimeoutError, TransportError } from './errors.js'; import { VERSION } from '../version.js'; import type { TriggerRunBody, @@ -14,6 +14,7 @@ import type { BatchRunFreshResponse, ListRunsQuery, ListRunsResponse, + CancelRunResponse, } from './runs.types.js'; export type FetchImpl = typeof globalThis.fetch; @@ -74,6 +75,14 @@ export interface HttpClientOptions { * Stays silent when absent; wired to stderr at `--verbose` level. */ onTransition?: (msg: string) => void; + /** + * Optional callback fired with the backend's supported-floor header + * (`X-TestSprite-CLI-Min-Version`) when present on a response. Fires on both + * success and error responses. The client forwards the value verbatim — it + * applies no policy itself; the caller (client-factory) decides whether to + * warn the user. + */ + onServerVersion?: (info: { minVersion?: string }) => void; /** * Per-request wall-clock timeout in milliseconds applied to every outgoing * fetch. The signal fires independently of any caller-supplied signal — the @@ -87,6 +96,16 @@ export interface HttpClientOptions { * request is well within the 120s default. */ requestTimeoutMs?: number; + /** + * Process-lifetime shutdown signal (DEV-331 piece 1). Composed into every + * outgoing fetch so an armed SIGINT/SIGTERM aborts an in-flight request + * (a `--wait` long-poll can sit inside a single fetch for minutes) instead + * of waiting out its window. Aborts with an `InterruptError` reason, which + * is classified before the timeout-vs-caller logic and is never retried or + * re-wrapped as `TransportError`. Defaults off; production wiring passes + * `globalShutdown.signal` via the client factory. + */ + shutdownSignal?: AbortSignal; } export interface RequestOptions { @@ -167,19 +186,42 @@ export class HttpClient { private readonly random: () => number; private readonly onDebug?: (event: DebugEvent) => void; private readonly onTransition?: (msg: string) => void; + private readonly onServerVersion?: (info: { minVersion?: string }) => void; private readonly requestTimeoutMs: number; + private readonly shutdownSignal?: AbortSignal; constructor(options: HttpClientOptions) { this.baseUrl = trimTrailingSlash(options.baseUrl); this.apiKey = options.apiKey; + this.shutdownSignal = options.shutdownSignal; this.fetchImpl = options.fetchImpl ?? globalThis.fetch.bind(globalThis); this.sleep = options.sleep ?? defaultSleep; this.random = options.random ?? Math.random; this.onDebug = options.onDebug; this.onTransition = options.onTransition; + this.onServerVersion = options.onServerVersion; this.requestTimeoutMs = options.requestTimeoutMs ?? REQUEST_TIMEOUT_DEFAULT_MS; } + /** + * Read the backend's supported-floor header off a response and forward it to + * `onServerVersion` when present. Never throws — a bad header must not break + * the request. Called on every response (success or error) so the advisory + * covers all paths. (The backend advertises only the floor; "latest" is + * resolved client-side via the npm update-notice.) + */ + private captureServerVersion(response: Response): void { + if (!this.onServerVersion) return; + try { + const minVersion = response.headers.get('x-testsprite-cli-min-version') ?? undefined; + if (minVersion !== undefined) { + this.onServerVersion({ minVersion }); + } + } catch { + // Header parsing is best-effort; never let it affect the request. + } + } + async get(path: string, options: RequestOptions = {}): Promise { return this.requestWithMeta('GET', path, options).then(r => r.body); } @@ -382,6 +424,24 @@ export class HttpClient { }); } + /** + * POST /api/cli/v1/runs/{runId}/cancel + * User-initiated cancel of a queued/running run (DEV-331 piece 3). No + * body, no `Idempotency-Key` — the endpoint is naturally idempotent (D10): + * re-cancel → 200 `alreadyCancelled: true`; already-terminal (passed/ + * failed/blocked) → 409 CONFLICT; unknown/cross-tenant runId → 404. + * + * `retryOnConflict: false` — same rationale as `triggerRun`: a 409 here is + * a truthful terminal answer ("already finished"), not a transient + * snapshot conflict to paper over with a retry. + */ + async cancelRun(runId: string, options?: { signal?: AbortSignal }): Promise { + return this.post(`/runs/${encodeURIComponent(runId)}/cancel`, { + signal: options?.signal, + retryOnConflict: false, + }); + } + /** * Classify an error thrown while issuing OR reading a request. When it is an * abort/timeout and our per-request timeout signal fired (and the caller had @@ -395,9 +455,23 @@ export class HttpClient { timeoutSignal: AbortSignal, callerSignal: AbortSignal | undefined, requestId: string, + effectiveSignal: AbortSignal = timeoutSignal, ): void { + // A user interrupt (DEV-331) outranks every other classification: once the + // shutdown signal fired, whatever error surfaced from the aborted fetch or + // body read is the interrupt. Throw its InterruptError reason so the wait + // paths can render the honest detach UX — never a RequestTimeoutError, and + // never fall through to the transport retry loop. + if (this.shutdownSignal?.aborted) { + throw this.shutdownSignal.reason; + } if (isAbortError(err) || isTimeoutError(err)) { - if (timeoutSignal.aborted && (callerSignal == null || !callerSignal.aborted)) { + const timeoutWon = + timeoutSignal.aborted && + (callerSignal == null || + !callerSignal.aborted || + effectiveSignal.reason === timeoutSignal.reason); + if (timeoutWon) { throw new RequestTimeoutError(this.requestTimeoutMs, requestId); } throw err; @@ -413,9 +487,14 @@ export class HttpClient { const url = buildUrl(this.baseUrl, path, options.query); const requestId = options.requestId ?? newRequestId(); + const allowTransportRetry = canRetryTransport(method, options); let attempt = 0; while (true) { + // Bail before issuing (or re-issuing after a retry sleep) a request the + // user has already interrupted; the composed signal below would abort it + // immediately anyway, this just skips the wasted dispatch. + if (this.shutdownSignal?.aborted) throw this.shutdownSignal.reason; attempt += 1; this.debug({ kind: 'request', method, url, attempt, requestId }); const startedAt = Date.now(); @@ -428,184 +507,225 @@ export class HttpClient { // signal. The polling path supplies its own deadline-aware signal per // iteration — this timeout (120s default) is safely larger than any single // long-poll window (<=25s via ?waitSeconds), so it never bites polling. - const timeoutSignal = AbortSignal.timeout(this.requestTimeoutMs); + const requestTimeout = createRequestTimeout(this.requestTimeoutMs); + const timeoutSignal = requestTimeout.signal; + const composedSignals = [timeoutSignal]; + if (options.signal != null) composedSignals.push(options.signal); + // Shutdown composition (DEV-331): an armed SIGINT/SIGTERM aborts the + // in-flight fetch immediately (reason: InterruptError) instead of + // letting a long-poll drain its window before the interrupt surfaces. + if (this.shutdownSignal != null) composedSignals.push(this.shutdownSignal); const effectiveSignal = - options.signal != null ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal; + composedSignals.length > 1 ? AbortSignal.any(composedSignals) : timeoutSignal; try { - response = await this.fetchImpl(url, { - method, - headers: this.buildHeaders(requestId, options), - body: options.body !== undefined ? JSON.stringify(options.body) : undefined, - signal: effectiveSignal, - }); - } catch (err) { - // Distinguish a client-side request timeout from a caller-supplied abort. - // - // Node 22 `AbortSignal.timeout()` throws a `DOMException` with - // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. - // A caller-supplied abort sets `name === 'AbortError'`. - // We treat both abort variants together: if the timeout signal fired and - // the caller hadn't already aborted, surface a clear RequestTimeoutError. - // A timeout/abort during the fetch itself: classify it (RequestTimeoutError - // when our deadline fired; otherwise rethrow the caller's abort unmodified). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - // If a RequestTimeoutError already propagated from somewhere (e.g. from a - // nested call or from a test-injected fetchImpl), pass it through unchanged - // rather than re-wrapping it as a TransportError. - if (err instanceof RequestTimeoutError) throw err; - const message = err instanceof Error ? err.message : String(err); - this.debug({ - kind: 'error', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - durationMs: Date.now() - startedAt, - }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) throw new TransportError(message, requestId); - this.transition( - `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, - ); - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: 'TRANSPORT', - delayMs: decision.delayMs, - }); - await this.sleep(decision.delayMs); - continue; - } + try { + response = await this.fetchImpl(url, { + method, + headers: this.buildHeaders(requestId, options), + body: options.body !== undefined ? JSON.stringify(options.body) : undefined, + signal: effectiveSignal, + }); + } catch (err) { + // Distinguish a client-side request timeout from a caller-supplied abort. + // + // The request-timeout controller aborts with an Error/DOMException whose + // `name === 'TimeoutError'` (not 'AbortError') when the signal fires. + // A caller-supplied abort sets `name === 'AbortError'`. + // We treat both abort variants together: if the timeout signal fired and + // the caller hadn't already aborted, surface a clear RequestTimeoutError. + // A user interrupt is never retried and never re-wrapped as a + // TransportError (same passthrough discipline as RequestTimeoutError). + // The instanceof check matters even without `this.shutdownSignal`: + // the polling path composes the shutdown signal into its per- + // iteration caller signal, so the fetch can reject with the + // InterruptError reason directly (DEV-331). + if (err instanceof InterruptError) throw err; + // A timeout/abort during the fetch itself: classify it (RequestTimeoutError + // when our deadline fired; otherwise rethrow the caller's abort unmodified). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + // If a RequestTimeoutError already propagated from somewhere (e.g. from a + // nested call or from a test-injected fetchImpl), pass it through unchanged + // rather than re-wrapping it as a TransportError. + if (err instanceof RequestTimeoutError) throw err; + const message = err instanceof Error ? err.message : String(err); + this.debug({ + kind: 'error', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + durationMs: Date.now() - startedAt, + }); + const decision = allowTransportRetry + ? transportRetryDecision(attempt, this.random) + : { retry: false, delayMs: 0 }; + if (!decision.retry) throw new TransportError(message, requestId); + this.transition( + `Network error on ${shortPath(path)} — retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + requestTimeout.clear(); + await this.sleepBeforeRetry(decision.delayMs); + continue; + } - const durationMs = Date.now() - startedAt; - if (response.ok) { - this.debug({ - kind: 'response', - method, - url, - attempt, - status: response.status, - requestId, - durationMs, - }); + const durationMs = Date.now() - startedAt; + // Surface the backend version-compatibility headers on every response + // (success or error) before branching, so the caller's advisory covers + // all paths. + this.captureServerVersion(response); + if (response.ok) { + this.debug({ + kind: 'response', + method, + url, + attempt, + status: response.status, + requestId, + durationMs, + }); + try { + return { body: (await response.json()) as T, requestId, status: response.status }; + } catch (err) { + // Interrupt passthrough (see the fetch catch above). + if (err instanceof InterruptError) throw err; + // A timeout/abort can fire mid-body-read (headers received, stream stalls). + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); + // Otherwise the successful response body was not valid JSON — a + // misconfigured endpoint, a proxy / captive-portal / login page that + // returns HTML with a 200 status, or an empty body. Surface a typed + // error carrying the requestId instead of letting the raw SyntaxError + // escape to index.ts, where it would print a bare `{"error":"..."}` + // and break the --output json envelope contract. + throw malformedResponseError(response, requestId, err); + } + } + + let rawBody: unknown; try { - return { body: (await response.json()) as T, requestId, status: response.status }; + rawBody = await safeReadJson(response); } catch (err) { - // A timeout/abort can fire mid-body-read (headers received, stream stalls). - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); + // Interrupt passthrough (see the fetch catch above). + if (err instanceof InterruptError) throw err; + // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse + // errors), so a timeout fired mid-body-read on a non-OK response lands here. + this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId, effectiveSignal); throw err; } - } - let rawBody: unknown; - try { - rawBody = await safeReadJson(response); - } catch (err) { - // safeReadJson rethrows aborts/timeouts (it swallows only non-abort parse - // errors), so a timeout fired mid-body-read on a non-OK response lands here. - this.rethrowIfAbort(err, timeoutSignal, options.signal, requestId); - throw err; - } + // Edge proxies / load balancers return 408/502/504 without our error + // envelope on transient outages. These are transport-level retries, + // not facade errors — fold them in here so we get the bounded backoff + // budget instead of a single INTERNAL bail. + if (rawBody === null && isTransportEdgeStatus(response.status)) { + this.debug({ + kind: 'error', + method, + url, + attempt, + status: response.status, + requestId, + errorCode: 'TRANSPORT', + durationMs, + }); + const decision = allowTransportRetry + ? transportRetryDecision(attempt, this.random) + : { retry: false, delayMs: 0 }; + if (!decision.retry) { + throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); + } + this.transition( + `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + ); + this.debug({ + kind: 'retry', + method, + url, + attempt, + requestId, + errorCode: 'TRANSPORT', + delayMs: decision.delayMs, + }); + requestTimeout.clear(); + await this.sleepBeforeRetry(decision.delayMs); + continue; + } - // Edge proxies / load balancers return 408/502/504 without our error - // envelope on transient outages. Per the CLI error spec §7 these are - // transport-level retries, not facade errors — fold them in here so - // we get the bounded backoff budget instead of a single INTERNAL bail. - if (rawBody === null && isTransportEdgeStatus(response.status)) { + const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); + // Clamp server-directed Retry-After to [1s, 300s] and surface on the + // thrown error so outer callers (e.g. runBatchRun outer retry loop) + // can honor it without re-reading the now-consumed HTTP response. + const retryAfterMsForError = + retryAfterSec !== undefined + ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 + : undefined; + const apiError = ApiError.fromEnvelope( + rawBody, + response.status, + retryAfterMsForError, + // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing + // links) resolve the environment-correct portal domain. + this.baseUrl, + ); this.debug({ kind: 'error', method, url, attempt, status: response.status, + errorCode: apiError.code, requestId, - errorCode: 'TRANSPORT', durationMs, }); - const decision = transportRetryDecision(attempt, this.random); - if (!decision.retry) { - throw new TransportError(`HTTP ${response.status} from ${url}`, requestId); - } - this.transition( - `HTTP ${response.status} from ${shortPath(path)} — transport error, retrying in ${Math.round(decision.delayMs / 1000)}s (attempt ${attempt})`, + const retryOnConflict = options.retryOnConflict !== false; + const retryOnRateLimit = options.retryOnRateLimit !== false; + const decision = apiRetryDecision( + apiError.code, + attempt, + retryAfterSec, + this.random, + retryOnConflict, + retryOnRateLimit, ); + if (!decision.retry) throw apiError; + const delaySec = Math.round(decision.delayMs / 1000); + if (apiError.code === 'RATE_LIMITED') { + this.transition( + `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, + ); + } else if (apiError.code === 'INTERNAL') { + this.transition( + `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } else if (apiError.code === 'UNAVAILABLE') { + this.transition( + `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, + ); + } this.debug({ kind: 'retry', method, url, attempt, requestId, - errorCode: 'TRANSPORT', + errorCode: apiError.code, delayMs: decision.delayMs, }); - await this.sleep(decision.delayMs); - continue; + requestTimeout.clear(); + await this.sleepBeforeRetry(decision.delayMs); + } finally { + requestTimeout.clear(); } - - const retryAfterSec = parseRetryAfter(response.headers.get('retry-after')); - // Clamp server-directed Retry-After to [1s, 300s] and surface on the - // thrown error so outer callers (e.g. runBatchRun outer retry loop) - // can honor it without re-reading the now-consumed HTTP response. - const retryAfterMsForError = - retryAfterSec !== undefined ? Math.min(Math.max(retryAfterSec, 1), 300) * 1000 : undefined; - const apiError = ApiError.fromEnvelope( - rawBody, - response.status, - retryAfterMsForError, - // Lets synthesized nextAction text (e.g. INSUFFICIENT_CREDITS billing - // links) resolve the environment-correct portal domain. - this.baseUrl, - ); - this.debug({ - kind: 'error', - method, - url, - attempt, - status: response.status, - errorCode: apiError.code, - requestId, - durationMs, - }); - const retryOnConflict = options.retryOnConflict !== false; - const retryOnRateLimit = options.retryOnRateLimit !== false; - const decision = apiRetryDecision( - apiError.code, - attempt, - retryAfterSec, - this.random, - retryOnConflict, - retryOnRateLimit, - ); - if (!decision.retry) throw apiError; - const delaySec = Math.round(decision.delayMs / 1000); - if (apiError.code === 'RATE_LIMITED') { - this.transition( - `Rate limited (HTTP 429) — waiting ${delaySec}s before retry (attempt ${attempt})`, - ); - } else if (apiError.code === 'INTERNAL') { - this.transition( - `Server error (HTTP 5xx, requestId: ${requestId}) — retrying in ${delaySec}s (attempt ${attempt})`, - ); - } else if (apiError.code === 'UNAVAILABLE') { - this.transition( - `Service unavailable (HTTP 503) — retrying in ${delaySec}s (attempt ${attempt})`, - ); - } - this.debug({ - kind: 'retry', - method, - url, - attempt, - requestId, - errorCode: apiError.code, - delayMs: decision.delayMs, - }); - await this.sleep(decision.delayMs); } } @@ -630,6 +750,33 @@ export class HttpClient { return headers; } + /** + * Retry-delay sleep that bails the moment the shutdown signal fires + * (DEV-331, codex finding 1): a RATE_LIMITED `Retry-After: 60` or a + * transport backoff must not delay the honest-detach exit by up to a + * minute — reject with the InterruptError reason immediately. Mirrors + * poll.ts::sleepUnlessInterrupted. + */ + private sleepBeforeRetry(ms: number): Promise { + const signal = this.shutdownSignal; + if (signal == null) return this.sleep(ms); + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + this.sleep(ms).then( + () => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, + err => { + signal.removeEventListener('abort', onAbort); + reject(err instanceof Error ? err : new Error(String(err))); + }, + ); + }); + } + private debug(event: DebugEvent): void { if (this.onDebug) this.onDebug(event); } @@ -681,6 +828,39 @@ function newRequestId(): string { return `cli_${randomUUID()}`; } +interface RequestTimeoutHandle { + signal: AbortSignal; + clear: () => void; +} + +function createRequestTimeout(timeoutMs: number): RequestTimeoutHandle { + const controller = new AbortController(); + const timer = setTimeout(() => { + controller.abort(makeTimeoutReason()); + }, timeoutMs); + unrefTimer(timer); + + return { + signal: controller.signal, + clear: () => clearTimeout(timer), + }; +} + +function makeTimeoutReason(): Error { + if (typeof DOMException !== 'undefined') { + return new DOMException('The operation timed out.', 'TimeoutError'); + } + const err = new Error('The operation timed out.'); + err.name = 'TimeoutError'; + return err; +} + +function unrefTimer(timer: ReturnType): void { + if (typeof timer !== 'object' || timer === null || !('unref' in timer)) return; + const unref = (timer as { unref?: () => void }).unref; + if (typeof unref === 'function') unref.call(timer); +} + async function safeReadJson(response: Response): Promise { try { return await response.json(); @@ -692,6 +872,53 @@ async function safeReadJson(response: Response): Promise { } } +/** + * Build a typed error for a successful response whose body could not be parsed + * as JSON. + * + * The CLI expects every API response to be a JSON envelope. When a `200 OK` + * carries a non-JSON body — a misconfigured endpoint, a proxy / captive-portal + * / SSO login page returning HTML with a success status, or an empty body — + * `response.json()` throws a raw `SyntaxError`. Left unhandled it escapes to + * the top-level handler in `index.ts`, which prints a bare + * `{"error":""}` under `--output json` (breaking the + * typed-envelope contract every other error honors) and gives the operator no + * actionable context. + * + * This wraps it in a typed `INTERNAL` `ApiError` (exit 1, unchanged) that + * carries the `requestId`, names the likely cause, and points the operator at + * their endpoint configuration. `details` includes the HTTP status, the + * response `content-type` (when present), and the underlying parse message. + */ +export function malformedResponseError( + response: Response, + requestId: string, + cause: unknown, +): ApiError { + const contentType = response.headers.get('content-type') ?? undefined; + const parseError = cause instanceof Error ? cause.message : String(cause); + const contentTypeNote = contentType ? ` (content-type: ${contentType})` : ''; + return new ApiError( + { + code: 'INTERNAL', + message: + `The server returned a non-JSON response${contentTypeNote} for an HTTP ${response.status}. ` + + `This usually means the endpoint is not the TestSprite API — a proxy, captive portal, or ` + + `login page can return HTML with a success status.`, + nextAction: + 'Check that --endpoint-url / TESTSPRITE_API_URL points at the TestSprite API ' + + '(default https://api.testsprite.com), then retry.', + requestId, + details: { + httpStatus: response.status, + ...(contentType ? { contentType } : {}), + parseError, + }, + }, + response.status, + ); +} + export function parseRetryAfter(headerValue: string | null): number | undefined { if (!headerValue) return undefined; const numeric = Number(headerValue); @@ -723,6 +950,20 @@ function transportRetryDecision(attempt: number, random: () => number): RetryDec return { retry: true, delayMs: backoffDelay(attempt, random) }; } +function canRetryTransport(method: string, options: RequestOptions): boolean { + return isIdempotentMethod(method) || hasIdempotencyKey(options.headers); +} + +function isIdempotentMethod(method: string): boolean { + const normalized = method.toUpperCase(); + return normalized === 'GET' || normalized === 'HEAD'; +} + +function hasIdempotencyKey(headers: Record | undefined): boolean { + if (!headers) return false; + return Object.keys(headers).some(name => name.toLowerCase() === 'idempotency-key'); +} + function apiRetryDecision( code: ErrorCode, attempt: number, @@ -754,6 +995,9 @@ function apiRetryDecision( case 'UNSUPPORTED': case 'INSUFFICIENT_CREDITS': case 'FEATURE_GATED': + case 'CLIENT_TOO_OLD': + // CLIENT_TOO_OLD: retrying re-sends the same too-old client — it can only + // self-heal by upgrading, so fail fast with the upgrade guidance. return { retry: false, delayMs: 0 }; case 'CONFLICT': // Read paths (e.g. GET /failure) retry once: 409 = mid-mutation snapshot. diff --git a/src/lib/interrupt.test.ts b/src/lib/interrupt.test.ts new file mode 100644 index 0000000..bd20bba --- /dev/null +++ b/src/lib/interrupt.test.ts @@ -0,0 +1,172 @@ +import { EventEmitter } from 'node:events'; +import { writeSync } from 'node:fs'; +import { describe, expect, it, vi } from 'vitest'; +import { InterruptError } from './errors.js'; +import { + SIGINT_EXIT_CODE, + ShutdownController, + TERMINATION_EXIT_CODES, + formatInterruptMessage, + installBrokenPipeGuard, + installSignalHandlers, +} from './interrupt.js'; + +// installSignalHandlers' default stderr writes via fs.writeSync (synchronous, so +// the hint survives a piped stderr before exit); mock it to assert on that path. +vi.mock('node:fs', async importOriginal => { + const actual = (await importOriginal()) as Record; + return { ...actual, writeSync: vi.fn() }; +}); + +describe('formatInterruptMessage', () => { + it('defaults to SIGINT and explains the run continues server-side', () => { + const message = formatInterruptMessage(); + expect(message).toContain('Interrupted (SIGINT)'); + expect(message).toContain('test wait'); + expect(message).toContain('test list'); + }); + + it('names the specific signal when given one', () => { + expect(formatInterruptMessage('SIGTERM')).toContain('Interrupted (SIGTERM)'); + expect(formatInterruptMessage('SIGHUP')).toContain('Interrupted (SIGHUP)'); + }); +}); + +/** Fresh handler map + controller per case: the disarmed path exits on the + * FIRST signal, so sequential signals on one install take the second-signal + * hard-exit branch (by design — DEV-331 SIG-5). */ +function install(shutdown = new ShutdownController()) { + const handlers = new Map void>(); + const stderr: string[] = []; + const exit = vi.fn(); + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + stderr: line => stderr.push(line), + exit, + shutdown, + }); + return { handlers, stderr, exit, shutdown }; +} + +describe('installSignalHandlers', () => { + it('registers SIGINT, SIGTERM and SIGHUP with the conventional 128+signum exit codes', () => { + for (const [signal, code] of [ + ['SIGINT', 130], + ['SIGTERM', 143], + ['SIGHUP', 129], + ] as const) { + const { handlers, stderr, exit } = install(); + expect([...handlers.keys()].sort()).toEqual(['SIGHUP', 'SIGINT', 'SIGTERM']); + handlers.get(signal)!(); + expect(exit).toHaveBeenLastCalledWith(code); + // Disarmed handler emits a leading blank line then the explanation. + expect(stderr[0]).toBe(''); + expect(stderr.join('\n')).toContain(`Interrupted (${signal})`); + } + expect(SIGINT_EXIT_CODE).toBe(130); + expect(TERMINATION_EXIT_CODES.SIGTERM).toBe(143); + expect(TERMINATION_EXIT_CODES.SIGHUP).toBe(129); + }); + + it('writes the hint synchronously via writeSync before exit (survives a piped stderr)', () => { + vi.mocked(writeSync).mockClear(); + const handlers = new Map void>(); + const exit = vi.fn(); + // No stderr dep: exercise the synchronous default path. + installSignalHandlers({ + on: (signal, handler) => handlers.set(signal, handler), + exit, + shutdown: new ShutdownController(), + }); + handlers.get('SIGINT')!(); + expect(exit).toHaveBeenCalledWith(130); + const written = vi + .mocked(writeSync) + .mock.calls.map(call => String(call[1])) + .join(''); + expect(written).toContain('Interrupted (SIGINT)'); + }); + + it('armed scope: first signal aborts with InterruptError and does NOT exit or print', () => { + const { handlers, stderr, exit, shutdown } = install(); + const disarm = shutdown.arm(); + handlers.get('SIGINT')!(); + + expect(exit).not.toHaveBeenCalled(); + expect(stderr).toEqual([]); + expect(shutdown.signal.aborted).toBe(true); + expect(shutdown.received).toBe('SIGINT'); + const reason = shutdown.signal.reason as InterruptError; + expect(reason).toBeInstanceOf(InterruptError); + expect(reason.signal).toBe('SIGINT'); + expect(reason.exitCode).toBe(130); + disarm(); + }); + + it('second signal during armed cleanup hard-exits with the second signal code (SIG-5)', () => { + const { handlers, exit, shutdown } = install(); + shutdown.arm(); + handlers.get('SIGINT')!(); + expect(exit).not.toHaveBeenCalled(); + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenCalledWith(143); + }); + + it('disposed scope reverts to the disarmed immediate-exit behavior', () => { + const { handlers, stderr, exit, shutdown } = install(); + const disarm = shutdown.arm(); + disarm(); + disarm(); // idempotent — double dispose must not underflow the counter + handlers.get('SIGTERM')!(); + expect(exit).toHaveBeenCalledWith(143); + expect(stderr.join('\n')).toContain('Interrupted (SIGTERM)'); + }); + + it('nested arms (fan-out members) stay armed until the last disposer runs', () => { + const shutdown = new ShutdownController(); + const a = shutdown.arm(); + const b = shutdown.arm(); + a(); + expect(shutdown.isArmed).toBe(true); + b(); + expect(shutdown.isArmed).toBe(false); + }); +}); + +describe('installBrokenPipeGuard', () => { + function makeEpipe(): NodeJS.ErrnoException { + return Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }); + } + + it('exits 0 on stdout EPIPE (clean SIGPIPE-equivalent for `| head`)', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + stdout.emit('error', makeEpipe()); + expect(exit).toHaveBeenCalledWith(0); + }); + + it('re-throws a non-EPIPE stdout error instead of silently swallowing it', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + expect(() => + stdout.emit('error', Object.assign(new Error('boom'), { code: 'ENOSPC' })), + ).toThrow('boom'); + expect(exit).not.toHaveBeenCalled(); + }); + + it('swallows stderr EPIPE without exiting or throwing', () => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const exit = vi.fn(); + installBrokenPipeGuard({ stdout, stderr, exit }); + + expect(() => stderr.emit('error', makeEpipe())).not.toThrow(); + expect(exit).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/interrupt.ts b/src/lib/interrupt.ts new file mode 100644 index 0000000..19edb09 --- /dev/null +++ b/src/lib/interrupt.ts @@ -0,0 +1,230 @@ +/** + * Process lifecycle hardening: graceful termination signals and broken-pipe. + * + * Termination signals: without a handler, Node terminates the process abruptly + * with no output, so a user (Ctrl+C), a CI runner or `docker stop` (SIGTERM), or + * a closed terminal/SSH session (SIGHUP) that interrupts a long + * `test run --wait` is left unsure whether the run was cancelled or is still + * executing server-side (it is: the CLI only polls; the run lives on the + * backend). The handler prints a one-line explanation plus how to resume, then + * exits with the conventional `128 + signal` code. + * + * Broken pipe: when output is piped to a reader that closes early + * (`testsprite ... | head`), the kernel raises `EPIPE` on the next stdout write. + * Node turns an `'error'` with no listener into an uncaughtException and dumps a + * raw `write EPIPE` stack (exit 1). The guard swallows it and exits 0, the + * conventional SIGPIPE-equivalent result for "the reader went away". + * + * `process` and the streams are injectable so the wiring is unit-testable + * without spawning a subprocess or sending a real signal. + */ + +import { setMaxListeners } from 'node:events'; +import { writeSync } from 'node:fs'; +import { InterruptError, TERMINATION_EXIT_CODES, type TerminationSignal } from './errors.js'; + +export { TERMINATION_EXIT_CODES, type TerminationSignal } from './errors.js'; + +/** Back-compat alias: SIGINT's conventional exit code. */ +export const SIGINT_EXIT_CODE = TERMINATION_EXIT_CODES.SIGINT; + +/** + * Structural view of {@link ShutdownController} threaded through the DI + * surfaces (`TestDeps`, `PollOptions`) — commands and the polling loop need + * only these members, and tests can supply a lightweight fake. + */ +export interface ShutdownHandle { + /** Aborts (reason: `InterruptError`) when a termination signal arrives while armed. */ + readonly signal: AbortSignal; + /** Enter a graceful-detach scope. Returns the disposer that leaves it. */ + arm(): () => void; +} + +/** + * Process-lifetime coordinator between the signal handler and the `--wait` + * polling paths (DEV-331 piece 1). + * + * Two modes, chosen by whether a graceful-detach scope is armed when the + * signal arrives: + * + * - **Armed** (inside `pollRunUntilTerminal`): the handler only aborts + * `signal` with an `InterruptError` — no I/O, no exit. The in-flight fetch + * and every backoff sleep bail immediately; the `--wait` catch blocks own + * the cleanup (finalize the ticker, print the honest partial envelope + + * re-attach hint, rethrow to `index.ts` → exit 130/143/129). + * - **Disarmed** (no wait in progress — prompts, one-shot commands, local + * FS work): the handler prints the generic explanation and exits + * immediately, preserving the pre-DEV-331 behavior. An abort nobody + * observes must never leave the process hanging at e.g. a readline prompt. + * + * A second signal while the armed cleanup is in flight is the documented + * escape hatch: immediate hard exit. + */ +export class ShutdownController { + private readonly controller = new AbortController(); + private armedCount = 0; + private receivedSignal: TerminationSignal | null = null; + + constructor() { + // Every fetch and every poll iteration composes this signal via + // AbortSignal.any — a 50-run batch fan-out legitimately holds >10 + // concurrent listeners, so silence Node's MaxListeners warning. + setMaxListeners(0, this.controller.signal); + } + + get signal(): AbortSignal { + return this.controller.signal; + } + + /** The first termination signal received, or null if none yet. */ + get received(): TerminationSignal | null { + return this.receivedSignal; + } + + get isArmed(): boolean { + return this.armedCount > 0; + } + + /** + * Enter a graceful-detach scope (re-entrant: fan-out members overlap). + * Returns an idempotent disposer. + */ + arm(): () => void { + this.armedCount += 1; + let disposed = false; + return () => { + if (disposed) return; + disposed = true; + this.armedCount -= 1; + }; + } + + /** Record the signal and abort with an `InterruptError` carrying it. */ + interrupt(signal: TerminationSignal): void { + this.receivedSignal = signal; + this.controller.abort(new InterruptError(signal)); + } +} + +/** + * The process-wide instance: `index.ts` hands it to `installSignalHandlers`, + * and it is the default `shutdown` for `TestDeps` / `PollOptions` / + * `ClientFactoryDeps`, so production wiring is automatic. Tests inject their + * own `ShutdownController` (or a `ShutdownHandle` fake) instead. + */ +export const globalShutdown = new ShutdownController(); + +export function formatInterruptMessage(signal: TerminationSignal = 'SIGINT'): string { + return ( + `Interrupted (${signal}). Any run already started keeps executing on the server; ` + + 'check it with `testsprite test list` or `testsprite test wait `.' + ); +} + +export interface InterruptDeps { + /** Signal registrar. Defaults to `process.on`. */ + on?: (signal: TerminationSignal, handler: () => void) => void; + /** Line-oriented stderr writer (appends a newline). */ + stderr?: (line: string) => void; + /** Process exit. Defaults to `process.exit`. */ + exit?: (code: number) => void; + /** Shutdown coordinator. Defaults to {@link globalShutdown}. */ + shutdown?: ShutdownController; +} + +/** + * Register handlers for SIGINT, SIGTERM and SIGHUP. Idempotent enough for a + * single top-level call in `index.ts`; not designed to be installed twice. + * + * First signal, armed scope: abort-only — the `--wait` catch paths own the + * honest-detach UX and the exit (DEV-331 D1: Ctrl-C = detach, never cancel). + * First signal, disarmed: print the generic explanation + exit `128+signum`. + * Second signal (any mode): immediate hard exit — the escape hatch when the + * graceful cleanup itself wedges. + */ +export function installSignalHandlers(deps: InterruptDeps = {}): void { + const on = + deps.on ?? + ((signal: TerminationSignal, handler: () => void) => { + process.on(signal, handler); + }); + const stderr = + deps.stderr ?? + ((line: string) => { + // A signal handler calls process.exit() right after writing, which can + // truncate an async process.stderr.write() when stderr is a pipe. Write + // synchronously so the interrupt hint is flushed before the process exits. + try { + writeSync(process.stderr.fd, `${line}\n`); + } catch { + // Best-effort: if stderr is already gone (EPIPE), still exit cleanly. + } + }); + const exit = deps.exit ?? ((code: number) => process.exit(code)); + const shutdown = deps.shutdown ?? globalShutdown; + + for (const signal of Object.keys(TERMINATION_EXIT_CODES) as TerminationSignal[]) { + on(signal, () => { + if (shutdown.received !== null) { + // Second signal while graceful cleanup is in flight: hard exit now. + exit(TERMINATION_EXIT_CODES[signal]); + return; + } + if (shutdown.isArmed) { + // Graceful detach: abort only (sync, signal-safe — no I/O here so a + // pending stdout `drain` wait can settle); the armed catch paths + // finalize the ticker, print the partial + re-attach hint, and exit + // via index.ts with this signal's code. + shutdown.interrupt(signal); + return; + } + // Disarmed (no --wait in progress): legacy immediate exit. Record the + // signal first so a second one takes the hard-exit branch even when + // `exit` is injected and does not terminate (unit tests). + shutdown.interrupt(signal); + // Blank line first so the message starts on its own row rather than + // trailing the progress ticker's in-place line. + stderr(''); + stderr(formatInterruptMessage(signal)); + exit(TERMINATION_EXIT_CODES[signal]); + }); + } +} + +export interface BrokenPipeDeps { + /** stdout stream. Defaults to `process.stdout`. */ + stdout?: NodeJS.EventEmitter; + /** stderr stream. Defaults to `process.stderr`. */ + stderr?: NodeJS.EventEmitter; + /** Process exit. Defaults to `process.exit`. */ + exit?: (code: number) => void; +} + +/** + * Guard against `EPIPE` on stdout/stderr so piping to a reader that closes + * early (`testsprite ... | head`) exits cleanly instead of crashing with an + * unhandled `write EPIPE` stack. Only `EPIPE` is swallowed; any other stream + * error is left to surface normally. + */ +export function installBrokenPipeGuard(deps: BrokenPipeDeps = {}): void { + const stdout = deps.stdout ?? process.stdout; + const stderr = deps.stderr ?? process.stderr; + const exit = deps.exit ?? ((code: number) => process.exit(code)); + + stdout.on('error', (error: NodeJS.ErrnoException) => { + // Reader went away (`| head`, `| less` then q): exit cleanly like SIGPIPE + // rather than dumping an unhandled `write EPIPE` stack. Any other stdout + // error is a genuine, actionable failure, so re-throw it (Node's default). + if (error.code === 'EPIPE') { + exit(0); + return; + } + throw error; + }); + stderr.on('error', (error: NodeJS.ErrnoException) => { + // stderr closed: nothing can be reported over it, so swallow EPIPE. Any + // other error re-throws so a genuine failure is not silently hidden. + if (error.code === 'EPIPE') return; + throw error; + }); +} diff --git a/src/lib/junit-report.test.ts b/src/lib/junit-report.test.ts new file mode 100644 index 0000000..d504dce --- /dev/null +++ b/src/lib/junit-report.test.ts @@ -0,0 +1,330 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { ApiError } from './errors.js'; +import { + assertJUnitReportOptions, + buildJUnitReport, + escapeXml, + parseJUnitReportFormat, + resolveBatchReportProjectId, + writeJUnitReportFile, + type JUnitTestResult, +} from './junit-report.js'; + +function makeResult(overrides: Partial & { testId: string }): JUnitTestResult { + return { + status: 'passed', + ...overrides, + }; +} + +describe('escapeXml', () => { + it('escapes XML special characters', () => { + expect(escapeXml(`a&bd"e'f`)).toBe('a&b<c>d"e'f'); + }); + + it('leaves plain text unchanged', () => { + expect(escapeXml('test_abc123')).toBe('test_abc123'); + }); +}); + +describe('parseJUnitReportFormat', () => { + it('accepts junit', () => { + expect(parseJUnitReportFormat('junit')).toBe('junit'); + }); + + it('returns undefined for absent value', () => { + expect(parseJUnitReportFormat(undefined)).toBeUndefined(); + expect(parseJUnitReportFormat('')).toBeUndefined(); + }); + + it('rejects unknown formats', () => { + expect(() => parseJUnitReportFormat('html')).toThrowError(ApiError); + try { + parseJUnitReportFormat('html'); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('assertJUnitReportOptions', () => { + it('allows absent report flags', () => { + expect(() => assertJUnitReportOptions({ wait: false, batchPath: true })).not.toThrow(); + }); + + it('rejects report-file without report', () => { + expect(() => + assertJUnitReportOptions({ reportFile: './out.xml', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); + + it('rejects report-suite-name without report', () => { + expect(() => + assertJUnitReportOptions({ + reportSuiteName: 'my-suite', + wait: true, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report on non-batch paths', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: true, + batchPath: false, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without wait', () => { + expect(() => + assertJUnitReportOptions({ + report: 'junit', + reportFile: './out.xml', + wait: false, + batchPath: true, + }), + ).toThrowError(ApiError); + }); + + it('rejects report without report-file', () => { + expect(() => + assertJUnitReportOptions({ report: 'junit', wait: true, batchPath: true }), + ).toThrowError(ApiError); + }); +}); + +describe('resolveBatchReportProjectId', () => { + it('prefers explicit projectId', () => { + expect(resolveBatchReportProjectId({ projectId: 'proj_a' }, [])).toBe('proj_a'); + }); + + it('infers from polled run rows', () => { + expect(resolveBatchReportProjectId({}, [{ projectId: 'proj_from_run' }])).toBe('proj_from_run'); + }); + + it('requires --project when the project cannot be inferred', () => { + expect(() => resolveBatchReportProjectId({}, [])).toThrowError(ApiError); + try { + resolveBatchReportProjectId({}, []); + } catch (err) { + expect((err as ApiError).code).toBe('VALIDATION_ERROR'); + expect((err as ApiError).exitCode).toBe(5); + } + }); +}); + +describe('buildJUnitReport', () => { + it('renders an empty suite', () => { + const xml = buildJUnitReport({ + suiteName: 'Dry suite', + classname: 'proj_empty', + results: [], + }); + expect(xml).toContain( + ''); + }); + + it('counts passed tests without child elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [makeResult({ testId: 'test_a', status: 'passed' })], + }); + expect(xml).toContain(''); + expect(xml).not.toContain(' { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 'test_fail', + status: 'failed', + runId: 'run_1', + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('runId: run_1'); + expect(xml).toContain('failures="1"'); + }); + + it('maps blocked and cancelled to failures', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ testId: 't_blocked', status: 'blocked' }), + makeResult({ testId: 't_cancelled', status: 'cancelled' }), + ], + }); + expect(xml).toContain('failures="2"'); + expect(xml).toContain('type="blocked"'); + expect(xml).toContain('type="cancelled"'); + }); + + it('maps timeout to failure', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_timeout', + status: 'timeout', + error: { code: 'UNSUPPORTED', message: 'Timed out', exitCode: 7 }, + }), + ], + }); + expect(xml).toContain(''); + }); + + it('maps API error status to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_err', + status: 'error', + error: { code: 'NOT_FOUND', message: 'Run missing', exitCode: 4 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('errors="1"'); + }); + + it('maps auth failures to error elements', () => { + const xml = buildJUnitReport({ + suiteName: 'Batch', + classname: 'proj_1', + results: [ + makeResult({ + testId: 't_auth', + status: 'failed', + error: { code: 'AUTH_INVALID', message: 'Bad key', exitCode: 3 }, + }), + ], + }); + expect(xml).toContain(''); + expect(xml).toContain('failures="0" errors="1"'); + }); + + it('escapes special characters in testcase names and messages', () => { + const xml = buildJUnitReport({ + suiteName: 'Suite "A"', + classname: 'proj<&>', + results: [ + makeResult({ + testId: 'test<1>', + status: 'failed', + error: { code: 'ASSERT', message: 'expected & "ok"', exitCode: 1 }, + }), + ], + }); + expect(xml).toContain('name="test<1>"'); + expect(xml).toContain('classname="proj<&>"'); + expect(xml).toContain('message="expected <true> & "ok""'); + }); + + it('aggregates mixed outcomes', () => { + const xml = buildJUnitReport({ + suiteName: 'Mixed', + classname: 'proj_mix', + results: [ + makeResult({ testId: 'p', status: 'passed' }), + makeResult({ testId: 'f', status: 'failed' }), + makeResult({ + testId: 'e', + status: 'error', + error: { code: 'INTERNAL', message: 'boom', exitCode: 10 }, + }), + ], + }); + expect(xml).toContain('tests="3" failures="1" errors="1" skipped="0"'); + }); +}); + +describe('writeJUnitReportFile', () => { + it('writes XML atomically to the target path', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-')); + const target = join(dir, 'results.xml'); + const xml = buildJUnitReport({ + suiteName: 'Suite', + classname: 'proj_write', + results: [makeResult({ testId: 't1', status: 'passed' })], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects a directory target', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-dir-')); + await expect(writeJUnitReportFile(dir, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects missing parent directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-parent-')); + const missing = join(dir, 'missing', 'out.xml'); + await expect(writeJUnitReportFile(missing, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); + + it('overwrites an existing file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-overwrite-')); + const target = join(dir, 'results.xml'); + writeFileSync(target, 'old', 'utf8'); + const xml = buildJUnitReport({ + suiteName: 'New', + classname: 'proj_new', + results: [], + }); + + await writeJUnitReportFile(target, xml); + + expect(readFileSync(target, 'utf8')).toBe(xml); + rmSync(dir, { recursive: true, force: true }); + }); + + it('rejects empty path', async () => { + await expect(writeJUnitReportFile('', '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + }); + + it('rejects parent that is a file', async () => { + const dir = mkdtempSync(join(tmpdir(), 'junit-report-file-parent-')); + const parentFile = join(dir, 'not-a-dir'); + writeFileSync(parentFile, 'x', 'utf8'); + const target = join(parentFile, 'out.xml'); + await expect(writeJUnitReportFile(target, '')).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + }); + rmSync(dir, { recursive: true, force: true }); + }); +}); diff --git a/src/lib/junit-report.ts b/src/lib/junit-report.ts new file mode 100644 index 0000000..3b11442 --- /dev/null +++ b/src/lib/junit-report.ts @@ -0,0 +1,267 @@ +import { createWriteStream } from 'node:fs'; +import { rename, stat, unlink } from 'node:fs/promises'; +import { basename, dirname, isAbsolute, join, resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { localValidationError, TransportError } from './errors.js'; + +export type JUnitReportFormat = 'junit'; + +/** Minimal testcase input shared by batch run and batch rerun poll results. */ +export interface JUnitTestResult { + testId: string; + runId?: string; + status: string; + /** Observed on polled runs; used for classname when --project is omitted. */ + projectId?: string; + error?: { code: string; message: string; exitCode?: number }; +} + +export interface JUnitReportBuildOptions { + suiteName: string; + classname: string; + results: readonly JUnitTestResult[]; +} + +export interface JUnitReportFlagOptions { + report?: JUnitReportFormat; + reportFile?: string; + reportSuiteName?: string; + wait: boolean; + /** True when the invocation is a batch path (run --all or rerun batch). */ + batchPath: boolean; +} + +const XML_DECL = ''; + +/** + * Parse `--report `. Only `junit` is accepted in v1. + */ +export function parseJUnitReportFormat(raw: string | undefined): JUnitReportFormat | undefined { + if (raw === undefined || raw === '') return undefined; + if (raw === 'junit') return 'junit'; + throw localValidationError('report', `unsupported report format "${raw}" — accepted: junit`); +} + +/** + * Validate `--report` / `--report-file` / `--report-suite-name` combinations. + * Report export is a sidecar artifact for batch `--wait` runs only. + */ +export function assertJUnitReportOptions(opts: JUnitReportFlagOptions): void { + if (opts.report === undefined) { + if (opts.reportFile !== undefined && opts.reportFile !== '') { + throw localValidationError('report-file', '--report-file requires --report junit'); + } + if (opts.reportSuiteName !== undefined && opts.reportSuiteName !== '') { + throw localValidationError( + 'report-suite-name', + '--report-suite-name requires --report junit', + ); + } + return; + } + + if (!opts.batchPath) { + throw localValidationError( + 'report', + '--report junit only applies to batch --wait runs (test run --all, or test rerun --all / multiple test ids)', + ); + } + if (!opts.wait) { + throw localValidationError( + 'report', + '--report junit requires --wait (the report is written after batch polling completes)', + ); + } + if (opts.reportFile === undefined || opts.reportFile === '') { + throw localValidationError('report-file', '--report junit requires --report-file '); + } +} + +/** + * Resolve the project id used for JUnit classname / default suite naming. + * Prefer explicit `--project`, then ids observed on polled run rows. + */ +export function resolveBatchReportProjectId( + opts: { projectId?: string }, + results: ReadonlyArray<{ projectId?: string }>, +): string { + if (opts.projectId) return opts.projectId; + const fromPoll = results.map(r => r.projectId).find((id): id is string => !!id); + if (fromPoll) return fromPoll; + throw localValidationError( + 'project', + '--report junit requires --project when the project cannot be inferred from run results', + ); +} + +/** + * Escape text for inclusion in XML element bodies and double-quoted attributes. + */ +export function escapeXml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, '''); +} + +type JUnitOutcome = 'passed' | 'failure' | 'error' | 'skipped'; + +function classifyJUnitOutcome(status: string, error?: JUnitTestResult['error']): JUnitOutcome { + if (status === 'passed') return 'passed'; + if (status === 'skipped') return 'skipped'; + if (status === 'error' || error?.exitCode === 3) return 'error'; + return 'failure'; +} + +function failureMessage(result: JUnitTestResult): string { + if (result.error?.message) return result.error.message; + if (result.error?.code) return result.error.code; + return result.status; +} + +function renderTestcase(result: JUnitTestResult, classname: string): string { + const outcome = classifyJUnitOutcome(result.status, result.error); + const name = escapeXml(result.testId); + const cls = escapeXml(classname); + const lines = [` `]; + + if (outcome === 'failure') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.status); + const body = escapeXml( + [ + `status: ${result.status}`, + result.runId ? `runId: ${result.runId}` : undefined, + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'error') { + const message = escapeXml(failureMessage(result)); + const type = escapeXml(result.error?.code ?? result.status); + const body = escapeXml( + [ + result.error?.code ? `code: ${result.error.code}` : undefined, + result.error?.message ? `message: ${result.error.message}` : undefined, + result.runId ? `runId: ${result.runId}` : undefined, + ] + .filter(Boolean) + .join('\n'), + ); + lines.push(` ${body}`); + } else if (outcome === 'skipped') { + lines.push(` `); + } + + lines.push(' '); + return lines.join('\n'); +} + +/** + * Build a JUnit XML document from batch poll results. Duration is `0` in v1 + * because batch poll envelopes do not carry per-run timing. + */ +export function buildJUnitReport(opts: JUnitReportBuildOptions): string { + const results = opts.results; + let failures = 0; + let errors = 0; + let skipped = 0; + + for (const result of results) { + const outcome = classifyJUnitOutcome(result.status, result.error); + if (outcome === 'failure') failures++; + else if (outcome === 'error') errors++; + else if (outcome === 'skipped') skipped++; + } + + const suiteName = escapeXml(opts.suiteName); + const testcases = results.map(r => renderTestcase(r, opts.classname)).join('\n'); + + return [ + XML_DECL, + '', + ` `, + testcases, + ' ', + '', + '', + ].join('\n'); +} + +async function assertReportFileParent(rawPath: string): Promise { + if (typeof rawPath !== 'string' || rawPath.length === 0) { + throw localValidationError('report-file', 'must be a non-empty file path'); + } + const resolved = isAbsolute(rawPath) ? rawPath : resolve(process.cwd(), rawPath); + if (resolved.endsWith('/') || resolved.endsWith('\\')) { + throw localValidationError('report-file', 'must point to a file, not a directory'); + } + + const parent = dirname(resolved); + let parentStat; + try { + parentStat = await stat(parent); + } catch { + throw localValidationError('report-file', `parent directory does not exist: ${parent}`); + } + if (!parentStat.isDirectory()) { + throw localValidationError('report-file', `parent path is not a directory: ${parent}`); + } + + let targetStat; + try { + targetStat = await stat(resolved); + } catch { + return resolved; + } + if (targetStat.isDirectory()) { + throw localValidationError('report-file', `must point to a file, not a directory: ${resolved}`); + } + return resolved; +} + +/** + * Atomically write JUnit XML to `--report-file` (temp sibling + rename). + */ +export async function writeJUnitReportFile(rawPath: string, xml: string): Promise { + const resolved = await assertReportFileParent(rawPath); + const parent = dirname(resolved); + const tmpPath = join(parent, `.${basename(resolved)}.tmp-${randomUUID()}`); + + await new Promise((resolvePromise, reject) => { + const stream = createWriteStream(tmpPath, { encoding: 'utf8' }); + let streamError: Error | null = null; + stream.on('error', err => { + streamError = err instanceof Error ? err : new Error(String(err)); + }); + stream.write(xml, err => { + if (err) { + streamError = err instanceof Error ? err : new Error(String(err)); + } + stream.end(() => { + if (streamError) { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError(`Failed to write --report-file ${resolved}: ${streamError.message}`), + ); + return; + } + rename(tmpPath, resolved) + .then(() => resolvePromise()) + .catch(renameErr => { + unlink(tmpPath).catch(() => undefined); + reject( + new TransportError( + `Failed to write --report-file ${resolved}: ${renameErr instanceof Error ? renameErr.message : String(renameErr)}`, + ), + ); + }); + }); + }); + }); +} diff --git a/src/lib/output.test.ts b/src/lib/output.test.ts index b572bda..bab3b53 100644 --- a/src/lib/output.test.ts +++ b/src/lib/output.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { Output, isOutputMode } from './output.js'; +import { Output, isOutputMode, resolveOutputMode } from './output.js'; +import { ApiError } from './errors.js'; describe('isOutputMode', () => { it('accepts json and text', () => { @@ -15,6 +16,36 @@ describe('isOutputMode', () => { }); }); +describe('resolveOutputMode', () => { + it('returns the mode verbatim for valid values', () => { + expect(resolveOutputMode('json')).toBe('json'); + expect(resolveOutputMode('text')).toBe('text'); + }); + + it('defaults to text when the flag is omitted (undefined)', () => { + expect(resolveOutputMode(undefined)).toBe('text'); + }); + + it('throws a typed VALIDATION_ERROR (exit 5) instead of silently falling back to text', () => { + // The footgun this guards against: an agent that asks for `--output json` + // but mistypes it would otherwise receive a text payload and fail to parse + // it as JSON with no signal. Every command group must reject, not coerce. + for (const bad of ['josn', 'yaml', 'JSON', 'Text', '']) { + let caught: unknown; + try { + resolveOutputMode(bad); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(ApiError); + const apiErr = caught as ApiError; + expect(apiErr.code).toBe('VALIDATION_ERROR'); + expect(apiErr.exitCode).toBe(5); + expect(apiErr.nextAction).toContain('must be one of: json, text'); + } + }); +}); + describe('Output', () => { let logSpy: ReturnType; let errorSpy: ReturnType; diff --git a/src/lib/output.ts b/src/lib/output.ts index 2b2db39..4d59cca 100644 --- a/src/lib/output.ts +++ b/src/lib/output.ts @@ -1,18 +1,40 @@ +import { localValidationError } from './errors.js'; + export type OutputMode = 'json' | 'text'; /** * Help-text footer pointing at the global options surface so users * looking at any subcommand `--help` don't miss `--dry-run`, `--output`, - * `--profile`, `--endpoint-url`, `--debug`. + * `--profile`, `--endpoint-url`, `--request-timeout`, `--debug`. */ export const GLOBAL_OPTS_HINT = - '\nGlobal options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug):' + + '\nGlobal options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug):' + '\n testsprite --help'; export function isOutputMode(value: unknown): value is OutputMode { return value === 'json' || value === 'text'; } +/** + * Resolve a raw `--output` flag value to a concrete {@link OutputMode}. + * + * `undefined` (flag omitted) resolves to the default `'text'`. Any other + * value that is not `'json'` or `'text'` throws a typed VALIDATION_ERROR + * (exit 5) with an actionable message. + * + * The alternative — silently falling back to `'text'` — is a footgun for the + * CLI's primary consumer (coding agents): a caller that asks for + * `--output json` but mistypes it (`--output josn`) would otherwise receive a + * human-readable text payload and fail to parse it as JSON, with no signal as + * to why. Every command group routes its global-option resolution through this + * helper so the validation is uniform. + */ +export function resolveOutputMode(raw: unknown): OutputMode { + if (raw === undefined) return 'text'; + if (isOutputMode(raw)) return raw; + throw localValidationError('output', 'must be one of: json, text', ['json', 'text']); +} + export interface OutputStreams { /** * Line-oriented stdout writer. Each call is one logical line; the diff --git a/src/lib/pagination.test.ts b/src/lib/pagination.test.ts index 33fd735..8a20295 100644 --- a/src/lib/pagination.test.ts +++ b/src/lib/pagination.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest'; import { ApiError } from './errors.js'; -import { paginate, validatePaginationFlags, type FetchPage, type Page } from './pagination.js'; +import { + MAX_AUTO_PAGES, + paginate, + validatePaginationFlags, + type FetchPage, + type Page, +} from './pagination.js'; function makePages(pages: Page[]): { fetchPage: FetchPage; @@ -43,10 +49,18 @@ describe('validatePaginationFlags', () => { expect(() => validatePaginationFlags({ pageSize: Number.NaN })).toThrow(ApiError); }); + it('rejects fractional pageSize values', () => { + expect(() => validatePaginationFlags({ pageSize: 1.5 })).toThrow(ApiError); + }); + it('rejects maxItems=0', () => { expect(() => validatePaginationFlags({ maxItems: 0 })).toThrow(ApiError); }); + it('rejects fractional maxItems values', () => { + expect(() => validatePaginationFlags({ maxItems: 2.5 })).toThrow(ApiError); + }); + it('accepts pageSize=100 (the hard cap)', () => { expect(() => validatePaginationFlags({ pageSize: 100 })).not.toThrow(); }); @@ -113,4 +127,55 @@ describe('paginate', () => { await expect(paginate(fetchPage, { pageSize: 0 })).rejects.toBeInstanceOf(ApiError); expect(calls).toHaveLength(0); }); + + it('continues through empty cursor pages until later data', async () => { + const { fetchPage, calls } = makePages([ + { items: [], nextToken: 'cursor-1' }, + { items: [1], nextToken: null }, + ]); + + const page = await paginate(fetchPage); + + expect(page.items).toEqual([1]); + expect(page.nextToken).toBeNull(); + expect(calls).toHaveLength(2); + expect(calls[1]!.cursor).toBe('cursor-1'); + }); + + it('rejects when API repeats a non-null cursor without making progress', async () => { + const { fetchPage, calls } = makePages([{ items: [], nextToken: 'cursor-1' }]); + + await expect(paginate(fetchPage)).rejects.toMatchObject({ + code: 'UNAVAILABLE', + details: expect.objectContaining({ reason: 'repeated_next_token' }), + }); + + expect(calls).toHaveLength(2); + }); + + it('rejects when auto-pagination exceeds the page safety cap', async () => { + const calls: Array<{ pageSize: number; cursor: string | undefined }> = []; + const fetchPage: FetchPage = async args => { + calls.push(args); + return { + items: [], + nextToken: + args.cursor === undefined ? 'cursor-1' : `cursor-${Number(args.cursor.slice(7)) + 1}`, + }; + }; + + let thrown: unknown; + try { + await paginate(fetchPage); + } catch (err) { + thrown = err; + } + + expect(thrown).toBeInstanceOf(ApiError); + expect(thrown).toMatchObject({ + code: 'UNAVAILABLE', + details: expect.objectContaining({ reason: 'max_pages_exceeded' }), + }); + expect(calls).toHaveLength(MAX_AUTO_PAGES); + }); }); diff --git a/src/lib/pagination.ts b/src/lib/pagination.ts index 59dbc12..cae7581 100644 --- a/src/lib/pagination.ts +++ b/src/lib/pagination.ts @@ -1,5 +1,5 @@ import type { HttpClient } from './http.js'; -import { localValidationError } from './errors.js'; +import { ApiError, localValidationError } from './errors.js'; /** * Page shape returned by every list endpoint per @@ -22,14 +22,15 @@ export interface PaginationFlags { const HARD_PAGE_SIZE_CAP = 100; const DEFAULT_PAGE_SIZE = 25; +export const MAX_AUTO_PAGES = 1000; /** * Validates and normalizes pagination flags. Per the CLI OpenAPI spec * §components.parameters.PageSize the hard cap is 100. Values above the * cap are now rejected with exit 5 rather than silently clamped, giving - * callers fast feedback that their flag value is out of range. Sub-1 / - * NaN values also throw. `maxItems` is validated but not capped (it is a - * client-side cursor, not a server parameter). + * callers fast feedback that their flag value is out of range. Fractional, + * sub-1, and NaN values also throw. `maxItems` is validated but not capped + * (it is a client-side cursor, not a server parameter). * * NOTE: `runResultHistory` previously did its own silent clamp via * `Math.min(Math.max(1, n), 100)` — that was unified to this path by the @@ -38,7 +39,7 @@ const DEFAULT_PAGE_SIZE = 25; export function validatePaginationFlags(flags: PaginationFlags): PaginationFlags { const out: PaginationFlags = { ...flags }; if (out.pageSize !== undefined) { - if (!Number.isFinite(out.pageSize) || out.pageSize < 1) { + if (!Number.isFinite(out.pageSize) || !Number.isInteger(out.pageSize) || out.pageSize < 1) { throw localValidationError( 'page-size', `must be a positive integer between 1 and ${HARD_PAGE_SIZE_CAP}`, @@ -52,7 +53,7 @@ export function validatePaginationFlags(flags: PaginationFlags): PaginationFlags } } if (out.maxItems !== undefined) { - if (!Number.isFinite(out.maxItems) || out.maxItems < 1) { + if (!Number.isFinite(out.maxItems) || !Number.isInteger(out.maxItems) || out.maxItems < 1) { throw localValidationError('maxItems', 'must be a positive integer'); } } @@ -91,14 +92,24 @@ export async function paginate( const items: T[] = []; let cursor: string | undefined = flags.startingToken; let lastNextToken: string | null = null; + let pagesFetched = 0; + const seenNextTokens = new Set(); + if (cursor !== undefined) seenNextTokens.add(cursor); while (true) { const remaining = maxItems !== undefined ? maxItems - items.length : Infinity; if (remaining <= 0) break; + if (pagesFetched >= MAX_AUTO_PAGES) { + throw paginationSafetyError('max_pages_exceeded', { + maxPages: MAX_AUTO_PAGES, + lastCursor: cursor ?? null, + }); + } const callPageSize = Number.isFinite(remaining) ? Math.min(pageSize, remaining) : pageSize; const page = await fetchPage({ pageSize: callPageSize, cursor }); + pagesFetched += 1; lastNextToken = page.nextToken; for (const item of page.items) { @@ -107,12 +118,30 @@ export async function paginate( } if (page.nextToken === null) break; + if (seenNextTokens.has(page.nextToken)) { + throw paginationSafetyError('repeated_next_token', { + cursor: page.nextToken, + pagesFetched, + }); + } + seenNextTokens.add(page.nextToken); cursor = page.nextToken; } return { items, nextToken: lastNextToken }; } +function paginationSafetyError(reason: string, details: Record): ApiError { + return ApiError.fromEnvelope({ + code: 'UNAVAILABLE', + message: 'Pagination did not make progress safely.', + nextAction: + 'Retry later. If the problem continues, contact TestSprite support with the cursor details.', + requestId: 'local', + details: { reason, ...details }, + }); +} + /** * Drop-in helper for commands that take a single page and surface * the cursor verbatim (no auto-follow). Used when the caller passed diff --git a/src/lib/poll.spec.ts b/src/lib/poll.spec.ts index e6a14a1..93889eb 100644 --- a/src/lib/poll.spec.ts +++ b/src/lib/poll.spec.ts @@ -7,7 +7,8 @@ */ import { describe, expect, it } from 'vitest'; -import { ApiError } from './errors.js'; +import { ApiError, InterruptError } from './errors.js'; +import { ShutdownController } from './interrupt.js'; import { pollRunUntilTerminal, TimeoutError } from './poll.js'; import type { RunClient } from './poll.js'; import type { RunResponse } from './runs.types.js'; @@ -794,3 +795,104 @@ describe('pollRunUntilTerminal — resolveAlternate hook', () => { } }); }); + +// --------------------------------------------------------------------------- +// Graceful detach — shutdown handle (DEV-331 piece 1) +// --------------------------------------------------------------------------- + +describe('pollRunUntilTerminal — shutdown (SIGINT/SIGTERM graceful detach)', () => { + it('throws the InterruptError when the shutdown signal was already aborted (beats the deadline)', async () => { + const shutdown = new ShutdownController(); + shutdown.interrupt('SIGINT'); + // timeoutSeconds 0 → the deadline has also passed; the interrupt must win. + const err = await pollRunUntilTerminal(makeClient([makeRun('passed')]), RUN_ID, { + timeoutSeconds: 0, + sleep: instantSleep, + shutdown, + }).catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + }); + + it('aborts an in-flight long-poll fetch and surfaces InterruptError (not TimeoutError)', async () => { + // getRun hangs until the composed per-iteration signal aborts — the same + // contract as a real fetch. Flag-checking between iterations would never + // notice; only the signal composition can interrupt this. + const client: RunClient = { + getRun: (_runId, opts) => + new Promise((_resolve, reject) => { + opts?.signal?.addEventListener('abort', () => reject(opts.signal!.reason), { + once: true, + }); + }), + }; + const shutdown = new ShutdownController(); + const poll = pollRunUntilTerminal(client, RUN_ID, { + timeoutSeconds: 30, + sleep: instantSleep, + shutdown, + }); + queueMicrotask(() => shutdown.interrupt('SIGINT')); + const err = await poll.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGINT'); + expect((err as InterruptError).exitCode).toBe(130); + }); + + it('bails out of a retryAfterSeconds sleep immediately on interrupt', async () => { + // The injected sleep never resolves — only the shutdown race can end it. + const neverSleep = () => new Promise(() => {}); + const client = makeClient([makeRun('running', { retryAfterSeconds: 60 }), makeRun('running')]); + const shutdown = new ShutdownController(); + const poll = pollRunUntilTerminal(client, RUN_ID, { + timeoutSeconds: 300, + sleep: neverSleep, + shutdown, + }); + await new Promise(resolve => setTimeout(resolve, 10)); // let the loop reach the sleep + shutdown.interrupt('SIGTERM'); + const err = await poll.catch(e => e); + expect(err).toBeInstanceOf(InterruptError); + expect((err as InterruptError).signal).toBe('SIGTERM'); + expect((err as InterruptError).exitCode).toBe(143); + }); + + it('arms the graceful-detach scope for the poll duration and disarms after', async () => { + let armed = 0; + let disposedCount = 0; + const handle = { + signal: new AbortController().signal, + arm: () => { + armed += 1; + return () => { + disposedCount += 1; + }; + }, + }; + const run = await pollRunUntilTerminal(makeClient([makeRun('passed')]), RUN_ID, { + timeoutSeconds: 5, + sleep: instantSleep, + shutdown: handle, + }); + expect(run.status).toBe('passed'); + expect(armed).toBe(1); + expect(disposedCount).toBe(1); + }); + + it('disarms even when the poll throws (TimeoutError path)', async () => { + let disposedCount = 0; + const handle = { + signal: new AbortController().signal, + arm: () => () => { + disposedCount += 1; + }, + }; + const err = await pollRunUntilTerminal(makeClient([makeRun('running')]), RUN_ID, { + timeoutSeconds: 0, + sleep: instantSleep, + shutdown: handle, + }).catch(e => e); + expect(err).toBeInstanceOf(TimeoutError); + expect(disposedCount).toBe(1); + }); +}); diff --git a/src/lib/poll.ts b/src/lib/poll.ts index 0dfb0ac..92498e2 100644 --- a/src/lib/poll.ts +++ b/src/lib/poll.ts @@ -21,7 +21,8 @@ * Deadline exceeded → throw `TimeoutError`. */ -import { ApiError } from './errors.js'; +import { ApiError, InterruptError } from './errors.js'; +import type { ShutdownHandle } from './interrupt.js'; import type { RunResponse } from './runs.types.js'; import { isTerminalStatus } from './runs.types.js'; @@ -89,6 +90,16 @@ export interface PollOptions { elapsedMs: number, signal: AbortSignal, ) => Promise; + /** + * Graceful-detach coordinator (DEV-331 piece 1). While the poll runs, the + * scope is armed: a SIGINT/SIGTERM aborts `shutdown.signal` with an + * `InterruptError` instead of killing the process, and this loop surfaces + * it immediately — the in-flight long-poll fetch aborts (composed into the + * per-iteration signal) and every backoff/retry sleep bails early. The + * caller's catch block renders the honest partial + re-attach hint. + * Absent (tests, non-wait callers): behavior is unchanged. + */ + shutdown?: ShutdownHandle; } const LONG_POLL_WAIT_SECONDS = 25; @@ -108,9 +119,30 @@ export async function pollRunUntilTerminal( client: RunClient, runId: string, options: PollOptions, +): Promise { + // Arm the graceful-detach scope for the duration of the poll (DEV-331): + // while armed, a termination signal aborts instead of hard-killing the + // process, and the wait-path catch blocks own the honest detach UX. + const disarm = options.shutdown?.arm(); + try { + return await pollLoop(client, runId, options); + } finally { + disarm?.(); + } +} + +async function pollLoop( + client: RunClient, + runId: string, + options: PollOptions, ): Promise { const { timeoutSeconds, onTick, onTransition, resolveAlternate } = options; - const sleep = options.sleep ?? defaultSleep; + const shutdownSignal = options.shutdown?.signal; + const rawSleep = options.sleep ?? defaultSleep; + // Every sleep site (retryAfterSeconds, backoff schedule, not_yet_visible, + // 5xx retry) bails immediately on interrupt — a Ctrl-C must not sit out a + // 15s backoff before it is noticed. + const sleep = (ms: number): Promise => sleepUnlessInterrupted(rawSleep, ms, shutdownSignal); const startMs = Date.now(); const deadlineMs = startMs + timeoutSeconds * 1000; @@ -123,6 +155,9 @@ export async function pollRunUntilTerminal( let notYetVisibleRetries = 0; while (true) { + // Interrupt outranks the deadline: a Ctrl-C that raced the timeout must + // surface as the honest detach, not as a generic TimeoutError. + if (shutdownSignal?.aborted) throw shutdownSignal.reason; const now = Date.now(); if (now >= deadlineMs) { throw new TimeoutError(runId, timeoutSeconds); @@ -139,20 +174,33 @@ export async function pollRunUntilTerminal( const abortTimer = setTimeout(() => { abortController.abort(); }, remainingMs + TRANSPORT_CUSHION_MS); + // Compose the interrupt into the per-iteration signal: a `--wait` can sit + // inside one <=25s long-poll fetch (and the auto-raised per-request + // timeout means even longer for slow backends) — checking the flag + // between iterations is not enough, the in-flight fetch must abort. + const iterationSignal = + shutdownSignal != null + ? AbortSignal.any([abortController.signal, shutdownSignal]) + : abortController.signal; let run: RunResponse; try { if (useBackoff) { - run = await client.getRun(runId, { signal: abortController.signal }); + run = await client.getRun(runId, { signal: iterationSignal }); } else { const waitSeconds = Math.min(remainingSeconds, LONG_POLL_WAIT_SECONDS); - run = await client.getRun(runId, { waitSeconds, signal: abortController.signal }); + run = await client.getRun(runId, { waitSeconds, signal: iterationSignal }); } // Successful GET resets the consecutive-error counter. consecutiveErrors = 0; notYetVisibleRetries = 0; } catch (err) { clearTimeout(abortTimer); + // Interrupt classification precedes the timeout mapping: the composed + // signal makes the fetch reject on Ctrl-C, and that abort must surface + // as the InterruptError — not as a spurious TimeoutError. + if (err instanceof InterruptError) throw err; + if (shutdownSignal?.aborted) throw shutdownSignal.reason; // An AbortError from our per-iteration controller means the deadline // passed while the fetch was in flight — surface as TimeoutError. if (isAbortError(err)) { @@ -239,9 +287,16 @@ export async function pollRunUntilTerminal( } const altAbort = new AbortController(); const altTimer = setTimeout(() => altAbort.abort(), altRemainingMs); + // The alternate lookup aborts on interrupt too; its errors are swallowed + // by the fallback (best-effort), so the loop-top interrupt check above + // surfaces the InterruptError on the next iteration. + const altSignal = + shutdownSignal != null + ? AbortSignal.any([altAbort.signal, shutdownSignal]) + : altAbort.signal; let alternate: RunResponse | null = null; try { - alternate = await resolveAlternate(run, elapsedMs, altAbort.signal); + alternate = await resolveAlternate(run, elapsedMs, altSignal); } finally { clearTimeout(altTimer); } @@ -294,6 +349,36 @@ function defaultSleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } +/** + * Race a sleep against the shutdown signal: rejects with the signal's + * `InterruptError` reason the moment it fires, so no backoff/retry wait can + * delay the honest-detach UX. Wraps the injected `sleep` (tests keep their + * deterministic fakes); the underlying timer is left to fire harmlessly — + * the interrupt path exits the process long before it matters. + */ +function sleepUnlessInterrupted( + sleep: (ms: number) => Promise, + ms: number, + signal: AbortSignal | undefined, +): Promise { + if (signal == null) return sleep(ms); + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + sleep(ms).then( + () => { + signal.removeEventListener('abort', onAbort); + resolve(); + }, + err => { + signal.removeEventListener('abort', onAbort); + reject(err instanceof Error ? err : new Error(String(err))); + }, + ); + }); +} + /** * Detects an AbortError thrown when an AbortSignal fires. * Works for native fetch AbortErrors as well as `AbortController.abort()` diff --git a/src/lib/prompt.test.ts b/src/lib/prompt.test.ts index a552882..bb8cfb5 100644 --- a/src/lib/prompt.test.ts +++ b/src/lib/prompt.test.ts @@ -39,11 +39,63 @@ describe('promptText', () => { expect(await promptText('? ', { input, output })).toBe('ok'); }); + it('preserves buffered answers for sequential prompts on the same stream', async () => { + const input = Readable.from(['first\nsecond\nthird\n']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + await expect(promptText('Three: ', { input, output })).resolves.toBe('third'); + }); + + it('uses buffered tail input at EOF for a following prompt', async () => { + const input = Readable.from(['first\nsecond']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + }); + + it('preserves buffered CRLF answers for sequential prompts', async () => { + const input = Readable.from(['first\r\nsecond\r\n']); + const output = new CaptureStream(); + + await expect(promptText('One: ', { input, output })).resolves.toBe('first'); + await expect(promptText('Two: ', { input, output })).resolves.toBe('second'); + }); + it('returns the buffered input on stream end without newline', async () => { const input = Readable.from(['eof-no-newline']); const output = new CaptureStream(); expect(await promptText('? ', { input, output })).toBe('eof-no-newline'); }); + + it('writes the question to stderr by default — keeps stdout pure for --output json', async () => { + // Regression: prompts used to default to stdout, which polluted the JSON + // result on the interactive setup/configure path. Interactive UI belongs + // on stderr. Manually swap the global stream writers (prompt reads + // process.stderr/stdout at call time, so this is reliably intercepted). + const errChunks: string[] = []; + const outChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + (process.stdout as unknown as { write: (c: string) => boolean }).write = c => { + outChunks.push(String(c)); + return true; + }; + try { + await promptText('Q: ', { input: Readable.from(['x\n']) }); // no output → default + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + (process.stdout as unknown as { write: typeof origOut }).write = origOut; + } + expect(errChunks.join('')).toContain('Q: '); + expect(outChunks.join('')).not.toContain('Q: '); + }); }); describe('promptSecret (non-TTY behavior)', () => { @@ -63,6 +115,35 @@ describe('promptSecret (non-TTY behavior)', () => { expect(written).not.toContain('sk-hidden-12345'); }); + it('writes the prompt to stderr by default — keeps stdout pure for --output json', async () => { + // Same regression as promptText: the secret prompt is interactive UI and + // must default to stderr so stdout carries only the command result. Manual + // stream swap (vi.spyOn does not intercept process.stderr.write here). + const errChunks: string[] = []; + const outChunks: string[] = []; + const origErr = process.stderr.write.bind(process.stderr); + const origOut = process.stdout.write.bind(process.stdout); + (process.stderr as unknown as { write: (c: string) => boolean }).write = c => { + errChunks.push(String(c)); + return true; + }; + (process.stdout as unknown as { write: (c: string) => boolean }).write = c => { + outChunks.push(String(c)); + return true; + }; + try { + await promptSecret('Key: ', { input: Readable.from(['sk-x\n']) }); // no output → default + } finally { + (process.stderr as unknown as { write: typeof origErr }).write = origErr; + (process.stdout as unknown as { write: typeof origOut }).write = origOut; + } + expect(errChunks.join('')).toContain('Key: '); + expect(outChunks.join('')).not.toContain('Key: '); + // The typed secret is never echoed to either real stream. + expect(errChunks.join('')).not.toContain('sk-x'); + expect(outChunks.join('')).not.toContain('sk-x'); + }); + it('honors DEL/backspace before submission', async () => { const DEL = String.fromCharCode(0x7f); const input = Readable.from([`abc${DEL}d\n`]); @@ -70,6 +151,16 @@ describe('promptSecret (non-TTY behavior)', () => { expect(await promptSecret('? ', { input, output })).toBe('abd'); }); + it('preserves buffered secret answers for sequential prompts', async () => { + const input = Readable.from(['sk-one\nsk-two\n']); + const output = new CaptureStream(); + + await expect(promptSecret('First key: ', { input, output })).resolves.toBe('sk-one'); + await expect(promptSecret('Second key: ', { input, output })).resolves.toBe('sk-two'); + expect(output.text()).not.toContain('sk-one'); + expect(output.text()).not.toContain('sk-two'); + }); + it('rejects on Ctrl-C input', async () => { const ETX = String.fromCharCode(0x03); const input = Readable.from([`abc${ETX}`]); diff --git a/src/lib/prompt.ts b/src/lib/prompt.ts index 96fca29..bccead9 100644 --- a/src/lib/prompt.ts +++ b/src/lib/prompt.ts @@ -11,16 +11,24 @@ interface RawModeCapable { resume?: () => unknown; } +const pendingPromptInput = new WeakMap(); + export async function promptText(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; - const output = streams.output ?? process.stdout; + // Prompts are interactive UI, not data — write the question (and any echo) + // to stderr so stdout carries only the command's result. This keeps + // `--output json` stdout a single pure JSON document even on the interactive + // setup / configure path (§8.1 stdout purity). stderr is still the user's + // TTY, so the prompt remains visible. + const output = streams.output ?? process.stderr; output.write(question); return readLine(input, output, false); } export async function promptSecret(question: string, streams: PromptStreams = {}): Promise { const input = streams.input ?? process.stdin; - const output = streams.output ?? process.stdout; + // See promptText: interactive prompt + masking go to stderr, not stdout. + const output = streams.output ?? process.stderr; output.write(question); const inputAsTTY = input as Readable & RawModeCapable; @@ -41,18 +49,29 @@ function readLine( return new Promise((resolve, reject) => { let buffer = ''; let resolved = false; + let listening = false; const onData = (chunk: Buffer | string): void => { const str = typeof chunk === 'string' ? chunk : chunk.toString('utf-8'); - for (const ch of str) { - const code = ch.charCodeAt(0); + processText(str); + }; + + const processText = (str: string): void => { + for (let i = 0; i < str.length; i += 1) { + const code = str.charCodeAt(i); // Enter (CR or LF) if (code === 13 || code === 10) { + let nextIndex = i + 1; + if (code === 13 && str.charCodeAt(nextIndex) === 10) { + nextIndex += 1; + } + savePendingInput(str.slice(nextIndex)); finish(); return; } // Ctrl-C if (code === 3) { + savePendingInput(''); cleanup(); output.write('\n'); if (!resolved) { @@ -71,7 +90,7 @@ function readLine( } // Drop other control chars if (code < 32) continue; - buffer += ch; + buffer += str[i]; if (mask) output.write('*'); } }; @@ -89,9 +108,12 @@ function readLine( }; const cleanup = (): void => { - input.off('data', onData); - input.off('end', onEnd); - input.off('error', onError); + if (listening) { + input.off('data', onData); + input.off('end', onEnd); + input.off('error', onError); + listening = false; + } const pausable = input as { pause?: () => unknown }; if (typeof pausable.pause === 'function') pausable.pause(); }; @@ -105,12 +127,37 @@ function readLine( } }; + const savePendingInput = (text: string): void => { + if (text.length > 0) { + pendingPromptInput.set(input, text); + } else { + pendingPromptInput.delete(input); + } + }; + + const pending = pendingPromptInput.get(input); + if (pending !== undefined) { + pendingPromptInput.delete(input); + processText(pending); + if (resolved) return; + if (isInputEnded(input)) { + finish(); + return; + } + } + input.on('data', onData); input.on('end', onEnd); input.on('error', onError); + listening = true; const resumable = input as { resume?: () => unknown }; if (typeof resumable.resume === 'function') resumable.resume(); }); } +function isInputEnded(input: NodeJS.ReadableStream): boolean { + const state = input as { readableEnded?: boolean; destroyed?: boolean; closed?: boolean }; + return state.readableEnded === true || state.destroyed === true || state.closed === true; +} + export type { Writable }; diff --git a/src/lib/proxy.test.ts b/src/lib/proxy.test.ts new file mode 100644 index 0000000..5c686af --- /dev/null +++ b/src/lib/proxy.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from 'vitest'; +import { maybeInstallProxyAgent } from './proxy.js'; + +describe('maybeInstallProxyAgent', () => { + it('installs an agent when HTTPS_PROXY is set', () => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }); + + it.each(['https_proxy', 'HTTP_PROXY', 'http_proxy'] as const)( + 'also honors the %s spelling', + name => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { [name]: 'http://proxy.corp.example.com:8080' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }, + ); + + it('does nothing when no proxy variable is set (default path unchanged)', () => { + const install = vi.fn(); + expect(maybeInstallProxyAgent({ env: {}, install })).toBe(false); + expect(maybeInstallProxyAgent({ env: { HTTPS_PROXY: '' }, install })).toBe(false); + expect(install).not.toHaveBeenCalled(); + }); + + it('falls back (returns false, warns, never throws) when installing the agent fails', () => { + // A malformed/unsupported proxy value makes the agent throw at startup; the + // CLI must degrade to a proxy-less dispatcher, not crash every command. + const errs: string[] = []; + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080' }, + install: () => { + throw new Error('unsupported proxy scheme'); + }, + stderr: line => errs.push(line), + }); + expect(installed).toBe(false); + expect(errs.join('\n')).toContain('ignoring proxy environment'); + }); + + it('still installs when NO_PROXY is set (exemptions are applied per request by undici)', () => { + const install = vi.fn(); + const installed = maybeInstallProxyAgent({ + env: { HTTPS_PROXY: 'http://proxy.corp.example.com:8080', NO_PROXY: 'localhost,127.0.0.1' }, + install, + }); + expect(installed).toBe(true); + expect(install).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/lib/proxy.ts b/src/lib/proxy.ts new file mode 100644 index 0000000..b536761 --- /dev/null +++ b/src/lib/proxy.ts @@ -0,0 +1,58 @@ +/** + * Proxy support (issue #119): honor HTTPS_PROXY / HTTP_PROXY / NO_PROXY. + * + * Node's built-in fetch (undici) deliberately ignores the proxy environment + * variables, so behind a corporate or CI proxy every request dies with + * `fetch failed` after a full retry cycle. Installing undici's + * `EnvHttpProxyAgent` as the global dispatcher restores the conventional + * behavior (including NO_PROXY exemptions) for every fetch the CLI makes. + * + * Only active when a proxy env var is actually present, so the default path + * stays byte-identical and pays zero startup cost. Dependency note for + * reviewers: this adds `undici` as an explicit runtime dependency (the same + * engine Node already bundles); the alternative, a hand-rolled CONNECT + * tunnel, would re-implement what undici ships and maintains. + */ +import type { Dispatcher } from 'undici'; +import { EnvHttpProxyAgent, setGlobalDispatcher } from 'undici'; + +export interface ProxyDeps { + env?: NodeJS.ProcessEnv; + /** Dispatcher installer. Defaults to undici's setGlobalDispatcher. */ + install?: (agent: Dispatcher) => void; + /** Warning sink. Defaults to `process.stderr`. */ + stderr?: (line: string) => void; +} + +/** + * Install the env-driven proxy dispatcher when any proxy variable is set + * (both canonical upper-case and conventional lower-case spellings). + * Returns whether an agent was installed (observable for tests/debugging). + */ +export function maybeInstallProxyAgent(deps: ProxyDeps = {}): boolean { + const env = deps.env ?? process.env; + const hasProxy = [env.HTTPS_PROXY, env.https_proxy, env.HTTP_PROXY, env.http_proxy].some( + value => typeof value === 'string' && value.length > 0, + ); + if (!hasProxy) return false; + const install = deps.install ?? setGlobalDispatcher; + // EnvHttpProxyAgent reads HTTPS_PROXY/HTTP_PROXY/NO_PROXY itself, per + // request, so NO_PROXY exemptions apply without extra plumbing here. + // + // A malformed or unsupported proxy value (e.g. `socks5://...`) makes the + // agent throw. Because this runs at startup, an unguarded throw would abort + // every command before the CLI's own error handling — so fall back to the + // default (proxy-less) dispatcher and warn instead of crashing. + try { + install(new EnvHttpProxyAgent()); + return true; + } catch (error) { + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr( + `warning: ignoring proxy environment (could not initialize proxy agent): ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return false; + } +} diff --git a/src/lib/render-error.test.ts b/src/lib/render-error.test.ts index 2b1ced7..9635f2b 100644 --- a/src/lib/render-error.test.ts +++ b/src/lib/render-error.test.ts @@ -37,6 +37,12 @@ describe('rephraseUnknownOption', () => { expect(result).toContain('--endpoint-url'); }); + it('rephrases --request-timeout placed after subcommand', () => { + const result = rephraseUnknownOption("error: unknown option '--request-timeout'"); + expect(result).not.toBeNull(); + expect(result).toContain('--request-timeout'); + }); + it('rephrases --debug placed after subcommand', () => { const result = rephraseUnknownOption("error: unknown option '--debug'"); expect(result).not.toBeNull(); @@ -87,6 +93,12 @@ describe('rephraseUnknownOption', () => { expect(result).not.toBeNull(); expect(result).toContain('testsprite --endpoint-url '); }); + + it('value flag (--request-timeout) example DOES include a placeholder', () => { + const result = rephraseUnknownOption("error: unknown option '--request-timeout'"); + expect(result).not.toBeNull(); + expect(result).toContain('testsprite --request-timeout '); + }); }); describe('renderCommanderError', () => { diff --git a/src/lib/render-error.ts b/src/lib/render-error.ts index ebb4208..1898acd 100644 --- a/src/lib/render-error.ts +++ b/src/lib/render-error.ts @@ -11,13 +11,15 @@ import type { OutputMode } from './output.js'; * * `boolean` flags (--dry-run, --debug, --verbose) take no value; emit * example without a placeholder. `value` flags (--output, --profile, - * --endpoint-url) take a single argument; emit example with ``. + * --endpoint-url, --request-timeout) take a single argument; emit example + * with ``. */ const GLOBAL_FLAG_ARITY: Record = { 'dry-run': 'boolean', output: 'value', profile: 'value', 'endpoint-url': 'value', + 'request-timeout': 'value', debug: 'boolean', verbose: 'boolean', }; diff --git a/src/lib/runs.types.ts b/src/lib/runs.types.ts index 7e556fe..15501c9 100644 --- a/src/lib/runs.types.ts +++ b/src/lib/runs.types.ts @@ -225,6 +225,20 @@ export interface RunResponse { steps?: RunStepDto[] | null; } +// --------------------------------------------------------------------------- +// DEV-331 piece 3 — cancel wire types +// --------------------------------------------------------------------------- + +/** + * Response from `POST /api/cli/v1/runs/{runId}/cancel`. + * Same shape as `GET /runs/{runId}` (`status: "cancelled"`, verdict + * untouched) plus `alreadyCancelled` distinguishing a fresh cancel + * (naturally idempotent) from a no-op re-cancel. + */ +export interface CancelRunResponse extends RunResponse { + alreadyCancelled: boolean; +} + /** Terminal states from the RunStatus union. */ export const TERMINAL_RUN_STATUSES: ReadonlySet = new Set([ 'passed', diff --git a/src/lib/skill-nudge.test.ts b/src/lib/skill-nudge.test.ts index 34f043b..2b26c0e 100644 --- a/src/lib/skill-nudge.test.ts +++ b/src/lib/skill-nudge.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { MANAGED_SECTION_BEGIN, TARGETS } from './agent-targets.js'; +import { MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, TARGETS } from './agent-targets.js'; import type { OutputMode } from './output.js'; import { SKILL_NUDGE_COMMANDS, @@ -13,33 +13,46 @@ import { // isVerifySkillInstalled // --------------------------------------------------------------------------- +// The implementation joins paths with the native separator; normalize so the +// fakes below match on Windows (backslashes) as well as POSIX. +const toPosix = (p: string) => p.replaceAll('\\', '/'); + describe('isVerifySkillInstalled', () => { it('true when the claude own-file SKILL.md exists', () => { - const existsSync = (p: string) => p.endsWith('.claude/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => + toPosix(p).endsWith('.claude/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cursor .mdc landing file', () => { - const existsSync = (p: string) => p.endsWith('.cursor/rules/testsprite-verify.mdc'); + const existsSync = (p: string) => toPosix(p).endsWith('.cursor/rules/testsprite-verify.mdc'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the cline landing file', () => { - const existsSync = (p: string) => p.endsWith('.clinerules/testsprite-verify.md'); + const existsSync = (p: string) => toPosix(p).endsWith('.clinerules/testsprite-verify.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true for the antigravity landing file', () => { - const existsSync = (p: string) => p.endsWith('.agents/skills/testsprite-verify/SKILL.md'); + const existsSync = (p: string) => + toPosix(p).endsWith('.agents/skills/testsprite-verify/SKILL.md'); expect(isVerifySkillInstalled('/proj', { existsSync })).toBe(true); }); it('true when AGENTS.md exists AND carries our BEGIN sentinel', () => { const existsSync = (p: string) => p.endsWith('AGENTS.md'); - const readFileSync = () => `# project\n${MANAGED_SECTION_BEGIN}\n...skill...\n`; + const readFileSync = () => + `# project\n${MANAGED_SECTION_BEGIN}\n...skill...\n${MANAGED_SECTION_END}\n`; expect(isVerifySkillInstalled('/proj', { existsSync, readFileSync })).toBe(true); }); + it('false when AGENTS.md has only the BEGIN sentinel without a complete managed section', () => { + const existsSync = (p: string) => p.endsWith('AGENTS.md'); + const readFileSync = () => `# project\n${MANAGED_SECTION_BEGIN}\n...partial skill...\n`; + expect(isVerifySkillInstalled('/proj', { existsSync, readFileSync })).toBe(false); + }); + it('false when only a bare AGENTS.md (no sentinel) exists', () => { const existsSync = (p: string) => p.endsWith('AGENTS.md'); const readFileSync = () => '# my project\nNothing TestSprite here.\n'; @@ -66,7 +79,7 @@ describe('isVerifySkillInstalled', () => { return false; }, }); - expect(seen.every(p => p.startsWith('/some/proj'))).toBe(true); + expect(seen.every(p => toPosix(p).startsWith('/some/proj'))).toBe(true); // One probe per target landing path. expect(seen).toHaveLength(Object.keys(TARGETS).length); }); @@ -191,6 +204,6 @@ describe('maybeEmitSkillNudge', () => { }); maybeEmitSkillNudge(ctx); expect(probed.length).toBeGreaterThan(0); - expect(probed.every(p => p.startsWith('/work/here'))).toBe(true); + expect(probed.every(p => toPosix(p).startsWith('/work/here'))).toBe(true); }); }); diff --git a/src/lib/skill-nudge.ts b/src/lib/skill-nudge.ts index 1afc863..0f67572 100644 --- a/src/lib/skill-nudge.ts +++ b/src/lib/skill-nudge.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { MANAGED_SECTION_BEGIN, TARGETS } from './agent-targets.js'; +import { MANAGED_SECTION_BEGIN, MANAGED_SECTION_END, TARGETS } from './agent-targets.js'; import { defaultCredentialsPath, readProfile } from './credentials.js'; import type { OutputMode } from './output.js'; @@ -44,8 +44,8 @@ export interface SkillPresenceDeps { * True if the `testsprite-verify` skill is installed for ANY supported agent in * `dir`. own-file targets (claude/cursor/cline/antigravity): the landing file * exists. managed-section target (codex / AGENTS.md): the file exists AND - * carries our BEGIN sentinel — a user-authored AGENTS.md without the sentinel - * does NOT count as our skill. + * carries one complete managed section — a user-authored AGENTS.md without the + * sentinels, or with a truncated section, does NOT count as our skill. * * The TARGETS table is the single source of truth for landing paths, so this * stays in lockstep with `agent install` without re-listing paths. Best-effort: @@ -59,7 +59,7 @@ export function isVerifySkillInstalled(dir: string, deps: SkillPresenceDeps = {} if (!exists(full)) continue; if (spec.mode === 'managed-section') { try { - if (read(full).includes(MANAGED_SECTION_BEGIN)) return true; + if (hasCompleteManagedSection(read(full))) return true; } catch { // unreadable AGENTS.md → treat this target as absent, keep checking } @@ -70,6 +70,14 @@ export function isVerifySkillInstalled(dir: string, deps: SkillPresenceDeps = {} return false; } +/** True when a managed-section file contains an ordered TestSprite BEGIN/END pair. */ +function hasCompleteManagedSection(content: string): boolean { + const begin = content.indexOf(MANAGED_SECTION_BEGIN); + if (begin === -1) return false; + const end = content.indexOf(MANAGED_SECTION_END, begin + MANAGED_SECTION_BEGIN.length); + return end !== -1; +} + export interface SkillNudgeContext { /** Full command path, e.g. "test run" / "auth whoami". */ commandPath: string; @@ -133,6 +141,7 @@ export function maybeEmitSkillNudge(ctx: SkillNudgeContext): void { } } +/** Interpret common env-var spellings for an enabled opt-out flag. */ function isTruthyEnv(v: string | undefined): boolean { if (v === undefined) return false; const s = v.trim().toLowerCase(); diff --git a/src/lib/target-url.spec.ts b/src/lib/target-url.spec.ts index 96f705b..4998bc3 100644 --- a/src/lib/target-url.spec.ts +++ b/src/lib/target-url.spec.ts @@ -210,6 +210,31 @@ describe('assertNotLocal — IPv6 hardening (SSRF bypass guard)', () => { }); }); +describe('assertNotLocal — trailing-dot FQDN normalization (SSRF bypass guard)', () => { + // `localhost.` is the fully-qualified form of `localhost` (RFC 6761 reserves + // both to resolve to loopback). It previously bypassed the + // `host === 'localhost'` check because the WHATWG URL parser keeps the + // trailing dot on named hosts (IP literals are dot-normalized, named hosts + // are not). + it('blocks http://localhost. (trailing-dot loopback)', () => { + expectBlocked('http://localhost.'); + }); + + it('blocks http://localhost.:8080 (trailing-dot loopback with port)', () => { + expectBlocked('http://localhost.:8080'); + }); + + it('blocks http://localhost%2e (percent-encoded trailing dot)', () => { + expectBlocked('http://localhost%2e'); + }); + + // A legitimate public FQDN with a trailing dot must still be allowed + // (no false positive from the dot strip). + it('allows https://example.com. (public FQDN with trailing dot)', () => { + expectAllowed('https://example.com.'); + }); +}); + describe('assertNotLocal — allowed public URLs', () => { it('allows https://example.com', () => { expectAllowed('https://example.com'); diff --git a/src/lib/target-url.ts b/src/lib/target-url.ts index 030fc93..895c0f0 100644 --- a/src/lib/target-url.ts +++ b/src/lib/target-url.ts @@ -41,7 +41,14 @@ export function assertNotLocal(rawUrl: string): void { throw localTargetError('target-url', 'must use http or https scheme'); } - const host = parsed.hostname.toLowerCase(); + // Normalize a single trailing dot in the hostname. `localhost.` is the + // fully-qualified form of `localhost` (RFC 6761 reserves both to resolve to + // loopback), so `http://localhost.` must be rejected just like + // `http://localhost`. Without this strip, the trailing-dot form (also + // reachable via `localhost%2e`) slips past the `host === 'localhost'` check. + // IP literals are already dot-normalized by the WHATWG URL parser, so this + // only affects named hosts. + const host = parsed.hostname.toLowerCase().replace(/\.$/, ''); // Loopback / unspecified. if (host === 'localhost' || host === '0.0.0.0') { diff --git a/src/lib/text-table.ts b/src/lib/text-table.ts new file mode 100644 index 0000000..bd66b5d --- /dev/null +++ b/src/lib/text-table.ts @@ -0,0 +1,94 @@ +import { localValidationError } from './errors.js'; + +export interface TextTableColumn { + header: string; + width: number | ((rows: readonly T[]) => number); + render: (row: T) => string; +} + +export interface TextTableOptions { + columns?: string; + noHeader?: boolean; + separator?: boolean; +} + +export function renderTextTable( + rows: readonly T[], + columns: readonly TextTableColumn[], + options: TextTableOptions = {}, +): string { + const selected = resolveTextColumns(options.columns, columns); + const widths = measureTextColumns(rows, selected); + const body = rows.map(row => + formatTextTableRow( + selected.map(column => column.render(row)), + widths, + ), + ); + + if (options.noHeader === true) return body.join('\n'); + + const header = formatTextTableRow( + selected.map(column => column.header), + widths, + ); + return [header, ...(options.separator === true ? ['-'.repeat(header.length)] : []), ...body].join( + '\n', + ); +} + +export function resolveTextColumns( + raw: string | undefined, + columns: readonly TextTableColumn[], +): readonly TextTableColumn[] { + if (raw === undefined || raw.trim() === '') return columns; + + const byKey = new Map(columns.map(column => [textColumnKey(column.header), column])); + const validKeys = columns.map(column => textColumnKey(column.header)); + const requested = raw.split(',').map(token => token.trim()); + + if (requested.some(token => token.length === 0)) { + throw localValidationError( + 'columns', + `must be a comma-separated list of: ${validKeys.join(', ')}`, + validKeys, + ); + } + + return requested.map(token => { + const key = textColumnKey(token); + const column = byKey.get(key); + if (column === undefined) { + throw localValidationError( + 'columns', + `unknown column "${token}"; must be one of: ${validKeys.join(', ')}`, + validKeys, + ); + } + return column; + }); +} + +export function measureTextColumns( + rows: readonly T[], + columns: readonly TextTableColumn[], +): number[] { + return columns.map(column => + typeof column.width === 'function' ? column.width(rows) : column.width, + ); +} + +export function formatTextTableRow(values: readonly string[], widths: readonly number[]): string { + return values + .map((value, index) => (index === values.length - 1 ? value : pad(value, widths[index] ?? 0))) + .join(' '); +} + +function textColumnKey(value: string): string { + return value.toLowerCase().replace(/[^a-z0-9]/g, ''); +} + +function pad(value: string, width: number): string { + if (value.length >= width) return value; + return value + ' '.repeat(width - value.length); +} diff --git a/src/lib/ticker.spec.ts b/src/lib/ticker.spec.ts index d449900..72760a9 100644 --- a/src/lib/ticker.spec.ts +++ b/src/lib/ticker.spec.ts @@ -3,7 +3,7 @@ */ import { describe, expect, it, vi } from 'vitest'; -import { createTicker } from './ticker.js'; +import { createTicker, isNoColor } from './ticker.js'; describe('createTicker — non-TTY (CI mode)', () => { it('update is a no-op (no writes)', () => { @@ -222,3 +222,66 @@ describe('createTicker — spy on process.stderr', () => { } }); }); + +describe('createTicker — NO_COLOR support', () => { + it('suppresses ANSI escape sequences when noColor=true on TTY', () => { + const lines: string[] = []; + const raw: string[] = []; + const ticker = createTicker( + line => lines.push(line), + true, // isTTY = true + text => raw.push(text), + true, // noColor = true + ); + ticker.update('progress'); + // Should use stderrWrite (line-oriented) instead of rawWrite with ANSI + expect(raw).toHaveLength(0); + expect(lines).toHaveLength(1); + expect(lines[0]!).not.toContain('\x1b[2K'); + expect(lines[0]!).not.toContain('\r'); + expect(lines[0]!).toContain('progress'); + }); + + it('finalize emits plain text without ANSI when noColor=true on TTY', () => { + const lines: string[] = []; + const raw: string[] = []; + const ticker = createTicker( + line => lines.push(line), + true, + text => raw.push(text), + true, // noColor = true + ); + ticker.finalize('done'); + expect(raw).toHaveLength(0); + expect(lines).toHaveLength(1); + expect(lines[0]!).not.toContain('\x1b[2K'); + expect(lines[0]!).toContain('done'); + }); + + it('normal ANSI output when noColor=false on TTY', () => { + const raw: string[] = []; + const ticker = createTicker( + () => {}, + true, + text => raw.push(text), + false, // noColor = false + ); + ticker.update('progress'); + expect(raw).toHaveLength(1); + expect(raw[0]!).toContain('\x1b[2K\r'); + }); +}); + +describe('isNoColor', () => { + it('returns true when NO_COLOR is set to a non-empty value', () => { + expect(isNoColor({ NO_COLOR: '1' })).toBe(true); + expect(isNoColor({ NO_COLOR: 'true' })).toBe(true); + }); + + it('returns false when NO_COLOR is absent or empty', () => { + expect(isNoColor({})).toBe(false); + expect(isNoColor({ OTHER_VAR: '1' })).toBe(false); + // Per https://no-color.org/, an empty NO_COLOR does NOT disable color. + expect(isNoColor({ NO_COLOR: '' })).toBe(false); + }); +}); diff --git a/src/lib/ticker.ts b/src/lib/ticker.ts index 4cae6b1..d9eaed2 100644 --- a/src/lib/ticker.ts +++ b/src/lib/ticker.ts @@ -8,6 +8,9 @@ * - Uses `\r` + ANSI clear-line to overwrite in place on TTY * - On terminal, emits one final line + newline then prints the result * - `--output json` disables the ticker (caller doesn't create one) + * - Respects the NO_COLOR env var (https://no-color.org/): when set, + * ANSI escape sequences are suppressed and updates are emitted as + * plain lines instead of in-place overwrites. * * Overhead: <2ms per update (no syscalls beyond a single write). * @@ -25,6 +28,15 @@ export interface Ticker { finalize(line?: string): void; } +/** + * Returns true when NO_COLOR is present in the environment and is not + * an empty string, per https://no-color.org/. + */ +export function isNoColor(env: NodeJS.ProcessEnv = process.env): boolean { + const value = env.NO_COLOR; + return typeof value === 'string' && value.length > 0; +} + /** * Create a ticker bound to the given stderr writer. Respects * `isTTY` to silently no-op in CI environments. @@ -35,11 +47,14 @@ export interface Ticker { * @param stderrRaw - optional raw writer (no \n appended); used for * the carriage-return + clear-line trick. Defaults to * `process.stderr.write.bind(process.stderr)`. + * @param noColor - whether to suppress ANSI escape sequences. + * Defaults to checking `NO_COLOR` env var per https://no-color.org/. */ export function createTicker( stderrWrite: (line: string) => void, isTTY?: boolean, stderrRaw?: (text: string) => void, + noColor?: boolean, ): Ticker { const tty = isTTY ?? (typeof process !== 'undefined' ? process.stderr.isTTY === true : false); const rawWrite = @@ -47,6 +62,7 @@ export function createTicker( (typeof process !== 'undefined' ? (text: string) => process.stderr.write(text) : (_text: string) => undefined); + const suppressAnsi = noColor ?? isNoColor(); let lastLength = 0; @@ -58,6 +74,25 @@ export function createTicker( }; } + if (suppressAnsi) { + // TTY but NO_COLOR: emit plain-text lines without ANSI escape sequences. + return { + update(line: string): void { + const stamped = `${new Date().toISOString()} ${line}`; + stderrWrite(stamped); + lastLength = stamped.length; + }, + finalize(line?: string): void { + if (line !== undefined) { + const stamped = `${new Date().toISOString()} ${line}`; + stderrWrite(stamped); + lastLength = stamped.length; + } + void stderrWrite; + }, + }; + } + return { update(line: string): void { // ANSI ESC[2K clears the entire line; \r moves to column 0. diff --git a/src/lib/update-check.test.ts b/src/lib/update-check.test.ts new file mode 100644 index 0000000..dbce172 --- /dev/null +++ b/src/lib/update-check.test.ts @@ -0,0 +1,231 @@ +/** + * Unit tests for the update notice (issue #122). Every effect is injected: + * no real network, filesystem, clock, or TTY is touched. + */ + +import { describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import type { UpdateCheckDeps } from './update-check.js'; +import { + UPDATE_CHECK_OPT_OUT_ENV, + UPDATE_CHECK_TTL_MS, + compareSemver, + fetchLatestVersion, + maybeNotifyUpdate, + shouldCheckForUpdate, +} from './update-check.js'; + +/** In-memory fs + deterministic clock harness for the cache round-trip. */ +function makeHarness(overrides: UpdateCheckDeps = {}) { + const files = new Map(); + const stderrLines: string[] = []; + const deps: UpdateCheckDeps = { + env: {}, + now: () => 1_000_000, + cachePath: '/fake/.testsprite/update-check.json', + readFile: path => { + const content = files.get(path); + if (content === undefined) throw new Error('ENOENT'); + return content; + }, + writeFile: (path, content) => { + files.set(path, content); + }, + mkdir: () => undefined, + isTTY: true, + stderr: line => stderrLines.push(line), + currentVersion: '0.2.0', + fetchImpl: async () => + new Response(JSON.stringify({ version: '0.2.0' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ...overrides, + }; + return { deps, files, stderrLines }; +} + +describe('shouldCheckForUpdate gates', () => { + it('opt-out env set to any non-empty value disables (even "0")', () => { + const { deps } = makeHarness({ env: { [UPDATE_CHECK_OPT_OUT_ENV]: '0' } }); + expect(shouldCheckForUpdate(deps)).toBe(false); + }); + + it('CI set disables; CI="false" re-enables', () => { + expect(shouldCheckForUpdate(makeHarness({ env: { CI: 'true' } }).deps)).toBe(false); + expect(shouldCheckForUpdate(makeHarness({ env: { CI: '' } }).deps)).toBe(false); + expect(shouldCheckForUpdate(makeHarness({ env: { CI: 'false' } }).deps)).toBe(true); + }); + + it('non-TTY stderr disables', () => { + expect(shouldCheckForUpdate(makeHarness({ isTTY: false }).deps)).toBe(false); + }); + + it('a fresh cache suppresses; a stale cache does not', () => { + const fresh = makeHarness(); + fresh.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 1_000_000 - UPDATE_CHECK_TTL_MS + 5_000 }), + ); + expect(shouldCheckForUpdate(fresh.deps)).toBe(false); + + const stale = makeHarness(); + stale.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 1_000_000 - UPDATE_CHECK_TTL_MS - 5_000 }), + ); + expect(shouldCheckForUpdate(stale.deps)).toBe(true); + }); + + it('missing, corrupt, wrong-shape, or future-stamped caches count as stale', () => { + expect(shouldCheckForUpdate(makeHarness().deps)).toBe(true); // missing + const corrupt = makeHarness(); + corrupt.files.set('/fake/.testsprite/update-check.json', '{not json'); + expect(shouldCheckForUpdate(corrupt.deps)).toBe(true); + const wrongShape = makeHarness(); + wrongShape.files.set('/fake/.testsprite/update-check.json', JSON.stringify({ nope: true })); + expect(shouldCheckForUpdate(wrongShape.deps)).toBe(true); + const future = makeHarness(); + future.files.set( + '/fake/.testsprite/update-check.json', + JSON.stringify({ lastCheckMs: 9_999_999_999 }), + ); + expect(shouldCheckForUpdate(future.deps)).toBe(true); + }); +}); + +describe('fetchLatestVersion', () => { + it('returns the version from a valid registry body', async () => { + const { deps } = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '1.2.3' }), { status: 200 }), + }); + await expect(fetchLatestVersion(deps)).resolves.toBe('1.2.3'); + }); + + it('returns undefined on non-2xx, thrown fetch, and wrong-shape body', async () => { + const notOk = makeHarness({ fetchImpl: async () => new Response('nope', { status: 500 }) }); + await expect(fetchLatestVersion(notOk.deps)).resolves.toBeUndefined(); + + const throwing = makeHarness({ + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + await expect(fetchLatestVersion(throwing.deps)).resolves.toBeUndefined(); + + const wrongShape = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ notVersion: 1 }), { status: 200 }), + }); + await expect(fetchLatestVersion(wrongShape.deps)).resolves.toBeUndefined(); + }); +}); + +describe('compareSemver', () => { + it('orders numerically and treats prerelease as older than its release', () => { + expect(compareSemver('0.2.0', '0.3.0')).toBe(-1); + expect(compareSemver('1.0.0', '0.9.9')).toBe(1); + expect(compareSemver('0.2.0', '0.2.0')).toBe(0); + expect(compareSemver('0.10.0', '0.9.0')).toBe(1); // numeric, not lexicographic + expect(compareSemver('1.0.0-rc.1', '1.0.0')).toBe(-1); + expect(compareSemver('1.0.0', '1.0.0-rc.1')).toBe(1); + }); + + it('unparseable input on either side compares as 0 (never a false notice)', () => { + expect(compareSemver('garbage', '1.0.0')).toBe(0); + expect(compareSemver('1.0.0', '')).toBe(0); + }); +}); + +describe('maybeNotifyUpdate', () => { + it('prints exactly one stderr line naming both versions when newer, and stamps the cache', async () => { + const harness = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '0.3.1' }), { status: 200 }), + }); + await maybeNotifyUpdate(harness.deps); + expect(harness.stderrLines).toHaveLength(1); + expect(harness.stderrLines[0]).toContain('0.2.0 -> 0.3.1'); + expect(harness.stderrLines[0]).toContain(UPDATE_CHECK_OPT_OUT_ENV); + const cache = JSON.parse(harness.files.get('/fake/.testsprite/update-check.json')!) as { + lastCheckMs: number; + latestKnown?: string; + }; + expect(cache.lastCheckMs).toBe(1_000_000); + expect(cache.latestKnown).toBe('0.3.1'); + }); + + it('stays silent on an equal or older registry version', async () => { + const equal = makeHarness(); + await maybeNotifyUpdate(equal.deps); + expect(equal.stderrLines).toHaveLength(0); + + const older = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '0.1.9' }), { status: 200 }), + }); + await maybeNotifyUpdate(older.deps); + expect(older.stderrLines).toHaveLength(0); + }); + + it('a failed probe stays silent but still stamps the cache (retry once per TTL)', async () => { + const harness = makeHarness({ + fetchImpl: async () => { + throw new TypeError('fetch failed'); + }, + }); + await maybeNotifyUpdate(harness.deps); + expect(harness.stderrLines).toHaveLength(0); + const cache = JSON.parse(harness.files.get('/fake/.testsprite/update-check.json')!) as { + lastCheckMs: number; + latestKnown?: string; + }; + expect(cache.lastCheckMs).toBe(1_000_000); + expect(cache.latestKnown).toBeUndefined(); + }); + + it('does nothing when a gate blocks (no fetch fired)', async () => { + const fetchImpl = vi.fn(async () => new Response('{}', { status: 200 })); + const harness = makeHarness({ env: { CI: '1' }, fetchImpl }); + await maybeNotifyUpdate(harness.deps); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(harness.stderrLines).toHaveLength(0); + }); + + it('never rejects, even when every injected dependency throws', async () => { + const harness = makeHarness({ + fetchImpl: async () => new Response(JSON.stringify({ version: '9.9.9' }), { status: 200 }), + writeFile: () => { + throw new Error('EROFS'); + }, + stderr: () => { + throw new Error('broken stderr sink'); + }, + }); + await expect(maybeNotifyUpdate(harness.deps)).resolves.toBeUndefined(); + }); + + it('uses the default fs readers/writers when none are injected (real cache round-trip)', async () => { + const dir = mkdtempSync(join(tmpdir(), 'update-check-')); + // Point cachePath at a not-yet-existing subdir so the default mkdir + // (recursive) and writeFile arrows both run, and the first read (file + // absent) exercises the default readFile arrow's ENOENT path. + const cachePath = join(dir, 'nested', 'update-check.json'); + const stderrLines: string[] = []; + try { + await maybeNotifyUpdate({ + env: {}, + now: () => 1_000_000, + isTTY: true, + currentVersion: '0.0.1', + cachePath, + stderr: line => stderrLines.push(line), + fetchImpl: async () => new Response(JSON.stringify({ version: '9.9.9' }), { status: 200 }), + }); + // Cache was persisted by the default writeFile through the default mkdir. + expect(JSON.parse(readFileSync(cachePath, 'utf8'))).toMatchObject({ latestKnown: '9.9.9' }); + expect(stderrLines.join('\n')).toContain('0.0.1 -> 9.9.9'); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/lib/update-check.ts b/src/lib/update-check.ts new file mode 100644 index 0000000..e928913 --- /dev/null +++ b/src/lib/update-check.ts @@ -0,0 +1,285 @@ +/** + * Non-blocking "new version available" notice (issue #122), following the + * pattern of the gh and npm CLIs: at most one npm-registry probe per 24 hours, + * result cached on disk, advisory printed to stderr so stdout stays parseable. + * + * Behavior (`maybeNotifyUpdate`): + * 1. Gate through `shouldCheckForUpdate`; every gate below must pass. + * 2. Probe the npm registry for the `latest` dist-tag (1.5s hard timeout). + * 3. Stamp the cache with `lastCheckMs` (plus `latestKnown` when the probe + * succeeded) so the next 24h of invocations skip the network entirely. + * The stamp happens even after a failed probe: a dead registry must not + * trigger a retry on every command. + * 4. When `latest` is strictly newer than the running version, write exactly + * one advisory line to stderr. The function never throws or rejects and + * never alters the exit status of the command it rides along with. + * + * Gates, in order (`shouldCheckForUpdate`): + * - `TESTSPRITE_NO_UPDATE_NOTIFIER` set to any non-empty value: opted out. + * Presence-style, mirroring gh's GH_NO_UPDATE_NOTIFIER: even "0" disables. + * - `CI` set to anything except the literal "false": CI logs are not the + * place for update nags. `CI=false` explicitly re-enables the notice. + * - stderr is not a TTY: piped or redirected output stays clean. + * - the on-disk cache is fresh (last probe within the TTL). A missing, + * unreadable, corrupt, or wrong-shape cache counts as stale, and so does a + * `lastCheckMs` in the future (clock rollback or corrupt data). + * + * Why not the npm `update-notifier` package: this CLI's runtime dependency + * budget is commander + valibot only (package.json). `update-notifier` would + * add a transitive dependency tree for what Node 20 already ships as + * primitives (fetch, AbortSignal.timeout, sync fs). Owning the ~100 lines + * keeps the install size flat and every effect injectable for tests. + * + * All effects (env, network, clock, fs, tty, stderr sink) are injectable via + * `UpdateCheckDeps`, the same dependency-injection style as `skill-nudge.ts`. + */ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { dirname, join } from 'node:path'; +import * as v from 'valibot'; +import { VERSION } from '../version.js'; +import type { FetchImpl } from './http.js'; + +/** Re-check interval: 24 hours, expressed in milliseconds. */ +export const UPDATE_CHECK_TTL_MS = 24 * 60 * 60 * 1000; + +/** + * Env var that disables the update notice entirely. Presence-style: any + * non-empty value (including "0") opts out. Deliberately stricter than the + * truthy-style `TESTSPRITE_NO_SKILL_WARNING` because it matches the + * convention users already know from gh's GH_NO_UPDATE_NOTIFIER. + */ +export const UPDATE_CHECK_OPT_OUT_ENV = 'TESTSPRITE_NO_UPDATE_NOTIFIER'; + +/** + * Hard cap for the registry probe, in milliseconds. The probe rides along a + * real command, so unlike the 120s API budget in `http.ts` it gets a tiny + * window: a slow registry means "no update info", never a visible delay. + */ +const REGISTRY_TIMEOUT_MS = 1_500; + +/** + * npm registry `latest` dist-tag endpoint for this package. Package name from + * package.json (`@testsprite/testsprite-cli`); the scope separator must be + * URL-encoded as %2F per the npm registry API. + */ +const REGISTRY_LATEST_URL = 'https://registry.npmjs.org/@testsprite%2Ftestsprite-cli/latest'; + +/** On-disk cache shape at `cachePath`; unknown keys are stripped on read. */ +const UPDATE_CHECK_CACHE_SCHEMA = v.object({ + lastCheckMs: v.number(), + latestKnown: v.optional(v.string()), +}); + +export type UpdateCheckCache = v.InferOutput; + +/** Minimal slice of the registry response the notice needs. */ +const REGISTRY_LATEST_BODY_SCHEMA = v.object({ version: v.string() }); + +export interface UpdateCheckDeps { + env?: NodeJS.ProcessEnv; + fetchImpl?: FetchImpl; + /** Clock, epoch milliseconds. */ + now?: () => number; + /** Cache file; lives next to credentials/config under ~/.testsprite. */ + cachePath?: string; + readFile?: (path: string) => string; + writeFile?: (path: string, content: string) => void; + /** Must create missing parent directories (recursive). */ + mkdir?: (dir: string) => void; + /** Whether stderr is an interactive terminal. */ + isTTY?: boolean; + /** Sink for the single advisory line. */ + stderr?: (line: string) => void; + /** Version the running binary reports. */ + currentVersion?: string; +} + +type ResolvedUpdateCheckDeps = Required; + +function resolveUpdateCheckDeps(deps: UpdateCheckDeps): ResolvedUpdateCheckDeps { + return { + env: deps.env ?? process.env, + fetchImpl: deps.fetchImpl ?? globalThis.fetch, + now: deps.now ?? Date.now, + cachePath: deps.cachePath ?? join(homedir(), '.testsprite', 'update-check.json'), + readFile: deps.readFile ?? ((path: string) => readFileSync(path, 'utf8')), + writeFile: + deps.writeFile ?? ((path: string, content: string) => writeFileSync(path, content, 'utf8')), + mkdir: + deps.mkdir ?? + ((dir: string) => { + mkdirSync(dir, { recursive: true }); + }), + isTTY: deps.isTTY ?? process.stderr.isTTY === true, + stderr: deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)), + currentVersion: deps.currentVersion ?? VERSION, + }; +} + +/** + * Read and validate the cache file. Every failure mode (missing file, + * unreadable file, invalid JSON, wrong shape) returns undefined, which the + * caller treats as "stale, probe again". + */ +function readUpdateCheckCache(resolved: ResolvedUpdateCheckDeps): UpdateCheckCache | undefined { + try { + const raw = resolved.readFile(resolved.cachePath); + const body: unknown = JSON.parse(raw); + const parsed = v.safeParse(UPDATE_CHECK_CACHE_SCHEMA, body); + return parsed.success ? parsed.output : undefined; + } catch { + // Missing or unreadable cache: treat as stale. + return undefined; + } +} + +/** + * Persist the cache, creating the parent directory when missing. Best-effort: + * every error (read-only home, quota, fs races) is swallowed. A failed write + * only means the next invocation probes the registry again. + */ +function writeUpdateCheckCache(resolved: ResolvedUpdateCheckDeps, cache: UpdateCheckCache): void { + try { + resolved.mkdir(dirname(resolved.cachePath)); + resolved.writeFile(resolved.cachePath, `${JSON.stringify(cache)}\n`); + } catch { + // Cache persistence is optional; never surface fs errors to the command. + } +} + +/** + * True when every gate documented in the module header passes: no opt-out + * env, not CI (unless CI=false), stderr is a TTY, and the cached check is + * stale or absent. Order matters: the cheap env gates run before any fs read. + */ +export function shouldCheckForUpdate(deps: UpdateCheckDeps = {}): boolean { + const resolved = resolveUpdateCheckDeps(deps); + + const optOutValue = resolved.env[UPDATE_CHECK_OPT_OUT_ENV]; + if (optOutValue !== undefined && optOutValue !== '') return false; + + // Any set CI value except the literal "false" counts as CI. The empty + // string still signals a CI-managed environment; silence wins in doubt. + const ciValue = resolved.env.CI; + if (ciValue !== undefined && ciValue !== 'false') return false; + + if (!resolved.isTTY) return false; + + const cache = readUpdateCheckCache(resolved); + if (cache !== undefined) { + const elapsedMs = resolved.now() - cache.lastCheckMs; + // A negative elapsed (lastCheckMs in the future) means clock rollback or + // corrupt data: treat as stale instead of suppressing the check forever. + // A NaN lastCheckMs fails both comparisons and lands on stale too. + if (elapsedMs >= 0 && elapsedMs < UPDATE_CHECK_TTL_MS) return false; + } + + return true; +} + +/** + * Probe the npm registry for the `latest` dist-tag version of this package. + * Hard 1.5s timeout via AbortSignal.timeout. ANY failure (network error, + * timeout, non-2xx status, invalid JSON, wrong shape) resolves to undefined; + * this function never rejects. + */ +export async function fetchLatestVersion(deps: UpdateCheckDeps = {}): Promise { + const resolved = resolveUpdateCheckDeps(deps); + try { + const response = await resolved.fetchImpl(REGISTRY_LATEST_URL, { + signal: AbortSignal.timeout(REGISTRY_TIMEOUT_MS), + }); + if (!response.ok) return undefined; + const body: unknown = await response.json(); + const parsed = v.safeParse(REGISTRY_LATEST_BODY_SCHEMA, body); + return parsed.success ? parsed.output.version : undefined; + } catch { + // Offline, DNS failure, abort, or a non-JSON body: no update info. + return undefined; + } +} + +interface ParsedSemver { + major: number; + minor: number; + patch: number; + hasPrerelease: boolean; +} + +/** + * x.y.z with optional prerelease (after "-") and optional build metadata + * (after "+"), per the semver 2.0.0 grammar. A leading "v" is tolerated + * because humans type it; the npm registry never returns one. + */ +const SEMVER_RE = /^v?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$/; + +function parseSemver(version: string): ParsedSemver | undefined { + const match = SEMVER_RE.exec(version.trim()); + if (!match) return undefined; + const [, majorRaw, minorRaw, patchRaw, prerelease] = match; + if (majorRaw === undefined || minorRaw === undefined || patchRaw === undefined) return undefined; + return { + major: Number(majorRaw), + minor: Number(minorRaw), + patch: Number(patchRaw), + hasPrerelease: prerelease !== undefined, + }; +} + +/** + * Compare two semver strings numerically. Returns -1 when versionA is older + * than versionB, 1 when newer, 0 when equal. A version carrying a prerelease + * tag sorts OLDER than the plain release with the same x.y.z core. Prerelease + * identifiers themselves are not ranked (two prereleases on the same core + * compare as 0): the registry `latest` tag points at releases, so identifier + * ordering never decides whether the notice fires. Unparseable input on + * either side compares as 0, so garbage can never produce a false notice. + */ +export function compareSemver(versionA: string, versionB: string): number { + const left = parseSemver(versionA); + const right = parseSemver(versionB); + if (left === undefined || right === undefined) return 0; + if (left.major !== right.major) return left.major < right.major ? -1 : 1; + if (left.minor !== right.minor) return left.minor < right.minor ? -1 : 1; + if (left.patch !== right.patch) return left.patch < right.patch ? -1 : 1; + if (left.hasPrerelease !== right.hasPrerelease) return left.hasPrerelease ? -1 : 1; + return 0; +} + +/** + * Fire-and-forget update notice. Gates, probes, stamps the cache, and prints + * at most one stderr line when the registry version is strictly newer than + * the running one. Never throws and never rejects: any failure in any + * injected dependency (clock, fs, network, the stderr sink itself) is + * swallowed, because an advisory must never break or delay a real command. + */ +export async function maybeNotifyUpdate(deps: UpdateCheckDeps = {}): Promise { + try { + const resolved = resolveUpdateCheckDeps(deps); + if (!shouldCheckForUpdate(resolved)) return; + + const latest = await fetchLatestVersion(resolved); + + // Stamp even on a failed probe so a dead registry is retried at most + // once per TTL window, not on every invocation. + writeUpdateCheckCache(resolved, { + lastCheckMs: resolved.now(), + ...(latest === undefined ? {} : { latestKnown: latest }), + }); + + if (latest === undefined) return; + if (compareSemver(latest, resolved.currentVersion) !== 1) return; + + // User-facing advisory copy (exact format specified by issue #122), not a + // diagnostic log line; stderr keeps stdout parseable for scripts. + resolved.stderr( + `A new version of testsprite-cli is available: ${resolved.currentVersion} -> ${latest}. ` + + `Run npm install -g @testsprite/testsprite-cli to update. ` + + `(Disable with ${UPDATE_CHECK_OPT_OUT_ENV}=1)`, + ); + } catch { + // An update notice must never break, delay, or alter the exit status of + // the command it accompanies. Swallow everything. + } +} diff --git a/src/lib/v3-advisory.test.ts b/src/lib/v3-advisory.test.ts new file mode 100644 index 0000000..124720f --- /dev/null +++ b/src/lib/v3-advisory.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; +import { routingLabel, V3_ROUTING_ADVISORY, emitV3RoutingAdvisory } from './v3-advisory.js'; + +describe('routingLabel', () => { + it('maps the boolean to v3 / v2', () => { + expect(routingLabel(true)).toBe('v3'); + expect(routingLabel(false)).toBe('v2'); + }); +}); + +describe('V3 routing advisory', () => { + it('names each open behavior gap (cancel, delete, target-url)', () => { + const text = V3_ROUTING_ADVISORY.join('\n'); + expect(text).toContain('test cancel'); + expect(text).toContain('test delete'); + expect(text).toContain('--target-url'); + }); + + it('emitV3RoutingAdvisory writes every line to the sink', () => { + const lines: string[] = []; + emitV3RoutingAdvisory(l => lines.push(l)); + expect(lines).toEqual(V3_ROUTING_ADVISORY); + }); +}); diff --git a/src/lib/v3-advisory.ts b/src/lib/v3-advisory.ts new file mode 100644 index 0000000..a8b75ed --- /dev/null +++ b/src/lib/v3-advisory.ts @@ -0,0 +1,25 @@ +/** + * Shared V3-routing text surfaces for `auth status` and `doctor`. + * + * `v3Enabled` on the `/me` response is the authoritative routing bit. When it + * is on, some commands behave differently while the V3 gaps stay open — the + * advisory names them. Copy lives here so both commands stay in sync. + */ + +/** One-word routing label for the text card. */ +export function routingLabel(v3Enabled: boolean): 'v3' | 'v2' { + return v3Enabled ? 'v3' : 'v2'; +} + +/** Consolidated advisory (stderr) emitted when V3 routing is on. */ +export const V3_ROUTING_ADVISORY: string[] = [ + '[advisory] V3 routing is on for this account. While these gaps are open:', + ' - `test cancel` may return 404', + ' - `test delete` may leave a zombie run', + ' - `--target-url` is ignored on frontend runs', +]; + +/** Write the advisory to a stderr sink, one line per call. */ +export function emitV3RoutingAdvisory(stderr: (line: string) => void): void { + for (const line of V3_ROUTING_ADVISORY) stderr(line); +} diff --git a/src/lib/version-notice.test.ts b/src/lib/version-notice.test.ts new file mode 100644 index 0000000..1ce473e --- /dev/null +++ b/src/lib/version-notice.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + formatBelowFloorNotice, + noteServerVersion, + resetBelowFloorNoticeState, + shouldWarnBelowFloor, + type VersionNoticeDeps, +} from './version-notice.js'; + +/** Baseline deps: all gates open, running version below the floor. */ +function baseDeps(overrides: Partial = {}): VersionNoticeDeps { + return { + currentVersion: '0.9.0', + env: {}, + isTTY: true, + outputMode: 'text', + dryRun: false, + ...overrides, + }; +} + +afterEach(() => { + resetBelowFloorNoticeState(); + vi.restoreAllMocks(); +}); + +describe('shouldWarnBelowFloor', () => { + it('warns when the running version is strictly below the floor', () => { + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps())).toBe(true); + }); + + it('does not warn when at or above the floor', () => { + expect(shouldWarnBelowFloor({ minVersion: '0.9.0' }, baseDeps())).toBe(false); // equal + expect(shouldWarnBelowFloor({ minVersion: '0.8.0' }, baseDeps())).toBe(false); // above + }); + + it('does not warn when no minVersion header was present', () => { + expect(shouldWarnBelowFloor({}, baseDeps())).toBe(false); + expect(shouldWarnBelowFloor({ minVersion: undefined }, baseDeps())).toBe(false); + }); + + it('does not warn on unparseable versions (garbage never warns)', () => { + expect(shouldWarnBelowFloor({ minVersion: 'not-a-version' }, baseDeps())).toBe(false); + expect( + shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ currentVersion: 'nope' })), + ).toBe(false); + }); + + it('is gated off by the opt-out env (any non-empty value)', () => { + expect( + shouldWarnBelowFloor( + { minVersion: '1.0.0' }, + baseDeps({ env: { TESTSPRITE_NO_UPDATE_NOTIFIER: '1' } }), + ), + ).toBe(false); + expect( + shouldWarnBelowFloor( + { minVersion: '1.0.0' }, + baseDeps({ env: { TESTSPRITE_NO_UPDATE_NOTIFIER: '0' } }), + ), + ).toBe(false); + }); + + it('is gated off under --output json, --dry-run, and non-TTY', () => { + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ outputMode: 'json' }))).toBe( + false, + ); + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ dryRun: true }))).toBe(false); + expect(shouldWarnBelowFloor({ minVersion: '1.0.0' }, baseDeps({ isTTY: false }))).toBe(false); + }); +}); + +describe('formatBelowFloorNotice', () => { + it('names the current version, the floor, and the npm upgrade command', () => { + const line = formatBelowFloorNotice('0.9.0', '1.0.0'); + expect(line).toContain('0.9.0'); + expect(line).toContain('minimum supported version 1.0.0'); + expect(line).toContain('npm install -g @testsprite/testsprite-cli'); + expect(line).toContain('TESTSPRITE_NO_UPDATE_NOTIFIER=1'); + }); + + it('does not name a target release (npm update-notice owns "latest")', () => { + const line = formatBelowFloorNotice('0.9.0', '1.0.0'); + expect(line).not.toContain('Upgrade to 1.'); + }); +}); + +describe('noteServerVersion', () => { + it('emits exactly one advisory line when below the floor', () => { + const stderr = vi.fn(); + noteServerVersion({ minVersion: '1.0.0' }, baseDeps({ stderr })); + expect(stderr).toHaveBeenCalledTimes(1); + expect(stderr).toHaveBeenCalledWith(expect.stringContaining('below the minimum supported')); + }); + + it('warns at most once per process', () => { + const stderr = vi.fn(); + const deps = baseDeps({ stderr }); + noteServerVersion({ minVersion: '1.0.0' }, deps); + noteServerVersion({ minVersion: '1.0.0' }, deps); + noteServerVersion({ minVersion: '1.0.0' }, deps); + expect(stderr).toHaveBeenCalledTimes(1); + }); + + it('stays silent when not below the floor', () => { + const stderr = vi.fn(); + noteServerVersion({ minVersion: '0.5.0' }, baseDeps({ stderr })); + expect(stderr).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/version-notice.ts b/src/lib/version-notice.ts new file mode 100644 index 0000000..0e492dc --- /dev/null +++ b/src/lib/version-notice.ts @@ -0,0 +1,111 @@ +/** + * Inline "your CLI is below the backend's minimum supported version" advisory. + * + * The backend advertises its supported floor on every `/api/cli/v1` response via + * the `X-TestSprite-CLI-Min-Version` header. Unlike the npm-registry update + * notice (`update-check.ts`, which runs in a Commander `preAction` hook before + * any HTTP request), this advisory reacts to a header observed *during* the + * request, so it is emitted from the HTTP layer's `onServerVersion` hook (wired + * in `client-factory.ts`) rather than the pre-request notifier — no cross-run + * cache needed. + * + * Distinct from the update notice on purpose, and non-redundant: this names the + * backend FLOOR and fires only when the running version is strictly below it (a + * serious "you will be rejected once enforcement is on" state); the update + * notice names the npm LATEST and fires whenever a newer release exists. Each + * owns its own number — the backend never advertises "latest". + * + * Reuses `compareSemver` and the `TESTSPRITE_NO_UPDATE_NOTIFIER` opt-out from + * `update-check.ts` so the gating and semver semantics stay consistent. + */ +import { compareSemver, UPDATE_CHECK_OPT_OUT_ENV } from './update-check.js'; +import { VERSION } from '../version.js'; + +/** Version-compatibility signal observed on a backend response (the floor). */ +export interface ServerVersionInfo { + minVersion?: string; +} + +export interface VersionNoticeDeps { + /** Version the running binary reports. Defaults to the built-in `VERSION`. */ + currentVersion?: string; + env?: NodeJS.ProcessEnv; + /** Whether stderr is an interactive terminal. */ + isTTY?: boolean; + /** The command's `--output` mode; `'json'` suppresses the advisory. */ + outputMode?: string; + /** Suppress under `--dry-run` (the dry-run fetch returns no real headers). */ + dryRun?: boolean; + /** Sink for the single advisory line. */ + stderr?: (line: string) => void; +} + +/** + * True when every gate passes and the running version is strictly below the + * advertised floor. Gates mirror the update notice: opt-out env, JSON output, + * dry-run, and non-TTY all suppress. Pure — no side effects, no process state. + */ +export function shouldWarnBelowFloor( + info: ServerVersionInfo, + deps: VersionNoticeDeps = {}, +): boolean { + const env = deps.env ?? process.env; + const currentVersion = deps.currentVersion ?? VERSION; + const isTTY = deps.isTTY ?? process.stderr.isTTY === true; + + const optOut = env[UPDATE_CHECK_OPT_OUT_ENV]; + if (optOut !== undefined && optOut !== '') return false; + if (deps.outputMode === 'json') return false; + if (deps.dryRun === true) return false; + if (!isTTY) return false; + + const minVersion = info.minVersion; + if (!minVersion) return false; + + // compareSemver returns -1 when the first arg is OLDER than the second. + // Unparseable input on either side compares as 0, so garbage never warns. + return compareSemver(currentVersion, minVersion) === -1; +} + +/** + * The single advisory line. Names the floor (the backend's authority) and the + * upgrade command — it deliberately does NOT name a target release; the npm + * update-notice is the single source of truth for the newest version. + */ +export function formatBelowFloorNotice(currentVersion: string, minVersion: string): string { + return ( + `Your testsprite-cli (${currentVersion}) is below the minimum supported version ${minVersion}. ` + + `Upgrade: npm install -g @testsprite/testsprite-cli ` + + `(disable this notice with ${UPDATE_CHECK_OPT_OUT_ENV}=1).` + ); +} + +/** Module-level guard: at most one below-floor advisory per process. */ +let warnedThisProcess = false; + +/** Test-only: reset the once-per-process guard between cases. */ +export function resetBelowFloorNoticeState(): void { + warnedThisProcess = false; +} + +/** + * Emit the below-floor advisory at most once per process. Wired to the HTTP + * client's `onServerVersion` hook. Never throws — an advisory must not break or + * delay the command it rides along with. + */ +export function noteServerVersion(info: ServerVersionInfo, deps: VersionNoticeDeps = {}): void { + try { + if (warnedThisProcess) return; + if (!shouldWarnBelowFloor(info, deps)) return; + + const minVersion = info.minVersion; + if (!minVersion) return; + + const currentVersion = deps.currentVersion ?? VERSION; + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + stderr(formatBelowFloorNotice(currentVersion, minVersion)); + warnedThisProcess = true; + } catch { + // Advisory is best-effort; never surface its failures to the command. + } +} diff --git a/src/version-guard.test.ts b/src/version-guard.test.ts new file mode 100644 index 0000000..19f0eab --- /dev/null +++ b/src/version-guard.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest'; +import { + MIN_SUPPORTED_NODE_MAJOR, + parseMajorVersion, + shouldRejectNodeVersion, +} from './version-guard.js'; + +// These tests exercise the REAL guard functions used by src/index.ts, +// imported here rather than re-declared, so a regression in the source is +// actually caught. + +describe('parseMajorVersion', () => { + it('extracts the leading major from a semver string', () => { + expect(parseMajorVersion('20.11.1')).toBe(20); + expect(parseMajorVersion('18.0.0')).toBe(18); + expect(parseMajorVersion('22.3.0')).toBe(22); + }); + + it('returns NaN for a non-numeric version string', () => { + expect(Number.isNaN(parseMajorVersion('not-a-version'))).toBe(true); + }); +}); + +describe('shouldRejectNodeVersion', () => { + it('rejects majors below the supported floor', () => { + expect(shouldRejectNodeVersion('18.19.1')).toBe(true); + expect(shouldRejectNodeVersion('16.20.2')).toBe(true); + expect(shouldRejectNodeVersion('14.21.3')).toBe(true); + }); + + it('accepts the supported floor and above', () => { + expect(shouldRejectNodeVersion('20.0.0')).toBe(false); + expect(shouldRejectNodeVersion('20.11.0')).toBe(false); + expect(shouldRejectNodeVersion('21.0.0')).toBe(false); + expect(shouldRejectNodeVersion('22.1.0')).toBe(false); + }); + + it(`treats exactly ${MIN_SUPPORTED_NODE_MAJOR} as supported (boundary)`, () => { + expect(shouldRejectNodeVersion(`${MIN_SUPPORTED_NODE_MAJOR}.0.0`)).toBe(false); + expect(shouldRejectNodeVersion(`${MIN_SUPPORTED_NODE_MAJOR - 1}.9.9`)).toBe(true); + }); + + it('does not reject an unparseable version (guard never blocks on garbage)', () => { + expect(shouldRejectNodeVersion('not-a-version')).toBe(false); + }); + + it('the running Node satisfies the guard (meta-test)', () => { + // The test suite itself runs on a supported Node, so the guard must pass. + expect(shouldRejectNodeVersion(process.versions.node)).toBe(false); + }); +}); diff --git a/src/version-guard.ts b/src/version-guard.ts new file mode 100644 index 0000000..a7fe464 --- /dev/null +++ b/src/version-guard.ts @@ -0,0 +1,41 @@ +/** + * Node.js runtime version guard. + * + * The CLI targets modern Node (see `engines.node` in package.json). Running on + * an older runtime tends to fail later with a cryptic ESM/syntax error, so the + * entrypoint (`src/index.ts`) uses {@link shouldRejectNodeVersion} to exit early + * with a clear, actionable message instead. + * + * The logic lives here (rather than inline) so it can be unit-tested against the + * real implementation the entrypoint uses — not a copy. + */ + +/** Minimum Node.js major version supported by the CLI (matches package.json `engines.node`). */ +export const MIN_SUPPORTED_NODE_MAJOR = 20; + +/** + * Parse the leading major version number from a Node.js version string. + * + * @param nodeVersion - a dot-separated version string such as `process.versions.node` + * (e.g. `"20.11.1"`). A leading `v` is not expected (Node does not include one here). + * @returns the major version as a number, or `NaN` if the string has no numeric leading segment. + */ +export function parseMajorVersion(nodeVersion: string): number { + return Number(nodeVersion.split('.')[0]); +} + +/** + * Decide whether the given Node.js version is too old to run the CLI. + * + * A version is rejected only when its major number is a real value below + * {@link MIN_SUPPORTED_NODE_MAJOR}. An unparseable string yields `NaN`, which is + * treated as "do not reject" so the guard never blocks on a version string it + * cannot understand (the runtime would surface any real incompatibility itself). + * + * @param nodeVersion - a `process.versions.node` style string (e.g. `"18.19.1"`). + * @returns `true` when the runtime is below the supported floor and should be rejected. + */ +export function shouldRejectNodeVersion(nodeVersion: string): boolean { + const major = parseMajorVersion(nodeVersion); + return !Number.isNaN(major) && major < MIN_SUPPORTED_NODE_MAJOR; +} diff --git a/src/version.ts b/src/version.ts index efa916d..5b301a6 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ // AUTO-GENERATED by scripts/generate-version.mjs — do not edit by hand. // Run `npm run build` (or `npm run generate:version`) to regenerate. -export const VERSION = '0.2.0'; +export const VERSION = '0.4.0'; diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 6f1f158..4724064 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -4,7 +4,7 @@ exports[`--help snapshots > agent 1`] = ` "Usage: testsprite agent [options] [command] Install TestSprite guidance into coding-agent config (Claude Code, Cursor, -Cline, Antigravity, Codex) +Cline, Antigravity, Kiro, Windsurf, Copilot, Codex) Options: -h, --help display help for command @@ -14,6 +14,9 @@ Commands: first-run onboarding) into a project for a coding agent list List supported agent targets and skills, their status, and landing paths + status [options] Check installed TestSprite skill files against this CLI + version: ok, stale, modified, unmarked, absent, or corrupt + (exits 1 when anything needs attention, so it can gate CI) help [command] display help for command " `; @@ -25,8 +28,9 @@ Write the TestSprite agent skills (verification loop + first-run onboarding) into a project for a coding agent Options: - --target Agent target(s): claude, cursor, cline, antigravity, codex - (comma-separated or repeated) (default: []) + --target Agent target(s): claude, cursor, cline, antigravity, kiro, + windsurf, copilot, codex (comma-separated or repeated) + (default: []) --skill Skill(s) to install: testsprite-verify, testsprite-onboard (comma-separated or repeated; default: all) (default: []) --dir Project root to write into (default: cwd) @@ -36,7 +40,7 @@ Options: destroyed. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -49,7 +53,7 @@ List supported agent targets and skills, their status, and landing paths Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -86,7 +90,7 @@ exports[`--help snapshots > auth logout 1`] = ` Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -97,7 +101,7 @@ exports[`--help snapshots > auth whoami 1`] = ` Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -108,17 +112,22 @@ exports[`--help snapshots > init 1`] = ` (deprecated) alias for \`setup\` Options: - --api-key API key to configure (skips the interactive prompt) - --from-env Read TESTSPRITE_API_KEY from the environment instead of - prompting (default: false) - --agent Coding-agent target to install: claude, antigravity, - cursor, cline, codex (default: claude) (default: "claude") - --no-agent Skip the agent skill install (configure credentials only) - --force Overwrite an existing skill file (a .bak backup is kept) - --dir Project root for the skill install (default: current - directory) - -y, --yes Non-interactive: accept all defaults, never prompt - -h, --help display help for command + --api-key API key to configure (skips the interactive prompt) + --from-env Read TESTSPRITE_API_KEY from the environment instead of + prompting (default: false) + --agent Coding-agent target to install: claude, antigravity, + cursor, cline, kiro, windsurf, copilot, codex (default: + claude) (default: "claude") + --no-agent Skip the agent skill install (configure credentials + only) + --force Overwrite an existing skill file (a .bak backup is + kept) + --dir Project root for the skill install (default: current + directory) + -y, --yes Non-interactive: accept all defaults, never prompt + --skip-if-configured Skip the API key prompt when credentials already exist + for this profile (CI-safe idempotent re-run) + -h, --help display help for command " `; @@ -128,20 +137,37 @@ exports[`--help snapshots > project 1`] = ` Manage TestSprite projects Options: - -h, --help display help for command + -h, --help display help for command Commands: - list [options] List projects visible to the API key + list [options] List projects visible to the API key Exit codes: 0 success 3 auth error 5 validation error (e.g., bad --page-size) 10 transport/network failure (UNAVAILABLE) — retry the command - get Get a project by id - create [options] Create a new project - update [options] Update project metadata - help [command] display help for command + get Get a project by id + create [options] Create a new project + update [options] Update project metadata + delete [options] Permanently delete a project and everything under it (sub-projects, + their tests, and backend fixtures). Requires --confirm. + + Exit codes: + 0 success + 3 auth error + 4 project not found (or already deleted) + 5 validation error (e.g., missing --confirm) + credential [options] Set the static backend credential injected + into every backend test + (Bearer token / API key / Basic token / + public). Free tier. + auto-auth [options] Configure the recurring-token + (auto-refresh login) for backend tests + (Pro). + A fresh token is fetched on each run and + injected into every backend test. + help [command] display help for command " `; @@ -153,7 +179,7 @@ Get a project by id Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -173,9 +199,12 @@ Options: --page-size service page-size hint (1-100, default 25) --starting-token opaque cursor from a previous list response --max-items stop after this many items across auto-paged pages + --columns select/reorder text table columns (comma-separated + keys) + --no-header suppress the text table header row -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -196,9 +225,27 @@ Commands: (--plan-from, FE-only, M3.2 piece-5) create-batch [options] Create multiple FE tests from a JSONL of plan specs (FE-only) + scaffold [options] Emit a schema-correct starter test + definition (frontend plan JSON by + default, or a backend Python skeleton). + Pure-local: no network, no credentials. + open [options] Open the test in the TestSprite + dashboard: prints the deep-link URL, + then spawns your default browser unless + --no-browser. steps [options] List the steps for a test (server returns the cumulative log across every run; use --run-id to scope to one run) + diff Compare two runs and print what + regressed: verdict, failureKind, + failedStepIndex, per-step status flips, + codeVersion drift. Exit 0 when verdicts + match, 1 when they differ. + lint [options] Validate plan/steps files offline with + the same validators \`create\` runs, + collecting EVERY problem. No network, + no credentials. Exit 0 when all valid, + 5 otherwise. result [options] Get the latest result for a test (default) or list prior runs (--history). --output json shape differs by mode: @@ -223,7 +270,7 @@ Commands: Note: a 404 "not found" response is counted as skipped in the summary, not an error. run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. - Use --all --project for a wave-ordered batch run of all BE tests (M4). + Use --all --project for a wave-ordered batch run of all tests in a project (M4). Exit codes: 0 passed (or queued without --wait) @@ -232,22 +279,34 @@ Commands: 4 test not found 5 validation error (e.g., bad --target-url, or positional + --all both set) 6 conflict (already running — see nextAction for the active runId) - 7 timeout — resume with: testsprite test wait + 7 timeout — resume with: testsprite test wait , or stop it with: testsprite test cancel 10 transport/network failure (UNAVAILABLE) — retry the command 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get - wait [options] Wait for a run to reach a terminal status. + + Ctrl-C during --wait detaches only (the run keeps executing and billing); + stop it for real with: testsprite test cancel + wait [options] Wait for one or more runs to reach a terminal status. + + With several run-ids the runs are polled concurrently under one shared + --timeout and a {results, summary} envelope is printed (worst status wins + the exit code), so every re-attach hint the CLI prints can be pasted as + ONE command. Exit codes: 0 passed 1 failed / blocked / cancelled 3 auth error - 4 run not found - 7 timeout — resume with: testsprite test wait + 4 run not found (single run-id; with several ids a per-member poll error + is recorded as error: in its row and folded into exit 7) + 7 timeout or per-member poll error — resume with: testsprite test wait 10 transport/network failure (UNAVAILABLE) — retry the command On failure/blocked/cancelled, run: testsprite test artifact get + + Ctrl-C detaches only (the run keeps executing and billing); stop it for + real with: testsprite test cancel rerun [options] [test-ids...] Re-execute a test (or multiple) as a cheap replay — FE replays the saved script (no credit), BE re-runs the dependency closure. Exit codes: @@ -257,19 +316,72 @@ Commands: 4 test not found 5 validation error 6 conflict (already running — see nextAction for the active runId) - 7 timeout or deferred — resume with: testsprite test wait + 7 timeout or deferred — resume with: testsprite test wait , or stop it with: testsprite test cancel 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get + + Ctrl-C during --wait detaches only (the run keeps executing and billing); + stop it for real with: testsprite test cancel + flaky [options] Repeatedly replay a test to measure stability and surface flakiness. + Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. + + Exit codes: + 0 stable (every attempt passed) + 1 flaky or failing (at least one attempt did not pass) + 3 auth error + 4 test not found (no replayable run — trigger \`testsprite test run \` first) + 5 validation error code Inspect and edit generated test code plan Manage FE test plan-steps (FE-only) failure Export the latest-failure agent bundle artifact Download run-scoped artifact bundles (M3.3 piece-4) + cancel Cancel one or more queued/running runs. + + Ctrl-C during --wait only detaches — it does NOT cancel the server-side + run. This is the real stop button. No refund is issued for the credits + already charged at trigger time (D3); an in-flight Lambda finishes on its + own and its result is discarded once cancelled. + + Exit codes: + 0 cancelled (fresh or already-cancelled — naturally idempotent) + 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict) + 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict + + Multi-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}. help [command] display help for command " `; +exports[`--help snapshots > test cancel 1`] = ` +"Usage: testsprite test cancel [options] + +Cancel one or more queued/running runs. + +Ctrl-C during --wait only detaches — it does NOT cancel the server-side +run. This is the real stop button. No refund is issued for the credits +already charged at trigger time (D3); an in-flight Lambda finishes on its +own and its result is discarded once cancelled. + +Exit codes: + 0 cancelled (fresh or already-cancelled — naturally idempotent) + 4 run id not found (single id), or ANY id not found (multi-id — outranks conflict) + 6 run already terminal (passed/failed/blocked) — single id: 409; multi-id: any conflict + +Multi-id output is a summary: {cancelled, alreadyCancelled, conflicts, notFound}. + +Arguments: + run-id one or more run ids to cancel + +Options: + -h, --help display help for command + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test code get 1`] = ` "Usage: testsprite test code get [options] @@ -280,7 +392,7 @@ Options: source body; json mode: wire envelope) -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -315,7 +427,7 @@ Options: per invocation; pin one yourself for safe retries. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -328,10 +440,44 @@ Write a self-contained failure-context bundle for a test's latest failing run Options: --out Directory to write the §7 disk layout into (default: print wire envelope to stdout) - --failed-only Keep only the failed step plus its immediate neighbors (±1) + --failed-only Trim to the failed step ±1. The bundle is already + failure-focused server-side, so this is usually a no-op; use + \`test steps \` for the full run trail. + -h, --help display help for command + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + +exports[`--help snapshots > test flaky 1`] = ` +"Usage: testsprite test flaky [options] + +Repeatedly replay a test to measure stability and surface flakiness. +Replays run with auto-heal OFF (strict verbatim) so healed drift cannot mask nondeterministic pass/fail. + +Exit codes: + 0 stable (every attempt passed) + 1 flaky or failing (at least one attempt did not pass) + 3 auth error + 4 test not found (no replayable run — trigger \`testsprite test run \` first) + 5 validation error + +Options: + --runs number of replays to run (1-10, default 5) + --until-fail stop at the first non-passing attempt (fast "is it flaky at + all?" check) (default: false) + --timeout per-attempt max seconds to wait (1-3600, default 600) -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Notes: + • Frontend replays are free verbatim script replays (no credit); backend replays + re-run the dependency closure and may cost credits — a one-line advisory is printed. + • Replays use auto-heal OFF so a flaky test is not silently "healed" into a pass; + this measures replay stability of the saved script against the configured URL. + • \`--output json\` emits a machine-readable stability report for CI gating. + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -344,7 +490,7 @@ Get a test by id Options: -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -367,9 +513,12 @@ Options: --cursor alias for --starting-token; accepted for parity with \`test result --history\` --max-items stop after this many items across auto-paged pages + --columns select/reorder text table columns (comma-separated + keys) + --no-header suppress the text table header row -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -386,38 +535,49 @@ Exit codes: 4 test not found 5 validation error 6 conflict (already running — see nextAction for the active runId) - 7 timeout or deferred — resume with: testsprite test wait + 7 timeout or deferred — resume with: testsprite test wait , or stop it with: testsprite test cancel 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get +Ctrl-C during --wait detaches only (the run keeps executing and billing); +stop it for real with: testsprite test cancel + Options: - --all rerun all tests in the resolved project (requires - --project) (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --skip-terminal with --all: skip tests already in a terminal status - (passed|failed|blocked|cancelled) (default: false) - --status with --all: only dispatch tests whose status matches - one of these values (comma-separated; accepted: - draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) - --filter with --all: only rerun tests whose name contains - this substring (case-insensitive) - --wait block until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --no-auto-heal opt out of AI heal-on-drift for this FE rerun - (default: auto-heal is ON). Costs 0.2 credits per - engage when a step has drifted. Ignored for backend - tests. - --skip-dependencies BE only: rerun only the named test without expanding - the producer/teardown closure (default: false) - --max-concurrency with --wait, max in-flight polls at once (1-100, - default: 50) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --verbose if auto-generated. - -h, --help display help for command + --all rerun all tests in the resolved project (requires + --project) (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --skip-terminal with --all: skip tests already in a terminal + status (passed|failed|blocked|cancelled) + (default: false) + --status with --all: only dispatch tests whose status + matches one of these values (comma-separated; + accepted: + draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) + --filter with --all: only rerun tests whose name contains + this substring (case-insensitive) + --wait block until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --no-auto-heal opt out of AI heal-on-drift for this FE rerun + (default: auto-heal is ON). Costs 0.2 credits per + engage when a step has drifted. Ignored for + backend tests. + --skip-dependencies BE only: rerun only the named test without + expanding the producer/teardown closure (default: + false) + --max-concurrency with --wait, max in-flight polls at once (1-100, + default: 50) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --verbose if auto-generated. + --report with batch --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Notes: • rerun replays a saved run/script and is MORE LENIENT than a fresh \`test run\` @@ -433,7 +593,7 @@ Dry-run shape notes: omit \`closure\` (or return it as null) since there is no dependency expansion. • \`autoHeal\` defaults true for FE reruns; BE reruns ignore the field entirely. -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -463,9 +623,11 @@ Options: --page-size with --history: number of runs per page (1–100, default 20) --cursor with --history: opaque cursor from a prior page + --columns with --history: select/reorder text table columns + --no-header with --history: suppress the text table header row -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -474,7 +636,7 @@ exports[`--help snapshots > test run 1`] = ` "Usage: testsprite test run [options] [test-id] Trigger a test run. With --wait, polls until terminal status. -Use --all --project for a wave-ordered batch run of all BE tests (M4). +Use --all --project for a wave-ordered batch run of all tests in a project (M4). Exit codes: 0 passed (or queued without --wait) @@ -483,40 +645,53 @@ Exit codes: 4 test not found 5 validation error (e.g., bad --target-url, or positional + --all both set) 6 conflict (already running — see nextAction for the active runId) - 7 timeout — resume with: testsprite test wait + 7 timeout — resume with: testsprite test wait , or stop it with: testsprite test cancel 10 transport/network failure (UNAVAILABLE) — retry the command 11 rate limited — honor Retry-After On failure/blocked/cancelled, run: testsprite test artifact get +Ctrl-C during --wait detaches only (the run keeps executing and billing); +stop it for real with: testsprite test cancel + Options: - --target-url override the project default env URL for this run - (http/https only, no localhost/private IPs) - --wait poll until terminal status or --timeout elapses - (default: false) - --timeout with --wait, max seconds to wait (1–3600, default - 600) - --idempotency-key opaque key for safe retries (1–256 chars). Printed - to stderr at --debug if auto-generated. - --all run all BE tests in the project (wave-ordered fresh - run; requires --project). Mutually exclusive with - . (default: false) - --project project id (required with --all; returned by - \`testsprite project list\`) - --filter with --all: only run tests whose name contains this - substring (case-insensitive) - --max-concurrency with --all --wait, max in-flight polls at once - (1-100, default: 50) - -h, --help display help for command + --target-url override the project default env URL for this run + (http/https only, no localhost/private IPs) + --wait poll until terminal status or --timeout elapses + (default: false) + --timeout with --wait, max seconds to wait (1–3600, default + 600) + --idempotency-key opaque key for safe retries (1–256 chars). + Printed to stderr at --debug if auto-generated. + --all run all tests in the project (wave-ordered fresh + run; requires --project). Mutually exclusive with + . (default: false) + --project project id (required with --all; returned by + \`testsprite project list\`) + --filter with --all: only run tests whose name contains + this substring (case-insensitive) + --max-concurrency with --all --wait, max in-flight polls at once + (1-100, default: 50) + --report with --all --wait: write a JUnit XML sidecar + report after polling (accepted: junit) + --report-file output path for --report (atomic write) + --report-suite-name optional JUnit override + (default: testsprite:) + -h, --help display help for command Dependency-aware fresh run (M4): - testsprite test run --all --project run all BE tests in wave order + testsprite test run --all --project run all project tests in wave order testsprite test run --all --project --filter name-glob subset + testsprite test run --all --project --wait --report junit --report-file ./results.xml BE tests can declare --produces/--needs at create time to drive wave ordering (see \`testsprite test create --help\` for details). -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Frontend tests: the current unified engine runs FE tests too (they are billed +like any run). On the legacy backend-only engine FE tests cannot run — they are +reported under skippedFrontend with an advisory; run those with 'test run '. + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -539,7 +714,7 @@ Options: excluded when this flag is set. -h, --help display help for command -Global options (--dry-run, --output, --profile, --endpoint-url, --verbose, --debug): +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): testsprite --help " `; @@ -560,10 +735,13 @@ Options: without the full trace. --debug Print HTTP method/path, request id, latency, retry decisions to stderr - --dry-run Skip the network, credentials, and filesystem; - emit a canned sample matching the OpenAPI - contract. Useful for learning the CLI surface - without an API key. + --dry-run Skip the network and credentials; emit a canned + sample matching the OpenAPI contract. Useful for + learning the CLI surface without an API key. + Note: file inputs you pass + (--plan-from/--plans/--steps) are still read and + validated locally; only --code-file uses a + placeholder. --request-timeout Client-side per-request timeout in seconds (default: 120). Aborts any single fetch that does not complete within this deadline. Override @@ -582,9 +760,12 @@ Commands: test Inspect TestSprite tests agent Install TestSprite guidance into coding-agent config (Claude Code, Cursor, Cline, Antigravity, - Codex) + Kiro, Windsurf, Copilot, Codex) usage|credits Show credit balance and plan/entitlement info (proactive pre-flight before a large test run) + doctor Diagnose CLI setup: version, Node, profile, + endpoint, credentials, connectivity, skill + completion [shell] Print a shell completion script (bash|zsh|fish) help [command] display help for command " `; diff --git a/test/cli.subprocess.test.ts b/test/cli.subprocess.test.ts index 21d7cfe..3b04712 100644 --- a/test/cli.subprocess.test.ts +++ b/test/cli.subprocess.test.ts @@ -7,14 +7,15 @@ * and runs `auth whoami` against the mock." */ -import { execFileSync, spawn } from 'node:child_process'; -import { existsSync, mkdtempSync, statSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs'; import type { IncomingMessage, Server, ServerResponse } from 'node:http'; import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { execNpm } from './helpers/execNpm.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, '..'); @@ -37,7 +38,7 @@ beforeAll(async () => { // existsSync skip we used to do here let `dist` rot under // refactors and gave false-green on `project list` once // already. - execFileSync('npm', ['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); server = createServer((req: IncomingMessage, res: ServerResponse) => { const url = req.url ?? '/'; if (url.startsWith('/api/cli/v1/projects/')) { @@ -364,7 +365,10 @@ function runCli(args: string[], envOverrides: Record = {}): Prom cwd: REPO_ROOT, env: { ...process.env, + // os.homedir() reads HOME on POSIX but USERPROFILE on Windows — + // set both so the child never sees the real ~/.testsprite. HOME: tmpHome, + USERPROFILE: tmpHome, TESTSPRITE_API_KEY: undefined, TESTSPRITE_API_URL: undefined, ...envOverrides, @@ -499,6 +503,24 @@ describe('project list subprocess', () => { const parsed = JSON.parse(result.stderr) as { error: { code: string } }; expect(parsed.error.code).toBe('VALIDATION_ERROR'); }, 30_000); + + it('--request-timeout 30s exits 5 (VALIDATION_ERROR), not a silent fallback to 120s', async () => { + // Previously an invalid flag value resolved to `undefined` and the command + // silently ran with the default 120s deadline — the operator believed they + // had set a timeout but had not. Now the explicit flag is validated like + // every other flag. + const result = await runCli( + ['--output', 'json', '--request-timeout', '30s', 'project', 'list'], + { + TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_URL: baseUrl, + }, + ); + expect(result.exitCode).toBe(5); + const parsed = JSON.parse(result.stderr) as { error: { code: string; nextAction: string } }; + expect(parsed.error.code).toBe('VALIDATION_ERROR'); + expect(parsed.error.nextAction).toContain('request-timeout'); + }, 30_000); }); describe('malformed --endpoint-url is rejected (exit 5), not retried as a network error', () => { @@ -530,6 +552,33 @@ describe('malformed --endpoint-url is rejected (exit 5), not retried as a networ }, 30_000); }); +describe('invalid --output is rejected uniformly (exit 5)', () => { + // Regression: previously only `test` and `project` validated `--output`; + // `auth`, `usage`, `agent`, and `init` silently coerced an unknown value to + // text mode. An agent that asked for `--output json` but mistyped it then + // received a text payload it could not parse, with no signal as to why. Every + // command group now routes through resolveOutputMode (exit 5 on bad input). + + // Note: when `--output` itself is the invalid value, the requested mode is + // unusable for the error envelope, so it is rendered in text mode (the catch + // block in index.ts falls back to text for an unrecognised --output). + + it('agent list --output josn exits 5 with an actionable message (offline command)', async () => { + const result = await runCli(['--output', 'josn', 'agent', 'list'], {}); + expect(result.exitCode).toBe(5); + expect(result.stderr).toContain('must be one of: json, text'); + }, 30_000); + + it('auth status --output yaml exits 5 before any network call', async () => { + const result = await runCli(['--output', 'yaml', 'auth', 'status'], { + TESTSPRITE_API_KEY: 'sk-subproc', + TESTSPRITE_API_URL: baseUrl, + }); + expect(result.exitCode).toBe(5); + expect(result.stderr).toContain('must be one of: json, text'); + }, 30_000); +}); + describe('a malformed --profile is rejected (exit 5), not silently corrupting credentials', () => { // A profile name becomes an INI section header (`[name]`). `prod]` would // serialise to `[prod]]`, which the parser cannot read back — `setup` would @@ -852,7 +901,10 @@ describe('setup --from-env subprocess', () => { expect(result.exitCode).toBe(0); const credentialsPath = join(tmpHome, '.testsprite', 'credentials'); expect(existsSync(credentialsPath)).toBe(true); - expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); + // POSIX file modes don't exist on Windows (stat reports 0666). + if (process.platform !== 'win32') { + expect(statSync(credentialsPath).mode & 0o777).toBe(0o600); + } }, 30_000); it('exits 5 with VALIDATION_ERROR when --from-env is set without TESTSPRITE_API_KEY', async () => { @@ -896,6 +948,22 @@ describe('--dry-run subprocess smoke', () => { expect(parsed.id).toBeTruthy(); }, 30_000); + it('project update --dry-run does not read a missing --password-file', async () => { + const result = await runCli([ + 'project', + 'update', + 'proj_anything', + '--password-file', + '/tmp/definitely-not-here-testsprite', + '--dry-run', + '--output', + 'json', + ]); + expect(result.exitCode).toBe(0); + const parsed = JSON.parse(result.stdout) as { updatedFields: string[] }; + expect(parsed.updatedFields).toContain('password'); + }, 30_000); + it('test list --dry-run returns canned TestList', async () => { const result = await runCli([ 'test', @@ -983,7 +1051,7 @@ describe('--dry-run subprocess smoke', () => { // skipped the prompt. const credPath = join(tmpHome, '.testsprite', 'credentials'); // Make sure any previous test didn't leave one behind. - if (existsSync(credPath)) execFileSync('rm', [credPath]); + rmSync(credPath, { force: true }); const result = await runCli(['setup', '--dry-run', '--no-agent', '--output', 'json']); expect(result.exitCode).toBe(0); expect(existsSync(credPath)).toBe(false); diff --git a/test/e2e/agent-install.e2e.test.ts b/test/e2e/agent-install.e2e.test.ts index 1cbf4fc..4fb4584 100644 --- a/test/e2e/agent-install.e2e.test.ts +++ b/test/e2e/agent-install.e2e.test.ts @@ -169,11 +169,25 @@ describe('content integrity', () => { content.trimStart().startsWith('#'), `cline: should start with a markdown heading`, ).toBe(true); + } else if (target === 'windsurf') { + // Windsurf Cascade frontmatter: trigger + description (no name/alwaysApply) + expect(content.startsWith('---'), `windsurf: should start with ---`).toBe(true); + expect(content).toContain('trigger: model_decision'); + expect(content).toContain('description:'); + } else if (target === 'copilot') { + // GitHub Copilot instructions frontmatter: applyTo glob + description + expect(content.startsWith('---'), `copilot: should start with ---`).toBe(true); + expect(content).toContain("applyTo: '**'"); + expect(content).toContain('description:'); } - // (b) branding — the renamed H1 must be present + // (b) branding — the renamed H1 must be present in every body variant expect(content).toContain('TestSprite Verification Loop'); - expect(content).toContain('The verification loop that flies'); + // The full-body intro line lives only in the FULL body; compact-body targets + // (e.g. windsurf, budget-capped) ship the trimmed verify body and omit it. + if (!TARGETS[target].compactBody) { + expect(content).toContain('The verification loop that flies'); + } // (c) Load-bearing command strings expect(content, `${target}: missing 'testsprite test run'`).toContain('testsprite test run'); @@ -198,6 +212,13 @@ describe('content integrity', () => { expect(content).toContain('alwaysApply: false'); } else if (target === 'cline') { expect(content.startsWith('---'), `cline/onboard: must NOT start with ---`).toBe(false); + } else if (target === 'windsurf') { + expect(content.startsWith('---'), `windsurf/onboard: should start with ---`).toBe(true); + expect(content).toContain('trigger: model_decision'); + expect(content).toContain('description:'); + } else if (target === 'copilot') { + expect(content.startsWith('---'), `copilot/onboard: should start with ---`).toBe(true); + expect(content).toContain("applyTo: '**'"); } // Load-bearing onboard string: the skill body must reference setup @@ -430,13 +451,13 @@ describe('dry-run', () => { // --------------------------------------------------------------------------- describe('multi-target install', () => { - it('--target=claude,cursor,cline,antigravity,codex writes all targets + skills, exit 0', () => { + it('--target=claude,cursor,cline,antigravity,kiro,codex writes all targets + skills, exit 0', () => { const tmpDir = freshTmpDir(); const result = runCli([ 'agent', 'install', - '--target=claude,cursor,cline,antigravity,codex', + '--target=claude,cursor,cline,antigravity,kiro,codex', '--dir', tmpDir, '--output', @@ -449,7 +470,7 @@ describe('multi-target install', () => { action: string; path: string; }>; - const allTargets: AgentTarget[] = ['claude', 'cursor', 'cline', 'antigravity', 'codex']; + const allTargets: AgentTarget[] = ['claude', 'cursor', 'cline', 'antigravity', 'kiro', 'codex']; for (const target of allTargets) { if (TARGETS[target].mode === 'managed-section') { @@ -790,7 +811,7 @@ describe('agent list', () => { }>; expect(Array.isArray(parsed)).toBe(true); - // Expected: 5 targets × 2 skills = 10 rows + // Expected: 8 targets × 2 skills = 16 rows const expectedCount = Object.keys(TARGETS).length * DEFAULT_SKILLS.length; expect(parsed.length).toBe(expectedCount); @@ -819,7 +840,16 @@ describe('agent list', () => { // --------------------------------------------------------------------------- describe('matrix coverage guard', () => { it('TARGETS matches the documented, e2e-covered set (update this list when adding a target)', () => { - expect(Object.keys(TARGETS)).toEqual(['claude', 'antigravity', 'cursor', 'cline', 'codex']); + expect(Object.keys(TARGETS)).toEqual([ + 'claude', + 'antigravity', + 'cursor', + 'cline', + 'kiro', + 'windsurf', + 'copilot', + 'codex', + ]); }); it('SKILLS matches the documented, e2e-covered set (update this list when adding a skill)', () => { diff --git a/test/e2e/setup.e2e.test.ts b/test/e2e/setup.e2e.test.ts index b5d3963..5a07434 100644 --- a/test/e2e/setup.e2e.test.ts +++ b/test/e2e/setup.e2e.test.ts @@ -222,7 +222,16 @@ describe('deprecated `init` alias', () => { describe('matrix coverage guard', () => { it('TARGETS matches the documented set (update this list when adding a target)', () => { - expect(Object.keys(TARGETS)).toEqual(['claude', 'antigravity', 'cursor', 'cline', 'codex']); + expect(Object.keys(TARGETS)).toEqual([ + 'claude', + 'antigravity', + 'cursor', + 'cline', + 'kiro', + 'windsurf', + 'copilot', + 'codex', + ]); }); }); diff --git a/test/e2e/signal.e2e.test.ts b/test/e2e/signal.e2e.test.ts new file mode 100644 index 0000000..6e03743 --- /dev/null +++ b/test/e2e/signal.e2e.test.ts @@ -0,0 +1,188 @@ +/** + * Local e2e tests for SIGINT/SIGTERM graceful detach during `--wait` + * (exit 130/143/129 per the documented signal contract). + * + * Spawns the real built binary (`dist/index.js`) against a local HTTP stub + * whose `GET /runs/{id}` long-poll hangs forever, sends a real signal to the + * child, and asserts the honest-detach contract: + * + * - stdout: parseable partial `{runId, status:"running"}` (JSON mode) + * - stderr: "keeps running (and billing)" + re-attach hint (+ INTERRUPTED + * envelope in JSON mode) + * - exit code 130 (SIGINT) / 143 (SIGTERM) + * + * Run via: `npm run test:e2e` (builds first). Excluded from `npm test`. + */ + +import { spawn } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import { createServer, type Server } from 'node:http'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = resolve(__dirname, '../..'); +const BIN_PATH = join(REPO_ROOT, 'dist', 'index.js'); + +const RUN_ID = 'run_sig_e2e_01'; + +let server: Server; +let baseUrl = ''; +/** Resolvers waiting for the next hanging /runs request to arrive. */ +const runRequestWaiters: Array<() => void> = []; + +beforeAll(async () => { + if (!existsSync(BIN_PATH)) { + throw new Error('dist/index.js not found — run `npm run test:e2e` which builds first.'); + } + server = createServer((req, res) => { + // Hang every request (long-poll / stalled-backend simulation): the CLI's + // abort must cut it. Signal any test waiting for the request to arrive. + runRequestWaiters.splice(0).forEach(fn => fn()); + req.on('close', () => res.destroy()); + }); + await new Promise(resolveListen => { + server.listen(0, '127.0.0.1', () => resolveListen()); + }); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('no server address'); + baseUrl = `http://127.0.0.1:${address.port}`; +}); + +afterAll(async () => { + await new Promise(resolveClose => { + server.close(() => resolveClose()); + server.closeAllConnections(); + }); +}); + +/** Resolves when the stub receives the next hanging GET /runs request. */ +function nextRunRequest(): Promise { + return new Promise(resolveWait => runRequestWaiters.push(resolveWait)); +} + +interface SpawnResult { + code: number | null; + signal: NodeJS.Signals | null; + stdout: string; + stderr: string; +} + +/** + * Spawn `testsprite test wait` against the hanging stub, deliver `signal` + * once the long-poll request is in flight, and collect the outcome. + */ +async function waitAndInterrupt( + signal: NodeJS.Signals, + extraArgs: string[] = [], +): Promise { + const child = spawn( + process.execPath, + [BIN_PATH, 'test', 'wait', RUN_ID, '--timeout', '120', ...extraArgs], + { + env: { + ...process.env, + TESTSPRITE_API_KEY: 'sk-e2e-signal', + TESTSPRITE_API_URL: baseUrl, + TESTSPRITE_NO_SKILL_WARNING: '1', + TESTSPRITE_NO_UPDATE_NOTIFIER: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString())); + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString())); + + const arrived = nextRunRequest(); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + resolveExit => { + child.on('exit', (code, exitSignal) => resolveExit({ code, signal: exitSignal })); + }, + ); + + await arrived; // the long-poll fetch is in flight — the armed window is open + await new Promise(r => setTimeout(r, 150)); // let the request settle into the poll loop + child.kill(signal); + + const { code, signal: exitSignal } = await exited; + return { code, signal: exitSignal, stdout, stderr }; +} + +describe('signal e2e — graceful detach during test wait (DEV-331)', () => { + it('SIG-1/SIG-2: SIGINT → exit 130, partial JSON on stdout, honest stderr hint', async () => { + const result = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(result.code).toBe(130); + + // stdout is a parseable partial naming the runId (file redirects never 0-byte). + const partial = JSON.parse(result.stdout) as { runId: string; status: string }; + expect(partial.runId).toBe(RUN_ID); + expect(partial.status).toBe('running'); + + // stderr: honest detach line + machine-readable INTERRUPTED envelope. + expect(result.stderr).toContain('Interrupted (SIGINT)'); + expect(result.stderr).toContain('billing'); + expect(result.stderr).toContain(`testsprite test wait ${RUN_ID}`); + expect(result.stderr).toContain('"code": "INTERRUPTED"'); + expect(result.stderr).toContain('"signal": "SIGINT"'); + }, 30_000); + + it('SIG-1 (text mode): SIGINT → exit 130, human-readable partial + hint', async () => { + const result = await waitAndInterrupt('SIGINT'); + expect(result.code).toBe(130); + expect(result.stdout).toContain(RUN_ID); + expect(result.stdout).toContain('running (interrupted)'); + expect(result.stderr).toContain('Interrupted (SIGINT)'); + expect(result.stderr).toContain('Error: Interrupted by SIGINT.'); + }, 30_000); + + it('SIG-3: SIGTERM → exit 143', async () => { + const result = await waitAndInterrupt('SIGTERM', ['--output', 'json']); + expect(result.code).toBe(143); + expect(result.stderr).toContain('Interrupted (SIGTERM)'); + expect(result.stderr).toContain('"signal": "SIGTERM"'); + }, 30_000); + + it('SIG-7: SIGINT during a non-wait command → immediate exit 130 with the generic explanation', async () => { + // `test list` is outside any armed --wait scope. The stub hangs its fetch; + // the disarmed handler must exit immediately with the generic explanation + // (no partial envelope — there is no runId to re-attach to). + const child = spawn(process.execPath, [BIN_PATH, 'test', 'list', '--project', 'p1'], { + env: { + ...process.env, + TESTSPRITE_API_KEY: 'sk-e2e-signal', + TESTSPRITE_API_URL: baseUrl, + TESTSPRITE_NO_SKILL_WARNING: '1', + TESTSPRITE_NO_UPDATE_NOTIFIER: '1', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString())); + const exited = new Promise<{ code: number | null }>(resolveExit => { + child.on('exit', code => resolveExit({ code })); + }); + const arrived = nextRunRequest(); + await arrived; // the list fetch is in flight (disarmed — no poll running) + child.kill('SIGINT'); + const { code } = await exited; + expect(code).toBe(130); + expect(stderr).toContain('Interrupted (SIGINT)'); + expect(stderr).toContain('test wait'); + expect(stderr).not.toContain(' at '); // no stack trace / corrupted output + }, 30_000); + + it('SIG-8: detach then re-attach — the same runId can be waited on again (server unaffected)', async () => { + // First wait: interrupted. + const first = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(first.code).toBe(130); + // Re-attach: the stub receives a fresh long-poll for the SAME runId — + // proof the detach was client-side only. (We interrupt again to end it.) + const second = await waitAndInterrupt('SIGINT', ['--output', 'json']); + expect(second.code).toBe(130); + const partial = JSON.parse(second.stdout) as { runId: string }; + expect(partial.runId).toBe(RUN_ID); + }, 60_000); +}); diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index 2448c91..b6c23b4 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -13,6 +13,7 @@ import { execFileSync } from 'node:child_process'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { beforeAll, describe, expect, it } from 'vitest'; +import { execNpm } from './helpers/execNpm.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = resolve(__dirname, '..'); @@ -39,14 +40,17 @@ const cases: Array<[string, string[]]> = [ ['test result', ['test', 'result', '--help']], ['test failure get', ['test', 'failure', 'get', '--help']], ['test rerun', ['test', 'rerun', '--help']], + ['test flaky', ['test', 'flaky', '--help']], // R5: regression guard for commands that gained new flag wording ['test create-batch', ['test', 'create-batch', '--help']], ['test run', ['test', 'run', '--help']], + // DEV-331 piece 3 + ['test cancel', ['test', 'cancel', '--help']], ]; describe('--help snapshots', () => { beforeAll(() => { - execFileSync('npm', ['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); + execNpm(['run', 'build'], { cwd: REPO_ROOT, stdio: 'pipe' }); }); for (const [name, args] of cases) { diff --git a/test/helpers/execNpm.ts b/test/helpers/execNpm.ts new file mode 100644 index 0000000..0535112 --- /dev/null +++ b/test/helpers/execNpm.ts @@ -0,0 +1,17 @@ +import { execFileSync } from 'node:child_process'; + +/** + * Cross-platform `npm` invocation for test `beforeAll` build steps. + * On Windows, `npm` is a `.cmd` shim rather than a directly executable + * binary — `execFileSync('npm', ...)` fails with `ENOENT` there unless + * `shell: true` lets the OS resolve the shim through PATHEXT. + */ +export function execNpm( + args: string[], + options: { cwd: string; stdio?: 'pipe' | 'inherit' | 'ignore' }, +): Buffer | string { + return execFileSync('npm', args, { + ...options, + shell: process.platform === 'win32', + }); +} diff --git a/test/helpers/hermetic-env.ts b/test/helpers/hermetic-env.ts new file mode 100644 index 0000000..c798bb3 --- /dev/null +++ b/test/helpers/hermetic-env.ts @@ -0,0 +1,41 @@ +/** + * Unit-test env hermeticity (vitest `setupFiles`, runs before each test file). + * + * Two leaks this closes, both of which made results depend on the + * developer's machine: + * + * 1. Real `TESTSPRITE_*` env vars. `loadConfig` gives `TESTSPRITE_API_KEY` + * precedence over the credentials file, so a key exported in the + * developer's shell silently overrode test fixtures. + * 2. The real home directory. `os.homedir()` reads `HOME` on POSIX but + * `USERPROFILE` on Windows, so the documented `HOME=$(mktemp -d)` + * recipe never isolated Windows runs. Both vars are redirected to a + * throwaway dir so no test can read or write `~/.testsprite`. + * + * Tests that need these vars set them explicitly (on `process.env` or via + * injected `env` deps) after this runs. + */ +import { existsSync, mkdirSync, mkdtempSync } from 'node:fs'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const realHome = homedir(); +const hermeticHome = mkdtempSync(join(tmpdir(), 'testsprite-unit-home-')); +if (process.platform === 'win32') { + // Node version shims (Volta) resolve LocalAppData under USERPROFILE and + // abort if it's missing, which would break the `npm run build` beforeAll + // in the subprocess/snapshot suites. + mkdirSync(join(hermeticHome, 'AppData', 'Local'), { recursive: true }); +} +// Same shim concern on macOS/Linux: Volta derives ~/.volta from HOME unless +// VOLTA_HOME is set. Pin it to the real install before redirecting HOME. +const realVoltaHome = join(realHome, '.volta'); +if (!process.env.VOLTA_HOME && existsSync(realVoltaHome)) { + process.env.VOLTA_HOME = realVoltaHome; +} +process.env.HOME = hermeticHome; +process.env.USERPROFILE = hermeticHome; + +for (const key of Object.keys(process.env)) { + if (key.startsWith('TESTSPRITE_')) delete process.env[key]; +} diff --git a/vitest.config.ts b/vitest.config.ts index 18ea00f..bd9b2ad 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,12 @@ export default defineConfig({ test: { include: ['src/**/*.{test,spec}.ts', 'test/**/*.{test,spec}.ts'], exclude: ['test/dev-e2e/**', 'test/e2e/**', 'node_modules/**', 'dist/**'], + // Strip real TESTSPRITE_* env vars and redirect the home dir so results + // never depend on the developer's shell or ~/.testsprite (see the file). + setupFiles: ['./test/helpers/hermetic-env.ts'], + // Subprocess/snapshot suites each run `npm run build` in beforeAll; parallel + // file workers can race on dist/ and produce a stale binary (exit 1 vs 5 flakes). + fileParallelism: false, coverage: { provider: 'v8', reporter: ['text', 'text-summary', 'json-summary', 'html'],