From bf923b65f75628105eaa59797ddcb83b910ffd1c Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 11:15:51 +0200 Subject: [PATCH 01/15] ci(relayflows): add PR-specific red-green proof gate --- .../2026-08/traj_yr3f5u3r6zdz/summary.md | 54 +++ .../2026-08/traj_yr3f5u3r6zdz/trajectory.json | 128 ++++++++ .github/pull_request_template.md | 9 + .github/workflows/relayflow-pr-proof.yml | 96 ++++++ .gitignore | 2 + scripts/pr-proof/contract.mjs | 309 ++++++++++++++++++ scripts/pr-proof/prepare.mjs | 198 +++++++++++ scripts/pr-proof/run-arm.mjs | 202 ++++++++++++ scripts/pr-proof/run-cloud.mjs | 108 ++++++ scripts/pr-proof/verify-evidence.mjs | 82 +++++ tests/fixtures/pr-proof-contract.test.ts | 232 +++++++++++++ tests/relayflows/cases/README.md | 108 ++++++ workflows/pr-proof.ts | 86 +++++ 13 files changed, 1614 insertions(+) create mode 100644 .agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md create mode 100644 .agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json create mode 100644 .github/workflows/relayflow-pr-proof.yml create mode 100644 scripts/pr-proof/contract.mjs create mode 100644 scripts/pr-proof/prepare.mjs create mode 100644 scripts/pr-proof/run-arm.mjs create mode 100644 scripts/pr-proof/run-cloud.mjs create mode 100644 scripts/pr-proof/verify-evidence.mjs create mode 100644 tests/fixtures/pr-proof-contract.test.ts create mode 100644 tests/relayflows/cases/README.md create mode 100644 workflows/pr-proof.ts diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md new file mode 100644 index 000000000..7fc138e4b --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md @@ -0,0 +1,54 @@ +# Trajectory: Add PR-specific Cloud RelayFlow red-green proof infrastructure + +> **Status:** ✅ Completed +> **Confidence:** 92% +> **Started:** August 25, 2026 at 10:45 AM +> **Completed:** August 25, 2026 at 11:14 AM + +--- + +## Summary + +Implemented trusted PR classification and single-case dispatch, a fail-closed Cloud RelayFlow that proves base then head in distinct per-step sandboxes, provenance/evidence validation, GitHub Action wiring, tests, template guidance, and credential rollout documentation. + +**Approach:** Kept pull_request_target as a data-only trusted dispatcher, ran PR code only in Cloud, represented expected-red as a structured successful observation, disabled repair retries, validated exact base/head SHAs and sandbox IDs, and tested both the contracts and the complete repository. + +--- + +## Key Decisions + +### Require exactly one PR proof case and a structured observation result +- **Chose:** Require exactly one PR proof case and a structured observation result +- **Reasoning:** One feature or fix should trigger only its own case. The case runner must exit successfully after observing behavior and write bug/absent/fixed plus an exact signature; test crashes, missing tests, skips, and build failures remain infrastructure failures and cannot count as expected red. + +### Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes +- **Chose:** Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes +- **Reasoning:** The GitHub runner must never checkout or execute untrusted head code while holding Cloud credentials. It will fetch and validate one declarative case manifest, stage exact base/head SHAs, and submit the trusted base-branch RelayFlow. + +### Disable automatic repair and retries for PR proof workflows +- **Chose:** Disable automatic repair and retries for PR proof workflows +- **Rejected:** Default retry policy, Continue on failure +- **Reasoning:** A red/green gate must preserve the first observation and fail closed. RelayFlow defaults to agent-assisted repair retries when agents exist, so the proof workflow explicitly uses fail-fast with zero step retries. + +### Stage the validated generated input in the disposable CI git index +- **Chose:** Stage the validated generated input in the disposable CI git index +- **Rejected:** Change global code-sync semantics, Commit a placeholder input file +- **Reasoning:** Cloud code sync uploads git-known files. The per-PR input is generated after checkout, so the action must git-add it without committing or pushing or the Cloud workflow cannot see the exact case and SHAs. + +### Require explicit proof classification on every PR +- **Chose:** Require explicit proof classification on every PR +- **Rejected:** Infer only from conventional titles, Run Cloud for every PR +- **Reasoning:** Silently treating missing metadata as non-functional would let non-conventional feature and fix titles bypass the gate. Every PR must declare feature, bugfix, or non-functional; only the first two launch Cloud. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Require exactly one PR proof case and a structured observation result: Require exactly one PR proof case and a structured observation result +- Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes: Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes +- Disable automatic repair and retries for PR proof workflows: Disable automatic repair and retries for PR proof workflows +- Stage the validated generated input in the disposable CI git index: Stage the validated generated input in the disposable CI git index +- Require explicit proof classification on every PR: Require explicit proof classification on every PR diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json new file mode 100644 index 000000000..c8dc4482f --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json @@ -0,0 +1,128 @@ +{ + "id": "traj_yr3f5u3r6zdz", + "version": 1, + "task": { + "title": "Add PR-specific Cloud RelayFlow red-green proof infrastructure" + }, + "status": "completed", + "startedAt": "2026-08-25T08:45:32.835Z", + "completedAt": "2026-08-25T09:14:31.949Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-25T08:50:30.315Z" + } + ], + "chapters": [ + { + "id": "chap_cco5mxx3i8f5", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-25T08:50:30.315Z", + "endedAt": "2026-08-25T09:14:31.949Z", + "events": [ + { + "ts": 1787647830316, + "type": "decision", + "content": "Require exactly one PR proof case and a structured observation result: Require exactly one PR proof case and a structured observation result", + "raw": { + "question": "Require exactly one PR proof case and a structured observation result", + "chosen": "Require exactly one PR proof case and a structured observation result", + "alternatives": [], + "reasoning": "One feature or fix should trigger only its own case. The case runner must exit successfully after observing behavior and write bug/absent/fixed plus an exact signature; test crashes, missing tests, skips, and build failures remain infrastructure failures and cannot count as expected red." + }, + "significance": "high" + }, + { + "ts": 1787647830374, + "type": "decision", + "content": "Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes: Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes", + "raw": { + "question": "Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes", + "chosen": "Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes", + "alternatives": [], + "reasoning": "The GitHub runner must never checkout or execute untrusted head code while holding Cloud credentials. It will fetch and validate one declarative case manifest, stage exact base/head SHAs, and submit the trusted base-branch RelayFlow." + }, + "significance": "high" + }, + { + "ts": 1787649023999, + "type": "decision", + "content": "Disable automatic repair and retries for PR proof workflows: Disable automatic repair and retries for PR proof workflows", + "raw": { + "question": "Disable automatic repair and retries for PR proof workflows", + "chosen": "Disable automatic repair and retries for PR proof workflows", + "alternatives": [ + { + "option": "Default retry policy", + "reason": "" + }, + { + "option": "Continue on failure", + "reason": "" + } + ], + "reasoning": "A red/green gate must preserve the first observation and fail closed. RelayFlow defaults to agent-assisted repair retries when agents exist, so the proof workflow explicitly uses fail-fast with zero step retries." + }, + "significance": "high" + }, + { + "ts": 1787649024705, + "type": "decision", + "content": "Stage the validated generated input in the disposable CI git index: Stage the validated generated input in the disposable CI git index", + "raw": { + "question": "Stage the validated generated input in the disposable CI git index", + "chosen": "Stage the validated generated input in the disposable CI git index", + "alternatives": [ + { + "option": "Change global code-sync semantics", + "reason": "" + }, + { + "option": "Commit a placeholder input file", + "reason": "" + } + ], + "reasoning": "Cloud code sync uploads git-known files. The per-PR input is generated after checkout, so the action must git-add it without committing or pushing or the Cloud workflow cannot see the exact case and SHAs." + }, + "significance": "high" + }, + { + "ts": 1787649025196, + "type": "decision", + "content": "Require explicit proof classification on every PR: Require explicit proof classification on every PR", + "raw": { + "question": "Require explicit proof classification on every PR", + "chosen": "Require explicit proof classification on every PR", + "alternatives": [ + { + "option": "Infer only from conventional titles", + "reason": "" + }, + { + "option": "Run Cloud for every PR", + "reason": "" + } + ], + "reasoning": "Silently treating missing metadata as non-functional would let non-conventional feature and fix titles bypass the gate. Every PR must declare feature, bugfix, or non-functional; only the first two launch Cloud." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Implemented trusted PR classification and single-case dispatch, a fail-closed Cloud RelayFlow that proves base then head in distinct per-step sandboxes, provenance/evidence validation, GitHub Action wiring, tests, template guidance, and credential rollout documentation.", + "approach": "Kept pull_request_target as a data-only trusted dispatcher, ran PR code only in Cloud, represented expected-red as a structured successful observation, disabled repair retries, validated exact base/head SHAs and sandbox IDs, and tested both the contracts and the complete repository.", + "confidence": 0.92 + }, + "commits": [], + "filesChanged": [], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "191e5f14a343431ec42282c1103c9f014854ac0b", + "endRef": "191e5f14a343431ec42282c1103c9f014854ac0b" + } +} \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ae2c91eb3..59f7513df 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,6 +7,15 @@ - [ ] Tests added/updated - [ ] Manual testing completed +## RelayFlow Proof + +Replace both values below. Use `feature` or `bugfix` for user-visible behavior +changes and add exactly one case under `tests/relayflows/cases//`. +Use `non-functional` and `n/a` only when runtime behavior is unchanged. + +- Change type: `replace-me` +- RelayFlow case: `replace-me` + ## Screenshots diff --git a/.github/workflows/relayflow-pr-proof.yml b/.github/workflows/relayflow-pr-proof.yml new file mode 100644 index 000000000..78b4c6613 --- /dev/null +++ b/.github/workflows/relayflow-pr-proof.yml @@ -0,0 +1,96 @@ +name: RelayFlow PR Proof + +# Security boundary: pull_request_target gives this trusted base-branch +# dispatcher access to Cloud credentials. It MUST NOT checkout, import, or run +# PR-head code on the GitHub runner. The exact head is cloned only inside the +# isolated Cloud proof sandboxes by scripts/pr-proof/run-arm.mjs. +on: + pull_request_target: + types: [opened, synchronize, reopened, edited, ready_for_review] + workflow_dispatch: + inputs: + pr_number: + description: Pull request number to prove + required: true + type: number + +permissions: + contents: read + pull-requests: read + +concurrency: + group: relayflow-pr-proof-${{ github.event.pull_request.number || inputs.pr_number }} + cancel-in-progress: true + +jobs: + proof: + name: RelayFlow PR proof + runs-on: ubuntu-latest + timeout-minutes: 70 + steps: + - name: Checkout trusted base + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22.14.0 + + - name: Classify PR and validate declared case + id: prepare + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node scripts/pr-proof/prepare.mjs + --event "$GITHUB_EVENT_PATH" + --output .relayflow/pr-proof-input.json + --github-output "$GITHUB_OUTPUT" + --summary "$GITHUB_STEP_SUMMARY" + + - name: Confirm Cloud proof credential + if: steps.prepare.outputs.required == 'true' + env: + CLOUD_API_URL: ${{ secrets.CLOUD_API_URL }} + CLOUD_API_ACCESS_TOKEN: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN }} + CLOUD_API_REFRESH_TOKEN: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN }} + CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN_EXPIRES_AT }} + CLOUD_API_REFRESH_TOKEN_EXPIRES_AT: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN_EXPIRES_AT }} + run: | + test -n "$CLOUD_API_URL" + test -n "$CLOUD_API_ACCESS_TOKEN" + test -n "$CLOUD_API_REFRESH_TOKEN" + test -n "$CLOUD_API_ACCESS_TOKEN_EXPIRES_AT" + + # Cloud code sync intentionally uploads only paths known to git. Add the + # generated, validated input to this disposable runner's index so it is + # present in the Cloud workspace; this does not create a commit or push. + - name: Include proof input in Cloud code sync + if: steps.prepare.outputs.required == 'true' + run: git add -f -- .relayflow/pr-proof-input.json + + - name: Install the released Agent Relay CLI + if: steps.prepare.outputs.required == 'true' + run: npm install --global "agent-relay@$(node -p "require('./package.json').version")" + + - name: Run isolated Cloud red-green proof + if: steps.prepare.outputs.required == 'true' + id: cloud + env: + CLOUD_API_URL: ${{ secrets.CLOUD_API_URL }} + CLOUD_API_ACCESS_TOKEN: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN }} + CLOUD_API_REFRESH_TOKEN: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN }} + CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN_EXPIRES_AT }} + CLOUD_API_REFRESH_TOKEN_EXPIRES_AT: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN_EXPIRES_AT }} + PR_PROOF_CLOUD_TIMEOUT_MS: '3600000' + run: node scripts/pr-proof/run-cloud.mjs workflows/pr-proof.ts + + - name: Upload Cloud proof logs + if: always() && steps.prepare.outputs.required == 'true' + uses: actions/upload-artifact@v4 + with: + name: relayflow-pr-proof-${{ github.event.pull_request.number || inputs.pr_number }} + path: .workflow-artifacts/pr-proof/cloud.log + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 60dbfe742..dfa0f28b2 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,7 @@ __pycache__/ .agentworkforce/relay/ .relay/ .agent-relay/ +.relayflow/pr-proof-input.json .msd/ /workflows/* @@ -91,6 +92,7 @@ __pycache__/ !/workflows/cloud-connect/ !/workflows/verify-features.ts !/workflows/audit-feature-manifest.ts +!/workflows/pr-proof.ts # Eval harness JSON reports (generated per run) tests/integration/broker/evals-reports/ diff --git a/scripts/pr-proof/contract.mjs b/scripts/pr-proof/contract.mjs new file mode 100644 index 000000000..b9b4dbd87 --- /dev/null +++ b/scripts/pr-proof/contract.mjs @@ -0,0 +1,309 @@ +import path from 'node:path'; + +export const PR_PROOF_VERSION = 1; +export const CASE_ROOT = 'tests/relayflows/cases'; +export const INPUT_PATH = '.relayflow/pr-proof-input.json'; +export const ARTIFACT_ROOT = '.workflow-artifacts/pr-proof'; + +const CASE_ID_RE = /^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$/; +const SHA_RE = /^[0-9a-f]{40}$/; +const REPOSITORY_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const SIGNATURE_RE = /^[a-z0-9](?:[a-z0-9._:-]{0,127})$/; +const REQUIRED_TITLE_RE = /^(feat|fix)(?:\([^)]*\))?!?:/i; +const TYPE_MARKER_RE = /^\s*-\s*Change type:\s*`([^`]+)`\s*\s*$/im; +const CASE_MARKER_RE = /^\s*-\s*RelayFlow case:\s*`([^`]+)`\s*\s*$/im; + +export class PrProofContractError extends Error { + constructor(message, details = []) { + super(message); + this.name = 'PrProofContractError'; + this.details = details; + } +} + +function nonEmptyString(value) { + return typeof value === 'string' && value.trim() ? value.trim() : null; +} + +function assertObject(value, label, errors) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + errors.push(`${label} must be an object`); + return null; + } + return value; +} + +export function caseManifestPath(caseId) { + if (!CASE_ID_RE.test(caseId)) { + throw new PrProofContractError(`Invalid RelayFlow case id: ${caseId}`); + } + return `${CASE_ROOT}/${caseId}/case.json`; +} + +export function changedRelayFlowCaseIds(files = []) { + const prefix = `${CASE_ROOT}/`; + return [ + ...new Set( + files + .filter((file) => typeof file === 'string' && file.startsWith(prefix)) + .map((file) => file.slice(prefix.length).split('/')) + .filter((parts) => parts.length > 1 && CASE_ID_RE.test(parts[0])) + .map((parts) => parts[0]) + ), + ].sort(); +} + +export function parsePrProofMetadata(body = '') { + const valuesFor = (pattern) => + [...body.matchAll(new RegExp(pattern.source, `${pattern.flags}g`))].map((match) => + match[1].trim().toLowerCase() + ); + const changeTypes = valuesFor(TYPE_MARKER_RE); + const caseIds = valuesFor(CASE_MARKER_RE); + return { + changeType: changeTypes[0] ?? null, + caseId: caseIds[0] ?? null, + changeTypeCount: changeTypes.length, + caseIdCount: caseIds.length, + }; +} + +export function classifyPullRequest({ title = '', body = '' }) { + const metadata = parsePrProofMetadata(body); + const conventionalKind = REQUIRED_TITLE_RE.exec(title)?.[1]?.toLowerCase() ?? null; + const titleKind = conventionalKind === 'feat' ? 'feature' : conventionalKind === 'fix' ? 'bugfix' : null; + const errors = []; + + if (metadata.changeTypeCount === 0) { + errors.push('PR body must declare a RelayFlow Proof change type'); + } + if (metadata.changeTypeCount > 1) { + errors.push('PR body must contain exactly one RelayFlow Proof change type marker'); + } + if (metadata.caseIdCount === 0) { + errors.push('PR body must declare a RelayFlow Proof case'); + } + if (metadata.caseIdCount > 1) { + errors.push('PR body must contain exactly one RelayFlow Proof case marker'); + } + + if (!metadata.changeType) { + return { + required: Boolean(titleKind), + kind: titleKind, + caseId: null, + metadata, + errors, + reason: 'missing proof metadata', + }; + } + + if (!['feature', 'bugfix', 'non-functional'].includes(metadata.changeType)) { + errors.push( + `Unsupported RelayFlow Proof change type ${JSON.stringify(metadata.changeType)}; expected feature, bugfix, or non-functional` + ); + } + + const metadataKind = + metadata.changeType === 'feature' || metadata.changeType === 'bugfix' ? metadata.changeType : null; + if (titleKind && metadata.changeType === 'non-functional') { + errors.push(`PR title declares a ${titleKind}, so the change type cannot be non-functional`); + } + if (titleKind && metadataKind && titleKind !== metadataKind) { + errors.push(`PR title declares ${titleKind}, but the PR body declares ${metadataKind}`); + } + + const required = Boolean(metadataKind || titleKind); + const kind = metadataKind ?? titleKind; + if (required) { + if (!metadata.caseId || metadata.caseId === 'n/a') { + errors.push('Feature and bug-fix PRs must declare exactly one RelayFlow case id'); + } else if (!CASE_ID_RE.test(metadata.caseId)) { + errors.push(`Invalid RelayFlow case id: ${metadata.caseId}`); + } + } else if (metadata.caseId !== 'n/a') { + errors.push('Non-functional PRs must declare the RelayFlow case as n/a'); + } + + return { + required, + kind, + caseId: required && metadata.caseId !== 'n/a' ? metadata.caseId : null, + metadata, + errors, + reason: required ? `declared ${kind}` : 'declared non-functional', + }; +} + +function validateExpectation(value, label, allowedOutcome, errors) { + const expectation = assertObject(value, label, errors); + if (!expectation) return null; + const outcome = nonEmptyString(expectation.outcome); + const signature = nonEmptyString(expectation.signature); + if (outcome !== allowedOutcome) { + errors.push(`${label}.outcome must be ${allowedOutcome}`); + } + if (!signature || !SIGNATURE_RE.test(signature)) { + errors.push(`${label}.signature must match ${SIGNATURE_RE}`); + } + return outcome && signature ? { outcome, signature } : null; +} + +export function validateCaseManifest(value, { caseId, kind } = {}) { + const errors = []; + const manifest = assertObject(value, 'case manifest', errors); + if (!manifest) throw new PrProofContractError('Invalid RelayFlow case manifest', errors); + + const id = nonEmptyString(manifest.id); + const manifestKind = nonEmptyString(manifest.kind); + const title = nonEmptyString(manifest.title); + if (manifest.version !== PR_PROOF_VERSION) { + errors.push(`case manifest version must be ${PR_PROOF_VERSION}`); + } + if (!id || !CASE_ID_RE.test(id)) errors.push('case manifest id is invalid'); + if (caseId && id !== caseId) errors.push(`case manifest id ${id} does not match declared case ${caseId}`); + if (!['feature', 'bugfix'].includes(manifestKind)) { + errors.push('case manifest kind must be feature or bugfix'); + } + if (kind && manifestKind !== kind) { + errors.push(`case manifest kind ${manifestKind} does not match PR kind ${kind}`); + } + if (!title || title.length > 160) errors.push('case manifest title must be 1-160 characters'); + + const runner = assertObject(manifest.runner, 'case manifest runner', errors); + const command = Array.isArray(runner?.command) ? runner.command : null; + if (!command || command.length < 2 || command.some((entry) => !nonEmptyString(entry))) { + errors.push('case manifest runner.command must contain an executable and case script path'); + } else { + const executable = command[0]; + if (!['node', 'bash'].includes(executable)) { + errors.push('case manifest runner.command executable must be node or bash'); + } + const expectedRoot = `${CASE_ROOT}/${id}/`; + const scriptPath = command[1].replaceAll('\\', '/'); + if ( + path.posix.isAbsolute(scriptPath) || + scriptPath.includes('..') || + !scriptPath.startsWith(expectedRoot) + ) { + errors.push(`case runner script must stay under ${expectedRoot}`); + } + } + + const timeoutSeconds = Number(manifest.timeoutSeconds); + if (!Number.isInteger(timeoutSeconds) || timeoutSeconds < 30 || timeoutSeconds > 1800) { + errors.push('case manifest timeoutSeconds must be an integer between 30 and 1800'); + } + + const expected = assertObject(manifest.expected, 'case manifest expected', errors); + const baseOutcome = manifestKind === 'feature' ? 'absent' : 'bug'; + const base = validateExpectation(expected?.base, 'expected.base', baseOutcome, errors); + const head = validateExpectation(expected?.head, 'expected.head', 'fixed', errors); + + if (errors.length > 0) { + throw new PrProofContractError('Invalid RelayFlow case manifest', errors); + } + + return { + version: PR_PROOF_VERSION, + id, + kind: manifestKind, + title, + runner: { command: command.map((entry) => entry.trim()) }, + timeoutSeconds, + expected: { base, head }, + }; +} + +export function validateProofInput(value) { + const errors = []; + const input = assertObject(value, 'proof input', errors); + if (!input) throw new PrProofContractError('Invalid PR proof input', errors); + + const repository = nonEmptyString(input.repository); + const baseSha = nonEmptyString(input.baseSha); + const headSha = nonEmptyString(input.headSha); + const caseId = nonEmptyString(input.caseId); + const kind = nonEmptyString(input.kind); + const pullRequest = Number(input.pullRequest); + if (input.version !== PR_PROOF_VERSION) errors.push(`proof input version must be ${PR_PROOF_VERSION}`); + if (!repository || !REPOSITORY_RE.test(repository)) errors.push('proof input repository is invalid'); + if (!Number.isInteger(pullRequest) || pullRequest < 1) { + errors.push('proof input pullRequest must be a positive integer'); + } + if (!baseSha || !SHA_RE.test(baseSha)) errors.push('proof input baseSha must be a full lowercase SHA'); + if (!headSha || !SHA_RE.test(headSha)) errors.push('proof input headSha must be a full lowercase SHA'); + if (baseSha && headSha && baseSha === headSha) errors.push('proof input baseSha and headSha must differ'); + if (!caseId || !CASE_ID_RE.test(caseId)) errors.push('proof input caseId is invalid'); + if (!['feature', 'bugfix'].includes(kind)) errors.push('proof input kind must be feature or bugfix'); + + let manifest = null; + try { + manifest = validateCaseManifest(input.manifest, { caseId, kind }); + } catch (error) { + if (error instanceof PrProofContractError) errors.push(...error.details); + else throw error; + } + if (errors.length > 0) throw new PrProofContractError('Invalid PR proof input', errors); + + return { + version: PR_PROOF_VERSION, + repository, + pullRequest, + baseSha, + headSha, + caseId, + kind, + manifest, + }; +} + +export function validateObservation(value, { caseId, arm, expected }) { + const errors = []; + const observation = assertObject(value, 'case observation', errors); + if (!observation) throw new PrProofContractError('Invalid case observation', errors); + if (observation.version !== PR_PROOF_VERSION) + errors.push(`observation version must be ${PR_PROOF_VERSION}`); + if (observation.caseId !== caseId) errors.push('observation caseId does not match the dispatched case'); + if (observation.arm !== arm) errors.push('observation arm does not match the dispatched arm'); + if (observation.outcome !== expected.outcome) { + errors.push(`observation outcome ${observation.outcome} does not match expected ${expected.outcome}`); + } + if (observation.signature !== expected.signature) { + errors.push( + `observation signature ${observation.signature} does not match expected ${expected.signature}` + ); + } + if (errors.length > 0) throw new PrProofContractError('Invalid case observation', errors); + return { + version: PR_PROOF_VERSION, + caseId, + arm, + outcome: observation.outcome, + signature: observation.signature, + details: nonEmptyString(observation.details) ?? '', + }; +} + +export function validateEvidence(value, input, arm) { + const errors = []; + const evidence = assertObject(value, `${arm} evidence`, errors); + if (!evidence) throw new PrProofContractError(`Invalid ${arm} evidence`, errors); + const expectedSha = arm === 'base' ? input.baseSha : input.headSha; + const expected = input.manifest.expected[arm]; + if (evidence.version !== PR_PROOF_VERSION) errors.push(`${arm} evidence version is invalid`); + if (evidence.caseId !== input.caseId) errors.push(`${arm} evidence caseId is invalid`); + if (evidence.arm !== arm) errors.push(`${arm} evidence arm is invalid`); + if (evidence.repository !== input.repository) errors.push(`${arm} evidence repository is invalid`); + if (evidence.pullRequest !== input.pullRequest) { + errors.push(`${arm} evidence pull request is invalid`); + } + if (evidence.targetSha !== expectedSha) errors.push(`${arm} evidence target SHA is invalid`); + if (evidence.harnessSha !== input.headSha) errors.push(`${arm} evidence harness SHA is invalid`); + if (!nonEmptyString(evidence.sandboxId)) errors.push(`${arm} evidence sandboxId is missing`); + if (evidence.runnerExitCode !== 0) errors.push(`${arm} evidence runner exit code is not zero`); + if (evidence.outcome !== expected.outcome) errors.push(`${arm} evidence outcome is invalid`); + if (evidence.signature !== expected.signature) errors.push(`${arm} evidence signature is invalid`); + if (errors.length > 0) throw new PrProofContractError(`Invalid ${arm} evidence`, errors); + return evidence; +} diff --git a/scripts/pr-proof/prepare.mjs b/scripts/pr-proof/prepare.mjs new file mode 100644 index 000000000..2f80be989 --- /dev/null +++ b/scripts/pr-proof/prepare.mjs @@ -0,0 +1,198 @@ +#!/usr/bin/env node + +import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; + +import { + INPUT_PATH, + PR_PROOF_VERSION, + PrProofContractError, + caseManifestPath, + changedRelayFlowCaseIds, + classifyPullRequest, + validateCaseManifest, +} from './contract.mjs'; + +function option(name, fallback = null) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : fallback; +} + +function githubHeaders(token) { + return { + accept: 'application/vnd.github+json', + authorization: `Bearer ${token}`, + 'x-github-api-version': '2022-11-28', + 'user-agent': 'relay-pr-proof-dispatcher', + }; +} + +async function githubJson(url, token) { + const response = await fetch(url, { headers: githubHeaders(token) }); + if (!response.ok) { + throw new Error(`GitHub API ${response.status} for ${url}`); + } + return response.json(); +} + +async function pullRequestFiles(apiUrl, repository, number, token) { + const files = []; + for (let page = 1; page <= 30; page += 1) { + const batch = await githubJson( + `${apiUrl}/repos/${repository}/pulls/${number}/files?per_page=100&page=${page}`, + token + ); + if (!Array.isArray(batch)) throw new Error('GitHub pull request files response was not an array'); + files.push(...batch.map((entry) => entry.filename).filter((entry) => typeof entry === 'string')); + if (batch.length < 100) return files; + } + throw new Error('PR changes more than 3,000 files; RelayFlow proof dispatch refuses ambiguous scope'); +} + +async function readHeadFile(apiUrl, repository, filePath, headSha, token) { + const payload = await githubJson( + `${apiUrl}/repos/${repository}/contents/${filePath}?ref=${encodeURIComponent(headSha)}`, + token + ); + if (payload?.encoding !== 'base64' || typeof payload.content !== 'string') { + throw new Error(`GitHub did not return base64 content for ${filePath}`); + } + return Buffer.from(payload.content.replaceAll('\n', ''), 'base64').toString('utf8'); +} + +function pullRequestFromPayload(payload) { + if (payload.pull_request) return payload.pull_request; + return null; +} + +async function writeGithubOutput(values, outputPath) { + if (!outputPath) return; + await appendFile( + outputPath, + Object.entries(values) + .map(([key, value]) => `${key}=${String(value)}`) + .join('\n') + '\n' + ); +} + +async function writeSummary(lines, summaryPath) { + if (!summaryPath) return; + await appendFile(summaryPath, `${lines.join('\n')}\n`); +} + +async function main() { + const eventPath = option('--event', process.env.GITHUB_EVENT_PATH); + const outputPath = option('--output', INPUT_PATH); + const githubOutput = option('--github-output', process.env.GITHUB_OUTPUT); + const summaryPath = option('--summary', process.env.GITHUB_STEP_SUMMARY); + const token = process.env.GITHUB_TOKEN?.trim(); + const repository = process.env.GITHUB_REPOSITORY?.trim(); + const apiUrl = (process.env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + if (!eventPath || !token || !repository) { + throw new Error('GITHUB_EVENT_PATH, GITHUB_TOKEN, and GITHUB_REPOSITORY are required'); + } + + const payload = JSON.parse(await readFile(eventPath, 'utf8')); + let pullRequest = pullRequestFromPayload(payload); + const manualNumber = Number(payload.inputs?.pr_number); + if (!pullRequest && Number.isInteger(manualNumber) && manualNumber > 0) { + pullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${manualNumber}`, token); + } + if (!pullRequest) throw new Error('The event does not identify a pull request'); + + const number = Number(pullRequest.number); + const headSha = pullRequest.head?.sha; + const baseSha = pullRequest.base?.sha; + const headRepository = pullRequest.head?.repo?.full_name; + if (!Number.isInteger(number) || !headSha || !baseSha || !headRepository) { + throw new Error('Pull request payload is missing number, repository, or exact SHAs'); + } + + const classification = classifyPullRequest({ title: pullRequest.title, body: pullRequest.body ?? '' }); + if (classification.errors.length > 0) { + throw new PrProofContractError( + 'Pull request does not satisfy the RelayFlow proof contract', + classification.errors + ); + } + if (!classification.required) { + await writeGithubOutput({ required: false, case_id: 'n/a' }, githubOutput); + await writeSummary( + ['## RelayFlow PR proof', '', 'Cloud proof is not required for this non-functional change.'], + summaryPath + ); + return; + } + if (headRepository !== repository) { + throw new PrProofContractError('Fork pull requests cannot receive the credential-bearing Cloud proof', [ + `head repository ${headRepository} is not the trusted repository ${repository}`, + 'A maintainer must reproduce the change on a same-repository branch before merge.', + ]); + } + + const caseId = classification.caseId; + const manifestPath = caseManifestPath(caseId); + const changedFiles = await pullRequestFiles(apiUrl, repository, number, token); + const caseRoot = path.posix.dirname(manifestPath) + '/'; + const changedCaseIds = changedRelayFlowCaseIds(changedFiles); + if (!changedFiles.some((file) => file === manifestPath || file.startsWith(caseRoot))) { + throw new PrProofContractError('The declared RelayFlow case is not changed by this PR', [ + `expected a changed file under ${caseRoot}`, + ]); + } + if (changedCaseIds.length !== 1 || changedCaseIds[0] !== caseId) { + throw new PrProofContractError('A PR must change exactly its one declared RelayFlow case', [ + `declared ${caseId}; changed ${changedCaseIds.join(', ') || 'none'}`, + ]); + } + + const manifestSource = await readHeadFile(apiUrl, repository, manifestPath, headSha, token); + let manifestJson; + try { + manifestJson = JSON.parse(manifestSource); + } catch (error) { + throw new PrProofContractError(`RelayFlow case manifest is not valid JSON: ${manifestPath}`, [ + error instanceof Error ? error.message : String(error), + ]); + } + const manifest = validateCaseManifest(manifestJson, { + caseId, + kind: classification.kind, + }); + + const proofInput = { + version: PR_PROOF_VERSION, + repository, + pullRequest: number, + baseSha, + headSha, + caseId, + kind: classification.kind, + manifestPath, + manifest, + }; + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(proofInput, null, 2)}\n`); + await writeGithubOutput({ required: true, case_id: caseId, input_path: outputPath }, githubOutput); + await writeSummary( + [ + '## RelayFlow PR proof', + '', + `- Case: \`${caseId}\``, + `- Base: \`${baseSha}\``, + `- Head: \`${headSha}\``, + '- Order: reproduce on base, then verify on head', + ], + summaryPath + ); +} + +main().catch(async (error) => { + const details = error instanceof PrProofContractError ? error.details : []; + const lines = ['## RelayFlow PR proof', '', `**Contract failure:** ${error.message}`]; + if (details.length > 0) lines.push('', ...details.map((detail) => `- ${detail}`)); + await writeSummary(lines, option('--summary', process.env.GITHUB_STEP_SUMMARY)).catch(() => {}); + console.error(error.message); + for (const detail of details) console.error(`- ${detail}`); + process.exitCode = 1; +}); diff --git a/scripts/pr-proof/run-arm.mjs b/scripts/pr-proof/run-arm.mjs new file mode 100644 index 000000000..e505c2205 --- /dev/null +++ b/scripts/pr-proof/run-arm.mjs @@ -0,0 +1,202 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { + ARTIFACT_ROOT, + INPUT_PATH, + PR_PROOF_VERSION, + PrProofContractError, + validateCaseManifest, + validateObservation, + validateProofInput, +} from './contract.mjs'; + +const MAX_CAPTURE_BYTES = 128 * 1024; + +function appendBounded(current, chunk) { + const next = current + chunk; + return next.length <= MAX_CAPTURE_BYTES ? next : next.slice(next.length - MAX_CAPTURE_BYTES); +} + +async function run(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { + const text = chunk.toString(); + stdout = appendBounded(stdout, text); + process.stdout.write(text); + }); + child.stderr.on('data', (chunk) => { + const text = chunk.toString(); + stderr = appendBounded(stderr, text); + process.stderr.write(text); + }); + const timeout = options.timeoutMs + ? setTimeout(() => { + child.kill('SIGTERM'); + setTimeout(() => child.kill('SIGKILL'), 5_000).unref(); + }, options.timeoutMs) + : null; + child.on('error', reject); + child.on('close', (code, signal) => { + if (timeout) clearTimeout(timeout); + resolve({ exitCode: code ?? 1, signal, stdout, stderr }); + }); + }); +} + +async function runChecked(command, args, options = {}) { + const result = await run(command, args, options); + if (result.exitCode !== 0) { + throw new Error( + `${command} ${args.join(' ')} failed with exit ${result.exitCode}${result.signal ? ` (${result.signal})` : ''}` + ); + } + return result; +} + +async function checkout(repository, sha, destination) { + const remote = `https://github.com/${repository}.git`; + await mkdir(destination, { recursive: true }); + await runChecked('git', ['init', '--quiet', destination]); + await runChecked('git', ['-C', destination, 'remote', 'add', 'origin', remote]); + await runChecked('git', ['-C', destination, 'fetch', '--quiet', '--depth=1', 'origin', sha], { + timeoutMs: 5 * 60_000, + }); + await runChecked('git', ['-C', destination, 'checkout', '--quiet', '--detach', 'FETCH_HEAD']); + const result = await runChecked('git', ['-C', destination, 'rev-parse', 'HEAD']); + const actual = result.stdout.trim(); + if (actual !== sha) throw new Error(`checkout provenance mismatch: expected ${sha}, got ${actual}`); + return actual; +} + +function sanitizedCaseEnvironment({ temporaryHome, targetDir, harnessDir, resultPath, input, arm }) { + const allowed = ['PATH', 'LANG', 'LC_ALL', 'TMPDIR', 'TMP', 'TEMP', 'SYSTEMROOT', 'WINDIR']; + const env = Object.fromEntries( + allowed.map((key) => [key, process.env[key]]).filter((entry) => typeof entry[1] === 'string' && entry[1]) + ); + return { + ...env, + HOME: temporaryHome, + CI: '1', + AGENT_RELAY_TELEMETRY_DISABLED: '1', + RELAY_PR_PROOF_ARM: arm, + RELAY_PR_PROOF_CASE_ID: input.caseId, + RELAY_PR_PROOF_BASE_SHA: input.baseSha, + RELAY_PR_PROOF_HEAD_SHA: input.headSha, + RELAY_PR_PROOF_TARGET_SHA: arm === 'base' ? input.baseSha : input.headSha, + RELAY_PR_PROOF_TARGET_DIR: targetDir, + RELAY_PR_PROOF_HARNESS_DIR: harnessDir, + RELAY_PR_PROOF_RESULT_PATH: resultPath, + }; +} + +function armFromArg() { + const arm = process.argv[2]; + if (arm !== 'base' && arm !== 'head') throw new Error('Usage: run-arm.mjs [input-path]'); + return arm; +} + +async function main() { + const arm = armFromArg(); + const inputPath = process.argv[3] ?? process.env.RELAY_PR_PROOF_INPUT ?? INPUT_PATH; + const input = validateProofInput(JSON.parse(await readFile(inputPath, 'utf8'))); + const sandboxId = process.env.SANDBOX_ID?.trim(); + if (!sandboxId) throw new Error('SANDBOX_ID is required; this proof arm must run as a Cloud step'); + + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), `relay-pr-proof-${arm}-`)); + const harnessDir = path.join(temporaryRoot, 'harness'); + const targetDir = path.join(temporaryRoot, 'target'); + const temporaryHome = path.join(temporaryRoot, 'home'); + const resultPath = path.join(temporaryRoot, 'observation.json'); + const targetSha = arm === 'base' ? input.baseSha : input.headSha; + const evidencePath = path.join(ARTIFACT_ROOT, `${arm}.json`); + + try { + await mkdir(temporaryHome, { recursive: true }); + const harnessSha = await checkout(input.repository, input.headSha, harnessDir); + const actualTargetSha = await checkout(input.repository, targetSha, targetDir); + + const manifestPath = path.join(harnessDir, 'tests', 'relayflows', 'cases', input.caseId, 'case.json'); + const headManifest = validateCaseManifest(JSON.parse(await readFile(manifestPath, 'utf8')), { + caseId: input.caseId, + kind: input.kind, + }); + if (JSON.stringify(headManifest) !== JSON.stringify(input.manifest)) { + throw new Error('Staged case manifest does not match the exact PR head checkout'); + } + + const [command, ...args] = input.manifest.runner.command; + const result = await run(command, args, { + cwd: harnessDir, + env: sanitizedCaseEnvironment({ + temporaryHome, + targetDir, + harnessDir, + resultPath, + input, + arm, + }), + timeoutMs: input.manifest.timeoutSeconds * 1000, + }); + if (result.exitCode !== 0) { + throw new Error( + `Case runner failed with exit ${result.exitCode}; expected-red behavior must be reported as a successful structured observation` + ); + } + + let observationJson; + try { + observationJson = JSON.parse(await readFile(resultPath, 'utf8')); + } catch (error) { + throw new PrProofContractError('Case runner did not write a valid observation JSON file', [ + error instanceof Error ? error.message : String(error), + ]); + } + const observation = validateObservation(observationJson, { + caseId: input.caseId, + arm, + expected: input.manifest.expected[arm], + }); + const evidence = { + version: PR_PROOF_VERSION, + caseId: input.caseId, + arm, + repository: input.repository, + pullRequest: input.pullRequest, + targetSha: actualTargetSha, + harnessSha, + sandboxId, + runnerExitCode: result.exitCode, + outcome: observation.outcome, + signature: observation.signature, + details: observation.details, + capturedStdout: result.stdout.slice(-8_000), + capturedStderr: result.stderr.slice(-8_000), + completedAt: new Date().toISOString(), + }; + await mkdir(path.dirname(evidencePath), { recursive: true }); + await writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + console.log(`PR_PROOF_ARM_COMPLETE arm=${arm} case=${input.caseId} sandbox=${sandboxId}`); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } +} + +main().catch((error) => { + console.error(error.message); + if (error instanceof PrProofContractError) { + for (const detail of error.details) console.error(`- ${detail}`); + } + process.exitCode = 1; +}); diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs new file mode 100644 index 000000000..a5e5eb513 --- /dev/null +++ b/scripts/pr-proof/run-cloud.mjs @@ -0,0 +1,108 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { appendFile, mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; + +const TERMINAL_SUCCESS = new Set(['completed', 'succeeded', 'success']); +const TERMINAL_FAILURE = new Set(['failed', 'cancelled', 'canceled', 'timed_out', 'error']); + +function run(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + env: options.env ?? process.env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + if (!options.quiet) process.stdout.write(chunk); + }); + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + if (!options.quiet) process.stderr.write(chunk); + }); + child.on('error', reject); + child.on('close', (code) => resolve({ exitCode: code ?? 1, stdout, stderr })); + }); +} + +function parseJsonOutput(output, label) { + try { + return JSON.parse(output); + } catch { + const first = output.indexOf('{'); + const last = output.lastIndexOf('}'); + if (first >= 0 && last > first) return JSON.parse(output.slice(first, last + 1)); + throw new Error(`${label} did not return JSON`); + } +} + +function statusFrom(payload) { + for (const candidate of [payload.status, payload.run?.status, payload.workflowRun?.status]) { + if (typeof candidate === 'string') return candidate.toLowerCase(); + } + throw new Error('Cloud status response did not contain a status'); +} + +async function main() { + const cli = process.env.PR_PROOF_AGENT_RELAY_BIN ?? 'agent-relay'; + const workflowPath = process.argv[2] ?? 'workflows/pr-proof.ts'; + const logsPath = process.env.PR_PROOF_CLOUD_LOG_PATH ?? '.workflow-artifacts/pr-proof/cloud.log'; + const pollMs = Number(process.env.PR_PROOF_POLL_MS ?? '15000'); + const timeoutMs = Number(process.env.PR_PROOF_CLOUD_TIMEOUT_MS ?? String(60 * 60_000)); + const launch = await run(cli, ['cloud', 'run', workflowPath, '--sync-code', '--json'], { quiet: true }); + if (launch.exitCode !== 0) { + process.stderr.write(launch.stderr); + throw new Error(`Cloud workflow submission failed with exit ${launch.exitCode}`); + } + const launchPayload = parseJsonOutput(launch.stdout, 'Cloud run'); + const runId = launchPayload.runId; + if (typeof runId !== 'string' || !runId) throw new Error('Cloud run response did not contain runId'); + console.log(`Cloud RelayFlow run: ${runId}`); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `run_id=${runId}\n`); + + const deadline = Date.now() + timeoutMs; + let terminalStatus = null; + while (Date.now() < deadline) { + await delay(pollMs); + const statusResult = await run(cli, ['cloud', 'status', runId, '--json'], { quiet: true }); + if (statusResult.exitCode !== 0) { + console.warn(`Cloud status poll failed (${statusResult.exitCode}); retrying`); + continue; + } + const status = statusFrom(parseJsonOutput(statusResult.stdout, 'Cloud status')); + console.log(`Cloud RelayFlow status: ${status}`); + if (TERMINAL_SUCCESS.has(status) || TERMINAL_FAILURE.has(status)) { + terminalStatus = status; + break; + } + } + if (!terminalStatus) { + await run(cli, ['cloud', 'cancel', runId], { quiet: true }); + throw new Error(`Cloud RelayFlow exceeded ${timeoutMs}ms`); + } + + await mkdir(path.dirname(logsPath), { recursive: true }); + const logs = await run(cli, ['cloud', 'logs', runId], { quiet: true }); + await writeFile(logsPath, logs.stdout + logs.stderr); + if (logs.stdout) process.stdout.write(logs.stdout); + if (logs.stderr) process.stderr.write(logs.stderr); + + if (!TERMINAL_SUCCESS.has(terminalStatus)) { + throw new Error(`Cloud RelayFlow finished with status ${terminalStatus}`); + } + if (process.env.GITHUB_STEP_SUMMARY) { + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `\n- Cloud run: \`${runId}\`\n- Cloud status: **${terminalStatus}**\n` + ); + } +} + +main().catch((error) => { + console.error(error.message); + process.exitCode = 1; +}); diff --git a/scripts/pr-proof/verify-evidence.mjs b/scripts/pr-proof/verify-evidence.mjs new file mode 100644 index 000000000..a06de778f --- /dev/null +++ b/scripts/pr-proof/verify-evidence.mjs @@ -0,0 +1,82 @@ +#!/usr/bin/env node + +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + ARTIFACT_ROOT, + INPUT_PATH, + PR_PROOF_VERSION, + PrProofContractError, + validateEvidence, + validateProofInput, +} from './contract.mjs'; + +function option(name, fallback = null) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : fallback; +} + +async function readJson(filePath) { + return JSON.parse(await readFile(filePath, 'utf8')); +} + +export async function verifyEvidenceFiles({ inputPath, arm, artifactRoot = ARTIFACT_ROOT }) { + const input = validateProofInput(await readJson(inputPath)); + const base = validateEvidence(await readJson(path.join(artifactRoot, 'base.json')), input, 'base'); + if (arm === 'base') return { input, base, head: null }; + + const head = validateEvidence(await readJson(path.join(artifactRoot, 'head.json')), input, 'head'); + if (base.sandboxId === head.sandboxId) { + throw new PrProofContractError('Base and head proofs did not run in distinct sandboxes', [ + `both evidence files report sandbox ${base.sandboxId}`, + ]); + } + return { input, base, head }; +} + +async function main() { + const arm = option('--arm', 'both'); + if (arm !== 'base' && arm !== 'both') throw new Error('--arm must be base or both'); + const inputPath = option('--input', process.env.RELAY_PR_PROOF_INPUT ?? INPUT_PATH); + const artifactRoot = option('--artifacts', ARTIFACT_ROOT); + const { input, base, head } = await verifyEvidenceFiles({ inputPath, arm, artifactRoot }); + console.log(`PR_PROOF_BASE_VALID case=${input.caseId} outcome=${base.outcome} sandbox=${base.sandboxId}`); + if (!head) return; + + const verdict = { + version: PR_PROOF_VERSION, + verdict: 'PASS', + caseId: input.caseId, + pullRequest: input.pullRequest, + base: { + sha: base.targetSha, + outcome: base.outcome, + signature: base.signature, + sandboxId: base.sandboxId, + }, + head: { + sha: head.targetSha, + outcome: head.outcome, + signature: head.signature, + sandboxId: head.sandboxId, + }, + completedAt: new Date().toISOString(), + }; + await mkdir(artifactRoot, { recursive: true }); + await writeFile(path.join(artifactRoot, 'verdict.json'), `${JSON.stringify(verdict, null, 2)}\n`); + console.log( + `PR_PROOF_PASS case=${input.caseId} base_sandbox=${base.sandboxId} head_sandbox=${head.sandboxId}` + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error.message); + if (error instanceof PrProofContractError) { + for (const detail of error.details) console.error(`- ${detail}`); + } + process.exitCode = 1; + }); +} diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts new file mode 100644 index 000000000..67a320621 --- /dev/null +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -0,0 +1,232 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +// Action-facing code intentionally stays dependency-free ESM so GitHub can run +// it before npm install. Vitest can import it directly for contract coverage. +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { + PrProofContractError, + changedRelayFlowCaseIds, + classifyPullRequest, + validateCaseManifest, + validateEvidence, + validateProofInput, +} from '../../scripts/pr-proof/contract.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { verifyEvidenceFiles } from '../../scripts/pr-proof/verify-evidence.mjs'; + +const BASE_SHA = '1'.repeat(40); +const HEAD_SHA = '2'.repeat(40); +const CASE_ID = '1591-application-ack-reconnect'; + +function proofBody(type = 'bugfix', caseId = CASE_ID) { + return [ + '## RelayFlow Proof', + '', + `- Change type: \`${type}\` `, + `- RelayFlow case: \`${caseId}\` `, + ].join('\n'); +} + +function manifest(kind = 'bugfix') { + return { + version: 1, + id: CASE_ID, + kind, + title: 'Reconnect when application acknowledgements stop', + runner: { + command: ['node', `tests/relayflows/cases/${CASE_ID}/run.mjs`], + }, + timeoutSeconds: 900, + expected: { + base: { + outcome: kind === 'feature' ? 'absent' : 'bug', + signature: 'application_ack_stall_not_detected', + }, + head: { + outcome: 'fixed', + signature: 'application_ack_stall_reconnects', + }, + }, + }; +} + +function input() { + return validateProofInput({ + version: 1, + repository: 'AgentWorkforce/relay', + pullRequest: 1610, + baseSha: BASE_SHA, + headSha: HEAD_SHA, + caseId: CASE_ID, + kind: 'bugfix', + manifest: manifest(), + }); +} + +function evidence(arm: 'base' | 'head', sandboxId: string) { + const proofInput = input(); + const expected = proofInput.manifest.expected[arm]; + return { + version: 1, + caseId: CASE_ID, + arm, + repository: 'AgentWorkforce/relay', + pullRequest: 1610, + targetSha: arm === 'base' ? BASE_SHA : HEAD_SHA, + harnessSha: HEAD_SHA, + sandboxId, + runnerExitCode: 0, + outcome: expected.outcome, + signature: expected.signature, + }; +} + +describe('RelayFlow PR proof classification', () => { + it('requires proof metadata for a conventional fix PR', () => { + const result = classifyPullRequest({ title: 'fix(broker): reconnect dead links', body: '' }); + expect(result.required).toBe(true); + expect(result.errors).toContainEqual( + expect.stringContaining('must declare a RelayFlow Proof change type') + ); + }); + + it('requires explicit classification even without a conventional title', () => { + const result = classifyPullRequest({ title: 'Update reconnect documentation', body: '' }); + expect(result.required).toBe(false); + expect(result.errors).toContainEqual( + expect.stringContaining('must declare a RelayFlow Proof change type') + ); + expect(result.errors).toContainEqual(expect.stringContaining('must declare a RelayFlow Proof case')); + }); + + it('selects exactly one declared bug-fix case', () => { + const result = classifyPullRequest({ + title: 'fix(broker): reconnect dead links', + body: proofBody(), + }); + expect(result).toMatchObject({ required: true, kind: 'bugfix', caseId: CASE_ID, errors: [] }); + }); + + it('passes a non-functional PR without Cloud work', () => { + const result = classifyPullRequest({ + title: 'docs: explain reconnect behavior', + body: proofBody('non-functional', 'n/a'), + }); + expect(result).toMatchObject({ required: false, caseId: null, errors: [] }); + }); + + it('rejects attempts to mark a fix title non-functional', () => { + const result = classifyPullRequest({ + title: 'fix(broker): reconnect dead links', + body: proofBody('non-functional', 'n/a'), + }); + expect(result.errors).toContainEqual(expect.stringContaining('cannot be non-functional')); + }); + + it('rejects duplicate proof selectors', () => { + const result = classifyPullRequest({ + title: 'fix(broker): reconnect dead links', + body: `${proofBody()}\n${proofBody()}`, + }); + expect(result.errors).toContainEqual(expect.stringContaining('exactly one RelayFlow Proof case')); + expect(result.errors).toContainEqual(expect.stringContaining('exactly one RelayFlow Proof change type')); + }); +}); + +describe('RelayFlow case manifest', () => { + it('identifies case directories without treating shared case docs as cases', () => { + expect( + changedRelayFlowCaseIds([ + 'tests/relayflows/cases/README.md', + `tests/relayflows/cases/${CASE_ID}/case.json`, + `tests/relayflows/cases/${CASE_ID}/run.mjs`, + ]) + ).toEqual([CASE_ID]); + }); + + it('accepts a structured external case runner', () => { + expect(validateCaseManifest(manifest(), { caseId: CASE_ID, kind: 'bugfix' })).toEqual(manifest()); + }); + + it('rejects a runner outside its case directory', () => { + const invalid = manifest(); + invalid.runner.command = ['node', 'scripts/untrusted.mjs']; + expect(() => validateCaseManifest(invalid, { caseId: CASE_ID, kind: 'bugfix' })).toThrow( + PrProofContractError + ); + }); + + it('requires an explicit absent outcome for feature bases', () => { + const invalid = manifest('feature'); + invalid.expected.base.outcome = 'bug'; + expect(() => validateCaseManifest(invalid, { caseId: CASE_ID, kind: 'feature' })).toThrow( + /Invalid RelayFlow case manifest/ + ); + }); +}); + +describe('RelayFlow evidence gates', () => { + it('accepts exact SHA, outcome, and signature provenance', () => { + const proofInput = input(); + expect(validateEvidence(evidence('base', 'sandbox-base'), proofInput, 'base')).toMatchObject({ + targetSha: BASE_SHA, + outcome: 'bug', + }); + }); + + it('rejects a base crash masquerading as expected red', () => { + const invalid = { ...evidence('base', 'sandbox-base'), runnerExitCode: 1 }; + try { + validateEvidence(invalid, input(), 'base'); + throw new Error('expected invalid evidence to fail'); + } catch (error) { + expect(error).toBeInstanceOf(PrProofContractError); + expect((error as { details: string[] }).details).toContainEqual( + expect.stringContaining('runner exit code') + ); + } + }); + + it('rejects base and head evidence from the same sandbox', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-test-')); + const inputPath = path.join(root, 'input.json'); + await writeFile(inputPath, JSON.stringify(input())); + await writeFile(path.join(root, 'base.json'), JSON.stringify(evidence('base', 'same-sandbox'))); + await writeFile(path.join(root, 'head.json'), JSON.stringify(evidence('head', 'same-sandbox'))); + await expect(verifyEvidenceFiles({ inputPath, arm: 'both', artifactRoot: root })).rejects.toThrow( + /distinct sandboxes/ + ); + await rm(root, { recursive: true, force: true }); + }); +}); + +describe('trusted dispatcher source contract', () => { + it('never checks out PR head code on the credential-bearing GitHub runner', async () => { + const source = await readFile('.github/workflows/relayflow-pr-proof.yml', 'utf8'); + expect(source).toContain('pull_request_target:'); + expect(source).toContain('ref: ${{ github.event.pull_request.base.sha || github.sha }}'); + expect(source).toContain('persist-credentials: false'); + expect(source).toContain('git add -f -- .relayflow/pr-proof-input.json'); + expect(source).not.toContain('github.event.pull_request.head.sha }}'); + }); + + it('gates head execution on deterministic base evidence', async () => { + const source = await readFile('workflows/pr-proof.ts', 'utf8'); + expect(source).toContain(".onError('fail-fast')"); + expect(source).toContain(".step('gate-base'"); + expect(source).toContain("dependsOn: ['gate-base']"); + expect(source).toContain(".step('gate-red-green'"); + expect(source).toContain('PR_PROOF_ARM_COMPLETE arm=base'); + expect(source).toContain('PR_PROOF_ARM_COMPLETE arm=head'); + }); + + it('records the per-step Cloud sandbox id instead of the orchestrator id', async () => { + const source = await readFile('scripts/pr-proof/run-arm.mjs', 'utf8'); + expect(source).toContain('process.env.SANDBOX_ID'); + expect(source).not.toContain('process.env.DAYTONA_SANDBOX_ID'); + }); +}); diff --git a/tests/relayflows/cases/README.md b/tests/relayflows/cases/README.md new file mode 100644 index 000000000..e48ddfc1b --- /dev/null +++ b/tests/relayflows/cases/README.md @@ -0,0 +1,108 @@ +# PR-specific RelayFlow cases + +Every Relay feature or bug-fix PR owns exactly one Cloud proof case in this +directory. The case is selected from the PR body and is the only expensive +Cloud scenario the PR check runs. + +Every PR must explicitly classify itself in the PR template. Runtime-neutral +changes declare `non-functional` and `n/a`; missing classification fails the +check instead of being silently treated as runtime-neutral. + +## Required files + +Create `tests/relayflows/cases//case.json` plus the runner named by its +`runner.command`: + +```json +{ + "version": 1, + "id": "1591-application-ack-reconnect", + "kind": "bugfix", + "title": "Reconnect when application acknowledgements stop", + "runner": { + "command": ["node", "tests/relayflows/cases/1591-application-ack-reconnect/run.mjs"] + }, + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "bug", + "signature": "application_ack_stall_not_detected" + }, + "head": { + "outcome": "fixed", + "signature": "application_ack_stall_reconnects" + } + } +} +``` + +Feature cases use `"absent"` for the base outcome and `"fixed"` for head. +Bug-fix cases use `"bug"` and `"fixed"`. + +## Runner contract + +The same runner from the exact PR head is executed against both target SHAs. +The wrapper supplies: + +- `RELAY_PR_PROOF_ARM`: `base` or `head` +- `RELAY_PR_PROOF_TARGET_DIR`: checkout of the target SHA +- `RELAY_PR_PROOF_HARNESS_DIR`: checkout of the exact head SHA +- `RELAY_PR_PROOF_RESULT_PATH`: destination for the observation JSON +- `RELAY_PR_PROOF_BASE_SHA` and `RELAY_PR_PROOF_HEAD_SHA` + +The runner must finish with exit code zero after observing behavior and write: + +```json +{ + "version": 1, + "caseId": "1591-application-ack-reconnect", + "arm": "base", + "outcome": "bug", + "signature": "application_ack_stall_not_detected", + "details": "WebSocket pongs continued but inventory.sync was never acknowledged." +} +``` + +Expected-red is data, not a failing process. A non-zero runner exit, missing +result, timeout, skipped test, missing test name, or build error is an +infrastructure failure and cannot prove the bug. + +Cases should exercise public/production behavior. A unit test added only on the +head cannot prove the base is broken because “test not found” is not a valid +observation. Prefer an external harness that builds the target checkout and +drives its CLI, broker, protocol, or API. + +## Execution and security + +The `pull_request_target` workflow checks out only the trusted base. It fetches +and validates the case manifest as data, then submits the trusted +`workflows/pr-proof.ts` RelayFlow. PR code runs only in the two Cloud agent +sandboxes. The base proof must pass its deterministic evidence gate before the +head sandbox is launched, and the final gate rejects identical sandbox IDs or +incorrect commit provenance. + +The RelayFlow is explicitly fail-fast with zero retries. Automatic repair +agents are disabled: a rejected observation is evidence to report, never an +invitation to edit the harness or artifacts until the gate passes. + +The action temporarily adds the generated `.relayflow/pr-proof-input.json` to +the runner's git index because Cloud code sync uploads git-known paths. It does +not commit or push the generated file. + +Fork PRs do not receive Cloud credentials. A maintainer must reproduce the +change and its case on a same-repository branch before merge. + +## Enabling the required check + +The repository needs a dedicated Cloud CLI session in these GitHub secrets: + +- `CLOUD_API_URL` +- `RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN` +- `RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN` +- `RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN_EXPIRES_AT` +- `RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN_EXPIRES_AT` (optional until refresh) + +Use a workspace-scoped, least-privilege credential that can prepare, invoke, +read, and cancel workflow runs. Do not copy a human laptop session into CI. +After a live canary PR proves the check, require the stable `RelayFlow PR proof` +job on `main` branch protection. diff --git a/workflows/pr-proof.ts b/workflows/pr-proof.ts new file mode 100644 index 000000000..4dff57b34 --- /dev/null +++ b/workflows/pr-proof.ts @@ -0,0 +1,86 @@ +/** + * PR-specific Cloud red/green proof. + * + * The trusted GitHub dispatcher runs this file from the PR base branch and + * stages `.relayflow/pr-proof-input.json`. PR head code is never executed on + * the GitHub runner. Each agent step receives a fresh Daytona sandbox from the + * Cloud SandboxedStepExecutor. + * + * Order is deliberate: + * 1. base-prover observes the exact bug/capability absence on the base SHA; + * 2. a deterministic gate rejects crashes, skips, missing tests, and the + * wrong failure signature; + * 3. head-verifier observes the declared fixed behavior on the head SHA; + * 4. a deterministic gate verifies exact SHAs and distinct sandbox IDs. + */ + +import { workflow } from '@relayflows/core'; + +const result = await workflow('relay-pr-proof') + .description( + 'Prove one declared Relay feature or bug-fix case red on the exact PR base SHA and green on the exact head SHA in distinct Cloud sandboxes.' + ) + .pattern('dag') + .channel('relay-pr-proof') + .maxConcurrency(1) + // Proof failures are evidence, not repair assignments. This also opts out + // of RelayFlow's default repair-agent retries so no agent can edit the + // harness or artifacts after a gate rejects them. + .onError('fail-fast') + .timeout(3_600_000) + .agent('base-prover', { + cli: 'codex', + preset: 'worker', + role: 'Run the trusted PR proof base-arm command exactly once without editing repository files.', + interactive: true, + retries: 0, + }) + .agent('head-verifier', { + cli: 'codex', + preset: 'worker', + role: 'Run the trusted PR proof head-arm command exactly once without editing repository files.', + interactive: true, + retries: 0, + }) + .step('prove-base', { + agent: 'base-prover', + task: [ + 'This is a deterministic verification assignment. Do not edit any files.', + 'Run exactly: node scripts/pr-proof/run-arm.mjs base .relayflow/pr-proof-input.json', + 'Wait for it to finish. If it succeeds, print its PR_PROOF_ARM_COMPLETE line and then print DONE.', + 'If it fails, preserve the failure and exit non-zero. Do not reinterpret a crash as bug reproduction.', + ].join('\n'), + verification: { type: 'output_contains', value: 'PR_PROOF_ARM_COMPLETE arm=base' }, + retries: 0, + }) + .step('gate-base', { + type: 'deterministic', + dependsOn: ['prove-base'], + command: 'node scripts/pr-proof/verify-evidence.mjs --arm base --input .relayflow/pr-proof-input.json', + captureOutput: true, + failOnError: true, + }) + .step('verify-head', { + agent: 'head-verifier', + dependsOn: ['gate-base'], + task: [ + 'This is a deterministic verification assignment. Do not edit any files.', + 'Run exactly: node scripts/pr-proof/run-arm.mjs head .relayflow/pr-proof-input.json', + 'Wait for it to finish. If it succeeds, print its PR_PROOF_ARM_COMPLETE line and then print DONE.', + 'If it fails, preserve the failure and exit non-zero. Do not manufacture or alter evidence.', + ].join('\n'), + verification: { type: 'output_contains', value: 'PR_PROOF_ARM_COMPLETE arm=head' }, + retries: 0, + }) + .step('gate-red-green', { + type: 'deterministic', + dependsOn: ['verify-head'], + command: 'node scripts/pr-proof/verify-evidence.mjs --arm both --input .relayflow/pr-proof-input.json', + captureOutput: true, + failOnError: true, + }) + .run({ cwd: process.cwd() }); + +if ('status' in result && result.status === 'failed') { + process.exitCode = 1; +} From fe507da32d93fea7f82aa77c0fa415691f3f97d2 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 11:58:34 +0200 Subject: [PATCH 02/15] fix(ci): harden RelayFlow PR proof gate --- .github/workflows/relayflow-pr-proof.yml | 32 ++- scripts/pr-proof/cloud-storage.mjs | 65 +++++ scripts/pr-proof/contract.mjs | 25 +- scripts/pr-proof/prepare.mjs | 73 +++++- scripts/pr-proof/process-runner.mjs | 89 +++++++ scripts/pr-proof/report-status.mjs | 156 ++++++++++++ scripts/pr-proof/run-arm.mjs | 85 +++---- scripts/pr-proof/run-cloud.mjs | 294 +++++++++++++++++------ scripts/pr-proof/verify-evidence.mjs | 25 +- tests/fixtures/pr-proof-contract.test.ts | 244 ++++++++++++++++++- tests/relayflows/cases/README.md | 39 ++- workflows/pr-proof.ts | 16 +- 12 files changed, 988 insertions(+), 155 deletions(-) create mode 100644 scripts/pr-proof/cloud-storage.mjs create mode 100644 scripts/pr-proof/process-runner.mjs create mode 100644 scripts/pr-proof/report-status.mjs diff --git a/.github/workflows/relayflow-pr-proof.yml b/.github/workflows/relayflow-pr-proof.yml index 78b4c6613..ef2715828 100644 --- a/.github/workflows/relayflow-pr-proof.yml +++ b/.github/workflows/relayflow-pr-proof.yml @@ -17,6 +17,7 @@ on: permissions: contents: read pull-requests: read + statuses: write concurrency: group: relayflow-pr-proof-${{ github.event.pull_request.number || inputs.pr_number }} @@ -24,21 +25,32 @@ concurrency: jobs: proof: - name: RelayFlow PR proof + name: RelayFlow PR proof dispatcher runs-on: ubuntu-latest timeout-minutes: 70 steps: - name: Checkout trusted base - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: ref: ${{ github.event.pull_request.base.sha || github.sha }} persist-credentials: false - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22.14.0 + # pull_request_target runs are attached to the base SHA. Publish the + # stable required context explicitly on the exact validated PR head. + - name: Start required status on PR head + id: status + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node scripts/pr-proof/report-status.mjs start + --event "$GITHUB_EVENT_PATH" + --github-output "$GITHUB_OUTPUT" + - name: Classify PR and validate declared case id: prepare env: @@ -47,6 +59,7 @@ jobs: node scripts/pr-proof/prepare.mjs --event "$GITHUB_EVENT_PATH" --output .relayflow/pr-proof-input.json + --expected-head-sha "${{ steps.status.outputs.head_sha }}" --github-output "$GITHUB_OUTPUT" --summary "$GITHUB_STEP_SUMMARY" @@ -89,8 +102,19 @@ jobs: - name: Upload Cloud proof logs if: always() && steps.prepare.outputs.required == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: relayflow-pr-proof-${{ github.event.pull_request.number || inputs.pr_number }} path: .workflow-artifacts/pr-proof/cloud.log if-no-files-found: warn + + - name: Finish required status on PR head + # A replacement run has already published pending on this same status + # context. A cancelled predecessor must not overwrite that newer state. + if: always() && !cancelled() && steps.status.outputs.head_sha != '' + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node scripts/pr-proof/report-status.mjs finish + --sha "${{ steps.status.outputs.head_sha }}" + --job-status "${{ job.status }}" diff --git a/scripts/pr-proof/cloud-storage.mjs b/scripts/pr-proof/cloud-storage.mjs new file mode 100644 index 000000000..c1340b4a0 --- /dev/null +++ b/scripts/pr-proof/cloud-storage.mjs @@ -0,0 +1,65 @@ +#!/usr/bin/env node + +const ARM_RE = /^(base|head)$/; +const NONCE_RE = /^[0-9a-f]{32}$/; + +function requiredEnvironment(env, name) { + const value = env[name]?.trim(); + if (!value) throw new Error(`${name} is required for Cloud proof evidence storage`); + return value; +} + +function storageUrl(env, input, arm) { + if (!ARM_RE.test(arm)) throw new Error(`Invalid proof arm: ${arm}`); + if (!NONCE_RE.test(input.handoffNonce ?? '')) throw new Error('Invalid proof handoff nonce'); + const apiUrl = requiredEnvironment(env, 'CLOUD_API_URL').replace(/\/$/, '') + '/'; + const runId = encodeURIComponent(requiredEnvironment(env, 'RUN_ID')); + const objectKey = ['pr-proof', input.handoffNonce, `${arm}.json`] + .map((segment) => encodeURIComponent(segment)) + .join('/'); + return new URL(`api/v1/workflows/runs/${runId}/storage/${objectKey}`, apiUrl); +} + +function authorization(env) { + return `Bearer ${requiredEnvironment(env, 'CLOUD_API_ACCESS_TOKEN')}`; +} + +export async function uploadCloudEvidence(input, arm, evidence, options = {}) { + const env = options.env ?? process.env; + const fetchImpl = options.fetchImpl ?? fetch; + const response = await fetchImpl(storageUrl(env, input, arm), { + method: 'PUT', + headers: { + accept: 'application/json', + authorization: authorization(env), + 'content-type': 'application/json', + }, + body: JSON.stringify(evidence), + }); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new Error(`Cloud evidence upload failed (${response.status})${detail ? `: ${detail}` : ''}`); + } +} + +export async function downloadCloudEvidence(input, arm, options = {}) { + const env = options.env ?? process.env; + const fetchImpl = options.fetchImpl ?? fetch; + const response = await fetchImpl(storageUrl(env, input, arm), { + headers: { + accept: 'application/json', + authorization: authorization(env), + }, + }); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new Error(`Cloud evidence download failed (${response.status})${detail ? `: ${detail}` : ''}`); + } + try { + return JSON.parse(await response.text()); + } catch (error) { + throw new Error( + `Cloud evidence response was not valid JSON: ${error instanceof Error ? error.message : String(error)}` + ); + } +} diff --git a/scripts/pr-proof/contract.mjs b/scripts/pr-proof/contract.mjs index b9b4dbd87..b1128947f 100644 --- a/scripts/pr-proof/contract.mjs +++ b/scripts/pr-proof/contract.mjs @@ -9,6 +9,8 @@ const CASE_ID_RE = /^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$/; const SHA_RE = /^[0-9a-f]{40}$/; const REPOSITORY_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; const SIGNATURE_RE = /^[a-z0-9](?:[a-z0-9._:-]{0,127})$/; +const HANDOFF_NONCE_RE = /^[0-9a-f]{32}$/; +const MAX_OBSERVATION_DETAILS_LENGTH = 4_000; const REQUIRED_TITLE_RE = /^(feat|fix)(?:\([^)]*\))?!?:/i; const TYPE_MARKER_RE = /^\s*-\s*Change type:\s*`([^`]+)`\s*\s*$/im; const CASE_MARKER_RE = /^\s*-\s*RelayFlow case:\s*`([^`]+)`\s*\s*$/im; @@ -47,7 +49,10 @@ export function changedRelayFlowCaseIds(files = []) { files .filter((file) => typeof file === 'string' && file.startsWith(prefix)) .map((file) => file.slice(prefix.length).split('/')) - .filter((parts) => parts.length > 1 && CASE_ID_RE.test(parts[0])) + // Keep malformed directory names in the result. The caller compares + // this set with the one declared valid case, so dropping an invalid + // sibling here would turn an ambiguous PR into a false single-case. + .filter((parts) => parts.length > 1 && parts[0]) .map((parts) => parts[0]) ), ].sort(); @@ -225,10 +230,11 @@ export function validateProofInput(value) { const headSha = nonEmptyString(input.headSha); const caseId = nonEmptyString(input.caseId); const kind = nonEmptyString(input.kind); - const pullRequest = Number(input.pullRequest); + const handoffNonce = nonEmptyString(input.handoffNonce); + const pullRequest = input.pullRequest; if (input.version !== PR_PROOF_VERSION) errors.push(`proof input version must be ${PR_PROOF_VERSION}`); if (!repository || !REPOSITORY_RE.test(repository)) errors.push('proof input repository is invalid'); - if (!Number.isInteger(pullRequest) || pullRequest < 1) { + if (typeof pullRequest !== 'number' || !Number.isInteger(pullRequest) || pullRequest < 1) { errors.push('proof input pullRequest must be a positive integer'); } if (!baseSha || !SHA_RE.test(baseSha)) errors.push('proof input baseSha must be a full lowercase SHA'); @@ -236,6 +242,9 @@ export function validateProofInput(value) { if (baseSha && headSha && baseSha === headSha) errors.push('proof input baseSha and headSha must differ'); if (!caseId || !CASE_ID_RE.test(caseId)) errors.push('proof input caseId is invalid'); if (!['feature', 'bugfix'].includes(kind)) errors.push('proof input kind must be feature or bugfix'); + if (!handoffNonce || !HANDOFF_NONCE_RE.test(handoffNonce)) { + errors.push('proof input handoffNonce must be 32 lowercase hexadecimal characters'); + } let manifest = null; try { @@ -254,6 +263,7 @@ export function validateProofInput(value) { headSha, caseId, kind, + handoffNonce, manifest, }; } @@ -274,6 +284,10 @@ export function validateObservation(value, { caseId, arm, expected }) { `observation signature ${observation.signature} does not match expected ${expected.signature}` ); } + const details = nonEmptyString(observation.details) ?? ''; + if (details.length > MAX_OBSERVATION_DETAILS_LENGTH) { + errors.push(`observation details must not exceed ${MAX_OBSERVATION_DETAILS_LENGTH} characters`); + } if (errors.length > 0) throw new PrProofContractError('Invalid case observation', errors); return { version: PR_PROOF_VERSION, @@ -281,7 +295,7 @@ export function validateObservation(value, { caseId, arm, expected }) { arm, outcome: observation.outcome, signature: observation.signature, - details: nonEmptyString(observation.details) ?? '', + details, }; } @@ -300,6 +314,9 @@ export function validateEvidence(value, input, arm) { } if (evidence.targetSha !== expectedSha) errors.push(`${arm} evidence target SHA is invalid`); if (evidence.harnessSha !== input.headSha) errors.push(`${arm} evidence harness SHA is invalid`); + if (evidence.handoffNonce !== input.handoffNonce) { + errors.push(`${arm} evidence handoff nonce is invalid`); + } if (!nonEmptyString(evidence.sandboxId)) errors.push(`${arm} evidence sandboxId is missing`); if (evidence.runnerExitCode !== 0) errors.push(`${arm} evidence runner exit code is not zero`); if (evidence.outcome !== expected.outcome) errors.push(`${arm} evidence outcome is invalid`); diff --git a/scripts/pr-proof/prepare.mjs b/scripts/pr-proof/prepare.mjs index 2f80be989..91e9bef6b 100644 --- a/scripts/pr-proof/prepare.mjs +++ b/scripts/pr-proof/prepare.mjs @@ -1,7 +1,9 @@ #!/usr/bin/env node import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { randomBytes } from 'node:crypto'; import path from 'node:path'; +import { pathToFileURL } from 'node:url'; import { INPUT_PATH, @@ -49,6 +51,28 @@ async function pullRequestFiles(apiUrl, repository, number, token) { throw new Error('PR changes more than 3,000 files; RelayFlow proof dispatch refuses ambiguous scope'); } +export function pullRequestSnapshot(pullRequest) { + return { + number: pullRequest?.number, + headSha: pullRequest?.head?.sha, + baseSha: pullRequest?.base?.sha, + headRepository: pullRequest?.head?.repo?.full_name, + title: pullRequest?.title, + body: pullRequest?.body ?? '', + }; +} + +export function assertSamePullRequestSnapshot(expected, actual) { + const next = pullRequestSnapshot(actual); + for (const key of Object.keys(expected)) { + if (next[key] !== expected[key]) { + throw new Error( + `Pull request changed during proof preparation (${key}: ${JSON.stringify(expected[key])} -> ${JSON.stringify(next[key])}); dispatch the latest event instead` + ); + } + } +} + async function readHeadFile(apiUrl, repository, filePath, headSha, token) { const payload = await githubJson( `${apiUrl}/repos/${repository}/contents/${filePath}?ref=${encodeURIComponent(headSha)}`, @@ -80,11 +104,12 @@ async function writeSummary(lines, summaryPath) { await appendFile(summaryPath, `${lines.join('\n')}\n`); } -async function main() { +export async function main() { const eventPath = option('--event', process.env.GITHUB_EVENT_PATH); const outputPath = option('--output', INPUT_PATH); const githubOutput = option('--github-output', process.env.GITHUB_OUTPUT); const summaryPath = option('--summary', process.env.GITHUB_STEP_SUMMARY); + const expectedHeadSha = option('--expected-head-sha'); const token = process.env.GITHUB_TOKEN?.trim(); const repository = process.env.GITHUB_REPOSITORY?.trim(); const apiUrl = (process.env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); @@ -93,20 +118,36 @@ async function main() { } const payload = JSON.parse(await readFile(eventPath, 'utf8')); - let pullRequest = pullRequestFromPayload(payload); + const eventPullRequest = pullRequestFromPayload(payload); + let pullRequest = eventPullRequest; const manualNumber = Number(payload.inputs?.pr_number); if (!pullRequest && Number.isInteger(manualNumber) && manualNumber > 0) { pullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${manualNumber}`, token); } if (!pullRequest) throw new Error('The event does not identify a pull request'); - const number = Number(pullRequest.number); + const eventNumber = Number(pullRequest.number); + if (!Number.isInteger(eventNumber) || eventNumber < 1) { + throw new Error('Pull request payload is missing a valid number'); + } + // The event can be stale after a synchronize/edited cancellation. Resolve a + // live snapshot before reading the live files endpoint, then compare the + // same fields again after all file and manifest reads. + pullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${eventNumber}`, token); + + const number = pullRequest.number; const headSha = pullRequest.head?.sha; const baseSha = pullRequest.base?.sha; const headRepository = pullRequest.head?.repo?.full_name; if (!Number.isInteger(number) || !headSha || !baseSha || !headRepository) { throw new Error('Pull request payload is missing number, repository, or exact SHAs'); } + if (expectedHeadSha && headSha !== expectedHeadSha) { + throw new Error( + `Pull request head changed during dispatch: status targets ${expectedHeadSha}, preparation resolved ${headSha}` + ); + } + const snapshot = pullRequestSnapshot(pullRequest); const classification = classifyPullRequest({ title: pullRequest.title, body: pullRequest.body ?? '' }); if (classification.errors.length > 0) { @@ -116,6 +157,8 @@ async function main() { ); } if (!classification.required) { + const finalPullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${number}`, token); + assertSamePullRequestSnapshot(snapshot, finalPullRequest); await writeGithubOutput({ required: false, case_id: 'n/a' }, githubOutput); await writeSummary( ['## RelayFlow PR proof', '', 'Cloud proof is not required for this non-functional change.'], @@ -160,6 +203,9 @@ async function main() { kind: classification.kind, }); + const finalPullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${number}`, token); + assertSamePullRequestSnapshot(snapshot, finalPullRequest); + const proofInput = { version: PR_PROOF_VERSION, repository, @@ -168,6 +214,7 @@ async function main() { headSha, caseId, kind: classification.kind, + handoffNonce: randomBytes(16).toString('hex'), manifestPath, manifest, }; @@ -187,12 +234,14 @@ async function main() { ); } -main().catch(async (error) => { - const details = error instanceof PrProofContractError ? error.details : []; - const lines = ['## RelayFlow PR proof', '', `**Contract failure:** ${error.message}`]; - if (details.length > 0) lines.push('', ...details.map((detail) => `- ${detail}`)); - await writeSummary(lines, option('--summary', process.env.GITHUB_STEP_SUMMARY)).catch(() => {}); - console.error(error.message); - for (const detail of details) console.error(`- ${detail}`); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(async (error) => { + const details = error instanceof PrProofContractError ? error.details : []; + const lines = ['## RelayFlow PR proof', '', `**Contract failure:** ${error.message}`]; + if (details.length > 0) lines.push('', ...details.map((detail) => `- ${detail}`)); + await writeSummary(lines, option('--summary', process.env.GITHUB_STEP_SUMMARY)).catch(() => {}); + console.error(error.message); + for (const detail of details) console.error(`- ${detail}`); + process.exitCode = 1; + }); +} diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs new file mode 100644 index 000000000..2cf3ba0fd --- /dev/null +++ b/scripts/pr-proof/process-runner.mjs @@ -0,0 +1,89 @@ +import { spawn } from 'node:child_process'; + +const DEFAULT_CAPTURE_BYTES = 128 * 1024; +const DEFAULT_TERMINATION_GRACE_MS = 5_000; + +function appendBounded(current, chunk, maximum) { + const next = current + chunk; + return next.length <= maximum ? next : next.slice(next.length - maximum); +} + +function signalProcessTree(child, signal) { + if (process.platform !== 'win32' && child.pid) { + try { + process.kill(-child.pid, signal); + return; + } catch (error) { + if (error?.code !== 'ESRCH') throw error; + } + } + try { + child.kill(signal); + } catch (error) { + if (error?.code !== 'ESRCH') throw error; + } +} + +/** + * Run a subprocess with bounded output and a process-tree timeout. The child + * owns a process group on POSIX so descendants that inherit stdout/stderr are + * terminated with it. After the grace period, the parent-side pipes are also + * closed so an escaped descendant cannot keep this promise pending forever. + */ +export function runBoundedProcess(command, args, options = {}) { + return new Promise((resolve, reject) => { + const maximum = options.maxCaptureBytes ?? DEFAULT_CAPTURE_BYTES; + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env ?? process.env, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + let timedOut = false; + let settled = false; + let hardKill = null; + + const timeout = options.timeoutMs + ? setTimeout(() => { + timedOut = true; + signalProcessTree(child, 'SIGTERM'); + hardKill = setTimeout(() => { + signalProcessTree(child, 'SIGKILL'); + child.stdout.destroy(); + child.stderr.destroy(); + }, options.terminationGraceMs ?? DEFAULT_TERMINATION_GRACE_MS); + hardKill.unref(); + }, options.timeoutMs) + : null; + + const cleanup = () => { + if (timeout) clearTimeout(timeout); + if (hardKill) clearTimeout(hardKill); + }; + + child.stdout.on('data', (chunk) => { + const text = chunk.toString(); + stdout = appendBounded(stdout, text, maximum); + if (options.echo !== false) process.stdout.write(text); + }); + child.stderr.on('data', (chunk) => { + const text = chunk.toString(); + stderr = appendBounded(stderr, text, maximum); + if (options.echo !== false) process.stderr.write(text); + }); + child.on('error', (error) => { + if (settled) return; + settled = true; + cleanup(); + reject(error); + }); + child.on('close', (code, signal) => { + if (settled) return; + settled = true; + cleanup(); + resolve({ exitCode: code ?? 1, signal, stdout, stderr, timedOut }); + }); + }); +} diff --git a/scripts/pr-proof/report-status.mjs b/scripts/pr-proof/report-status.mjs new file mode 100644 index 000000000..ba54c0d2a --- /dev/null +++ b/scripts/pr-proof/report-status.mjs @@ -0,0 +1,156 @@ +#!/usr/bin/env node + +import { appendFile, readFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +const SHA_RE = /^[0-9a-f]{40}$/; +const REPOSITORY_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const CONTEXT = 'RelayFlow PR proof'; + +function option(name, fallback = null) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : fallback; +} + +function headers(token) { + return { + accept: 'application/vnd.github+json', + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + 'x-github-api-version': '2022-11-28', + 'user-agent': 'relay-pr-proof-status', + }; +} + +async function githubJson(url, token, init = {}, fetchImpl = fetch) { + const response = await fetchImpl(url, { + ...init, + headers: { ...headers(token), ...(init.headers ?? {}) }, + }); + const payload = await response.json().catch(() => null); + if (!response.ok) { + const detail = payload?.message ? `: ${payload.message}` : ''; + throw new Error(`GitHub API ${response.status} for ${url}${detail}`); + } + return payload; +} + +function validateRepository(value) { + if (!REPOSITORY_RE.test(value ?? '')) throw new Error('GITHUB_REPOSITORY is invalid'); + return value; +} + +async function resolvePullRequest({ payload, repository, apiUrl, token, fetchImpl }) { + let pullRequest = payload.pull_request ?? null; + const manualNumber = Number(payload.inputs?.pr_number); + if (!pullRequest && Number.isInteger(manualNumber) && manualNumber > 0) { + pullRequest = await githubJson( + `${apiUrl}/repos/${repository}/pulls/${manualNumber}`, + token, + {}, + fetchImpl + ); + } + const number = Number(pullRequest?.number); + const headSha = pullRequest?.head?.sha; + const headRepository = pullRequest?.head?.repo?.full_name; + if (!Number.isInteger(number) || number < 1 || !SHA_RE.test(headSha ?? '')) { + throw new Error('The event does not identify a pull request with an exact head SHA'); + } + if (headRepository !== repository) { + throw new Error( + `Fork pull request ${headRepository ?? 'unknown'} cannot receive the credential-bearing proof status` + ); + } + return { number, headSha }; +} + +function targetUrl(env) { + const server = env.GITHUB_SERVER_URL?.replace(/\/$/, ''); + const repository = env.GITHUB_REPOSITORY; + const runId = env.GITHUB_RUN_ID; + return server && repository && runId ? `${server}/${repository}/actions/runs/${runId}` : undefined; +} + +export async function publishCommitStatus({ sha, state, description, env = process.env, fetchImpl = fetch }) { + const repository = validateRepository(env.GITHUB_REPOSITORY?.trim()); + const token = env.GITHUB_TOKEN?.trim(); + const apiUrl = (env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + if (!token) throw new Error('GITHUB_TOKEN is required to publish the PR proof status'); + if (!SHA_RE.test(sha ?? '')) throw new Error('PR proof status SHA must be a full lowercase SHA'); + if (!['pending', 'success', 'failure', 'error'].includes(state)) { + throw new Error(`Invalid PR proof status state: ${state}`); + } + return githubJson( + `${apiUrl}/repos/${repository}/statuses/${sha}`, + token, + { + method: 'POST', + body: JSON.stringify({ + state, + context: CONTEXT, + description: description.slice(0, 140), + target_url: targetUrl(env), + }), + }, + fetchImpl + ); +} + +function conclusionState(jobStatus) { + if (jobStatus === 'success') return 'success'; + if (jobStatus === 'failure') return 'failure'; + return 'error'; +} + +export async function main() { + const command = process.argv[2]; + const env = process.env; + if (command === 'start') { + const eventPath = option('--event', env.GITHUB_EVENT_PATH); + const outputPath = option('--github-output', env.GITHUB_OUTPUT); + const repository = validateRepository(env.GITHUB_REPOSITORY?.trim()); + const token = env.GITHUB_TOKEN?.trim(); + const apiUrl = (env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + if (!eventPath || !token) throw new Error('GITHUB_EVENT_PATH and GITHUB_TOKEN are required'); + const payload = JSON.parse(await readFile(eventPath, 'utf8')); + const resolved = await resolvePullRequest({ payload, repository, apiUrl, token, fetchImpl: fetch }); + await publishCommitStatus({ + sha: resolved.headSha, + state: 'pending', + description: `Cloud red/green proof started for PR #${resolved.number}`, + }); + if (outputPath) { + await appendFile(outputPath, `head_sha=${resolved.headSha}\npr_number=${resolved.number}\n`); + } + console.log(`PR_PROOF_STATUS_PENDING pr=${resolved.number} head=${resolved.headSha}`); + return; + } + + if (command === 'finish') { + const sha = option('--sha'); + const jobStatus = option('--job-status'); + const state = conclusionState(jobStatus); + await publishCommitStatus({ + sha, + state, + description: + state === 'success' + ? 'Declared Cloud red/green proof passed' + : state === 'failure' + ? 'Declared Cloud red/green proof failed' + : `Cloud red/green proof ended with ${jobStatus ?? 'unknown'} status`, + }); + console.log(`PR_PROOF_STATUS_FINAL head=${sha} state=${state}`); + return; + } + + throw new Error('Usage: report-status.mjs '); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/pr-proof/run-arm.mjs b/scripts/pr-proof/run-arm.mjs index e505c2205..e68eb8464 100644 --- a/scripts/pr-proof/run-arm.mjs +++ b/scripts/pr-proof/run-arm.mjs @@ -1,9 +1,9 @@ #!/usr/bin/env node -import { spawn } from 'node:child_process'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; +import { pathToFileURL } from 'node:url'; import { ARTIFACT_ROOT, @@ -14,52 +14,20 @@ import { validateObservation, validateProofInput, } from './contract.mjs'; +import { uploadCloudEvidence } from './cloud-storage.mjs'; +import { runBoundedProcess } from './process-runner.mjs'; -const MAX_CAPTURE_BYTES = 128 * 1024; +const MAX_OBSERVATION_FILE_BYTES = 64 * 1024; -function appendBounded(current, chunk) { - const next = current + chunk; - return next.length <= MAX_CAPTURE_BYTES ? next : next.slice(next.length - MAX_CAPTURE_BYTES); -} - -async function run(command, args, options = {}) { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd: options.cwd, - env: options.env, - stdio: ['ignore', 'pipe', 'pipe'], - }); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (chunk) => { - const text = chunk.toString(); - stdout = appendBounded(stdout, text); - process.stdout.write(text); - }); - child.stderr.on('data', (chunk) => { - const text = chunk.toString(); - stderr = appendBounded(stderr, text); - process.stderr.write(text); - }); - const timeout = options.timeoutMs - ? setTimeout(() => { - child.kill('SIGTERM'); - setTimeout(() => child.kill('SIGKILL'), 5_000).unref(); - }, options.timeoutMs) - : null; - child.on('error', reject); - child.on('close', (code, signal) => { - if (timeout) clearTimeout(timeout); - resolve({ exitCode: code ?? 1, signal, stdout, stderr }); - }); - }); +export async function runProcess(command, args, options = {}) { + return runBoundedProcess(command, args, options); } async function runChecked(command, args, options = {}) { - const result = await run(command, args, options); - if (result.exitCode !== 0) { + const result = await runProcess(command, args, options); + if (result.exitCode !== 0 || result.timedOut) { throw new Error( - `${command} ${args.join(' ')} failed with exit ${result.exitCode}${result.signal ? ` (${result.signal})` : ''}` + `${command} ${args.join(' ')} ${result.timedOut ? 'timed out' : `failed with exit ${result.exitCode}`}${result.signal ? ` (${result.signal})` : ''}` ); } return result; @@ -107,7 +75,7 @@ function armFromArg() { return arm; } -async function main() { +export async function main() { const arm = armFromArg(); const inputPath = process.argv[3] ?? process.env.RELAY_PR_PROOF_INPUT ?? INPUT_PATH; const input = validateProofInput(JSON.parse(await readFile(inputPath, 'utf8'))); @@ -137,7 +105,7 @@ async function main() { } const [command, ...args] = input.manifest.runner.command; - const result = await run(command, args, { + const result = await runProcess(command, args, { cwd: harnessDir, env: sanitizedCaseEnvironment({ temporaryHome, @@ -149,6 +117,11 @@ async function main() { }), timeoutMs: input.manifest.timeoutSeconds * 1000, }); + if (result.timedOut) { + throw new Error( + `Case runner exceeded ${input.manifest.timeoutSeconds}s; a timeout cannot count as expected-red evidence` + ); + } if (result.exitCode !== 0) { throw new Error( `Case runner failed with exit ${result.exitCode}; expected-red behavior must be reported as a successful structured observation` @@ -157,6 +130,12 @@ async function main() { let observationJson; try { + const resultStat = await stat(resultPath); + if (!resultStat.isFile() || resultStat.size > MAX_OBSERVATION_FILE_BYTES) { + throw new Error( + `observation must be a regular file no larger than ${MAX_OBSERVATION_FILE_BYTES} bytes` + ); + } observationJson = JSON.parse(await readFile(resultPath, 'utf8')); } catch (error) { throw new PrProofContractError('Case runner did not write a valid observation JSON file', [ @@ -176,6 +155,7 @@ async function main() { pullRequest: input.pullRequest, targetSha: actualTargetSha, harnessSha, + handoffNonce: input.handoffNonce, sandboxId, runnerExitCode: result.exitCode, outcome: observation.outcome, @@ -187,16 +167,19 @@ async function main() { }; await mkdir(path.dirname(evidencePath), { recursive: true }); await writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + await uploadCloudEvidence(input, arm, evidence); console.log(`PR_PROOF_ARM_COMPLETE arm=${arm} case=${input.caseId} sandbox=${sandboxId}`); } finally { await rm(temporaryRoot, { recursive: true, force: true }); } } -main().catch((error) => { - console.error(error.message); - if (error instanceof PrProofContractError) { - for (const detail of error.details) console.error(`- ${detail}`); - } - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error.message); + if (error instanceof PrProofContractError) { + for (const detail of error.details) console.error(`- ${detail}`); + } + process.exitCode = 1; + }); +} diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index a5e5eb513..7e5fd262c 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -1,34 +1,44 @@ #!/usr/bin/env node -import { spawn } from 'node:child_process'; -import { appendFile, mkdir, writeFile } from 'node:fs/promises'; +import { appendFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; import path from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; +import { pathToFileURL } from 'node:url'; + +import { runBoundedProcess } from './process-runner.mjs'; const TERMINAL_SUCCESS = new Set(['completed', 'succeeded', 'success']); const TERMINAL_FAILURE = new Set(['failed', 'cancelled', 'canceled', 'timed_out', 'error']); +const CLOUD_AUTH_KEYS = [ + 'CLOUD_API_URL', + 'CLOUD_API_ACCESS_TOKEN', + 'CLOUD_API_REFRESH_TOKEN', + 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT', + 'CLOUD_API_REFRESH_TOKEN_EXPIRES_AT', +]; +const CREDENTIAL_WINDOW_BUFFER_MS = 15 * 60_000; +const MAX_CAPTURE_BYTES = 2 * 1024 * 1024; +const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; function run(command, args, options = {}) { - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - env: options.env ?? process.env, - stdio: ['ignore', 'pipe', 'pipe'], - }); - let stdout = ''; - let stderr = ''; - child.stdout.on('data', (chunk) => { - stdout += chunk.toString(); - if (!options.quiet) process.stdout.write(chunk); - }); - child.stderr.on('data', (chunk) => { - stderr += chunk.toString(); - if (!options.quiet) process.stderr.write(chunk); - }); - child.on('error', reject); - child.on('close', (code) => resolve({ exitCode: code ?? 1, stdout, stderr })); + return runBoundedProcess(command, args, { + env: options.env, + echo: !options.quiet, + maxCaptureBytes: MAX_CAPTURE_BYTES, + timeoutMs: options.timeoutMs, }); } +export function boundedDuration(value, { fallback, minimum, maximum, label }) { + if (value === undefined || value === null || value === '') return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new Error(`${label} must be an integer between ${minimum} and ${maximum} milliseconds`); + } + return parsed; +} + function parseJsonOutput(output, label) { try { return JSON.parse(output); @@ -47,62 +57,210 @@ function statusFrom(payload) { throw new Error('Cloud status response did not contain a status'); } -async function main() { +function requiredCredential(env, name) { + const value = env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +export function validateCredentialWindow(env, timeoutMs, now = Date.now()) { + const accessExpiry = Date.parse(requiredCredential(env, 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT')); + if (!Number.isFinite(accessExpiry)) throw new Error('CLOUD_API_ACCESS_TOKEN_EXPIRES_AT is invalid'); + const requiredUntil = now + timeoutMs + CREDENTIAL_WINDOW_BUFFER_MS; + if (accessExpiry <= requiredUntil) { + throw new Error( + `Cloud CI access token expires before the proof deadline; reprovision the Relay secrets before ${new Date(requiredUntil).toISOString()}` + ); + } + const refreshExpirySource = env.CLOUD_API_REFRESH_TOKEN_EXPIRES_AT?.trim(); + if (refreshExpirySource) { + const refreshExpiry = Date.parse(refreshExpirySource); + if (!Number.isFinite(refreshExpiry) || refreshExpiry <= requiredUntil) { + throw new Error('Cloud CI refresh token expires before the proof deadline; reprovision Relay secrets'); + } + } +} + +export async function createCliAuthEnvironment(env = process.env) { + const apiUrl = requiredCredential(env, 'CLOUD_API_URL'); + const accessToken = requiredCredential(env, 'CLOUD_API_ACCESS_TOKEN'); + const refreshToken = requiredCredential(env, 'CLOUD_API_REFRESH_TOKEN'); + const accessTokenExpiresAt = requiredCredential(env, 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT'); + const refreshTokenExpiresAt = env.CLOUD_API_REFRESH_TOKEN_EXPIRES_AT?.trim(); + new URL(apiUrl); + + const temporaryHome = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-auth-')); + const authDir = path.join(temporaryHome, '.agentworkforce', 'relay'); + const authPath = path.join(authDir, 'cloud-auth.json'); + await mkdir(authDir, { recursive: true, mode: 0o700 }); + await writeFile( + authPath, + `${JSON.stringify({ + apiUrl, + accessToken, + refreshToken, + accessTokenExpiresAt, + ...(refreshTokenExpiresAt ? { refreshTokenExpiresAt } : {}), + })}\n`, + { mode: 0o600 } + ); + + const cliEnv = { ...env, HOME: temporaryHome }; + for (const key of CLOUD_AUTH_KEYS) delete cliEnv[key]; + return { cliEnv, temporaryHome, authPath, originalAccessToken: accessToken }; +} + +export async function assertCredentialDidNotRotate(authPath, originalAccessToken) { + const stored = JSON.parse(await readFile(authPath, 'utf8')); + if (stored.accessToken !== originalAccessToken) { + throw new Error( + 'Cloud CI credential refreshed during the job; the rotated tuple cannot be persisted to GitHub secrets. Reprovision the Relay secrets before rerunning.' + ); + } +} + +export async function main() { const cli = process.env.PR_PROOF_AGENT_RELAY_BIN ?? 'agent-relay'; const workflowPath = process.argv[2] ?? 'workflows/pr-proof.ts'; const logsPath = process.env.PR_PROOF_CLOUD_LOG_PATH ?? '.workflow-artifacts/pr-proof/cloud.log'; - const pollMs = Number(process.env.PR_PROOF_POLL_MS ?? '15000'); - const timeoutMs = Number(process.env.PR_PROOF_CLOUD_TIMEOUT_MS ?? String(60 * 60_000)); - const launch = await run(cli, ['cloud', 'run', workflowPath, '--sync-code', '--json'], { quiet: true }); - if (launch.exitCode !== 0) { - process.stderr.write(launch.stderr); - throw new Error(`Cloud workflow submission failed with exit ${launch.exitCode}`); - } - const launchPayload = parseJsonOutput(launch.stdout, 'Cloud run'); - const runId = launchPayload.runId; - if (typeof runId !== 'string' || !runId) throw new Error('Cloud run response did not contain runId'); - console.log(`Cloud RelayFlow run: ${runId}`); - if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `run_id=${runId}\n`); - - const deadline = Date.now() + timeoutMs; - let terminalStatus = null; - while (Date.now() < deadline) { - await delay(pollMs); - const statusResult = await run(cli, ['cloud', 'status', runId, '--json'], { quiet: true }); - if (statusResult.exitCode !== 0) { - console.warn(`Cloud status poll failed (${statusResult.exitCode}); retrying`); - continue; + const pollMs = boundedDuration(process.env.PR_PROOF_POLL_MS, { + fallback: 15_000, + minimum: 100, + maximum: 60_000, + label: 'PR_PROOF_POLL_MS', + }); + const timeoutMs = boundedDuration(process.env.PR_PROOF_CLOUD_TIMEOUT_MS, { + fallback: 60 * 60_000, + minimum: 60_000, + maximum: 65 * 60_000, + label: 'PR_PROOF_CLOUD_TIMEOUT_MS', + }); + const commandTimeoutMs = boundedDuration(process.env.PR_PROOF_CLOUD_COMMAND_TIMEOUT_MS, { + fallback: DEFAULT_COMMAND_TIMEOUT_MS, + minimum: 1_000, + maximum: 5 * 60_000, + label: 'PR_PROOF_CLOUD_COMMAND_TIMEOUT_MS', + }); + validateCredentialWindow(process.env, timeoutMs); + const auth = await createCliAuthEnvironment(process.env); + let runId = null; + let terminal = false; + let cancelPromise = null; + let shuttingDown = false; + + const cancelRemote = async (reason) => { + if (!runId || terminal) return; + cancelPromise ??= (async () => { + console.warn(`Cancelling Cloud RelayFlow run ${runId} (${reason})`); + const result = await run(cli, ['cloud', 'cancel', runId, '--json'], { + env: auth.cliEnv, + quiet: true, + timeoutMs: commandTimeoutMs, + }); + if (result.exitCode !== 0 || result.timedOut) { + console.warn(`Cloud cancellation failed with exit ${result.exitCode}: ${result.stderr.trim()}`); + } + })(); + await cancelPromise; + }; + + const signalHandler = (signal) => { + if (shuttingDown) return; + shuttingDown = true; + void (async () => { + await cancelRemote(signal).catch((error) => console.warn(error.message)); + await rm(auth.temporaryHome, { recursive: true, force: true }); + process.exit(signal === 'SIGINT' ? 130 : 143); + })(); + }; + process.once('SIGINT', signalHandler); + process.once('SIGTERM', signalHandler); + + try { + const launch = await run(cli, ['cloud', 'run', workflowPath, '--sync-code', '--json'], { + env: auth.cliEnv, + quiet: true, + timeoutMs: commandTimeoutMs, + }); + if (launch.timedOut) { + throw new Error( + 'Cloud workflow submission command timed out; it is not retried because submission may have completed' + ); } - const status = statusFrom(parseJsonOutput(statusResult.stdout, 'Cloud status')); - console.log(`Cloud RelayFlow status: ${status}`); - if (TERMINAL_SUCCESS.has(status) || TERMINAL_FAILURE.has(status)) { - terminalStatus = status; - break; + if (launch.exitCode !== 0) { + process.stderr.write(launch.stderr); + throw new Error(`Cloud workflow submission failed with exit ${launch.exitCode}`); } - } - if (!terminalStatus) { - await run(cli, ['cloud', 'cancel', runId], { quiet: true }); - throw new Error(`Cloud RelayFlow exceeded ${timeoutMs}ms`); - } + const launchPayload = parseJsonOutput(launch.stdout, 'Cloud run'); + runId = launchPayload.runId; + if (typeof runId !== 'string' || !runId) throw new Error('Cloud run response did not contain runId'); + console.log(`Cloud RelayFlow run: ${runId}`); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `run_id=${runId}\n`); - await mkdir(path.dirname(logsPath), { recursive: true }); - const logs = await run(cli, ['cloud', 'logs', runId], { quiet: true }); - await writeFile(logsPath, logs.stdout + logs.stderr); - if (logs.stdout) process.stdout.write(logs.stdout); - if (logs.stderr) process.stderr.write(logs.stderr); + const deadline = Date.now() + timeoutMs; + let terminalStatus = null; + while (Date.now() < deadline) { + await delay(pollMs); + const statusResult = await run(cli, ['cloud', 'status', runId, '--json'], { + env: auth.cliEnv, + quiet: true, + timeoutMs: commandTimeoutMs, + }); + if (statusResult.timedOut) { + throw new Error(`Cloud status command timed out for run ${runId}`); + } + if (statusResult.exitCode !== 0) { + console.warn(`Cloud status poll failed (${statusResult.exitCode}); retrying`); + continue; + } + const status = statusFrom(parseJsonOutput(statusResult.stdout, 'Cloud status')); + console.log(`Cloud RelayFlow status: ${status}`); + if (TERMINAL_SUCCESS.has(status) || TERMINAL_FAILURE.has(status)) { + terminalStatus = status; + terminal = true; + break; + } + } + if (!terminalStatus) { + await cancelRemote('deadline exceeded'); + terminal = true; + throw new Error(`Cloud RelayFlow exceeded ${timeoutMs}ms`); + } - if (!TERMINAL_SUCCESS.has(terminalStatus)) { - throw new Error(`Cloud RelayFlow finished with status ${terminalStatus}`); - } - if (process.env.GITHUB_STEP_SUMMARY) { - await appendFile( - process.env.GITHUB_STEP_SUMMARY, - `\n- Cloud run: \`${runId}\`\n- Cloud status: **${terminalStatus}**\n` - ); + await mkdir(path.dirname(logsPath), { recursive: true }); + const logs = await run(cli, ['cloud', 'logs', runId], { + env: auth.cliEnv, + quiet: true, + timeoutMs: commandTimeoutMs, + }); + await writeFile(logsPath, logs.stdout + logs.stderr); + if (logs.stdout) process.stdout.write(logs.stdout); + if (logs.stderr) process.stderr.write(logs.stderr); + if (logs.timedOut) throw new Error(`Cloud log retrieval timed out for run ${runId}`); + if (logs.exitCode !== 0) throw new Error(`Cloud log retrieval failed with exit ${logs.exitCode}`); + + if (!TERMINAL_SUCCESS.has(terminalStatus)) { + throw new Error(`Cloud RelayFlow finished with status ${terminalStatus}`); + } + await assertCredentialDidNotRotate(auth.authPath, auth.originalAccessToken); + if (process.env.GITHUB_STEP_SUMMARY) { + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `\n- Cloud run: \`${runId}\`\n- Cloud status: **${terminalStatus}**\n` + ); + } + } finally { + process.removeListener('SIGINT', signalHandler); + process.removeListener('SIGTERM', signalHandler); + if (runId && !terminal) + await cancelRemote('dispatcher exiting').catch((error) => console.warn(error.message)); + await rm(auth.temporaryHome, { recursive: true, force: true }); } } -main().catch((error) => { - console.error(error.message); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/pr-proof/verify-evidence.mjs b/scripts/pr-proof/verify-evidence.mjs index a06de778f..3cd339539 100644 --- a/scripts/pr-proof/verify-evidence.mjs +++ b/scripts/pr-proof/verify-evidence.mjs @@ -12,6 +12,7 @@ import { validateEvidence, validateProofInput, } from './contract.mjs'; +import { downloadCloudEvidence } from './cloud-storage.mjs'; function option(name, fallback = null) { const index = process.argv.indexOf(name); @@ -24,10 +25,16 @@ async function readJson(filePath) { export async function verifyEvidenceFiles({ inputPath, arm, artifactRoot = ARTIFACT_ROOT }) { const input = validateProofInput(await readJson(inputPath)); - const base = validateEvidence(await readJson(path.join(artifactRoot, 'base.json')), input, 'base'); + const base = await readJson(path.join(artifactRoot, 'base.json')); + const head = arm === 'base' ? null : await readJson(path.join(artifactRoot, 'head.json')); + return verifyEvidenceRecords({ input, arm, base, head }); +} + +export function verifyEvidenceRecords({ input, arm, base: baseRecord, head: headRecord }) { + const base = validateEvidence(baseRecord, input, 'base'); if (arm === 'base') return { input, base, head: null }; - const head = validateEvidence(await readJson(path.join(artifactRoot, 'head.json')), input, 'head'); + const head = validateEvidence(headRecord, input, 'head'); if (base.sandboxId === head.sandboxId) { throw new PrProofContractError('Base and head proofs did not run in distinct sandboxes', [ `both evidence files report sandbox ${base.sandboxId}`, @@ -36,12 +43,24 @@ export async function verifyEvidenceFiles({ inputPath, arm, artifactRoot = ARTIF return { input, base, head }; } +export async function verifyCloudEvidence({ inputPath, arm, env = process.env, fetchImpl = fetch }) { + const input = validateProofInput(await readJson(inputPath)); + const base = await downloadCloudEvidence(input, 'base', { env, fetchImpl }); + const head = arm === 'base' ? null : await downloadCloudEvidence(input, 'head', { env, fetchImpl }); + return verifyEvidenceRecords({ input, arm, base, head }); +} + async function main() { const arm = option('--arm', 'both'); if (arm !== 'base' && arm !== 'both') throw new Error('--arm must be base or both'); const inputPath = option('--input', process.env.RELAY_PR_PROOF_INPUT ?? INPUT_PATH); const artifactRoot = option('--artifacts', ARTIFACT_ROOT); - const { input, base, head } = await verifyEvidenceFiles({ inputPath, arm, artifactRoot }); + const source = option('--source', 'files'); + if (source !== 'files' && source !== 'cloud') throw new Error('--source must be files or cloud'); + const { input, base, head } = + source === 'cloud' + ? await verifyCloudEvidence({ inputPath, arm }) + : await verifyEvidenceFiles({ inputPath, arm, artifactRoot }); console.log(`PR_PROOF_BASE_VALID case=${input.caseId} outcome=${base.outcome} sandbox=${base.sandboxId}`); if (!head) return; diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 67a320621..beaba092c 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -13,14 +13,31 @@ import { classifyPullRequest, validateCaseManifest, validateEvidence, + validateObservation, validateProofInput, } from '../../scripts/pr-proof/contract.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. import { verifyEvidenceFiles } from '../../scripts/pr-proof/verify-evidence.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { downloadCloudEvidence, uploadCloudEvidence } from '../../scripts/pr-proof/cloud-storage.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { publishCommitStatus } from '../../scripts/pr-proof/report-status.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { assertSamePullRequestSnapshot, pullRequestSnapshot } from '../../scripts/pr-proof/prepare.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { + assertCredentialDidNotRotate, + boundedDuration, + createCliAuthEnvironment, + validateCredentialWindow, +} from '../../scripts/pr-proof/run-cloud.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { runProcess } from '../../scripts/pr-proof/run-arm.mjs'; const BASE_SHA = '1'.repeat(40); const HEAD_SHA = '2'.repeat(40); const CASE_ID = '1591-application-ack-reconnect'; +const HANDOFF_NONCE = 'a'.repeat(32); function proofBody(type = 'bugfix', caseId = CASE_ID) { return [ @@ -63,6 +80,7 @@ function input() { headSha: HEAD_SHA, caseId: CASE_ID, kind: 'bugfix', + handoffNonce: HANDOFF_NONCE, manifest: manifest(), }); } @@ -78,6 +96,7 @@ function evidence(arm: 'base' | 'head', sandboxId: string) { pullRequest: 1610, targetSha: arm === 'base' ? BASE_SHA : HEAD_SHA, harnessSha: HEAD_SHA, + handoffNonce: HANDOFF_NONCE, sandboxId, runnerExitCode: 0, outcome: expected.outcome, @@ -148,6 +167,15 @@ describe('RelayFlow case manifest', () => { ).toEqual([CASE_ID]); }); + it('keeps malformed sibling case directories visible so selection fails closed', () => { + expect( + changedRelayFlowCaseIds([ + `tests/relayflows/cases/${CASE_ID}/case.json`, + 'tests/relayflows/cases/INVALID CASE/run.mjs', + ]) + ).toEqual([CASE_ID, 'INVALID CASE']); + }); + it('accepts a structured external case runner', () => { expect(validateCaseManifest(manifest(), { caseId: CASE_ID, kind: 'bugfix' })).toEqual(manifest()); }); @@ -170,6 +198,22 @@ describe('RelayFlow case manifest', () => { }); describe('RelayFlow evidence gates', () => { + it('rejects coercible non-numeric pull request provenance', () => { + const invalid = { + ...input(), + pullRequest: true, + }; + try { + validateProofInput(invalid); + throw new Error('expected invalid proof input to fail'); + } catch (error) { + expect(error).toBeInstanceOf(PrProofContractError); + expect((error as { details: string[] }).details).toContain( + 'proof input pullRequest must be a positive integer' + ); + } + }); + it('accepts exact SHA, outcome, and signature provenance', () => { const proofInput = input(); expect(validateEvidence(evidence('base', 'sandbox-base'), proofInput, 'base')).toMatchObject({ @@ -191,6 +235,27 @@ describe('RelayFlow evidence gates', () => { } }); + it('rejects evidence from a stale or different handoff', () => { + const invalid = { ...evidence('base', 'sandbox-base'), handoffNonce: 'b'.repeat(32) }; + expect(() => validateEvidence(invalid, input(), 'base')).toThrow(/Invalid base evidence/); + }); + + it('bounds PR-authored observation details before evidence upload', () => { + expect(() => + validateObservation( + { + version: 1, + caseId: CASE_ID, + arm: 'base', + outcome: 'bug', + signature: manifest().expected.base.signature, + details: 'x'.repeat(4_001), + }, + { caseId: CASE_ID, arm: 'base', expected: manifest().expected.base } + ) + ).toThrow(/Invalid case observation/); + }); + it('rejects base and head evidence from the same sandbox', async () => { const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-test-')); const inputPath = path.join(root, 'input.json'); @@ -204,6 +269,165 @@ describe('RelayFlow evidence gates', () => { }); }); +describe('Cloud evidence handoff', () => { + const env = { + CLOUD_API_URL: 'https://cloud.test/cloud', + CLOUD_API_ACCESS_TOKEN: 'sandbox-access-token', + RUN_ID: 'run-123', + }; + + it('uploads evidence to nonce-bound run storage without exposing credentials in the URL', async () => { + let requestUrl = ''; + let requestInit: RequestInit | undefined; + const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { + requestUrl = String(url); + requestInit = init; + return Response.json({ ok: true }); + }; + const record = evidence('base', 'sandbox-base'); + await uploadCloudEvidence(input(), 'base', record, { env, fetchImpl }); + expect(requestUrl).toBe( + `https://cloud.test/cloud/api/v1/workflows/runs/run-123/storage/pr-proof/${HANDOFF_NONCE}/base.json` + ); + expect(requestUrl).not.toContain('sandbox-access-token'); + expect(requestInit?.method).toBe('PUT'); + expect(JSON.parse(String(requestInit?.body))).toMatchObject(record); + }); + + it('downloads structured evidence from the same nonce-bound object', async () => { + const record = evidence('head', 'sandbox-head'); + const fetchImpl = async () => new Response(JSON.stringify(record), { status: 200 }); + await expect(downloadCloudEvidence(input(), 'head', { env, fetchImpl })).resolves.toEqual(record); + }); +}); + +describe('Cloud dispatcher credential lifecycle', () => { + const now = Date.parse('2026-08-25T10:00:00.000Z'); + const credentialEnv = { + CLOUD_API_URL: 'https://cloud.test/cloud', + CLOUD_API_ACCESS_TOKEN: 'ci-access-token', + CLOUD_API_REFRESH_TOKEN: 'ci-refresh-token', + CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: '2026-08-25T13:00:00.000Z', + CLOUD_API_REFRESH_TOKEN_EXPIRES_AT: '2026-08-26T10:00:00.000Z', + }; + + it('fails before dispatch when the static access token cannot cover the proof window', () => { + expect(() => + validateCredentialWindow( + { ...credentialEnv, CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: '2026-08-25T10:30:00.000Z' }, + 60 * 60_000, + now + ) + ).toThrow(/expires before the proof deadline/); + }); + + it('rejects unbounded or malformed dispatcher durations', () => { + const options = { fallback: 15_000, minimum: 100, maximum: 60_000, label: 'poll' }; + expect(boundedDuration(undefined, options)).toBe(15_000); + expect(() => boundedDuration('not-a-number', options)).toThrow(/between 100 and 60000/); + expect(() => boundedDuration('60001', options)).toThrow(/between 100 and 60000/); + }); + + it('shares an owner-only temporary auth file across CLI subprocesses and detects rotation', async () => { + validateCredentialWindow(credentialEnv, 60 * 60_000, now); + const auth = await createCliAuthEnvironment({ ...process.env, ...credentialEnv }); + try { + expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN).toBeUndefined(); + expect(auth.cliEnv.HOME).toBe(auth.temporaryHome); + expect((await stat(auth.authPath)).mode & 0o777).toBe(0o600); + await expect( + assertCredentialDidNotRotate(auth.authPath, credentialEnv.CLOUD_API_ACCESS_TOKEN) + ).resolves.toBeUndefined(); + const stored = JSON.parse(await readFile(auth.authPath, 'utf8')); + stored.accessToken = 'rotated-token'; + await writeFile(auth.authPath, JSON.stringify(stored), { mode: 0o600 }); + await expect( + assertCredentialDidNotRotate(auth.authPath, credentialEnv.CLOUD_API_ACCESS_TOKEN) + ).rejects.toThrow(/cannot be persisted/); + } finally { + await rm(auth.temporaryHome, { recursive: true, force: true }); + } + }); +}); + +describe('process timeout contract', () => { + it('marks a process timed out even when it exits zero after SIGTERM', async () => { + const result = await runProcess( + process.execPath, + ['-e', "process.on('SIGTERM', () => process.exit(0)); setInterval(() => {}, 1000)"], + { timeoutMs: 100 } + ); + expect(result.timedOut).toBe(true); + }); + + it('terminates descendants that inherit output pipes instead of hanging after the parent exits', async () => { + const startedAt = Date.now(); + const script = [ + "const { spawn } = require('node:child_process');", + "spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: ['ignore', 'inherit', 'inherit'] });", + "process.on('SIGTERM', () => process.exit(0));", + 'setInterval(() => {}, 1000);', + ].join(''); + const result = await runProcess(process.execPath, ['-e', script], { timeoutMs: 100 }); + expect(result.timedOut).toBe(true); + expect(Date.now() - startedAt).toBeLessThan(2_000); + }); +}); + +describe('required head status', () => { + it('publishes the stable proof context on the exact head SHA', async () => { + let requestUrl = ''; + let requestBody: Record = {}; + const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { + requestUrl = String(url); + requestBody = JSON.parse(String(init?.body)); + return Response.json({ id: 1 }); + }; + await publishCommitStatus({ + sha: HEAD_SHA, + state: 'success', + description: 'proof passed', + env: { + GITHUB_REPOSITORY: 'AgentWorkforce/relay', + GITHUB_TOKEN: 'github-token', + GITHUB_API_URL: 'https://api.github.test', + }, + fetchImpl, + }); + expect(requestUrl).toBe(`https://api.github.test/repos/AgentWorkforce/relay/statuses/${HEAD_SHA}`); + expect(requestBody).toMatchObject({ context: 'RelayFlow PR proof', state: 'success' }); + }); +}); + +describe('pull request snapshot consistency', () => { + const pullRequest = { + number: 1612, + title: 'feat(ci): prove one case', + body: proofBody('feature'), + head: { sha: HEAD_SHA, repo: { full_name: 'AgentWorkforce/relay' } }, + base: { sha: BASE_SHA }, + }; + + it('accepts the same live metadata and exact base/head pair', () => { + expect(() => + assertSamePullRequestSnapshot(pullRequestSnapshot(pullRequest), structuredClone(pullRequest)) + ).not.toThrow(); + }); + + it('rejects a head or proof-metadata edit during file enumeration', () => { + const snapshot = pullRequestSnapshot(pullRequest); + expect(() => + assertSamePullRequestSnapshot(snapshot, { + ...pullRequest, + head: { ...pullRequest.head, sha: '3'.repeat(40) }, + }) + ).toThrow(/headSha/); + expect(() => + assertSamePullRequestSnapshot(snapshot, { ...pullRequest, body: proofBody('non-functional', 'n/a') }) + ).toThrow(/body/); + }); +}); + describe('trusted dispatcher source contract', () => { it('never checks out PR head code on the credential-bearing GitHub runner', async () => { const source = await readFile('.github/workflows/relayflow-pr-proof.yml', 'utf8'); @@ -211,6 +435,14 @@ describe('trusted dispatcher source contract', () => { expect(source).toContain('ref: ${{ github.event.pull_request.base.sha || github.sha }}'); expect(source).toContain('persist-credentials: false'); expect(source).toContain('git add -f -- .relayflow/pr-proof-input.json'); + expect(source).toContain('statuses: write'); + expect(source).toContain('report-status.mjs start'); + expect(source).toContain('report-status.mjs finish'); + expect(source).toContain('always() && !cancelled()'); + expect(source).toContain('--expected-head-sha "${{ steps.status.outputs.head_sha }}"'); + expect(source).toContain('actions/checkout@11d5960a326750d5838078e36cf38b85af677262'); + expect(source).toContain('actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020'); + expect(source).toContain('actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02'); expect(source).not.toContain('github.event.pull_request.head.sha }}'); }); @@ -222,6 +454,8 @@ describe('trusted dispatcher source contract', () => { expect(source).toContain(".step('gate-red-green'"); expect(source).toContain('PR_PROOF_ARM_COMPLETE arm=base'); expect(source).toContain('PR_PROOF_ARM_COMPLETE arm=head'); + expect(source).toContain('--source cloud'); + expect(source).toContain("result.status !== 'completed'"); }); it('records the per-step Cloud sandbox id instead of the orchestrator id', async () => { @@ -229,4 +463,12 @@ describe('trusted dispatcher source contract', () => { expect(source).toContain('process.env.SANDBOX_ID'); expect(source).not.toContain('process.env.DAYTONA_SANDBOX_ID'); }); + + it('shares refreshes only inside the job and cancels remote work on termination', async () => { + const source = await readFile('scripts/pr-proof/run-cloud.mjs', 'utf8'); + expect(source).toContain("process.once('SIGTERM', signalHandler)"); + expect(source).toContain("['cloud', 'cancel', runId, '--json']"); + expect(source).toContain("path.join(authDir, 'cloud-auth.json')"); + expect(source).toContain('assertCredentialDidNotRotate'); + }); }); diff --git a/tests/relayflows/cases/README.md b/tests/relayflows/cases/README.md index e48ddfc1b..c9469abd4 100644 --- a/tests/relayflows/cases/README.md +++ b/tests/relayflows/cases/README.md @@ -67,19 +67,35 @@ Expected-red is data, not a failing process. A non-zero runner exit, missing result, timeout, skipped test, missing test name, or build error is an infrastructure failure and cannot prove the bug. +Keep the observation file below 64 KiB and `details` at or below 4,000 +characters. Put verbose diagnostics in runner stdout/stderr; the wrapper keeps +bounded tails of both streams in the evidence record. + Cases should exercise public/production behavior. A unit test added only on the head cannot prove the base is broken because “test not found” is not a valid observation. Prefer an external harness that builds the target checkout and drives its CLI, broker, protocol, or API. +The wrapper independently attests the exact harness and target checkouts, the +runner process exit, structured observation, handoff nonce, and Cloud sandbox +identity. The case program and its semantic assertion are still PR-authored +test code. As with any new test in a pull request, a reviewer must inspect the +runner and confirm that it actually drives `RELAY_PR_PROOF_TARGET_DIR` and that +its red/green signatures express the claimed behavior. No generic runner can +prove the honesty of arbitrary test code; this check supplies reproducible +review evidence and is not a replacement for required code review. + ## Execution and security The `pull_request_target` workflow checks out only the trusted base. It fetches and validates the case manifest as data, then submits the trusted `workflows/pr-proof.ts` RelayFlow. PR code runs only in the two Cloud agent -sandboxes. The base proof must pass its deterministic evidence gate before the -head sandbox is launched, and the final gate rejects identical sandbox IDs or -incorrect commit provenance. +sandboxes. Each prover uploads structured evidence through the run-scoped Cloud +storage API; the case runner never receives Cloud credentials. A cryptographic +nonce generated by the trusted dispatcher binds both handoffs to the current +run. The base proof must pass its deterministic evidence gate before the head +sandbox is launched, and the final gate rejects a stale nonce, identical +sandbox IDs, or incorrect commit provenance. The RelayFlow is explicitly fail-fast with zero retries. Automatic repair agents are disabled: a rejected observation is evidence to report, never an @@ -104,5 +120,18 @@ The repository needs a dedicated Cloud CLI session in these GitHub secrets: Use a workspace-scoped, least-privilege credential that can prepare, invoke, read, and cancel workflow runs. Do not copy a human laptop session into CI. -After a live canary PR proves the check, require the stable `RelayFlow PR proof` -job on `main` branch protection. +The dispatcher stores the tuple in an owner-only temporary auth file so a +refresh is visible to every CLI subprocess in the same job. It refuses tokens +that expire inside the proof window and fails if a refresh rotates the tuple, +because Actions cannot write rotated values back to repository secrets. Rotate +the dedicated credential before that window. + +`pull_request_target` itself is attached to the base SHA, so the dispatcher +publishes a separate stable commit-status context named `RelayFlow PR proof` on +the exact PR head SHA. Require that context—not the dispatcher job name—on +`main` branch protection. + +GitHub loads `pull_request_target` workflow code from the default branch. Merge +this infrastructure PR before running the live canary on a subsequent feature +or bug-fix PR. Require the status context only after that canary proves the +credential, Cloud handoff, and red/green case end to end. diff --git a/workflows/pr-proof.ts b/workflows/pr-proof.ts index 4dff57b34..4664d76af 100644 --- a/workflows/pr-proof.ts +++ b/workflows/pr-proof.ts @@ -8,8 +8,8 @@ * * Order is deliberate: * 1. base-prover observes the exact bug/capability absence on the base SHA; - * 2. a deterministic gate rejects crashes, skips, missing tests, and the - * wrong failure signature; + * 2. run-scoped Cloud storage hands nonce-bound evidence to a deterministic + * gate, which rejects crashes, skips, and wrong signatures; * 3. head-verifier observes the declared fixed behavior on the head SHA; * 4. a deterministic gate verifies exact SHAs and distinct sandbox IDs. */ @@ -32,14 +32,14 @@ const result = await workflow('relay-pr-proof') cli: 'codex', preset: 'worker', role: 'Run the trusted PR proof base-arm command exactly once without editing repository files.', - interactive: true, + interactive: false, retries: 0, }) .agent('head-verifier', { cli: 'codex', preset: 'worker', role: 'Run the trusted PR proof head-arm command exactly once without editing repository files.', - interactive: true, + interactive: false, retries: 0, }) .step('prove-base', { @@ -56,7 +56,8 @@ const result = await workflow('relay-pr-proof') .step('gate-base', { type: 'deterministic', dependsOn: ['prove-base'], - command: 'node scripts/pr-proof/verify-evidence.mjs --arm base --input .relayflow/pr-proof-input.json', + command: + 'node scripts/pr-proof/verify-evidence.mjs --source cloud --arm base --input .relayflow/pr-proof-input.json', captureOutput: true, failOnError: true, }) @@ -75,12 +76,13 @@ const result = await workflow('relay-pr-proof') .step('gate-red-green', { type: 'deterministic', dependsOn: ['verify-head'], - command: 'node scripts/pr-proof/verify-evidence.mjs --arm both --input .relayflow/pr-proof-input.json', + command: + 'node scripts/pr-proof/verify-evidence.mjs --source cloud --arm both --input .relayflow/pr-proof-input.json', captureOutput: true, failOnError: true, }) .run({ cwd: process.cwd() }); -if ('status' in result && result.status === 'failed') { +if ('status' in result && result.status !== 'completed') { process.exitCode = 1; } From 803aa78740b2ddd7a6f524db4c67ef0921312587 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 12:03:45 +0200 Subject: [PATCH 03/15] fix(ci): resolve PR proof CodeQL findings --- scripts/pr-proof/cloud-storage.mjs | 9 +++++++++ scripts/pr-proof/prepare.mjs | 6 ++++++ scripts/pr-proof/report-status.mjs | 3 +++ scripts/pr-proof/run-arm.mjs | 19 ++++++++++++------- scripts/pr-proof/verify-evidence.mjs | 3 +++ tests/fixtures/pr-proof-contract.test.ts | 22 ++++++++++++++-------- 6 files changed, 47 insertions(+), 15 deletions(-) diff --git a/scripts/pr-proof/cloud-storage.mjs b/scripts/pr-proof/cloud-storage.mjs index c1340b4a0..05cf81267 100644 --- a/scripts/pr-proof/cloud-storage.mjs +++ b/scripts/pr-proof/cloud-storage.mjs @@ -27,6 +27,9 @@ function authorization(env) { export async function uploadCloudEvidence(input, arm, evidence, options = {}) { const env = options.env ?? process.env; const fetchImpl = options.fetchImpl ?? fetch; + // The origin is the trusted Cloud runtime URL; run ID, nonce, and arm are + // strictly validated/encoded run-scoped identifiers, not arbitrary hosts. + // codeql[js/file-access-to-http] const response = await fetchImpl(storageUrl(env, input, arm), { method: 'PUT', headers: { @@ -34,6 +37,9 @@ export async function uploadCloudEvidence(input, arm, evidence, options = {}) { authorization: authorization(env), 'content-type': 'application/json', }, + // Evidence is intentionally sent to this run's authenticated storage + // object after the trusted wrapper has validated its complete contract. + // codeql[js/file-access-to-http] body: JSON.stringify(evidence), }); if (!response.ok) { @@ -45,6 +51,9 @@ export async function uploadCloudEvidence(input, arm, evidence, options = {}) { export async function downloadCloudEvidence(input, arm, options = {}) { const env = options.env ?? process.env; const fetchImpl = options.fetchImpl ?? fetch; + // The trusted Cloud runtime origin and encoded run-scoped identifiers above + // prevent file-derived data from selecting an arbitrary destination. + // codeql[js/file-access-to-http] const response = await fetchImpl(storageUrl(env, input, arm), { headers: { accept: 'application/json', diff --git a/scripts/pr-proof/prepare.mjs b/scripts/pr-proof/prepare.mjs index 91e9bef6b..416b947d3 100644 --- a/scripts/pr-proof/prepare.mjs +++ b/scripts/pr-proof/prepare.mjs @@ -101,6 +101,9 @@ async function writeGithubOutput(values, outputPath) { async function writeSummary(lines, summaryPath) { if (!summaryPath) return; + // GitHub supplies the summary destination; PR fields are rendered only as + // inert Markdown text after classification/snapshot validation. + // codeql[js/http-to-file-access] await appendFile(summaryPath, `${lines.join('\n')}\n`); } @@ -219,6 +222,9 @@ export async function main() { manifest, }; await mkdir(path.dirname(outputPath), { recursive: true }); + // The workflow fixes outputPath, and every network-derived property has + // passed strict type/path/SHA/manifest validation plus a final PR snapshot. + // codeql[js/http-to-file-access] await writeFile(outputPath, `${JSON.stringify(proofInput, null, 2)}\n`); await writeGithubOutput({ required: true, case_id: caseId, input_path: outputPath }, githubOutput); await writeSummary( diff --git a/scripts/pr-proof/report-status.mjs b/scripts/pr-proof/report-status.mjs index ba54c0d2a..b6d92ef24 100644 --- a/scripts/pr-proof/report-status.mjs +++ b/scripts/pr-proof/report-status.mjs @@ -23,6 +23,9 @@ function headers(token) { } async function githubJson(url, token, init = {}, fetchImpl = fetch) { + // Callers construct only GitHub API URLs from the trusted API origin and + // validated repository, PR-number, or full-SHA components. + // codeql[js/file-access-to-http] const response = await fetchImpl(url, { ...init, headers: { ...headers(token), ...(init.headers ?? {}) }, diff --git a/scripts/pr-proof/run-arm.mjs b/scripts/pr-proof/run-arm.mjs index e68eb8464..224e68f4a 100644 --- a/scripts/pr-proof/run-arm.mjs +++ b/scripts/pr-proof/run-arm.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -130,13 +130,18 @@ export async function main() { let observationJson; try { - const resultStat = await stat(resultPath); - if (!resultStat.isFile() || resultStat.size > MAX_OBSERVATION_FILE_BYTES) { - throw new Error( - `observation must be a regular file no larger than ${MAX_OBSERVATION_FILE_BYTES} bytes` - ); + const resultFile = await open(resultPath, 'r'); + try { + const resultStat = await resultFile.stat(); + if (!resultStat.isFile() || resultStat.size > MAX_OBSERVATION_FILE_BYTES) { + throw new Error( + `observation must be a regular file no larger than ${MAX_OBSERVATION_FILE_BYTES} bytes` + ); + } + observationJson = JSON.parse(await resultFile.readFile('utf8')); + } finally { + await resultFile.close(); } - observationJson = JSON.parse(await readFile(resultPath, 'utf8')); } catch (error) { throw new PrProofContractError('Case runner did not write a valid observation JSON file', [ error instanceof Error ? error.message : String(error), diff --git a/scripts/pr-proof/verify-evidence.mjs b/scripts/pr-proof/verify-evidence.mjs index 3cd339539..1eb4d9c21 100644 --- a/scripts/pr-proof/verify-evidence.mjs +++ b/scripts/pr-proof/verify-evidence.mjs @@ -84,6 +84,9 @@ async function main() { completedAt: new Date().toISOString(), }; await mkdir(artifactRoot, { recursive: true }); + // artifactRoot is trusted workflow configuration; all Cloud-derived verdict + // fields passed the exact nonce/SHA/signature/sandbox evidence contract. + // codeql[js/http-to-file-access] await writeFile(path.join(artifactRoot, 'verdict.json'), `${JSON.stringify(verdict, null, 2)}\n`); console.log( `PR_PROOF_PASS case=${input.caseId} base_sandbox=${base.sandboxId} head_sandbox=${head.sandboxId}` diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index beaba092c..5bd08606b 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -334,13 +334,19 @@ describe('Cloud dispatcher credential lifecycle', () => { try { expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN).toBeUndefined(); expect(auth.cliEnv.HOME).toBe(auth.temporaryHome); - expect((await stat(auth.authPath)).mode & 0o777).toBe(0o600); - await expect( - assertCredentialDidNotRotate(auth.authPath, credentialEnv.CLOUD_API_ACCESS_TOKEN) - ).resolves.toBeUndefined(); - const stored = JSON.parse(await readFile(auth.authPath, 'utf8')); - stored.accessToken = 'rotated-token'; - await writeFile(auth.authPath, JSON.stringify(stored), { mode: 0o600 }); + const authFile = await open(auth.authPath, 'r+'); + try { + expect((await authFile.stat()).mode & 0o777).toBe(0o600); + await expect( + assertCredentialDidNotRotate(auth.authPath, credentialEnv.CLOUD_API_ACCESS_TOKEN) + ).resolves.toBeUndefined(); + const stored = JSON.parse(await authFile.readFile('utf8')); + stored.accessToken = 'rotated-token'; + await authFile.truncate(0); + await authFile.write(JSON.stringify(stored), 0, 'utf8'); + } finally { + await authFile.close(); + } await expect( assertCredentialDidNotRotate(auth.authPath, credentialEnv.CLOUD_API_ACCESS_TOKEN) ).rejects.toThrow(/cannot be persisted/); From 882ad0b4cbb19c445071add07a7761815ea1b191 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 12:05:42 +0200 Subject: [PATCH 04/15] fix(ci): bound observation reads by handle --- scripts/pr-proof/run-arm.mjs | 41 +++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/scripts/pr-proof/run-arm.mjs b/scripts/pr-proof/run-arm.mjs index 224e68f4a..b326e0ba3 100644 --- a/scripts/pr-proof/run-arm.mjs +++ b/scripts/pr-proof/run-arm.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node import { mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; +import { constants as fsConstants } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -19,6 +20,33 @@ import { runBoundedProcess } from './process-runner.mjs'; const MAX_OBSERVATION_FILE_BYTES = 64 * 1024; +async function readObservationFile(filePath) { + const resultFile = await open( + filePath, + fsConstants.O_RDONLY | fsConstants.O_NONBLOCK | fsConstants.O_NOFOLLOW + ); + try { + const resultStat = await resultFile.stat(); + if (!resultStat.isFile()) throw new Error('observation must be a regular, non-symlink file'); + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_OBSERVATION_FILE_BYTES) { + const chunk = Buffer.alloc(Math.min(16 * 1024, MAX_OBSERVATION_FILE_BYTES + 1 - totalBytes)); + const { bytesRead } = await resultFile.read(chunk, 0, chunk.length, null); + if (bytesRead === 0) break; + chunks.push(chunk.subarray(0, bytesRead)); + totalBytes += bytesRead; + } + if (totalBytes > MAX_OBSERVATION_FILE_BYTES) { + throw new Error(`observation must be no larger than ${MAX_OBSERVATION_FILE_BYTES} bytes`); + } + return Buffer.concat(chunks, totalBytes).toString('utf8'); + } finally { + await resultFile.close(); + } +} + export async function runProcess(command, args, options = {}) { return runBoundedProcess(command, args, options); } @@ -130,18 +158,7 @@ export async function main() { let observationJson; try { - const resultFile = await open(resultPath, 'r'); - try { - const resultStat = await resultFile.stat(); - if (!resultStat.isFile() || resultStat.size > MAX_OBSERVATION_FILE_BYTES) { - throw new Error( - `observation must be a regular file no larger than ${MAX_OBSERVATION_FILE_BYTES} bytes` - ); - } - observationJson = JSON.parse(await resultFile.readFile('utf8')); - } finally { - await resultFile.close(); - } + observationJson = JSON.parse(await readObservationFile(resultPath)); } catch (error) { throw new PrProofContractError('Case runner did not write a valid observation JSON file', [ error instanceof Error ? error.message : String(error), From 67290f4ef7c631bff8cdc877ff19a514f5a4c3f8 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 12:10:25 +0200 Subject: [PATCH 05/15] fix(pr-proof): fail closed before sandbox execution --- scripts/pr-proof/cloud-storage.mjs | 12 ++++++++++++ scripts/pr-proof/process-runner.mjs | 15 ++++++++++++-- scripts/pr-proof/run-arm.mjs | 5 ++++- tests/fixtures/pr-proof-contract.test.ts | 25 +++++++++++++++++++++++- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/scripts/pr-proof/cloud-storage.mjs b/scripts/pr-proof/cloud-storage.mjs index 05cf81267..ef6a57aef 100644 --- a/scripts/pr-proof/cloud-storage.mjs +++ b/scripts/pr-proof/cloud-storage.mjs @@ -2,6 +2,7 @@ const ARM_RE = /^(base|head)$/; const NONCE_RE = /^[0-9a-f]{32}$/; +const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; function requiredEnvironment(env, name) { const value = env[name]?.trim(); @@ -24,6 +25,15 @@ function authorization(env) { return `Bearer ${requiredEnvironment(env, 'CLOUD_API_ACCESS_TOKEN')}`; } +export function validateCloudEvidenceEnvironment(input, arm, env = process.env) { + storageUrl(env, input, arm); + authorization(env); +} + +function requestSignal(options) { + return options.signal ?? AbortSignal.timeout(options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS); +} + export async function uploadCloudEvidence(input, arm, evidence, options = {}) { const env = options.env ?? process.env; const fetchImpl = options.fetchImpl ?? fetch; @@ -32,6 +42,7 @@ export async function uploadCloudEvidence(input, arm, evidence, options = {}) { // codeql[js/file-access-to-http] const response = await fetchImpl(storageUrl(env, input, arm), { method: 'PUT', + signal: requestSignal(options), headers: { accept: 'application/json', authorization: authorization(env), @@ -55,6 +66,7 @@ export async function downloadCloudEvidence(input, arm, options = {}) { // prevent file-derived data from selecting an arbitrary destination. // codeql[js/file-access-to-http] const response = await fetchImpl(storageUrl(env, input, arm), { + signal: requestSignal(options), headers: { accept: 'application/json', authorization: authorization(env), diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs index 2cf3ba0fd..965147496 100644 --- a/scripts/pr-proof/process-runner.mjs +++ b/scripts/pr-proof/process-runner.mjs @@ -1,4 +1,5 @@ import { spawn } from 'node:child_process'; +import { StringDecoder } from 'node:string_decoder'; const DEFAULT_CAPTURE_BYTES = 128 * 1024; const DEFAULT_TERMINATION_GRACE_MS = 5_000; @@ -44,6 +45,8 @@ export function runBoundedProcess(command, args, options = {}) { let timedOut = false; let settled = false; let hardKill = null; + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); const timeout = options.timeoutMs ? setTimeout(() => { @@ -64,12 +67,14 @@ export function runBoundedProcess(command, args, options = {}) { }; child.stdout.on('data', (chunk) => { - const text = chunk.toString(); + const text = stdoutDecoder.write(chunk); + if (!text) return; stdout = appendBounded(stdout, text, maximum); if (options.echo !== false) process.stdout.write(text); }); child.stderr.on('data', (chunk) => { - const text = chunk.toString(); + const text = stderrDecoder.write(chunk); + if (!text) return; stderr = appendBounded(stderr, text, maximum); if (options.echo !== false) process.stderr.write(text); }); @@ -83,6 +88,12 @@ export function runBoundedProcess(command, args, options = {}) { if (settled) return; settled = true; cleanup(); + const stdoutTail = stdoutDecoder.end(); + const stderrTail = stderrDecoder.end(); + stdout = appendBounded(stdout, stdoutTail, maximum); + stderr = appendBounded(stderr, stderrTail, maximum); + if (options.echo !== false && stdoutTail) process.stdout.write(stdoutTail); + if (options.echo !== false && stderrTail) process.stderr.write(stderrTail); resolve({ exitCode: code ?? 1, signal, stdout, stderr, timedOut }); }); }); diff --git a/scripts/pr-proof/run-arm.mjs b/scripts/pr-proof/run-arm.mjs index b326e0ba3..f16d0d2b1 100644 --- a/scripts/pr-proof/run-arm.mjs +++ b/scripts/pr-proof/run-arm.mjs @@ -15,7 +15,7 @@ import { validateObservation, validateProofInput, } from './contract.mjs'; -import { uploadCloudEvidence } from './cloud-storage.mjs'; +import { uploadCloudEvidence, validateCloudEvidenceEnvironment } from './cloud-storage.mjs'; import { runBoundedProcess } from './process-runner.mjs'; const MAX_OBSERVATION_FILE_BYTES = 64 * 1024; @@ -109,6 +109,9 @@ export async function main() { const input = validateProofInput(JSON.parse(await readFile(inputPath, 'utf8'))); const sandboxId = process.env.SANDBOX_ID?.trim(); if (!sandboxId) throw new Error('SANDBOX_ID is required; this proof arm must run as a Cloud step'); + // Validate the Cloud evidence handoff before checking out or executing any + // PR-authored code, so a misconfigured proof cannot do work it cannot attest. + validateCloudEvidenceEnvironment(input, arm); const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), `relay-pr-proof-${arm}-`)); const harnessDir = path.join(temporaryRoot, 'harness'); diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 5bd08606b..23d9fbecf 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -19,7 +19,11 @@ import { // @ts-expect-error JavaScript module intentionally has no declaration file. import { verifyEvidenceFiles } from '../../scripts/pr-proof/verify-evidence.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. -import { downloadCloudEvidence, uploadCloudEvidence } from '../../scripts/pr-proof/cloud-storage.mjs'; +import { + downloadCloudEvidence, + uploadCloudEvidence, + validateCloudEvidenceEnvironment, +} from '../../scripts/pr-proof/cloud-storage.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. import { publishCommitStatus } from '../../scripts/pr-proof/report-status.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. @@ -291,6 +295,7 @@ describe('Cloud evidence handoff', () => { ); expect(requestUrl).not.toContain('sandbox-access-token'); expect(requestInit?.method).toBe('PUT'); + expect(requestInit?.signal).toBeInstanceOf(AbortSignal); expect(JSON.parse(String(requestInit?.body))).toMatchObject(record); }); @@ -299,6 +304,15 @@ describe('Cloud evidence handoff', () => { const fetchImpl = async () => new Response(JSON.stringify(record), { status: 200 }); await expect(downloadCloudEvidence(input(), 'head', { env, fetchImpl })).resolves.toEqual(record); }); + + it('rejects incomplete Cloud evidence configuration before proof execution', () => { + expect(() => + validateCloudEvidenceEnvironment(input(), 'base', { + CLOUD_API_URL: env.CLOUD_API_URL, + RUN_ID: env.RUN_ID, + }) + ).toThrow(/CLOUD_API_ACCESS_TOKEN/); + }); }); describe('Cloud dispatcher credential lifecycle', () => { @@ -378,6 +392,15 @@ describe('process timeout contract', () => { expect(result.timedOut).toBe(true); expect(Date.now() - startedAt).toBeLessThan(2_000); }); + + it('preserves UTF-8 characters split across output chunks', async () => { + const script = [ + 'process.stdout.write(Buffer.from([0xe2]));', + 'setTimeout(() => process.stdout.write(Buffer.from([0x82, 0xac])), 25);', + ].join(''); + const result = await runProcess(process.execPath, ['-e', script], { echo: false }); + expect(result.stdout).toBe('€'); + }); }); describe('required head status', () => { From 7cdbec3b64fc55c00d66538b668f5a307b660559 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 14:00:30 +0200 Subject: [PATCH 06/15] fix(pr-proof): close cancellation and output gaps --- .github/workflows/relayflow-pr-proof.yml | 7 +- packages/cloud/src/workflows.test.ts | 31 ++++ packages/cloud/src/workflows.ts | 9 + scripts/pr-proof/cloud-storage.mjs | 19 +- scripts/pr-proof/process-runner.mjs | 138 ++++++++++++-- scripts/pr-proof/report-status.mjs | 49 ++++- scripts/pr-proof/run-cloud.mjs | 75 +++++++- tests/fixtures/pr-proof-contract.test.ts | 223 ++++++++++++++++++++++- tests/relayflows/cases/README.md | 18 +- 9 files changed, 517 insertions(+), 52 deletions(-) diff --git a/.github/workflows/relayflow-pr-proof.yml b/.github/workflows/relayflow-pr-proof.yml index ef2715828..6703d6b75 100644 --- a/.github/workflows/relayflow-pr-proof.yml +++ b/.github/workflows/relayflow-pr-proof.yml @@ -109,9 +109,10 @@ jobs: if-no-files-found: warn - name: Finish required status on PR head - # A replacement run has already published pending on this same status - # context. A cancelled predecessor must not overwrite that newer state. - if: always() && !cancelled() && steps.status.outputs.head_sha != '' + # report-status verifies that this run still owns the latest pending + # context before publishing. That makes cancellation terminal without + # allowing a cancelled predecessor to overwrite a replacement run. + if: always() && steps.status.outputs.head_sha != '' env: GITHUB_TOKEN: ${{ github.token }} run: >- diff --git a/packages/cloud/src/workflows.test.ts b/packages/cloud/src/workflows.test.ts index 4919a8438..4f1418c9a 100644 --- a/packages/cloud/src/workflows.test.ts +++ b/packages/cloud/src/workflows.test.ts @@ -512,6 +512,37 @@ describe('runWorkflow code sync', () => { expect((runBodies[0] as { paths?: unknown }).paths).toBeUndefined(); }); + it('reports the prepared run id before upload when supervising automation opts in', async () => { + await writeFile('README.md', 'supervised\n'); + const workflowPath = path.join(tmpRoot, 'workflow.yaml'); + await writeFile( + workflowPath, + ['version: "1.0"', 'name: supervised', 'swarm:', ' pattern: dag', 'agents: []', 'workflows: []'].join( + '\n' + ) + ); + const runBodies: unknown[] = []; + mockPrepareAndRun(runBodies); + const errors: string[] = []; + const errorSpy = vi.spyOn(console, 'error').mockImplementation((value) => { + errors.push(String(value)); + }); + process.env.AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID = '1'; + try { + await runWorkflow(workflowPath); + } finally { + delete process.env.AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID; + errorSpy.mockRestore(); + } + + const preparedIndex = errors.indexOf('AGENT_RELAY_CLOUD_PREPARED_RUN_ID=run-1'); + const uploadIndex = errors.indexOf('Uploading to workflow storage...'); + const launchIndex = errors.indexOf('Launching workflow...'); + expect(preparedIndex).toBeGreaterThanOrEqual(0); + expect(preparedIndex).toBeLessThan(uploadIndex); + expect(preparedIndex).toBeLessThan(launchIndex); + }); + it('uploads code through the cloud API when prepare returns cloud-api storage', async () => { await writeFile('README.md', 'cloud-api\n'); const workflowPath = path.join(tmpRoot, 'workflow.yaml'); diff --git a/packages/cloud/src/workflows.ts b/packages/cloud/src/workflows.ts index 1d8792d4c..82b83a04c 100644 --- a/packages/cloud/src/workflows.ts +++ b/packages/cloud/src/workflows.ts @@ -58,6 +58,8 @@ type RunWorkflowOptions = { previousRunId?: string; }; +const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; + const CODE_SYNC_EXCLUDES = [ '.git', 'node_modules', @@ -289,6 +291,13 @@ export async function runWorkflow( } const prepared = prepPayload; + // Trusted automation can opt into this machine-readable progress record + // before code upload and final submission. It lets a supervising process + // cancel the prepared run if the CLI is interrupted after the launch POST + // has completed but before the final JSON response is observed. + if (process.env.AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID === '1') { + console.error(`${PREPARED_RUN_ID_MARKER}${prepared.runId}`); + } console.error(` Prepared in ${((Date.now() - t0) / 1000).toFixed(1)}s`); let s3Client: S3Client | null = null; diff --git a/scripts/pr-proof/cloud-storage.mjs b/scripts/pr-proof/cloud-storage.mjs index ef6a57aef..97132806e 100644 --- a/scripts/pr-proof/cloud-storage.mjs +++ b/scripts/pr-proof/cloud-storage.mjs @@ -3,6 +3,7 @@ const ARM_RE = /^(base|head)$/; const NONCE_RE = /^[0-9a-f]{32}$/; const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; +const MAX_TIMER_MS = 2_147_483_647; function requiredEnvironment(env, name) { const value = env[name]?.trim(); @@ -14,7 +15,12 @@ function storageUrl(env, input, arm) { if (!ARM_RE.test(arm)) throw new Error(`Invalid proof arm: ${arm}`); if (!NONCE_RE.test(input.handoffNonce ?? '')) throw new Error('Invalid proof handoff nonce'); const apiUrl = requiredEnvironment(env, 'CLOUD_API_URL').replace(/\/$/, '') + '/'; - const runId = encodeURIComponent(requiredEnvironment(env, 'RUN_ID')); + const orchestratorRunId = env.RUN_ID?.trim(); + const workerRunId = env.AGENT_RELAY_CLOUD_WORKER_RUN_ID?.trim(); + if (orchestratorRunId && workerRunId && orchestratorRunId !== workerRunId) { + throw new Error('Cloud proof runtime exposed conflicting workflow run IDs'); + } + const runId = encodeURIComponent(orchestratorRunId || workerRunId || requiredEnvironment(env, 'RUN_ID')); const objectKey = ['pr-proof', input.handoffNonce, `${arm}.json`] .map((segment) => encodeURIComponent(segment)) .join('/'); @@ -31,7 +37,16 @@ export function validateCloudEvidenceEnvironment(input, arm, env = process.env) } function requestSignal(options) { - return options.signal ?? AbortSignal.timeout(options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS); + if (options.signal) return options.signal; + const requested = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + if (!Number.isFinite(requested) || requested <= 0) { + throw new Error('Cloud evidence request timeout must be a positive finite number'); + } + const timeoutMs = Math.min(Math.floor(requested), MAX_TIMER_MS); + if (timeoutMs < 1) { + throw new Error('Cloud evidence request timeout must be at least one millisecond'); + } + return AbortSignal.timeout(timeoutMs); } export async function uploadCloudEvidence(input, arm, evidence, options = {}) { diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs index 965147496..44a95e32f 100644 --- a/scripts/pr-proof/process-runner.mjs +++ b/scripts/pr-proof/process-runner.mjs @@ -3,10 +3,36 @@ import { StringDecoder } from 'node:string_decoder'; const DEFAULT_CAPTURE_BYTES = 128 * 1024; const DEFAULT_TERMINATION_GRACE_MS = 5_000; +const MAX_TIMER_MS = 2_147_483_647; function appendBounded(current, chunk, maximum) { - const next = current + chunk; - return next.length <= maximum ? next : next.slice(next.length - maximum); + const bytes = Buffer.from(current + chunk, 'utf8'); + if (bytes.length <= maximum) return bytes.toString('utf8'); + + let start = bytes.length - maximum; + while (start < bytes.length && (bytes[start] & 0xc0) === 0x80) start += 1; + return bytes.subarray(start).toString('utf8'); +} + +function takeUtf8Prefix(value, maximum) { + const bytes = Buffer.from(value, 'utf8'); + if (bytes.length <= maximum) return value; + let end = maximum; + while (end > 0 && (bytes[end] & 0xc0) === 0x80) end -= 1; + return bytes.subarray(0, end).toString('utf8'); +} + +function boundedInteger(value, { fallback, minimum, label }) { + const candidate = value ?? fallback; + if ( + !Number.isFinite(candidate) || + !Number.isInteger(candidate) || + candidate < minimum || + candidate > MAX_TIMER_MS + ) { + throw new Error(`${label} must be an integer between ${minimum} and ${MAX_TIMER_MS}`); + } + return candidate; } function signalProcessTree(child, signal) { @@ -32,8 +58,34 @@ function signalProcessTree(child, signal) { * closed so an escaped descendant cannot keep this promise pending forever. */ export function runBoundedProcess(command, args, options = {}) { + const maximum = boundedInteger(options.maxCaptureBytes, { + fallback: DEFAULT_CAPTURE_BYTES, + minimum: 1, + label: 'maxCaptureBytes', + }); + const maximumLiveOutput = boundedInteger(options.maxLiveOutputBytes, { + fallback: maximum, + minimum: 0, + label: 'maxLiveOutputBytes', + }); + const timeoutMs = + options.timeoutMs === undefined || options.timeoutMs === null + ? null + : boundedInteger(options.timeoutMs, { + fallback: null, + minimum: 1, + label: 'timeoutMs', + }); + const terminationGraceMs = boundedInteger(options.terminationGraceMs, { + fallback: DEFAULT_TERMINATION_GRACE_MS, + minimum: 0, + label: 'terminationGraceMs', + }); + if (options.signal?.aborted) { + return Promise.reject(new Error('Subprocess aborted before launch')); + } + return new Promise((resolve, reject) => { - const maximum = options.maxCaptureBytes ?? DEFAULT_CAPTURE_BYTES; const child = spawn(command, args, { cwd: options.cwd, env: options.env ?? process.env, @@ -43,40 +95,76 @@ export function runBoundedProcess(command, args, options = {}) { let stdout = ''; let stderr = ''; let timedOut = false; + let aborted = false; let settled = false; let hardKill = null; + let forced = false; + let liveOutputBytes = 0; + let liveOutputTruncated = false; const stdoutDecoder = new StringDecoder('utf8'); const stderrDecoder = new StringDecoder('utf8'); - const timeout = options.timeoutMs - ? setTimeout(() => { - timedOut = true; - signalProcessTree(child, 'SIGTERM'); - hardKill = setTimeout(() => { - signalProcessTree(child, 'SIGKILL'); - child.stdout.destroy(); - child.stderr.destroy(); - }, options.terminationGraceMs ?? DEFAULT_TERMINATION_GRACE_MS); - hardKill.unref(); - }, options.timeoutMs) - : null; + const forceKill = () => { + if (forced) return; + forced = true; + signalProcessTree(child, 'SIGKILL'); + child.stdout.destroy(); + child.stderr.destroy(); + }; + + const beginTermination = (reason) => { + if (timedOut || aborted || settled) return; + timedOut = reason === 'timeout'; + aborted = reason === 'abort'; + signalProcessTree(child, 'SIGTERM'); + hardKill = setTimeout(forceKill, terminationGraceMs); + }; + + const timeout = timeoutMs ? setTimeout(() => beginTermination('timeout'), timeoutMs) : null; + const abortHandler = () => beginTermination('abort'); + options.signal?.addEventListener('abort', abortHandler, { once: true }); const cleanup = () => { if (timeout) clearTimeout(timeout); if (hardKill) clearTimeout(hardKill); + options.signal?.removeEventListener('abort', abortHandler); + }; + + const writeLiveOutput = (stream, text) => { + if (options.echo === false || !text) return; + if (liveOutputBytes >= maximumLiveOutput) { + if (!liveOutputTruncated) { + liveOutputTruncated = true; + stream.write('\n[... subprocess live output truncated ...]\n'); + } + return; + } + + const remaining = maximumLiveOutput - liveOutputBytes; + const prefix = takeUtf8Prefix(text, remaining); + const prefixBytes = Buffer.byteLength(prefix, 'utf8'); + if (prefix) stream.write(prefix); + liveOutputBytes += prefixBytes; + if (prefixBytes < Buffer.byteLength(text, 'utf8')) { + liveOutputBytes = maximumLiveOutput; + liveOutputTruncated = true; + stream.write('\n[... subprocess live output truncated ...]\n'); + } }; child.stdout.on('data', (chunk) => { const text = stdoutDecoder.write(chunk); if (!text) return; stdout = appendBounded(stdout, text, maximum); - if (options.echo !== false) process.stdout.write(text); + options.onStdout?.(text); + writeLiveOutput(process.stdout, text); }); child.stderr.on('data', (chunk) => { const text = stderrDecoder.write(chunk); if (!text) return; stderr = appendBounded(stderr, text, maximum); - if (options.echo !== false) process.stderr.write(text); + options.onStderr?.(text); + writeLiveOutput(process.stderr, text); }); child.on('error', (error) => { if (settled) return; @@ -87,14 +175,24 @@ export function runBoundedProcess(command, args, options = {}) { child.on('close', (code, signal) => { if (settled) return; settled = true; + // The process-group leader can exit after SIGTERM while a descendant + // with detached stdio remains alive. Force-kill the group before + // clearing the grace timer so that descendant cannot escape cleanup. + if (timedOut || aborted) forceKill(); cleanup(); const stdoutTail = stdoutDecoder.end(); const stderrTail = stderrDecoder.end(); stdout = appendBounded(stdout, stdoutTail, maximum); stderr = appendBounded(stderr, stderrTail, maximum); - if (options.echo !== false && stdoutTail) process.stdout.write(stdoutTail); - if (options.echo !== false && stderrTail) process.stderr.write(stderrTail); - resolve({ exitCode: code ?? 1, signal, stdout, stderr, timedOut }); + if (stdoutTail) { + options.onStdout?.(stdoutTail); + writeLiveOutput(process.stdout, stdoutTail); + } + if (stderrTail) { + options.onStderr?.(stderrTail); + writeLiveOutput(process.stderr, stderrTail); + } + resolve({ exitCode: code ?? 1, signal, stdout, stderr, timedOut, aborted }); }); }); } diff --git a/scripts/pr-proof/report-status.mjs b/scripts/pr-proof/report-status.mjs index b6d92ef24..40272ecb3 100644 --- a/scripts/pr-proof/report-status.mjs +++ b/scripts/pr-proof/report-status.mjs @@ -43,7 +43,7 @@ function validateRepository(value) { return value; } -async function resolvePullRequest({ payload, repository, apiUrl, token, fetchImpl }) { +export async function resolvePullRequest({ payload, repository, apiUrl, token, fetchImpl }) { let pullRequest = payload.pull_request ?? null; const manualNumber = Number(payload.inputs?.pr_number); if (!pullRequest && Number.isInteger(manualNumber) && manualNumber > 0) { @@ -56,15 +56,9 @@ async function resolvePullRequest({ payload, repository, apiUrl, token, fetchImp } const number = Number(pullRequest?.number); const headSha = pullRequest?.head?.sha; - const headRepository = pullRequest?.head?.repo?.full_name; if (!Number.isInteger(number) || number < 1 || !SHA_RE.test(headSha ?? '')) { throw new Error('The event does not identify a pull request with an exact head SHA'); } - if (headRepository !== repository) { - throw new Error( - `Fork pull request ${headRepository ?? 'unknown'} cannot receive the credential-bearing proof status` - ); - } return { number, headSha }; } @@ -100,6 +94,39 @@ export async function publishCommitStatus({ sha, state, description, env = proce ); } +export async function publishOwnedCommitStatus({ + sha, + state, + description, + env = process.env, + fetchImpl = fetch, +}) { + const repository = validateRepository(env.GITHUB_REPOSITORY?.trim()); + const token = env.GITHUB_TOKEN?.trim(); + const apiUrl = (env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + const ownerTargetUrl = targetUrl(env); + if (!token || !ownerTargetUrl) { + throw new Error('GITHUB_TOKEN, GITHUB_SERVER_URL, and GITHUB_RUN_ID are required to finish status'); + } + if (!SHA_RE.test(sha ?? '')) throw new Error('PR proof status SHA must be a full lowercase SHA'); + + const combined = await githubJson( + `${apiUrl}/repos/${repository}/commits/${sha}/status`, + token, + {}, + fetchImpl + ); + const latest = Array.isArray(combined?.statuses) + ? combined.statuses.find((status) => status?.context === CONTEXT) + : null; + if (latest?.state !== 'pending' || latest?.target_url !== ownerTargetUrl) { + return { published: false, ownerTargetUrl, latest }; + } + + await publishCommitStatus({ sha, state, description, env, fetchImpl }); + return { published: true, ownerTargetUrl, latest }; +} + function conclusionState(jobStatus) { if (jobStatus === 'success') return 'success'; if (jobStatus === 'failure') return 'failure'; @@ -134,7 +161,7 @@ export async function main() { const sha = option('--sha'); const jobStatus = option('--job-status'); const state = conclusionState(jobStatus); - await publishCommitStatus({ + const result = await publishOwnedCommitStatus({ sha, state, description: @@ -144,7 +171,11 @@ export async function main() { ? 'Declared Cloud red/green proof failed' : `Cloud red/green proof ended with ${jobStatus ?? 'unknown'} status`, }); - console.log(`PR_PROOF_STATUS_FINAL head=${sha} state=${state}`); + console.log( + result.published + ? `PR_PROOF_STATUS_FINAL head=${sha} state=${state}` + : `PR_PROOF_STATUS_FINAL_SKIPPED head=${sha} reason=status-owned-by-newer-run` + ); return; } diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 7e5fd262c..e2fbef5b9 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -19,14 +19,21 @@ const CLOUD_AUTH_KEYS = [ ]; const CREDENTIAL_WINDOW_BUFFER_MS = 15 * 60_000; const MAX_CAPTURE_BYTES = 2 * 1024 * 1024; +const MAX_LIVE_OUTPUT_BYTES = 256 * 1024; const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; +const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; +const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; function run(command, args, options = {}) { return runBoundedProcess(command, args, { env: options.env, echo: !options.quiet, maxCaptureBytes: MAX_CAPTURE_BYTES, + maxLiveOutputBytes: MAX_LIVE_OUTPUT_BYTES, timeoutMs: options.timeoutMs, + signal: options.signal, + onStdout: options.onStdout, + onStderr: options.onStderr, }); } @@ -63,6 +70,18 @@ function requiredCredential(env, name) { return value; } +export function preparedRunIdFromOutput(output) { + for (const line of output.split(/\r?\n/)) { + if (!line.startsWith(PREPARED_RUN_ID_MARKER)) continue; + const candidate = line.slice(PREPARED_RUN_ID_MARKER.length).trim(); + if (!RUN_ID_RE.test(candidate)) { + throw new Error('Cloud prepare progress contained an invalid run ID'); + } + return candidate; + } + return null; +} + export function validateCredentialWindow(env, timeoutMs, now = Date.now()) { const accessExpiry = Date.parse(requiredCredential(env, 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT')); if (!Number.isFinite(accessExpiry)) throw new Error('CLOUD_API_ACCESS_TOKEN_EXPIRES_AT is invalid'); @@ -147,6 +166,33 @@ export async function main() { let terminal = false; let cancelPromise = null; let shuttingDown = false; + let activeCommandController = null; + let launchProgress = ''; + let launchProgressError = null; + + const noteLaunchProgress = (text) => { + launchProgress = `${launchProgress}${text}`.slice(-8_192); + try { + const preparedRunId = preparedRunIdFromOutput(launchProgress); + if (!preparedRunId) return; + if (runId && runId !== preparedRunId) { + throw new Error(`Cloud prepare/run ID mismatch: ${runId} != ${preparedRunId}`); + } + runId = preparedRunId; + } catch (error) { + launchProgressError ??= error; + } + }; + + const runTracked = async (command, args, options = {}) => { + const controller = new AbortController(); + activeCommandController = controller; + try { + return await run(command, args, { ...options, signal: controller.signal }); + } finally { + if (activeCommandController === controller) activeCommandController = null; + } + }; const cancelRemote = async (reason) => { if (!runId || terminal) return; @@ -167,6 +213,7 @@ export async function main() { const signalHandler = (signal) => { if (shuttingDown) return; shuttingDown = true; + activeCommandController?.abort(); void (async () => { await cancelRemote(signal).catch((error) => console.warn(error.message)); await rm(auth.temporaryHome, { recursive: true, force: true }); @@ -177,14 +224,21 @@ export async function main() { process.once('SIGTERM', signalHandler); try { - const launch = await run(cli, ['cloud', 'run', workflowPath, '--sync-code', '--json'], { - env: auth.cliEnv, + const launch = await runTracked(cli, ['cloud', 'run', workflowPath, '--sync-code', '--json'], { + env: { + ...auth.cliEnv, + AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID: '1', + }, quiet: true, timeoutMs: commandTimeoutMs, + onStderr: noteLaunchProgress, }); + if (launchProgressError) throw launchProgressError; + if (launch.aborted) throw new Error('Cloud workflow submission was interrupted'); if (launch.timedOut) { + await cancelRemote('submission command timed out'); throw new Error( - 'Cloud workflow submission command timed out; it is not retried because submission may have completed' + 'Cloud workflow submission command timed out and its prepared run was cancelled; it is not retried' ); } if (launch.exitCode !== 0) { @@ -192,8 +246,14 @@ export async function main() { throw new Error(`Cloud workflow submission failed with exit ${launch.exitCode}`); } const launchPayload = parseJsonOutput(launch.stdout, 'Cloud run'); - runId = launchPayload.runId; - if (typeof runId !== 'string' || !runId) throw new Error('Cloud run response did not contain runId'); + const launchedRunId = launchPayload.runId; + if (typeof launchedRunId !== 'string' || !RUN_ID_RE.test(launchedRunId)) { + throw new Error('Cloud run response did not contain a valid runId'); + } + if (runId && runId !== launchedRunId) { + throw new Error(`Cloud prepare/run ID mismatch: ${runId} != ${launchedRunId}`); + } + runId = launchedRunId; console.log(`Cloud RelayFlow run: ${runId}`); if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `run_id=${runId}\n`); @@ -201,7 +261,7 @@ export async function main() { let terminalStatus = null; while (Date.now() < deadline) { await delay(pollMs); - const statusResult = await run(cli, ['cloud', 'status', runId, '--json'], { + const statusResult = await runTracked(cli, ['cloud', 'status', runId, '--json'], { env: auth.cliEnv, quiet: true, timeoutMs: commandTimeoutMs, @@ -228,7 +288,7 @@ export async function main() { } await mkdir(path.dirname(logsPath), { recursive: true }); - const logs = await run(cli, ['cloud', 'logs', runId], { + const logs = await runTracked(cli, ['cloud', 'logs', runId], { env: auth.cliEnv, quiet: true, timeoutMs: commandTimeoutMs, @@ -250,6 +310,7 @@ export async function main() { ); } } finally { + activeCommandController?.abort(); process.removeListener('SIGINT', signalHandler); process.removeListener('SIGTERM', signalHandler); if (runId && !terminal) diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 23d9fbecf..0977b0c01 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -25,7 +25,11 @@ import { validateCloudEvidenceEnvironment, } from '../../scripts/pr-proof/cloud-storage.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. -import { publishCommitStatus } from '../../scripts/pr-proof/report-status.mjs'; +import { + publishCommitStatus, + publishOwnedCommitStatus, + resolvePullRequest, +} from '../../scripts/pr-proof/report-status.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. import { assertSamePullRequestSnapshot, pullRequestSnapshot } from '../../scripts/pr-proof/prepare.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. @@ -33,6 +37,7 @@ import { assertCredentialDidNotRotate, boundedDuration, createCliAuthEnvironment, + preparedRunIdFromOutput, validateCredentialWindow, } from '../../scripts/pr-proof/run-cloud.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. @@ -305,6 +310,47 @@ describe('Cloud evidence handoff', () => { await expect(downloadCloudEvidence(input(), 'head', { env, fetchImpl })).resolves.toEqual(record); }); + it('accepts the Cloud worker run-id alias and rejects conflicting runtime provenance', async () => { + let requestUrl = ''; + const fetchImpl = async (url: string | URL | Request) => { + requestUrl = String(url); + return Response.json({ ok: true }); + }; + const workerEnv = { + CLOUD_API_URL: env.CLOUD_API_URL, + CLOUD_API_ACCESS_TOKEN: env.CLOUD_API_ACCESS_TOKEN, + AGENT_RELAY_CLOUD_WORKER_RUN_ID: env.RUN_ID, + }; + await uploadCloudEvidence(input(), 'base', evidence('base', 'sandbox-base'), { + env: workerEnv, + fetchImpl, + }); + expect(requestUrl).toContain('/workflows/runs/run-123/storage/'); + expect(() => + validateCloudEvidenceEnvironment(input(), 'base', { + ...workerEnv, + RUN_ID: 'different-run', + }) + ).toThrow(/conflicting workflow run IDs/); + }); + + it('rejects malformed request deadlines instead of disabling or collapsing the timeout', async () => { + await expect( + downloadCloudEvidence(input(), 'base', { + env, + requestTimeoutMs: Number.NaN, + fetchImpl: async () => Response.json({}), + }) + ).rejects.toThrow(/positive finite/); + await expect( + downloadCloudEvidence(input(), 'base', { + env, + requestTimeoutMs: 0.5, + fetchImpl: async () => Response.json({}), + }) + ).rejects.toThrow(/at least one millisecond/); + }); + it('rejects incomplete Cloud evidence configuration before proof execution', () => { expect(() => validateCloudEvidenceEnvironment(input(), 'base', { @@ -342,6 +388,17 @@ describe('Cloud dispatcher credential lifecycle', () => { expect(() => boundedDuration('60001', options)).toThrow(/between 100 and 60000/); }); + it('captures the prepared run before the final Cloud submission response', () => { + expect( + preparedRunIdFromOutput( + `Preparing run...\nAGENT_RELAY_CLOUD_PREPARED_RUN_ID=run-prepared-123\nUploading...\n` + ) + ).toBe('run-prepared-123'); + expect(() => preparedRunIdFromOutput('AGENT_RELAY_CLOUD_PREPARED_RUN_ID=invalid run\n')).toThrow( + /invalid run ID/ + ); + }); + it('shares an owner-only temporary auth file across CLI subprocesses and detects rotation', async () => { validateCredentialWindow(credentialEnv, 60 * 60_000, now); const auth = await createCliAuthEnvironment({ ...process.env, ...credentialEnv }); @@ -393,6 +450,34 @@ describe('process timeout contract', () => { expect(Date.now() - startedAt).toBeLessThan(2_000); }); + it('force-kills same-group descendants even when they do not inherit output pipes', async () => { + const script = [ + "const { spawn } = require('node:child_process');", + "const child = spawn(process.execPath, ['-e', \"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)\"], { stdio: 'ignore' });", + "process.stdout.write(String(child.pid) + '\\n');", + "process.on('SIGTERM', () => process.exit(0));", + 'setInterval(() => {}, 1000);', + ].join(''); + const result = await runProcess(process.execPath, ['-e', script], { + echo: false, + timeoutMs: 100, + terminationGraceMs: 100, + }); + const descendantPid = Number(result.stdout.trim()); + expect(result.timedOut).toBe(true); + expect(descendantPid).toBeGreaterThan(0); + await new Promise((resolve) => setTimeout(resolve, 50)); + let alive = true; + try { + process.kill(descendantPid, 0); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') alive = false; + else throw error; + } + if (alive) process.kill(descendantPid, 'SIGKILL'); + expect(alive).toBe(false); + }); + it('preserves UTF-8 characters split across output chunks', async () => { const script = [ 'process.stdout.write(Buffer.from([0xe2]));', @@ -401,9 +486,52 @@ describe('process timeout contract', () => { const result = await runProcess(process.execPath, ['-e', script], { echo: false }); expect(result.stdout).toBe('€'); }); + + it('caps captured and live output by UTF-8 bytes', async () => { + const captured = await runProcess(process.execPath, ['-e', "process.stdout.write('€'.repeat(10))"], { + echo: false, + maxCaptureBytes: 5, + }); + expect(Buffer.byteLength(captured.stdout, 'utf8')).toBeLessThanOrEqual(5); + expect(captured.stdout).not.toContain('�'); + + const writes: string[] = []; + const originalWrite = process.stdout.write; + process.stdout.write = ((chunk: string | Uint8Array) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stdout.write; + try { + await runProcess(process.execPath, ['-e', "process.stdout.write('abcdef')"], { + maxLiveOutputBytes: 4, + }); + } finally { + process.stdout.write = originalWrite; + } + expect(writes.join('')).toContain('abcd'); + expect(writes.join('')).not.toContain('abcdef'); + expect(writes.join('')).toContain('live output truncated'); + }); + + it('rejects malformed process bounds before spawning', async () => { + await expect( + runProcess(process.execPath, ['-e', 'process.exit(0)'], { timeoutMs: Number.NaN }) + ).rejects.toThrow(/timeoutMs must be an integer/); + await expect( + runProcess(process.execPath, ['-e', 'process.exit(0)'], { terminationGraceMs: -1 }) + ).rejects.toThrow(/terminationGraceMs must be an integer/); + }); }); describe('required head status', () => { + const statusEnv = { + GITHUB_REPOSITORY: 'AgentWorkforce/relay', + GITHUB_TOKEN: 'github-token', + GITHUB_API_URL: 'https://api.github.test', + GITHUB_SERVER_URL: 'https://github.test', + GITHUB_RUN_ID: '12345', + }; + it('publishes the stable proof context on the exact head SHA', async () => { let requestUrl = ''; let requestBody: Record = {}; @@ -416,16 +544,84 @@ describe('required head status', () => { sha: HEAD_SHA, state: 'success', description: 'proof passed', - env: { - GITHUB_REPOSITORY: 'AgentWorkforce/relay', - GITHUB_TOKEN: 'github-token', - GITHUB_API_URL: 'https://api.github.test', - }, + env: statusEnv, fetchImpl, }); expect(requestUrl).toBe(`https://api.github.test/repos/AgentWorkforce/relay/statuses/${HEAD_SHA}`); expect(requestBody).toMatchObject({ context: 'RelayFlow PR proof', state: 'success' }); }); + + it('publishes status for a fork head while leaving credential rejection to preparation', async () => { + await expect( + resolvePullRequest({ + payload: { + pull_request: { + number: 1612, + head: { sha: HEAD_SHA, repo: { full_name: 'outside/fork' } }, + }, + }, + repository: 'AgentWorkforce/relay', + apiUrl: 'https://api.github.test', + token: 'github-token', + fetchImpl: async () => Response.json({}), + }) + ).resolves.toEqual({ number: 1612, headSha: HEAD_SHA }); + }); + + it('finalizes only the pending status owned by the current workflow run', async () => { + const requests: Array<{ url: string; method: string; body?: Record }> = []; + const ownerTargetUrl = 'https://github.test/AgentWorkforce/relay/actions/runs/12345'; + const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { + requests.push({ + url: String(url), + method: init?.method ?? 'GET', + ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}), + }); + if (!init?.method) { + return Response.json({ + statuses: [{ context: 'RelayFlow PR proof', state: 'pending', target_url: ownerTargetUrl }], + }); + } + return Response.json({ id: 2 }); + }; + await expect( + publishOwnedCommitStatus({ + sha: HEAD_SHA, + state: 'error', + description: 'cancelled', + env: statusEnv, + fetchImpl, + }) + ).resolves.toMatchObject({ published: true }); + expect(requests).toHaveLength(2); + expect(requests[1]?.body).toMatchObject({ state: 'error' }); + }); + + it('does not let a cancelled predecessor overwrite a newer run', async () => { + let requests = 0; + const fetchImpl = async () => { + requests += 1; + return Response.json({ + statuses: [ + { + context: 'RelayFlow PR proof', + state: 'pending', + target_url: 'https://github.test/AgentWorkforce/relay/actions/runs/newer', + }, + ], + }); + }; + await expect( + publishOwnedCommitStatus({ + sha: HEAD_SHA, + state: 'error', + description: 'cancelled', + env: statusEnv, + fetchImpl, + }) + ).resolves.toMatchObject({ published: false }); + expect(requests).toBe(1); + }); }); describe('pull request snapshot consistency', () => { @@ -467,7 +663,8 @@ describe('trusted dispatcher source contract', () => { expect(source).toContain('statuses: write'); expect(source).toContain('report-status.mjs start'); expect(source).toContain('report-status.mjs finish'); - expect(source).toContain('always() && !cancelled()'); + expect(source).toContain("if: always() && steps.status.outputs.head_sha != ''"); + expect(source).not.toContain('always() && !cancelled()'); expect(source).toContain('--expected-head-sha "${{ steps.status.outputs.head_sha }}"'); expect(source).toContain('actions/checkout@11d5960a326750d5838078e36cf38b85af677262'); expect(source).toContain('actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020'); @@ -496,8 +693,20 @@ describe('trusted dispatcher source contract', () => { it('shares refreshes only inside the job and cancels remote work on termination', async () => { const source = await readFile('scripts/pr-proof/run-cloud.mjs', 'utf8'); expect(source).toContain("process.once('SIGTERM', signalHandler)"); + expect(source).toContain('activeCommandController?.abort()'); + expect(source).toContain('AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID'); expect(source).toContain("['cloud', 'cancel', runId, '--json']"); expect(source).toContain("path.join(authDir, 'cloud-auth.json')"); expect(source).toContain('assertCredentialDidNotRotate'); }); + + it('emits the prepared Cloud run id before upload and final submission', async () => { + const source = await readFile('packages/cloud/src/workflows.ts', 'utf8'); + const marker = source.indexOf("if (process.env.AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID === '1')"); + const upload = source.indexOf('Creating tarball...'); + const launch = source.indexOf('Launching workflow...'); + expect(marker).toBeGreaterThan(0); + expect(marker).toBeLessThan(upload); + expect(marker).toBeLessThan(launch); + }); }); diff --git a/tests/relayflows/cases/README.md b/tests/relayflows/cases/README.md index c9469abd4..f953d26ca 100644 --- a/tests/relayflows/cases/README.md +++ b/tests/relayflows/cases/README.md @@ -106,7 +106,9 @@ the runner's git index because Cloud code sync uploads git-known paths. It does not commit or push the generated file. Fork PRs do not receive Cloud credentials. A maintainer must reproduce the -change and its case on a same-repository branch before merge. +change and its case on a same-repository branch before merge. Non-functional +fork PRs can still complete the stable status without entering a +credential-bearing step. ## Enabling the required check @@ -132,6 +134,14 @@ the exact PR head SHA. Require that context—not the dispatcher job name—on `main` branch protection. GitHub loads `pull_request_target` workflow code from the default branch. Merge -this infrastructure PR before running the live canary on a subsequent feature -or bug-fix PR. Require the status context only after that canary proves the -credential, Cloud handoff, and red/green case end to end. +this infrastructure PR and publish the resulting Agent Relay package before +running the live canary on a subsequent feature or bug-fix PR. The published +CLI emits an opt-in prepared-run marker so the dispatcher can cancel remote +work even if submission is interrupted before its final JSON response. Require +the status context only after that canary proves the credential, Cloud handoff, +submission cancellation, and red/green case end to end. + +The final status step also runs during cancellation. Before it publishes a +terminal state, it verifies that its own Actions run still owns the latest +pending `RelayFlow PR proof` context; a cancelled predecessor cannot overwrite +a replacement run. From 62d445b80319f4e07aa31a04904ea8d399e01bb4 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 14:10:47 +0200 Subject: [PATCH 07/15] test(pr-proof): treat killed zombies as terminated --- tests/fixtures/pr-proof-contract.test.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 0977b0c01..384025bcc 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -1,4 +1,5 @@ import { mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; import os from 'node:os'; import path from 'node:path'; @@ -467,15 +468,18 @@ describe('process timeout contract', () => { expect(result.timedOut).toBe(true); expect(descendantPid).toBeGreaterThan(0); await new Promise((resolve) => setTimeout(resolve, 50)); - let alive = true; + let processState = ''; try { - process.kill(descendantPid, 0); + processState = execFileSync('ps', ['-o', 'stat=', '-p', String(descendantPid)], { + encoding: 'utf8', + }).trim(); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ESRCH') alive = false; - else throw error; + const status = (error as { status?: number }).status; + if (status !== 1) throw error; } - if (alive) process.kill(descendantPid, 'SIGKILL'); - expect(alive).toBe(false); + const running = processState.length > 0 && !processState.startsWith('Z'); + if (running) process.kill(descendantPid, 'SIGKILL'); + expect(running).toBe(false); }); it('preserves UTF-8 characters split across output chunks', async () => { From 7a66abefbf53d887f52790f8533a123aaefffbcd Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 14:20:38 +0200 Subject: [PATCH 08/15] fix(pr-proof): harden rerun ownership and progress parsing --- scripts/pr-proof/report-status.mjs | 11 +++- scripts/pr-proof/run-cloud.mjs | 41 ++++++++++++--- tests/fixtures/pr-proof-contract.test.ts | 66 ++++++++++++++++++++---- tests/relayflows/cases/README.md | 10 ++-- 4 files changed, 106 insertions(+), 22 deletions(-) diff --git a/scripts/pr-proof/report-status.mjs b/scripts/pr-proof/report-status.mjs index 40272ecb3..b706f5ebe 100644 --- a/scripts/pr-proof/report-status.mjs +++ b/scripts/pr-proof/report-status.mjs @@ -5,6 +5,7 @@ import { pathToFileURL } from 'node:url'; const SHA_RE = /^[0-9a-f]{40}$/; const REPOSITORY_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const RUN_ATTEMPT_RE = /^[1-9][0-9]*$/; const CONTEXT = 'RelayFlow PR proof'; function option(name, fallback = null) { @@ -66,7 +67,11 @@ function targetUrl(env) { const server = env.GITHUB_SERVER_URL?.replace(/\/$/, ''); const repository = env.GITHUB_REPOSITORY; const runId = env.GITHUB_RUN_ID; - return server && repository && runId ? `${server}/${repository}/actions/runs/${runId}` : undefined; + const runAttempt = env.GITHUB_RUN_ATTEMPT?.trim() || '1'; + if (!RUN_ATTEMPT_RE.test(runAttempt)) throw new Error('GITHUB_RUN_ATTEMPT is invalid'); + return server && repository && runId + ? `${server}/${repository}/actions/runs/${runId}/attempts/${runAttempt}` + : undefined; } export async function publishCommitStatus({ sha, state, description, env = process.env, fetchImpl = fetch }) { @@ -123,6 +128,10 @@ export async function publishOwnedCommitStatus({ return { published: false, ownerTargetUrl, latest }; } + // The workflow-level concurrency group is the serialization lock for this + // compare-and-write. GitHub permits only one running workflow in the PR's + // group, so a replacement cannot publish its pending status until this + // cancelled predecessor has finished this step and released the group. await publishCommitStatus({ sha, state, description, env, fetchImpl }); return { published: true, ownerTargetUrl, latest }; } diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index e2fbef5b9..f8e74a974 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -82,6 +82,37 @@ export function preparedRunIdFromOutput(output) { return null; } +export function createPreparedRunProgressParser(onRunId) { + let pending = ''; + + const inspect = (output) => { + for (const line of output.split(/\r?\n/)) { + const runId = preparedRunIdFromOutput(line); + if (runId) onRunId(runId); + } + }; + + return { + write(text) { + pending += text; + const lastNewline = pending.lastIndexOf('\n'); + if (lastNewline < 0) { + // The marker is a short, newline-terminated trusted CLI progress line. + // Bound unrelated unterminated stderr without parsing partial markers. + pending = pending.slice(-8_192); + return; + } + const complete = pending.slice(0, lastNewline + 1); + pending = pending.slice(lastNewline + 1); + inspect(complete); + }, + end() { + if (pending) inspect(pending); + pending = ''; + }, + }; +} + export function validateCredentialWindow(env, timeoutMs, now = Date.now()) { const accessExpiry = Date.parse(requiredCredential(env, 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT')); if (!Number.isFinite(accessExpiry)) throw new Error('CLOUD_API_ACCESS_TOKEN_EXPIRES_AT is invalid'); @@ -167,14 +198,10 @@ export async function main() { let cancelPromise = null; let shuttingDown = false; let activeCommandController = null; - let launchProgress = ''; let launchProgressError = null; - const noteLaunchProgress = (text) => { - launchProgress = `${launchProgress}${text}`.slice(-8_192); + const notePreparedRunId = (preparedRunId) => { try { - const preparedRunId = preparedRunIdFromOutput(launchProgress); - if (!preparedRunId) return; if (runId && runId !== preparedRunId) { throw new Error(`Cloud prepare/run ID mismatch: ${runId} != ${preparedRunId}`); } @@ -183,6 +210,7 @@ export async function main() { launchProgressError ??= error; } }; + const launchProgress = createPreparedRunProgressParser(notePreparedRunId); const runTracked = async (command, args, options = {}) => { const controller = new AbortController(); @@ -231,8 +259,9 @@ export async function main() { }, quiet: true, timeoutMs: commandTimeoutMs, - onStderr: noteLaunchProgress, + onStderr: (text) => launchProgress.write(text), }); + launchProgress.end(); if (launchProgressError) throw launchProgressError; if (launch.aborted) throw new Error('Cloud workflow submission was interrupted'); if (launch.timedOut) { diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 384025bcc..b83d0f5ee 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -37,6 +37,7 @@ import { assertSamePullRequestSnapshot, pullRequestSnapshot } from '../../script import { assertCredentialDidNotRotate, boundedDuration, + createPreparedRunProgressParser, createCliAuthEnvironment, preparedRunIdFromOutput, validateCredentialWindow, @@ -400,6 +401,16 @@ describe('Cloud dispatcher credential lifecycle', () => { ); }); + it('waits for a complete prepared-run progress line split across stderr chunks', () => { + const runIds: string[] = []; + const parser = createPreparedRunProgressParser((runId: string) => runIds.push(runId)); + parser.write('Preparing run...\nAGENT_RELAY_CLOUD_PREPARED_RUN_ID=run-pre'); + expect(runIds).toEqual([]); + parser.write('pared-123\nUploading...\n'); + parser.end(); + expect(runIds).toEqual(['run-prepared-123']); + }); + it('shares an owner-only temporary auth file across CLI subprocesses and detects rotation', async () => { validateCredentialWindow(credentialEnv, 60 * 60_000, now); const auth = await createCliAuthEnvironment({ ...process.env, ...credentialEnv }); @@ -467,17 +478,21 @@ describe('process timeout contract', () => { const descendantPid = Number(result.stdout.trim()); expect(result.timedOut).toBe(true); expect(descendantPid).toBeGreaterThan(0); - await new Promise((resolve) => setTimeout(resolve, 50)); - let processState = ''; - try { - processState = execFileSync('ps', ['-o', 'stat=', '-p', String(descendantPid)], { - encoding: 'utf8', - }).trim(); - } catch (error) { - const status = (error as { status?: number }).status; - if (status !== 1) throw error; + const deadline = Date.now() + 2_000; + let running = true; + while (running && Date.now() < deadline) { + let processState = ''; + try { + processState = execFileSync('ps', ['-o', 'stat=', '-p', String(descendantPid)], { + encoding: 'utf8', + }).trim(); + } catch (error) { + const status = (error as { status?: number }).status; + if (status !== 1) throw error; + } + running = processState.length > 0 && !processState.startsWith('Z'); + if (running) await new Promise((resolve) => setTimeout(resolve, 20)); } - const running = processState.length > 0 && !processState.startsWith('Z'); if (running) process.kill(descendantPid, 'SIGKILL'); expect(running).toBe(false); }); @@ -534,6 +549,7 @@ describe('required head status', () => { GITHUB_API_URL: 'https://api.github.test', GITHUB_SERVER_URL: 'https://github.test', GITHUB_RUN_ID: '12345', + GITHUB_RUN_ATTEMPT: '2', }; it('publishes the stable proof context on the exact head SHA', async () => { @@ -574,7 +590,7 @@ describe('required head status', () => { it('finalizes only the pending status owned by the current workflow run', async () => { const requests: Array<{ url: string; method: string; body?: Record }> = []; - const ownerTargetUrl = 'https://github.test/AgentWorkforce/relay/actions/runs/12345'; + const ownerTargetUrl = 'https://github.test/AgentWorkforce/relay/actions/runs/12345/attempts/2'; const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { requests.push({ url: String(url), @@ -626,6 +642,32 @@ describe('required head status', () => { ).resolves.toMatchObject({ published: false }); expect(requests).toBe(1); }); + + it('gives each rerun attempt a distinct status owner marker', async () => { + const targetUrls: string[] = []; + const fetchImpl = async (_url: string | URL | Request, init?: RequestInit) => { + targetUrls.push(JSON.parse(String(init?.body)).target_url); + return Response.json({ id: targetUrls.length }); + }; + await publishCommitStatus({ + sha: HEAD_SHA, + state: 'pending', + description: 'attempt 1', + env: { ...statusEnv, GITHUB_RUN_ATTEMPT: '1' }, + fetchImpl, + }); + await publishCommitStatus({ + sha: HEAD_SHA, + state: 'pending', + description: 'attempt 2', + env: statusEnv, + fetchImpl, + }); + expect(targetUrls).toEqual([ + 'https://github.test/AgentWorkforce/relay/actions/runs/12345/attempts/1', + 'https://github.test/AgentWorkforce/relay/actions/runs/12345/attempts/2', + ]); + }); }); describe('pull request snapshot consistency', () => { @@ -667,6 +709,8 @@ describe('trusted dispatcher source contract', () => { expect(source).toContain('statuses: write'); expect(source).toContain('report-status.mjs start'); expect(source).toContain('report-status.mjs finish'); + expect(source).toContain('concurrency:'); + expect(source).toContain('cancel-in-progress: true'); expect(source).toContain("if: always() && steps.status.outputs.head_sha != ''"); expect(source).not.toContain('always() && !cancelled()'); expect(source).toContain('--expected-head-sha "${{ steps.status.outputs.head_sha }}"'); diff --git a/tests/relayflows/cases/README.md b/tests/relayflows/cases/README.md index f953d26ca..3cf07de0a 100644 --- a/tests/relayflows/cases/README.md +++ b/tests/relayflows/cases/README.md @@ -141,7 +141,9 @@ work even if submission is interrupted before its final JSON response. Require the status context only after that canary proves the credential, Cloud handoff, submission cancellation, and red/green case end to end. -The final status step also runs during cancellation. Before it publishes a -terminal state, it verifies that its own Actions run still owns the latest -pending `RelayFlow PR proof` context; a cancelled predecessor cannot overwrite -a replacement run. +The final status step also runs during cancellation. Each run attempt gets a +distinct owner URL. Before publishing a terminal state, it verifies that the +current attempt still owns the latest pending `RelayFlow PR proof` context. +The workflow-level PR concurrency group serializes that compare-and-write with +the replacement's start step, so a cancelled predecessor cannot overwrite a +replacement run. From 08703fef665f05f53726ad23ce328e0823100175 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 14:28:56 +0200 Subject: [PATCH 09/15] test(pr-proof): skip POSIX process assertion without ps --- tests/fixtures/pr-proof-contract.test.ts | 69 +++++++++++++----------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index b83d0f5ee..b85eabb4b 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -1,5 +1,6 @@ import { mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; import { execFileSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -49,6 +50,7 @@ const BASE_SHA = '1'.repeat(40); const HEAD_SHA = '2'.repeat(40); const CASE_ID = '1591-application-ack-reconnect'; const HANDOFF_NONCE = 'a'.repeat(32); +const PS_PATH = ['/bin/ps', '/usr/bin/ps'].find((candidate) => existsSync(candidate)); function proofBody(type = 'bugfix', caseId = CASE_ID) { return [ @@ -462,40 +464,43 @@ describe('process timeout contract', () => { expect(Date.now() - startedAt).toBeLessThan(2_000); }); - it('force-kills same-group descendants even when they do not inherit output pipes', async () => { - const script = [ - "const { spawn } = require('node:child_process');", - "const child = spawn(process.execPath, ['-e', \"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)\"], { stdio: 'ignore' });", - "process.stdout.write(String(child.pid) + '\\n');", - "process.on('SIGTERM', () => process.exit(0));", - 'setInterval(() => {}, 1000);', - ].join(''); - const result = await runProcess(process.execPath, ['-e', script], { - echo: false, - timeoutMs: 100, - terminationGraceMs: 100, - }); - const descendantPid = Number(result.stdout.trim()); - expect(result.timedOut).toBe(true); - expect(descendantPid).toBeGreaterThan(0); - const deadline = Date.now() + 2_000; - let running = true; - while (running && Date.now() < deadline) { - let processState = ''; - try { - processState = execFileSync('ps', ['-o', 'stat=', '-p', String(descendantPid)], { - encoding: 'utf8', - }).trim(); - } catch (error) { - const status = (error as { status?: number }).status; - if (status !== 1) throw error; + it.skipIf(process.platform === 'win32' || !PS_PATH)( + 'force-kills same-group descendants even when they do not inherit output pipes', + async () => { + const script = [ + "const { spawn } = require('node:child_process');", + "const child = spawn(process.execPath, ['-e', \"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)\"], { stdio: 'ignore' });", + "process.stdout.write(String(child.pid) + '\\n');", + "process.on('SIGTERM', () => process.exit(0));", + 'setInterval(() => {}, 1000);', + ].join(''); + const result = await runProcess(process.execPath, ['-e', script], { + echo: false, + timeoutMs: 100, + terminationGraceMs: 100, + }); + const descendantPid = Number(result.stdout.trim()); + expect(result.timedOut).toBe(true); + expect(descendantPid).toBeGreaterThan(0); + const deadline = Date.now() + 2_000; + let running = true; + while (running && Date.now() < deadline) { + let processState = ''; + try { + processState = execFileSync(PS_PATH, ['-o', 'stat=', '-p', String(descendantPid)], { + encoding: 'utf8', + }).trim(); + } catch (error) { + const status = (error as { status?: number }).status; + if (status !== 1) throw error; + } + running = processState.length > 0 && !processState.startsWith('Z'); + if (running) await new Promise((resolve) => setTimeout(resolve, 20)); } - running = processState.length > 0 && !processState.startsWith('Z'); - if (running) await new Promise((resolve) => setTimeout(resolve, 20)); + if (running) process.kill(descendantPid, 'SIGKILL'); + expect(running).toBe(false); } - if (running) process.kill(descendantPid, 'SIGKILL'); - expect(running).toBe(false); - }); + ); it('preserves UTF-8 characters split across output chunks', async () => { const script = [ From 9b424f8d1fdaecbb0099e0db060115dc3a579a6f Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 14:33:39 +0200 Subject: [PATCH 10/15] fix(pr-proof): defer progress parse failures --- scripts/pr-proof/run-cloud.mjs | 11 +++++++++-- tests/fixtures/pr-proof-contract.test.ts | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index f8e74a974..4000d0fb8 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -211,6 +211,13 @@ export async function main() { } }; const launchProgress = createPreparedRunProgressParser(notePreparedRunId); + const captureLaunchProgressError = (action) => { + try { + action(); + } catch (error) { + launchProgressError ??= error; + } + }; const runTracked = async (command, args, options = {}) => { const controller = new AbortController(); @@ -259,9 +266,9 @@ export async function main() { }, quiet: true, timeoutMs: commandTimeoutMs, - onStderr: (text) => launchProgress.write(text), + onStderr: (text) => captureLaunchProgressError(() => launchProgress.write(text)), }); - launchProgress.end(); + captureLaunchProgressError(() => launchProgress.end()); if (launchProgressError) throw launchProgressError; if (launch.aborted) throw new Error('Cloud workflow submission was interrupted'); if (launch.timedOut) { diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index b85eabb4b..24fe93fea 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -748,6 +748,7 @@ describe('trusted dispatcher source contract', () => { expect(source).toContain("process.once('SIGTERM', signalHandler)"); expect(source).toContain('activeCommandController?.abort()'); expect(source).toContain('AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID'); + expect(source).toContain('captureLaunchProgressError(() => launchProgress.write(text))'); expect(source).toContain("['cloud', 'cancel', runId, '--json']"); expect(source).toContain("path.join(authDir, 'cloud-auth.json')"); expect(source).toContain('assertCredentialDidNotRotate'); From 54caebaa32d1121013a84ac21871ad532395403a Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 16:03:13 +0200 Subject: [PATCH 11/15] fix(ci): use one Cloud API key for PR proof --- .github/workflows/relayflow-pr-proof.yml | 14 +---- packages/cloud/src/auth.test.ts | 45 ++++++++++++++ packages/cloud/src/auth.ts | 43 ++++++++++++++ packages/cloud/src/types.ts | 1 + scripts/pr-proof/run-cloud.mjs | 69 +++------------------ tests/fixtures/pr-proof-contract.test.ts | 76 ++++++++++-------------- tests/relayflows/cases/README.md | 18 +++--- 7 files changed, 138 insertions(+), 128 deletions(-) diff --git a/.github/workflows/relayflow-pr-proof.yml b/.github/workflows/relayflow-pr-proof.yml index 6703d6b75..2b56d5e03 100644 --- a/.github/workflows/relayflow-pr-proof.yml +++ b/.github/workflows/relayflow-pr-proof.yml @@ -67,15 +67,10 @@ jobs: if: steps.prepare.outputs.required == 'true' env: CLOUD_API_URL: ${{ secrets.CLOUD_API_URL }} - CLOUD_API_ACCESS_TOKEN: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN }} - CLOUD_API_REFRESH_TOKEN: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN }} - CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN_EXPIRES_AT }} - CLOUD_API_REFRESH_TOKEN_EXPIRES_AT: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN_EXPIRES_AT }} + CLOUD_API_KEY: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_API_KEY }} run: | test -n "$CLOUD_API_URL" - test -n "$CLOUD_API_ACCESS_TOKEN" - test -n "$CLOUD_API_REFRESH_TOKEN" - test -n "$CLOUD_API_ACCESS_TOKEN_EXPIRES_AT" + test -n "$CLOUD_API_KEY" # Cloud code sync intentionally uploads only paths known to git. Add the # generated, validated input to this disposable runner's index so it is @@ -93,10 +88,7 @@ jobs: id: cloud env: CLOUD_API_URL: ${{ secrets.CLOUD_API_URL }} - CLOUD_API_ACCESS_TOKEN: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN }} - CLOUD_API_REFRESH_TOKEN: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN }} - CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN_EXPIRES_AT }} - CLOUD_API_REFRESH_TOKEN_EXPIRES_AT: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN_EXPIRES_AT }} + CLOUD_API_KEY: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_API_KEY }} PR_PROOF_CLOUD_TIMEOUT_MS: '3600000' run: node scripts/pr-proof/run-cloud.mjs workflows/pr-proof.ts diff --git a/packages/cloud/src/auth.test.ts b/packages/cloud/src/auth.test.ts index e070775b2..0a1286f40 100644 --- a/packages/cloud/src/auth.test.ts +++ b/packages/cloud/src/auth.test.ts @@ -219,6 +219,32 @@ describe('ensureAuthenticated', () => { return new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(); } + it('uses one non-refreshing Cloud API key without reading stored login state', async () => { + vi.stubEnv('CLOUD_API_URL', 'https://ci.example/cloud'); + vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); + fsMocks.readFile.mockResolvedValue(JSON.stringify(FILE_AUTH)); + + await expect(ensureAuthenticated('https://default.example/cloud')).resolves.toEqual({ + authMode: 'api-key', + apiUrl: 'https://ci.example/cloud', + accessToken: 'ci-api-key', + refreshToken: '', + accessTokenExpiresAt: '9999-12-31T23:59:59.999Z', + }); + expect(fsMocks.readFile).not.toHaveBeenCalled(); + }); + + it('fails closed when API-key auth has a malformed Cloud URL', async () => { + vi.stubEnv('CLOUD_API_URL', 'not-a-url'); + vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); + fsMocks.readFile.mockResolvedValue(JSON.stringify(FILE_AUTH)); + + await expect(ensureAuthenticated('https://default.example/cloud')).rejects.toMatchObject({ + code: 'AUTH_ENV_REPROVISION_REQUIRED', + }); + expect(fsMocks.readFile).not.toHaveBeenCalled(); + }); + it('returns stored file auth even when apiUrl differs from defaultApiUrl', async () => { // Regression: previously, any host mismatch between the CLI's default // apiUrl and the stored apiUrl forced a browser login on every cloud @@ -836,6 +862,25 @@ describe('authorizedApiFetch telemetry headers', () => { }); describe('authorizedApiFetch re-login', () => { + it('returns a Cloud 401 for API-key auth without refresh or interactive login', async () => { + const fetchSpy = vi.fn(async () => new Response('{}', { status: 401 })); + vi.stubGlobal('fetch', fetchSpy); + + const auth: StoredAuth = { + authMode: 'api-key', + apiUrl: 'https://api.example.test', + accessToken: 'ci-api-key', + refreshToken: '', + accessTokenExpiresAt: '9999-12-31T23:59:59.999Z', + }; + const result = await authorizedApiFetch(auth, '/api/v1/workflows/run', { method: 'POST' }); + + expect(result.response.status).toBe(401); + expect(result.auth).toBe(auth); + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(childProcessMocks.spawn).not.toHaveBeenCalled(); + }); + it('re-authenticates a headless host through the device flow, not the browser', async () => { // The steady state this feature exists for: barry logged in once over ssh // with `--device`, and now a request 401s with a refresh token the server diff --git a/packages/cloud/src/auth.ts b/packages/cloud/src/auth.ts index 6a052f73b..9bb942f36 100644 --- a/packages/cloud/src/auth.ts +++ b/packages/cloud/src/auth.ts @@ -36,6 +36,36 @@ const AUTH_LOCK_STALE_MS = 30_000; const AUTH_LOCK_TIMEOUT_MS = 30_000; const envBackedAuth = new WeakSet(); +const API_KEY_NON_REFRESHING_EXPIRY = '9999-12-31T23:59:59.999Z'; + +function isApiKeyAuth(auth: StoredAuth): boolean { + return auth.authMode === 'api-key'; +} + +function readApiKeyAuth(apiUrl: string, env: NodeJS.ProcessEnv = process.env): StoredAuth | null { + const apiKey = env.CLOUD_API_KEY?.trim(); + if (!apiKey) return null; + + const configuredApiUrl = env.CLOUD_API_URL?.trim() || apiUrl; + try { + new URL(configuredApiUrl); + } catch (error) { + throw new CloudAuthError('AUTH_ENV_REPROVISION_REQUIRED', 'CLOUD_API_URL is invalid for CLOUD_API_KEY', { + cause: error, + }); + } + + return { + authMode: 'api-key', + apiUrl: configuredApiUrl, + accessToken: apiKey, + // StoredAuth predates service credentials and models a refreshable login. + // These placeholders keep that internal shape stable; authMode guarantees + // neither value is consulted or sent over the network. + refreshToken: '', + accessTokenExpiresAt: API_KEY_NON_REFRESHING_EXPIRY, + }; +} function markEnvBackedAuth(auth: StoredAuth): StoredAuth { envBackedAuth.add(auth); @@ -99,6 +129,7 @@ function isValidStoredAuth(value: unknown): value is StoredAuth { const auth = value as Partial; return ( + auth.authMode === undefined && typeof auth.accessToken === 'string' && typeof auth.refreshToken === 'string' && typeof auth.accessTokenExpiresAt === 'string' && @@ -689,6 +720,11 @@ export async function ensureAuthenticated( refreshTimeoutMs?: number; } ): Promise { + if (options?.force !== true) { + const apiKeyAuth = readApiKeyAuth(apiUrl); + if (apiKeyAuth) return apiKeyAuth; + } + const session = await ensureCloudSession({ apiUrl, force: options?.force, @@ -825,6 +861,13 @@ export async function authorizedApiFetch( return { response, auth: activeAuth }; } + // API keys are deliberately non-refreshing. Return the Cloud 401 to the + // caller so unattended automation fails closed instead of attempting token + // rotation or opening an interactive login flow. + if (isApiKeyAuth(activeAuth)) { + return { response, auth: activeAuth }; + } + try { activeAuth = await refreshStoredAuth(activeAuth, { force: true, diff --git a/packages/cloud/src/types.ts b/packages/cloud/src/types.ts index 90c547617..867a545e5 100644 --- a/packages/cloud/src/types.ts +++ b/packages/cloud/src/types.ts @@ -2,6 +2,7 @@ import os from 'node:os'; import path from 'node:path'; export type StoredAuth = { + authMode?: 'api-key'; accessToken: string; refreshToken: string; accessTokenExpiresAt: string; diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs index 4000d0fb8..00cd83d22 100644 --- a/scripts/pr-proof/run-cloud.mjs +++ b/scripts/pr-proof/run-cloud.mjs @@ -1,7 +1,6 @@ #!/usr/bin/env node -import { appendFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import os from 'node:os'; +import { appendFile, mkdir, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; import { pathToFileURL } from 'node:url'; @@ -10,14 +9,12 @@ import { runBoundedProcess } from './process-runner.mjs'; const TERMINAL_SUCCESS = new Set(['completed', 'succeeded', 'success']); const TERMINAL_FAILURE = new Set(['failed', 'cancelled', 'canceled', 'timed_out', 'error']); -const CLOUD_AUTH_KEYS = [ - 'CLOUD_API_URL', +const LEGACY_REFRESHABLE_AUTH_KEYS = [ 'CLOUD_API_ACCESS_TOKEN', 'CLOUD_API_REFRESH_TOKEN', 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT', 'CLOUD_API_REFRESH_TOKEN_EXPIRES_AT', ]; -const CREDENTIAL_WINDOW_BUFFER_MS = 15 * 60_000; const MAX_CAPTURE_BYTES = 2 * 1024 * 1024; const MAX_LIVE_OUTPUT_BYTES = 256 * 1024; const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; @@ -113,60 +110,14 @@ export function createPreparedRunProgressParser(onRunId) { }; } -export function validateCredentialWindow(env, timeoutMs, now = Date.now()) { - const accessExpiry = Date.parse(requiredCredential(env, 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT')); - if (!Number.isFinite(accessExpiry)) throw new Error('CLOUD_API_ACCESS_TOKEN_EXPIRES_AT is invalid'); - const requiredUntil = now + timeoutMs + CREDENTIAL_WINDOW_BUFFER_MS; - if (accessExpiry <= requiredUntil) { - throw new Error( - `Cloud CI access token expires before the proof deadline; reprovision the Relay secrets before ${new Date(requiredUntil).toISOString()}` - ); - } - const refreshExpirySource = env.CLOUD_API_REFRESH_TOKEN_EXPIRES_AT?.trim(); - if (refreshExpirySource) { - const refreshExpiry = Date.parse(refreshExpirySource); - if (!Number.isFinite(refreshExpiry) || refreshExpiry <= requiredUntil) { - throw new Error('Cloud CI refresh token expires before the proof deadline; reprovision Relay secrets'); - } - } -} - -export async function createCliAuthEnvironment(env = process.env) { +export function createCliApiKeyEnvironment(env = process.env) { const apiUrl = requiredCredential(env, 'CLOUD_API_URL'); - const accessToken = requiredCredential(env, 'CLOUD_API_ACCESS_TOKEN'); - const refreshToken = requiredCredential(env, 'CLOUD_API_REFRESH_TOKEN'); - const accessTokenExpiresAt = requiredCredential(env, 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT'); - const refreshTokenExpiresAt = env.CLOUD_API_REFRESH_TOKEN_EXPIRES_AT?.trim(); + const apiKey = requiredCredential(env, 'CLOUD_API_KEY'); new URL(apiUrl); - const temporaryHome = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-auth-')); - const authDir = path.join(temporaryHome, '.agentworkforce', 'relay'); - const authPath = path.join(authDir, 'cloud-auth.json'); - await mkdir(authDir, { recursive: true, mode: 0o700 }); - await writeFile( - authPath, - `${JSON.stringify({ - apiUrl, - accessToken, - refreshToken, - accessTokenExpiresAt, - ...(refreshTokenExpiresAt ? { refreshTokenExpiresAt } : {}), - })}\n`, - { mode: 0o600 } - ); - - const cliEnv = { ...env, HOME: temporaryHome }; - for (const key of CLOUD_AUTH_KEYS) delete cliEnv[key]; - return { cliEnv, temporaryHome, authPath, originalAccessToken: accessToken }; -} - -export async function assertCredentialDidNotRotate(authPath, originalAccessToken) { - const stored = JSON.parse(await readFile(authPath, 'utf8')); - if (stored.accessToken !== originalAccessToken) { - throw new Error( - 'Cloud CI credential refreshed during the job; the rotated tuple cannot be persisted to GitHub secrets. Reprovision the Relay secrets before rerunning.' - ); - } + const cliEnv = { ...env, CLOUD_API_URL: apiUrl, CLOUD_API_KEY: apiKey }; + for (const key of LEGACY_REFRESHABLE_AUTH_KEYS) delete cliEnv[key]; + return { cliEnv }; } export async function main() { @@ -191,8 +142,7 @@ export async function main() { maximum: 5 * 60_000, label: 'PR_PROOF_CLOUD_COMMAND_TIMEOUT_MS', }); - validateCredentialWindow(process.env, timeoutMs); - const auth = await createCliAuthEnvironment(process.env); + const auth = createCliApiKeyEnvironment(process.env); let runId = null; let terminal = false; let cancelPromise = null; @@ -251,7 +201,6 @@ export async function main() { activeCommandController?.abort(); void (async () => { await cancelRemote(signal).catch((error) => console.warn(error.message)); - await rm(auth.temporaryHome, { recursive: true, force: true }); process.exit(signal === 'SIGINT' ? 130 : 143); })(); }; @@ -338,7 +287,6 @@ export async function main() { if (!TERMINAL_SUCCESS.has(terminalStatus)) { throw new Error(`Cloud RelayFlow finished with status ${terminalStatus}`); } - await assertCredentialDidNotRotate(auth.authPath, auth.originalAccessToken); if (process.env.GITHUB_STEP_SUMMARY) { await appendFile( process.env.GITHUB_STEP_SUMMARY, @@ -351,7 +299,6 @@ export async function main() { process.removeListener('SIGTERM', signalHandler); if (runId && !terminal) await cancelRemote('dispatcher exiting').catch((error) => console.warn(error.message)); - await rm(auth.temporaryHome, { recursive: true, force: true }); } } diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts index 24fe93fea..9a1ced05c 100644 --- a/tests/fixtures/pr-proof-contract.test.ts +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { execFileSync } from 'node:child_process'; import { existsSync } from 'node:fs'; import os from 'node:os'; @@ -36,12 +36,10 @@ import { import { assertSamePullRequestSnapshot, pullRequestSnapshot } from '../../scripts/pr-proof/prepare.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. import { - assertCredentialDidNotRotate, boundedDuration, createPreparedRunProgressParser, - createCliAuthEnvironment, + createCliApiKeyEnvironment, preparedRunIdFromOutput, - validateCredentialWindow, } from '../../scripts/pr-proof/run-cloud.mjs'; // @ts-expect-error JavaScript module intentionally has no declaration file. import { runProcess } from '../../scripts/pr-proof/run-arm.mjs'; @@ -365,24 +363,19 @@ describe('Cloud evidence handoff', () => { }); }); -describe('Cloud dispatcher credential lifecycle', () => { - const now = Date.parse('2026-08-25T10:00:00.000Z'); +describe('Cloud dispatcher API key lifecycle', () => { const credentialEnv = { CLOUD_API_URL: 'https://cloud.test/cloud', - CLOUD_API_ACCESS_TOKEN: 'ci-access-token', - CLOUD_API_REFRESH_TOKEN: 'ci-refresh-token', - CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: '2026-08-25T13:00:00.000Z', - CLOUD_API_REFRESH_TOKEN_EXPIRES_AT: '2026-08-26T10:00:00.000Z', + CLOUD_API_KEY: 'ci-api-key', }; - it('fails before dispatch when the static access token cannot cover the proof window', () => { - expect(() => - validateCredentialWindow( - { ...credentialEnv, CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: '2026-08-25T10:30:00.000Z' }, - 60 * 60_000, - now - ) - ).toThrow(/expires before the proof deadline/); + it('requires exactly the Cloud URL and API key before dispatch', () => { + expect(() => createCliApiKeyEnvironment({ CLOUD_API_URL: credentialEnv.CLOUD_API_URL })).toThrow( + /CLOUD_API_KEY is required/ + ); + expect(() => createCliApiKeyEnvironment({ CLOUD_API_KEY: credentialEnv.CLOUD_API_KEY })).toThrow( + /CLOUD_API_URL is required/ + ); }); it('rejects unbounded or malformed dispatcher durations', () => { @@ -413,31 +406,21 @@ describe('Cloud dispatcher credential lifecycle', () => { expect(runIds).toEqual(['run-prepared-123']); }); - it('shares an owner-only temporary auth file across CLI subprocesses and detects rotation', async () => { - validateCredentialWindow(credentialEnv, 60 * 60_000, now); - const auth = await createCliAuthEnvironment({ ...process.env, ...credentialEnv }); - try { - expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN).toBeUndefined(); - expect(auth.cliEnv.HOME).toBe(auth.temporaryHome); - const authFile = await open(auth.authPath, 'r+'); - try { - expect((await authFile.stat()).mode & 0o777).toBe(0o600); - await expect( - assertCredentialDidNotRotate(auth.authPath, credentialEnv.CLOUD_API_ACCESS_TOKEN) - ).resolves.toBeUndefined(); - const stored = JSON.parse(await authFile.readFile('utf8')); - stored.accessToken = 'rotated-token'; - await authFile.truncate(0); - await authFile.write(JSON.stringify(stored), 0, 'utf8'); - } finally { - await authFile.close(); - } - await expect( - assertCredentialDidNotRotate(auth.authPath, credentialEnv.CLOUD_API_ACCESS_TOKEN) - ).rejects.toThrow(/cannot be persisted/); - } finally { - await rm(auth.temporaryHome, { recursive: true, force: true }); - } + it('passes one API key to every CLI subprocess and removes legacy refresh credentials', () => { + const auth = createCliApiKeyEnvironment({ + ...credentialEnv, + CLOUD_API_ACCESS_TOKEN: 'legacy-access', + CLOUD_API_REFRESH_TOKEN: 'legacy-refresh', + CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: '2027-08-25T00:00:00.000Z', + CLOUD_API_REFRESH_TOKEN_EXPIRES_AT: '2027-08-25T00:00:00.000Z', + }); + + expect(auth.cliEnv.CLOUD_API_URL).toBe(credentialEnv.CLOUD_API_URL); + expect(auth.cliEnv.CLOUD_API_KEY).toBe(credentialEnv.CLOUD_API_KEY); + expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN).toBeUndefined(); + expect(auth.cliEnv.CLOUD_API_REFRESH_TOKEN).toBeUndefined(); + expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN_EXPIRES_AT).toBeUndefined(); + expect(auth.cliEnv.CLOUD_API_REFRESH_TOKEN_EXPIRES_AT).toBeUndefined(); }); }); @@ -743,15 +726,16 @@ describe('trusted dispatcher source contract', () => { expect(source).not.toContain('process.env.DAYTONA_SANDBOX_ID'); }); - it('shares refreshes only inside the job and cancels remote work on termination', async () => { + it('uses one non-refreshing API key and cancels remote work on termination', async () => { const source = await readFile('scripts/pr-proof/run-cloud.mjs', 'utf8'); expect(source).toContain("process.once('SIGTERM', signalHandler)"); expect(source).toContain('activeCommandController?.abort()'); expect(source).toContain('AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID'); expect(source).toContain('captureLaunchProgressError(() => launchProgress.write(text))'); expect(source).toContain("['cloud', 'cancel', runId, '--json']"); - expect(source).toContain("path.join(authDir, 'cloud-auth.json')"); - expect(source).toContain('assertCredentialDidNotRotate'); + expect(source).toContain("requiredCredential(env, 'CLOUD_API_KEY')"); + expect(source).not.toContain("path.join(authDir, 'cloud-auth.json')"); + expect(source).not.toContain('CLOUD_API_REFRESH_TOKEN='); }); it('emits the prepared Cloud run id before upload and final submission', async () => { diff --git a/tests/relayflows/cases/README.md b/tests/relayflows/cases/README.md index 3cf07de0a..68c598df4 100644 --- a/tests/relayflows/cases/README.md +++ b/tests/relayflows/cases/README.md @@ -112,21 +112,19 @@ credential-bearing step. ## Enabling the required check -The repository needs a dedicated Cloud CLI session in these GitHub secrets: +The repository needs a dedicated non-refreshing Cloud API key in these GitHub +secrets: - `CLOUD_API_URL` -- `RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN` -- `RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN` -- `RELAYFLOW_PR_PROOF_CLOUD_ACCESS_TOKEN_EXPIRES_AT` -- `RELAYFLOW_PR_PROOF_CLOUD_REFRESH_TOKEN_EXPIRES_AT` (optional until refresh) +- `RELAYFLOW_PR_PROOF_CLOUD_API_KEY` Use a workspace-scoped, least-privilege credential that can prepare, invoke, read, and cancel workflow runs. Do not copy a human laptop session into CI. -The dispatcher stores the tuple in an owner-only temporary auth file so a -refresh is visible to every CLI subprocess in the same job. It refuses tokens -that expire inside the proof window and fails if a refresh rotates the tuple, -because Actions cannot write rotated values back to repository secrets. Rotate -the dedicated credential before that window. +The dispatcher passes one API key to every CLI subprocess and removes any +legacy refreshable-auth environment variables. API-key auth never refreshes or +opens an interactive login; Cloud enforces its workspace binding, scopes, +expiry, and revocation server-side. A `401` fails the proof and requires an +operator to rotate the dedicated key. `pull_request_target` itself is attached to the base SHA, so the dispatcher publishes a separate stable commit-status context named `RelayFlow PR proof` on From 7afe0fe567b53ee9d4b9e4d3a2ad42238b2bf464 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 16:08:05 +0200 Subject: [PATCH 12/15] refactor(cloud): model API key auth explicitly --- packages/cloud/src/auth.test.ts | 14 +++++++------ packages/cloud/src/auth.ts | 37 +++++++++++++++------------------ packages/cloud/src/index.ts | 2 ++ packages/cloud/src/types.ts | 9 +++++++- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/packages/cloud/src/auth.test.ts b/packages/cloud/src/auth.test.ts index 0a1286f40..022945f73 100644 --- a/packages/cloud/src/auth.test.ts +++ b/packages/cloud/src/auth.test.ts @@ -36,7 +36,13 @@ import { toCloudIdentity, writeStoredAuth, } from './auth.js'; -import { AUTH_FILE_PATH, CloudAuthError, type StoredAuth, type WhoAmIResponse } from './types.js'; +import { + AUTH_FILE_PATH, + CloudAuthError, + type CloudApiKeyAuth, + type StoredAuth, + type WhoAmIResponse, +} from './types.js'; const AUTH_LOCK_PATH = `${AUTH_FILE_PATH}.lock`; @@ -228,8 +234,6 @@ describe('ensureAuthenticated', () => { authMode: 'api-key', apiUrl: 'https://ci.example/cloud', accessToken: 'ci-api-key', - refreshToken: '', - accessTokenExpiresAt: '9999-12-31T23:59:59.999Z', }); expect(fsMocks.readFile).not.toHaveBeenCalled(); }); @@ -866,12 +870,10 @@ describe('authorizedApiFetch re-login', () => { const fetchSpy = vi.fn(async () => new Response('{}', { status: 401 })); vi.stubGlobal('fetch', fetchSpy); - const auth: StoredAuth = { + const auth: CloudApiKeyAuth = { authMode: 'api-key', apiUrl: 'https://api.example.test', accessToken: 'ci-api-key', - refreshToken: '', - accessTokenExpiresAt: '9999-12-31T23:59:59.999Z', }; const result = await authorizedApiFetch(auth, '/api/v1/workflows/run', { method: 'POST' }); diff --git a/packages/cloud/src/auth.ts b/packages/cloud/src/auth.ts index 9bb942f36..ef6a55b7c 100644 --- a/packages/cloud/src/auth.ts +++ b/packages/cloud/src/auth.ts @@ -23,6 +23,8 @@ import { REFRESH_WINDOW_MS, CloudAuthError, defaultApiUrl, + type CloudApiKeyAuth, + type CloudRequestAuth, type CloudSession, type CloudSessionOptions, type StoredAuth, @@ -36,13 +38,12 @@ const AUTH_LOCK_STALE_MS = 30_000; const AUTH_LOCK_TIMEOUT_MS = 30_000; const envBackedAuth = new WeakSet(); -const API_KEY_NON_REFRESHING_EXPIRY = '9999-12-31T23:59:59.999Z'; -function isApiKeyAuth(auth: StoredAuth): boolean { - return auth.authMode === 'api-key'; +function isApiKeyAuth(auth: CloudRequestAuth): auth is CloudApiKeyAuth { + return 'authMode' in auth && auth.authMode === 'api-key'; } -function readApiKeyAuth(apiUrl: string, env: NodeJS.ProcessEnv = process.env): StoredAuth | null { +function readApiKeyAuth(apiUrl: string, env: NodeJS.ProcessEnv = process.env): CloudApiKeyAuth | null { const apiKey = env.CLOUD_API_KEY?.trim(); if (!apiKey) return null; @@ -59,11 +60,6 @@ function readApiKeyAuth(apiUrl: string, env: NodeJS.ProcessEnv = process.env): S authMode: 'api-key', apiUrl: configuredApiUrl, accessToken: apiKey, - // StoredAuth predates service credentials and models a refreshable login. - // These placeholders keep that internal shape stable; authMode guarantees - // neither value is consulted or sent over the network. - refreshToken: '', - accessTokenExpiresAt: API_KEY_NON_REFRESHING_EXPIRY, }; } @@ -129,7 +125,6 @@ function isValidStoredAuth(value: unknown): value is StoredAuth { const auth = value as Partial; return ( - auth.authMode === undefined && typeof auth.accessToken === 'string' && typeof auth.refreshToken === 'string' && typeof auth.accessTokenExpiresAt === 'string' && @@ -719,7 +714,7 @@ export async function ensureAuthenticated( device?: boolean; refreshTimeoutMs?: number; } -): Promise { +): Promise { if (options?.force !== true) { const apiKeyAuth = readApiKeyAuth(apiUrl); if (apiKeyAuth) return apiKeyAuth; @@ -838,8 +833,8 @@ function apiFetch( }); } -export async function authorizedApiFetch( - auth: StoredAuth, +export async function authorizedApiFetch( + auth: T, requestPath: string, init: RequestInit, options: { @@ -853,29 +848,31 @@ export async function authorizedApiFetch( device?: boolean; env?: NodeJS.ProcessEnv; } = {} -): Promise<{ response: Response; auth: StoredAuth }> { - let activeAuth = auth; +): Promise<{ response: Response; auth: T }> { + let activeAuth: CloudRequestAuth = auth; let response = await apiFetch(activeAuth.apiUrl, activeAuth.accessToken, requestPath, init); if (response.status !== 401) { - return { response, auth: activeAuth }; + return { response, auth: activeAuth as T }; } // API keys are deliberately non-refreshing. Return the Cloud 401 to the // caller so unattended automation fails closed instead of attempting token // rotation or opening an interactive login flow. if (isApiKeyAuth(activeAuth)) { - return { response, auth: activeAuth }; + return { response, auth: activeAuth as T }; } + let refreshableAuth: StoredAuth = activeAuth; try { - activeAuth = await refreshStoredAuth(activeAuth, { + refreshableAuth = await refreshStoredAuth(refreshableAuth, { force: true, refreshTimeoutMs: options.refreshTimeoutMs, signal: init.signal ?? undefined, }); + activeAuth = refreshableAuth; } catch (error) { - if (isEnvBackedAuth(activeAuth)) { + if (isEnvBackedAuth(refreshableAuth)) { throw toEnvAuthRefreshError(error); } @@ -902,5 +899,5 @@ export async function authorizedApiFetch( } response = await apiFetch(activeAuth.apiUrl, activeAuth.accessToken, requestPath, init); - return { response, auth: activeAuth }; + return { response, auth: activeAuth as T }; } diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 4f7f5f107..430b764a5 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -225,6 +225,8 @@ export { PermissionAuditLog, getDefaultPermissionAuditPath } from './audit.js'; export { type StoredAuth, + type CloudApiKeyAuth, + type CloudRequestAuth, CloudAuthError, type CloudAuthErrorCode, type CloudSession, diff --git a/packages/cloud/src/types.ts b/packages/cloud/src/types.ts index 867a545e5..a2ffd4acd 100644 --- a/packages/cloud/src/types.ts +++ b/packages/cloud/src/types.ts @@ -2,7 +2,6 @@ import os from 'node:os'; import path from 'node:path'; export type StoredAuth = { - authMode?: 'api-key'; accessToken: string; refreshToken: string; accessTokenExpiresAt: string; @@ -10,6 +9,14 @@ export type StoredAuth = { apiUrl: string; }; +export type CloudApiKeyAuth = { + authMode: 'api-key'; + accessToken: string; + apiUrl: string; +}; + +export type CloudRequestAuth = StoredAuth | CloudApiKeyAuth; + export type CloudAuthErrorCode = | 'AUTH_REFRESH_TIMEOUT' | 'AUTH_REFRESH_EXPIRED' From 3017a302e074d0ab04f009bda4254fbd2375b236 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 16:10:52 +0200 Subject: [PATCH 13/15] fix(cloud): preserve explicit API key endpoint --- packages/cloud/src/auth.test.ts | 9 ++++----- packages/cloud/src/auth.ts | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/cloud/src/auth.test.ts b/packages/cloud/src/auth.test.ts index 022945f73..64574ce36 100644 --- a/packages/cloud/src/auth.test.ts +++ b/packages/cloud/src/auth.test.ts @@ -225,25 +225,24 @@ describe('ensureAuthenticated', () => { return new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(); } - it('uses one non-refreshing Cloud API key without reading stored login state', async () => { + it('uses one non-refreshing Cloud API key and preserves explicit URL precedence', async () => { vi.stubEnv('CLOUD_API_URL', 'https://ci.example/cloud'); vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); fsMocks.readFile.mockResolvedValue(JSON.stringify(FILE_AUTH)); - await expect(ensureAuthenticated('https://default.example/cloud')).resolves.toEqual({ + await expect(ensureAuthenticated('https://explicit.example/cloud')).resolves.toEqual({ authMode: 'api-key', - apiUrl: 'https://ci.example/cloud', + apiUrl: 'https://explicit.example/cloud', accessToken: 'ci-api-key', }); expect(fsMocks.readFile).not.toHaveBeenCalled(); }); it('fails closed when API-key auth has a malformed Cloud URL', async () => { - vi.stubEnv('CLOUD_API_URL', 'not-a-url'); vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); fsMocks.readFile.mockResolvedValue(JSON.stringify(FILE_AUTH)); - await expect(ensureAuthenticated('https://default.example/cloud')).rejects.toMatchObject({ + await expect(ensureAuthenticated('not-a-url')).rejects.toMatchObject({ code: 'AUTH_ENV_REPROVISION_REQUIRED', }); expect(fsMocks.readFile).not.toHaveBeenCalled(); diff --git a/packages/cloud/src/auth.ts b/packages/cloud/src/auth.ts index ef6a55b7c..560c493a3 100644 --- a/packages/cloud/src/auth.ts +++ b/packages/cloud/src/auth.ts @@ -47,7 +47,7 @@ function readApiKeyAuth(apiUrl: string, env: NodeJS.ProcessEnv = process.env): C const apiKey = env.CLOUD_API_KEY?.trim(); if (!apiKey) return null; - const configuredApiUrl = env.CLOUD_API_URL?.trim() || apiUrl; + const configuredApiUrl = apiUrl; try { new URL(configuredApiUrl); } catch (error) { From f5283e401e0d9950f32a9af4eb534db2717b29bc Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 16:27:16 +0200 Subject: [PATCH 14/15] fix(cloud): scope API key auth to workflows --- packages/cloud/src/auth.test.ts | 24 ++++++++++++++++++++++-- packages/cloud/src/auth.ts | 18 ++++++++++++------ packages/cloud/src/workflows.test.ts | 8 ++++---- packages/cloud/src/workflows.ts | 16 ++++++++-------- 4 files changed, 46 insertions(+), 20 deletions(-) diff --git a/packages/cloud/src/auth.test.ts b/packages/cloud/src/auth.test.ts index 64574ce36..bdb970b75 100644 --- a/packages/cloud/src/auth.test.ts +++ b/packages/cloud/src/auth.test.ts @@ -30,6 +30,7 @@ import { clearStoredAuth, ensureAuthenticated, ensureCloudSession, + ensureWorkflowAuthenticated, readStoredAuth, refreshStoredAuth, refreshStoredCloudIdentity, @@ -230,7 +231,7 @@ describe('ensureAuthenticated', () => { vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); fsMocks.readFile.mockResolvedValue(JSON.stringify(FILE_AUTH)); - await expect(ensureAuthenticated('https://explicit.example/cloud')).resolves.toEqual({ + await expect(ensureWorkflowAuthenticated('https://explicit.example/cloud')).resolves.toEqual({ authMode: 'api-key', apiUrl: 'https://explicit.example/cloud', accessToken: 'ci-api-key', @@ -242,12 +243,31 @@ describe('ensureAuthenticated', () => { vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); fsMocks.readFile.mockResolvedValue(JSON.stringify(FILE_AUTH)); - await expect(ensureAuthenticated('not-a-url')).rejects.toMatchObject({ + await expect(ensureWorkflowAuthenticated('not-a-url')).rejects.toMatchObject({ code: 'AUTH_ENV_REPROVISION_REQUIRED', }); expect(fsMocks.readFile).not.toHaveBeenCalled(); }); + it('does not use workflow API-key auth for general Cloud authentication', async () => { + vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); + fsMocks.readFile.mockResolvedValue( + JSON.stringify({ + ...FILE_AUTH, + accessTokenExpiresAt: farFutureIso(), + }) + ); + + const result = await ensureAuthenticated('https://default.example/cloud'); + + expect(result).toMatchObject({ + apiUrl: FILE_AUTH.apiUrl, + accessToken: FILE_AUTH.accessToken, + refreshToken: FILE_AUTH.refreshToken, + }); + expect(result).not.toHaveProperty('authMode'); + }); + it('returns stored file auth even when apiUrl differs from defaultApiUrl', async () => { // Regression: previously, any host mismatch between the CLI's default // apiUrl and the stored apiUrl forced a browser login on every cloud diff --git a/packages/cloud/src/auth.ts b/packages/cloud/src/auth.ts index 560c493a3..d65a53e47 100644 --- a/packages/cloud/src/auth.ts +++ b/packages/cloud/src/auth.ts @@ -714,12 +714,7 @@ export async function ensureAuthenticated( device?: boolean; refreshTimeoutMs?: number; } -): Promise { - if (options?.force !== true) { - const apiKeyAuth = readApiKeyAuth(apiUrl); - if (apiKeyAuth) return apiKeyAuth; - } - +): Promise { const session = await ensureCloudSession({ apiUrl, force: options?.force, @@ -730,6 +725,17 @@ export async function ensureAuthenticated( return session.auth; } +/** + * Resolve authentication for the workflow control plane only. The API-key + * path is intentionally absent from general Cloud login, fleet, workspace, + * and integration commands even though Cloud also enforces its narrow scopes. + */ +export async function ensureWorkflowAuthenticated(apiUrl: string): Promise { + const apiKeyAuth = readApiKeyAuth(apiUrl); + if (apiKeyAuth) return apiKeyAuth; + return ensureAuthenticated(apiUrl); +} + export async function ensureCloudSession(options: CloudSessionOptions = {}): Promise { const env = options.env ?? process.env; const apiUrl = options.apiUrl || env.CLOUD_API_URL?.trim() || defaultApiUrl(); diff --git a/packages/cloud/src/workflows.test.ts b/packages/cloud/src/workflows.test.ts index 4f1418c9a..fef33be2b 100644 --- a/packages/cloud/src/workflows.test.ts +++ b/packages/cloud/src/workflows.test.ts @@ -6,7 +6,7 @@ import { mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; import type { RunWorkflowResponse } from './types.js'; const s3SendMock = vi.hoisted(() => vi.fn()); -const ensureAuthenticatedMock = vi.hoisted(() => vi.fn()); +const ensureWorkflowAuthenticatedMock = vi.hoisted(() => vi.fn()); const authorizedApiFetchMock = vi.hoisted(() => vi.fn()); vi.mock('@aws-sdk/client-s3', () => { @@ -25,7 +25,7 @@ vi.mock('@aws-sdk/client-s3', () => { }); vi.mock('./auth.js', () => ({ - ensureAuthenticated: (...args: unknown[]) => ensureAuthenticatedMock(...args), + ensureWorkflowAuthenticated: (...args: unknown[]) => ensureWorkflowAuthenticatedMock(...args), authorizedApiFetch: (...args: unknown[]) => authorizedApiFetchMock(...args), })); @@ -353,7 +353,7 @@ describe('runWorkflow code sync', () => { originalCwd = process.cwd(); tmpRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), 'cloud-run-workflow-'))); process.chdir(tmpRoot); - ensureAuthenticatedMock.mockResolvedValue({ accessToken: 'token' }); + ensureWorkflowAuthenticatedMock.mockResolvedValue({ accessToken: 'token' }); s3SendMock.mockResolvedValue({}); }); @@ -620,7 +620,7 @@ describe('workflow schedules', () => { originalCwd = process.cwd(); tmpRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), 'cloud-schedule-workflow-'))); process.chdir(tmpRoot); - ensureAuthenticatedMock.mockResolvedValue({ accessToken: 'token' }); + ensureWorkflowAuthenticatedMock.mockResolvedValue({ accessToken: 'token' }); }); afterEach(async () => { diff --git a/packages/cloud/src/workflows.ts b/packages/cloud/src/workflows.ts index 82b83a04c..0165fb835 100644 --- a/packages/cloud/src/workflows.ts +++ b/packages/cloud/src/workflows.ts @@ -6,7 +6,7 @@ import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import ignore from 'ignore'; import * as tar from 'tar'; -import { ensureAuthenticated, authorizedApiFetch } from './auth.js'; +import { ensureWorkflowAuthenticated, authorizedApiFetch } from './auth.js'; import { defaultApiUrl, type WorkflowFileType, @@ -240,7 +240,7 @@ export async function runWorkflow( options: RunWorkflowOptions = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - let auth = await ensureAuthenticated(apiUrl); + let auth = await ensureWorkflowAuthenticated(apiUrl); const input = await resolveWorkflowInput(workflowArg, options.fileType); if (input.fileType === 'ts') { @@ -459,7 +459,7 @@ export async function scheduleWorkflow( } const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); + const auth = await ensureWorkflowAuthenticated(apiUrl); const input = await resolveWorkflowInput(workflowArg, options.fileType); if (input.fileType === 'ts') { @@ -518,7 +518,7 @@ export async function scheduleWorkflow( export async function listWorkflowSchedules(options: { apiUrl?: string } = {}): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); + const auth = await ensureWorkflowAuthenticated(apiUrl); const { response } = await authorizedApiFetch(auth, '/api/v1/workflows/schedules', { headers: { Accept: 'application/json' }, }); @@ -544,7 +544,7 @@ export async function getRunStatus( options: { apiUrl?: string } = {} ): Promise> { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); + const auth = await ensureWorkflowAuthenticated(apiUrl); const { response } = await authorizedApiFetch(auth, `/api/v1/workflows/runs/${encodeURIComponent(runId)}`, { headers: { Accept: 'application/json' }, }); @@ -566,7 +566,7 @@ export async function cancelWorkflow( options: { apiUrl?: string } = {} ): Promise<{ runId: string; status: string }> { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); + const auth = await ensureWorkflowAuthenticated(apiUrl); const { response } = await authorizedApiFetch( auth, `/api/v1/workflows/runs/${encodeURIComponent(runId)}/cancel`, @@ -597,7 +597,7 @@ export async function getRunLogs( } = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); + const auth = await ensureWorkflowAuthenticated(apiUrl); const searchParams = new URLSearchParams(); if (typeof options.offset === 'number') { searchParams.set('offset', String(options.offset)); @@ -636,7 +636,7 @@ export async function syncWorkflowPatch( options: { apiUrl?: string } = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - let auth = await ensureAuthenticated(apiUrl); + let auth = await ensureWorkflowAuthenticated(apiUrl); // Verify the run is completed const { response: statusResponse, auth: a1 } = await authorizedApiFetch( From d4780007d251b0af52695bb468eeee45d6661009 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Tue, 25 Aug 2026 16:37:33 +0200 Subject: [PATCH 15/15] refactor(cloud): isolate workflow API key client --- packages/cloud/src/api-client.test.ts | 37 ++++++++- packages/cloud/src/api-client.ts | 50 +++++++++++- packages/cloud/src/auth.test.ts | 49 +----------- packages/cloud/src/auth.ts | 56 ++----------- packages/cloud/src/index.ts | 2 - packages/cloud/src/types.ts | 8 -- packages/cloud/src/workflows.test.ts | 35 +++++++- packages/cloud/src/workflows.ts | 110 +++++++++++++------------- 8 files changed, 175 insertions(+), 172 deletions(-) diff --git a/packages/cloud/src/api-client.test.ts b/packages/cloud/src/api-client.test.ts index d571ba594..3f51b66de 100644 --- a/packages/cloud/src/api-client.test.ts +++ b/packages/cloud/src/api-client.test.ts @@ -1,8 +1,41 @@ -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; -import { CloudApiClient } from './api-client.js'; +import { CloudApiClient, WorkflowApiKeyClient } from './api-client.js'; import { CloudAuthError } from './types.js'; +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('WorkflowApiKeyClient', () => { + it('uses one API key at the explicit URL and returns a 401 without retrying', async () => { + const fetchSpy = vi.fn(async () => new Response('{}', { status: 401 })); + vi.stubGlobal('fetch', fetchSpy); + + const client = WorkflowApiKeyClient.fromEnv('https://explicit.example/cloud', { + CLOUD_API_URL: 'https://ignored.example/cloud', + CLOUD_API_KEY: 'ci-api-key', + }); + expect(client).not.toBeNull(); + + const response = await client!.fetch('/api/v1/workflows/run', { method: 'POST' }); + + expect(response.status).toBe(401); + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(String(fetchSpy.mock.calls[0][0])).toBe('https://explicit.example/cloud/api/v1/workflows/run'); + const headers = new Headers(fetchSpy.mock.calls[0][1]?.headers); + expect(headers.get('authorization')).toBe('Bearer ci-api-key'); + }); + + it('fails closed when the API key has a malformed Cloud URL', () => { + expect(() => + WorkflowApiKeyClient.fromEnv('not-a-url', { + CLOUD_API_KEY: 'ci-api-key', + }) + ).toThrowError(expect.objectContaining({ code: 'AUTH_ENV_REPROVISION_REQUIRED' })); + }); +}); + describe('CloudApiClient', () => { it('refreshes before an otherwise-valid session reaches refresh-token expiry', async () => { const fetchSpy = vi.fn(async (input: string | URL) => { diff --git a/packages/cloud/src/api-client.ts b/packages/cloud/src/api-client.ts index 3f452c90e..41c59b33a 100644 --- a/packages/cloud/src/api-client.ts +++ b/packages/cloud/src/api-client.ts @@ -42,6 +42,52 @@ export function buildApiUrl(apiUrl: string, p: string): URL { return new URL(trimLeadingSlash(p), withTrailingSlash(apiUrl)); } +function bearerHeaders(headers: HeaderInput | undefined, accessToken: string, defaultJson: boolean): Headers { + const merged = new Headers(headers); + if (defaultJson && !merged.has('content-type')) { + merged.set('content-type', 'application/json'); + } + merged.set('Authorization', `Bearer ${accessToken}`); + return appendAgentRelayTelemetryHeaders(merged); +} + +/** + * A deliberately small, non-refreshing client for workflow automation. It has + * no path to stored sessions, token rotation, browser login, or device login. + */ +export class WorkflowApiKeyClient { + private constructor( + private readonly apiUrl: string, + private readonly apiKey: string + ) {} + + static fromEnv(apiUrl: string, env: NodeJS.ProcessEnv = process.env): WorkflowApiKeyClient | null { + const apiKey = env.CLOUD_API_KEY?.trim(); + if (!apiKey) return null; + + try { + new URL(apiUrl); + } catch (error) { + throw new CloudAuthError( + 'AUTH_ENV_REPROVISION_REQUIRED', + 'CLOUD_API_URL is invalid for CLOUD_API_KEY', + { + cause: error, + } + ); + } + + return new WorkflowApiKeyClient(apiUrl, apiKey); + } + + fetch(p: string, init: RequestInit = {}): Promise { + return fetch(buildApiUrl(this.apiUrl, p), { + ...init, + headers: bearerHeaders(init.headers, this.apiKey, true), + }); + } +} + export class CloudApiClient { private apiUrl: string; private accessToken: string; @@ -242,9 +288,7 @@ export class CloudApiClient { } private buildHeaders(headers: HeaderInput | undefined): Headers { - const merged = new Headers(headers); - merged.set('Authorization', `Bearer ${this.accessToken}`); - return appendAgentRelayTelemetryHeaders(merged); + return bearerHeaders(headers, this.accessToken, false); } private shouldRefresh(): boolean { diff --git a/packages/cloud/src/auth.test.ts b/packages/cloud/src/auth.test.ts index bdb970b75..d7e5c6d5f 100644 --- a/packages/cloud/src/auth.test.ts +++ b/packages/cloud/src/auth.test.ts @@ -30,20 +30,13 @@ import { clearStoredAuth, ensureAuthenticated, ensureCloudSession, - ensureWorkflowAuthenticated, readStoredAuth, refreshStoredAuth, refreshStoredCloudIdentity, toCloudIdentity, writeStoredAuth, } from './auth.js'; -import { - AUTH_FILE_PATH, - CloudAuthError, - type CloudApiKeyAuth, - type StoredAuth, - type WhoAmIResponse, -} from './types.js'; +import { AUTH_FILE_PATH, CloudAuthError, type StoredAuth, type WhoAmIResponse } from './types.js'; const AUTH_LOCK_PATH = `${AUTH_FILE_PATH}.lock`; @@ -226,29 +219,6 @@ describe('ensureAuthenticated', () => { return new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(); } - it('uses one non-refreshing Cloud API key and preserves explicit URL precedence', async () => { - vi.stubEnv('CLOUD_API_URL', 'https://ci.example/cloud'); - vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); - fsMocks.readFile.mockResolvedValue(JSON.stringify(FILE_AUTH)); - - await expect(ensureWorkflowAuthenticated('https://explicit.example/cloud')).resolves.toEqual({ - authMode: 'api-key', - apiUrl: 'https://explicit.example/cloud', - accessToken: 'ci-api-key', - }); - expect(fsMocks.readFile).not.toHaveBeenCalled(); - }); - - it('fails closed when API-key auth has a malformed Cloud URL', async () => { - vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); - fsMocks.readFile.mockResolvedValue(JSON.stringify(FILE_AUTH)); - - await expect(ensureWorkflowAuthenticated('not-a-url')).rejects.toMatchObject({ - code: 'AUTH_ENV_REPROVISION_REQUIRED', - }); - expect(fsMocks.readFile).not.toHaveBeenCalled(); - }); - it('does not use workflow API-key auth for general Cloud authentication', async () => { vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); fsMocks.readFile.mockResolvedValue( @@ -885,23 +855,6 @@ describe('authorizedApiFetch telemetry headers', () => { }); describe('authorizedApiFetch re-login', () => { - it('returns a Cloud 401 for API-key auth without refresh or interactive login', async () => { - const fetchSpy = vi.fn(async () => new Response('{}', { status: 401 })); - vi.stubGlobal('fetch', fetchSpy); - - const auth: CloudApiKeyAuth = { - authMode: 'api-key', - apiUrl: 'https://api.example.test', - accessToken: 'ci-api-key', - }; - const result = await authorizedApiFetch(auth, '/api/v1/workflows/run', { method: 'POST' }); - - expect(result.response.status).toBe(401); - expect(result.auth).toBe(auth); - expect(fetchSpy).toHaveBeenCalledOnce(); - expect(childProcessMocks.spawn).not.toHaveBeenCalled(); - }); - it('re-authenticates a headless host through the device flow, not the browser', async () => { // The steady state this feature exists for: barry logged in once over ssh // with `--device`, and now a request 401s with a refresh token the server diff --git a/packages/cloud/src/auth.ts b/packages/cloud/src/auth.ts index d65a53e47..2338b8021 100644 --- a/packages/cloud/src/auth.ts +++ b/packages/cloud/src/auth.ts @@ -23,8 +23,6 @@ import { REFRESH_WINDOW_MS, CloudAuthError, defaultApiUrl, - type CloudApiKeyAuth, - type CloudRequestAuth, type CloudSession, type CloudSessionOptions, type StoredAuth, @@ -39,30 +37,6 @@ const AUTH_LOCK_TIMEOUT_MS = 30_000; const envBackedAuth = new WeakSet(); -function isApiKeyAuth(auth: CloudRequestAuth): auth is CloudApiKeyAuth { - return 'authMode' in auth && auth.authMode === 'api-key'; -} - -function readApiKeyAuth(apiUrl: string, env: NodeJS.ProcessEnv = process.env): CloudApiKeyAuth | null { - const apiKey = env.CLOUD_API_KEY?.trim(); - if (!apiKey) return null; - - const configuredApiUrl = apiUrl; - try { - new URL(configuredApiUrl); - } catch (error) { - throw new CloudAuthError('AUTH_ENV_REPROVISION_REQUIRED', 'CLOUD_API_URL is invalid for CLOUD_API_KEY', { - cause: error, - }); - } - - return { - authMode: 'api-key', - apiUrl: configuredApiUrl, - accessToken: apiKey, - }; -} - function markEnvBackedAuth(auth: StoredAuth): StoredAuth { envBackedAuth.add(auth); return auth; @@ -725,17 +699,6 @@ export async function ensureAuthenticated( return session.auth; } -/** - * Resolve authentication for the workflow control plane only. The API-key - * path is intentionally absent from general Cloud login, fleet, workspace, - * and integration commands even though Cloud also enforces its narrow scopes. - */ -export async function ensureWorkflowAuthenticated(apiUrl: string): Promise { - const apiKeyAuth = readApiKeyAuth(apiUrl); - if (apiKeyAuth) return apiKeyAuth; - return ensureAuthenticated(apiUrl); -} - export async function ensureCloudSession(options: CloudSessionOptions = {}): Promise { const env = options.env ?? process.env; const apiUrl = options.apiUrl || env.CLOUD_API_URL?.trim() || defaultApiUrl(); @@ -839,8 +802,8 @@ function apiFetch( }); } -export async function authorizedApiFetch( - auth: T, +export async function authorizedApiFetch( + auth: StoredAuth, requestPath: string, init: RequestInit, options: { @@ -854,19 +817,12 @@ export async function authorizedApiFetch( device?: boolean; env?: NodeJS.ProcessEnv; } = {} -): Promise<{ response: Response; auth: T }> { - let activeAuth: CloudRequestAuth = auth; +): Promise<{ response: Response; auth: StoredAuth }> { + let activeAuth = auth; let response = await apiFetch(activeAuth.apiUrl, activeAuth.accessToken, requestPath, init); if (response.status !== 401) { - return { response, auth: activeAuth as T }; - } - - // API keys are deliberately non-refreshing. Return the Cloud 401 to the - // caller so unattended automation fails closed instead of attempting token - // rotation or opening an interactive login flow. - if (isApiKeyAuth(activeAuth)) { - return { response, auth: activeAuth as T }; + return { response, auth: activeAuth }; } let refreshableAuth: StoredAuth = activeAuth; @@ -905,5 +861,5 @@ export async function authorizedApiFetch( } response = await apiFetch(activeAuth.apiUrl, activeAuth.accessToken, requestPath, init); - return { response, auth: activeAuth as T }; + return { response, auth: activeAuth }; } diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 430b764a5..4f7f5f107 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -225,8 +225,6 @@ export { PermissionAuditLog, getDefaultPermissionAuditPath } from './audit.js'; export { type StoredAuth, - type CloudApiKeyAuth, - type CloudRequestAuth, CloudAuthError, type CloudAuthErrorCode, type CloudSession, diff --git a/packages/cloud/src/types.ts b/packages/cloud/src/types.ts index a2ffd4acd..90c547617 100644 --- a/packages/cloud/src/types.ts +++ b/packages/cloud/src/types.ts @@ -9,14 +9,6 @@ export type StoredAuth = { apiUrl: string; }; -export type CloudApiKeyAuth = { - authMode: 'api-key'; - accessToken: string; - apiUrl: string; -}; - -export type CloudRequestAuth = StoredAuth | CloudApiKeyAuth; - export type CloudAuthErrorCode = | 'AUTH_REFRESH_TIMEOUT' | 'AUTH_REFRESH_EXPIRED' diff --git a/packages/cloud/src/workflows.test.ts b/packages/cloud/src/workflows.test.ts index fef33be2b..57272e3f6 100644 --- a/packages/cloud/src/workflows.test.ts +++ b/packages/cloud/src/workflows.test.ts @@ -6,9 +6,14 @@ import { mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; import type { RunWorkflowResponse } from './types.js'; const s3SendMock = vi.hoisted(() => vi.fn()); -const ensureWorkflowAuthenticatedMock = vi.hoisted(() => vi.fn()); +const ensureAuthenticatedMock = vi.hoisted(() => vi.fn()); const authorizedApiFetchMock = vi.hoisted(() => vi.fn()); +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + vi.mock('@aws-sdk/client-s3', () => { class PutObjectCommand { input: unknown; @@ -25,7 +30,7 @@ vi.mock('@aws-sdk/client-s3', () => { }); vi.mock('./auth.js', () => ({ - ensureWorkflowAuthenticated: (...args: unknown[]) => ensureWorkflowAuthenticatedMock(...args), + ensureAuthenticated: (...args: unknown[]) => ensureAuthenticatedMock(...args), authorizedApiFetch: (...args: unknown[]) => authorizedApiFetchMock(...args), })); @@ -353,7 +358,7 @@ describe('runWorkflow code sync', () => { originalCwd = process.cwd(); tmpRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), 'cloud-run-workflow-'))); process.chdir(tmpRoot); - ensureWorkflowAuthenticatedMock.mockResolvedValue({ accessToken: 'token' }); + ensureAuthenticatedMock.mockResolvedValue({ accessToken: 'token' }); s3SendMock.mockResolvedValue({}); }); @@ -620,7 +625,7 @@ describe('workflow schedules', () => { originalCwd = process.cwd(); tmpRoot = await realpath(await mkdtemp(path.join(os.tmpdir(), 'cloud-schedule-workflow-'))); process.chdir(tmpRoot); - ensureWorkflowAuthenticatedMock.mockResolvedValue({ accessToken: 'token' }); + ensureAuthenticatedMock.mockResolvedValue({ accessToken: 'token' }); }); afterEach(async () => { @@ -792,4 +797,26 @@ describe('workflow schedules', () => { expect(schedules).toHaveLength(1); expect(schedules[0].id).toBe('sched-1'); }); + + it('uses the non-refreshing workflow API-key client without stored authentication', async () => { + vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); + const fetchSpy = vi.fn( + async () => + new Response(JSON.stringify({ schedules: [scheduleRecord()] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + vi.stubGlobal('fetch', fetchSpy); + + const schedules = await listWorkflowSchedules({ apiUrl: 'https://ci.example/cloud' }); + + expect(schedules).toHaveLength(1); + expect(ensureAuthenticatedMock).not.toHaveBeenCalled(); + expect(authorizedApiFetchMock).not.toHaveBeenCalled(); + expect(fetchSpy).toHaveBeenCalledOnce(); + const [url, init] = fetchSpy.mock.calls[0]; + expect(String(url)).toBe('https://ci.example/cloud/api/v1/workflows/schedules'); + expect(new Headers(init?.headers).get('authorization')).toBe('Bearer ci-api-key'); + }); }); diff --git a/packages/cloud/src/workflows.ts b/packages/cloud/src/workflows.ts index 0165fb835..9c22068ec 100644 --- a/packages/cloud/src/workflows.ts +++ b/packages/cloud/src/workflows.ts @@ -6,7 +6,8 @@ import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import ignore from 'ignore'; import * as tar from 'tar'; -import { ensureWorkflowAuthenticated, authorizedApiFetch } from './auth.js'; +import { ensureAuthenticated, authorizedApiFetch } from './auth.js'; +import { WorkflowApiKeyClient } from './api-client.js'; import { defaultApiUrl, type WorkflowFileType, @@ -240,7 +241,7 @@ export async function runWorkflow( options: RunWorkflowOptions = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - let auth = await ensureWorkflowAuthenticated(apiUrl); + const api = await workflowApiClient(apiUrl); const input = await resolveWorkflowInput(workflowArg, options.fileType); if (input.fileType === 'ts') { @@ -271,15 +272,10 @@ export async function runWorkflow( if (syncCode) { const t0 = Date.now(); console.error('Preparing run...'); - const { response: prepResponse, auth: prepAuth } = await authorizedApiFetch( - auth, - '/api/v1/workflows/prepare', - { - method: 'POST', - headers: { Accept: 'application/json' }, - } - ); - auth = prepAuth; + const prepResponse = await api.fetch('/api/v1/workflows/prepare', { + method: 'POST', + headers: { Accept: 'application/json' }, + }); const prepPayload = await readJsonResponse(prepResponse); if (!prepResponse.ok) { @@ -303,19 +299,14 @@ export async function runWorkflow( let s3Client: S3Client | null = null; const uploadCodeObject = async (objectKey: string, tarball: Buffer) => { if (isCloudApiWorkflowStorage(prepared)) { - const { response, auth: uploadAuth } = await authorizedApiFetch( - auth, - workflowStorageObjectPath(prepared.runId, objectKey), - { - method: 'PUT', - headers: { - 'content-type': 'application/gzip', - accept: 'application/json', - }, - body: tarball as unknown as BodyInit, - } - ); - auth = uploadAuth; + const response = await api.fetch(workflowStorageObjectPath(prepared.runId, objectKey), { + method: 'PUT', + headers: { + 'content-type': 'application/gzip', + accept: 'application/json', + }, + body: tarball as unknown as BodyInit, + }); const payload = await readJsonResponse(response); if (!response.ok) { throw new Error(`Workflow storage upload failed: ${describeResponseError(response, payload)}`); @@ -419,7 +410,7 @@ export async function runWorkflow( const t3 = Date.now(); console.error('Launching workflow...'); - const { response, auth: updatedAuth } = await authorizedApiFetch(auth, '/api/v1/workflows/run', { + const response = await api.fetch('/api/v1/workflows/run', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -427,7 +418,6 @@ export async function runWorkflow( }, body: JSON.stringify(requestBody), }); - auth = updatedAuth; console.error(` Launched in ${((Date.now() - t3) / 1000).toFixed(1)}s`); @@ -459,7 +449,7 @@ export async function scheduleWorkflow( } const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureWorkflowAuthenticated(apiUrl); + const api = await workflowApiClient(apiUrl); const input = await resolveWorkflowInput(workflowArg, options.fileType); if (input.fileType === 'ts') { @@ -495,7 +485,7 @@ export async function scheduleWorkflow( requestBody.scheduled_at = scheduledAt.toISOString(); } - const { response } = await authorizedApiFetch(auth, '/api/v1/workflows/schedules', { + const response = await api.fetch('/api/v1/workflows/schedules', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -518,8 +508,8 @@ export async function scheduleWorkflow( export async function listWorkflowSchedules(options: { apiUrl?: string } = {}): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureWorkflowAuthenticated(apiUrl); - const { response } = await authorizedApiFetch(auth, '/api/v1/workflows/schedules', { + const api = await workflowApiClient(apiUrl); + const response = await api.fetch('/api/v1/workflows/schedules', { headers: { Accept: 'application/json' }, }); @@ -544,8 +534,8 @@ export async function getRunStatus( options: { apiUrl?: string } = {} ): Promise> { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureWorkflowAuthenticated(apiUrl); - const { response } = await authorizedApiFetch(auth, `/api/v1/workflows/runs/${encodeURIComponent(runId)}`, { + const api = await workflowApiClient(apiUrl); + const response = await api.fetch(`/api/v1/workflows/runs/${encodeURIComponent(runId)}`, { headers: { Accept: 'application/json' }, }); @@ -566,15 +556,11 @@ export async function cancelWorkflow( options: { apiUrl?: string } = {} ): Promise<{ runId: string; status: string }> { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureWorkflowAuthenticated(apiUrl); - const { response } = await authorizedApiFetch( - auth, - `/api/v1/workflows/runs/${encodeURIComponent(runId)}/cancel`, - { - method: 'POST', - headers: { Accept: 'application/json' }, - } - ); + const api = await workflowApiClient(apiUrl); + const response = await api.fetch(`/api/v1/workflows/runs/${encodeURIComponent(runId)}/cancel`, { + method: 'POST', + headers: { Accept: 'application/json' }, + }); const payload = await readJsonResponse(response); if (!response.ok) { @@ -597,7 +583,7 @@ export async function getRunLogs( } = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureWorkflowAuthenticated(apiUrl); + const api = await workflowApiClient(apiUrl); const searchParams = new URLSearchParams(); if (typeof options.offset === 'number') { searchParams.set('offset', String(options.offset)); @@ -608,7 +594,7 @@ export async function getRunLogs( const requestPath = `/api/v1/workflows/runs/${encodeURIComponent(runId)}/logs${searchParams.size ? `?${searchParams.toString()}` : ''}`; - const { response } = await authorizedApiFetch(auth, requestPath, { + const response = await api.fetch(requestPath, { headers: { Accept: 'application/json' }, }); @@ -636,15 +622,12 @@ export async function syncWorkflowPatch( options: { apiUrl?: string } = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - let auth = await ensureWorkflowAuthenticated(apiUrl); + const api = await workflowApiClient(apiUrl); // Verify the run is completed - const { response: statusResponse, auth: a1 } = await authorizedApiFetch( - auth, - `/api/v1/workflows/runs/${encodeURIComponent(runId)}`, - { headers: { Accept: 'application/json' } } - ); - auth = a1; + const statusResponse = await api.fetch(`/api/v1/workflows/runs/${encodeURIComponent(runId)}`, { + headers: { Accept: 'application/json' }, + }); if (!statusResponse.ok) { const payload = await readJsonResponse(statusResponse); @@ -657,11 +640,9 @@ export async function syncWorkflowPatch( } // Download the patch - const { response } = await authorizedApiFetch( - auth, - `/api/v1/workflows/runs/${encodeURIComponent(runId)}/patch`, - { headers: { Accept: 'application/json' } } - ); + const response = await api.fetch(`/api/v1/workflows/runs/${encodeURIComponent(runId)}/patch`, { + headers: { Accept: 'application/json' }, + }); const payload = await readJsonResponse(response); if (!response.ok) { @@ -685,6 +666,25 @@ export async function syncWorkflowPatch( // ── Internal helpers ────────────────────────────────────────────────────────── +type WorkflowHttpClient = { + fetch(requestPath: string, init?: RequestInit): Promise; +}; + +async function storedWorkflowClient(apiUrl: string): Promise { + let auth = await ensureAuthenticated(apiUrl); + return { + async fetch(requestPath, init = {}) { + const result = await authorizedApiFetch(auth, requestPath, init); + auth = result.auth; + return result.response; + }, + }; +} + +async function workflowApiClient(apiUrl: string): Promise { + return WorkflowApiKeyClient.fromEnv(apiUrl) ?? storedWorkflowClient(apiUrl); +} + async function readJsonResponse(response: Response): Promise { const rawBody = await response.text(); if (!rawBody) {