diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md b/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md new file mode 100644 index 000000000..7fc138e4b --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/summary.md @@ -0,0 +1,54 @@ +# Trajectory: Add PR-specific Cloud RelayFlow red-green proof infrastructure + +> **Status:** ✅ Completed +> **Confidence:** 92% +> **Started:** August 25, 2026 at 10:45 AM +> **Completed:** August 25, 2026 at 11:14 AM + +--- + +## Summary + +Implemented trusted PR classification and single-case dispatch, a fail-closed Cloud RelayFlow that proves base then head in distinct per-step sandboxes, provenance/evidence validation, GitHub Action wiring, tests, template guidance, and credential rollout documentation. + +**Approach:** Kept pull_request_target as a data-only trusted dispatcher, ran PR code only in Cloud, represented expected-red as a structured successful observation, disabled repair retries, validated exact base/head SHAs and sandbox IDs, and tested both the contracts and the complete repository. + +--- + +## Key Decisions + +### Require exactly one PR proof case and a structured observation result +- **Chose:** Require exactly one PR proof case and a structured observation result +- **Reasoning:** One feature or fix should trigger only its own case. The case runner must exit successfully after observing behavior and write bug/absent/fixed plus an exact signature; test crashes, missing tests, skips, and build failures remain infrastructure failures and cannot count as expected red. + +### Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes +- **Chose:** Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes +- **Reasoning:** The GitHub runner must never checkout or execute untrusted head code while holding Cloud credentials. It will fetch and validate one declarative case manifest, stage exact base/head SHAs, and submit the trusted base-branch RelayFlow. + +### Disable automatic repair and retries for PR proof workflows +- **Chose:** Disable automatic repair and retries for PR proof workflows +- **Rejected:** Default retry policy, Continue on failure +- **Reasoning:** A red/green gate must preserve the first observation and fail closed. RelayFlow defaults to agent-assisted repair retries when agents exist, so the proof workflow explicitly uses fail-fast with zero step retries. + +### Stage the validated generated input in the disposable CI git index +- **Chose:** Stage the validated generated input in the disposable CI git index +- **Rejected:** Change global code-sync semantics, Commit a placeholder input file +- **Reasoning:** Cloud code sync uploads git-known files. The per-PR input is generated after checkout, so the action must git-add it without committing or pushing or the Cloud workflow cannot see the exact case and SHAs. + +### Require explicit proof classification on every PR +- **Chose:** Require explicit proof classification on every PR +- **Rejected:** Infer only from conventional titles, Run Cloud for every PR +- **Reasoning:** Silently treating missing metadata as non-functional would let non-conventional feature and fix titles bypass the gate. Every PR must declare feature, bugfix, or non-functional; only the first two launch Cloud. + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Require exactly one PR proof case and a structured observation result: Require exactly one PR proof case and a structured observation result +- Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes: Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes +- Disable automatic repair and retries for PR proof workflows: Disable automatic repair and retries for PR proof workflows +- Stage the validated generated input in the disposable CI git index: Stage the validated generated input in the disposable CI git index +- Require explicit proof classification on every PR: Require explicit proof classification on every PR diff --git a/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json b/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json new file mode 100644 index 000000000..c8dc4482f --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-08/traj_yr3f5u3r6zdz/trajectory.json @@ -0,0 +1,128 @@ +{ + "id": "traj_yr3f5u3r6zdz", + "version": 1, + "task": { + "title": "Add PR-specific Cloud RelayFlow red-green proof infrastructure" + }, + "status": "completed", + "startedAt": "2026-08-25T08:45:32.835Z", + "completedAt": "2026-08-25T09:14:31.949Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-08-25T08:50:30.315Z" + } + ], + "chapters": [ + { + "id": "chap_cco5mxx3i8f5", + "title": "Work", + "agentName": "default", + "startedAt": "2026-08-25T08:50:30.315Z", + "endedAt": "2026-08-25T09:14:31.949Z", + "events": [ + { + "ts": 1787647830316, + "type": "decision", + "content": "Require exactly one PR proof case and a structured observation result: Require exactly one PR proof case and a structured observation result", + "raw": { + "question": "Require exactly one PR proof case and a structured observation result", + "chosen": "Require exactly one PR proof case and a structured observation result", + "alternatives": [], + "reasoning": "One feature or fix should trigger only its own case. The case runner must exit successfully after observing behavior and write bug/absent/fixed plus an exact signature; test crashes, missing tests, skips, and build failures remain infrastructure failures and cannot count as expected red." + }, + "significance": "high" + }, + { + "ts": 1787647830374, + "type": "decision", + "content": "Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes: Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes", + "raw": { + "question": "Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes", + "chosen": "Use pull_request_target only as a trusted dispatcher; execute PR case code only inside Cloud sandboxes", + "alternatives": [], + "reasoning": "The GitHub runner must never checkout or execute untrusted head code while holding Cloud credentials. It will fetch and validate one declarative case manifest, stage exact base/head SHAs, and submit the trusted base-branch RelayFlow." + }, + "significance": "high" + }, + { + "ts": 1787649023999, + "type": "decision", + "content": "Disable automatic repair and retries for PR proof workflows: Disable automatic repair and retries for PR proof workflows", + "raw": { + "question": "Disable automatic repair and retries for PR proof workflows", + "chosen": "Disable automatic repair and retries for PR proof workflows", + "alternatives": [ + { + "option": "Default retry policy", + "reason": "" + }, + { + "option": "Continue on failure", + "reason": "" + } + ], + "reasoning": "A red/green gate must preserve the first observation and fail closed. RelayFlow defaults to agent-assisted repair retries when agents exist, so the proof workflow explicitly uses fail-fast with zero step retries." + }, + "significance": "high" + }, + { + "ts": 1787649024705, + "type": "decision", + "content": "Stage the validated generated input in the disposable CI git index: Stage the validated generated input in the disposable CI git index", + "raw": { + "question": "Stage the validated generated input in the disposable CI git index", + "chosen": "Stage the validated generated input in the disposable CI git index", + "alternatives": [ + { + "option": "Change global code-sync semantics", + "reason": "" + }, + { + "option": "Commit a placeholder input file", + "reason": "" + } + ], + "reasoning": "Cloud code sync uploads git-known files. The per-PR input is generated after checkout, so the action must git-add it without committing or pushing or the Cloud workflow cannot see the exact case and SHAs." + }, + "significance": "high" + }, + { + "ts": 1787649025196, + "type": "decision", + "content": "Require explicit proof classification on every PR: Require explicit proof classification on every PR", + "raw": { + "question": "Require explicit proof classification on every PR", + "chosen": "Require explicit proof classification on every PR", + "alternatives": [ + { + "option": "Infer only from conventional titles", + "reason": "" + }, + { + "option": "Run Cloud for every PR", + "reason": "" + } + ], + "reasoning": "Silently treating missing metadata as non-functional would let non-conventional feature and fix titles bypass the gate. Every PR must declare feature, bugfix, or non-functional; only the first two launch Cloud." + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Implemented trusted PR classification and single-case dispatch, a fail-closed Cloud RelayFlow that proves base then head in distinct per-step sandboxes, provenance/evidence validation, GitHub Action wiring, tests, template guidance, and credential rollout documentation.", + "approach": "Kept pull_request_target as a data-only trusted dispatcher, ran PR code only in Cloud, represented expected-red as a structured successful observation, disabled repair retries, validated exact base/head SHAs and sandbox IDs, and tested both the contracts and the complete repository.", + "confidence": 0.92 + }, + "commits": [], + "filesChanged": [], + "projectId": "AgentWorkforce/relay", + "tags": [], + "_trace": { + "startRef": "191e5f14a343431ec42282c1103c9f014854ac0b", + "endRef": "191e5f14a343431ec42282c1103c9f014854ac0b" + } +} \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ae2c91eb3..59f7513df 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,6 +7,15 @@ - [ ] Tests added/updated - [ ] Manual testing completed +## RelayFlow Proof + +Replace both values below. Use `feature` or `bugfix` for user-visible behavior +changes and add exactly one case under `tests/relayflows/cases//`. +Use `non-functional` and `n/a` only when runtime behavior is unchanged. + +- Change type: `replace-me` +- RelayFlow case: `replace-me` + ## Screenshots diff --git a/.github/workflows/relayflow-pr-proof.yml b/.github/workflows/relayflow-pr-proof.yml new file mode 100644 index 000000000..2b56d5e03 --- /dev/null +++ b/.github/workflows/relayflow-pr-proof.yml @@ -0,0 +1,113 @@ +name: RelayFlow PR Proof + +# Security boundary: pull_request_target gives this trusted base-branch +# dispatcher access to Cloud credentials. It MUST NOT checkout, import, or run +# PR-head code on the GitHub runner. The exact head is cloned only inside the +# isolated Cloud proof sandboxes by scripts/pr-proof/run-arm.mjs. +on: + pull_request_target: + types: [opened, synchronize, reopened, edited, ready_for_review] + workflow_dispatch: + inputs: + pr_number: + description: Pull request number to prove + required: true + type: number + +permissions: + contents: read + pull-requests: read + statuses: write + +concurrency: + group: relayflow-pr-proof-${{ github.event.pull_request.number || inputs.pr_number }} + cancel-in-progress: true + +jobs: + proof: + name: RelayFlow PR proof dispatcher + runs-on: ubuntu-latest + timeout-minutes: 70 + steps: + - name: Checkout trusted base + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 22.14.0 + + # pull_request_target runs are attached to the base SHA. Publish the + # stable required context explicitly on the exact validated PR head. + - name: Start required status on PR head + id: status + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node scripts/pr-proof/report-status.mjs start + --event "$GITHUB_EVENT_PATH" + --github-output "$GITHUB_OUTPUT" + + - name: Classify PR and validate declared case + id: prepare + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node scripts/pr-proof/prepare.mjs + --event "$GITHUB_EVENT_PATH" + --output .relayflow/pr-proof-input.json + --expected-head-sha "${{ steps.status.outputs.head_sha }}" + --github-output "$GITHUB_OUTPUT" + --summary "$GITHUB_STEP_SUMMARY" + + - name: Confirm Cloud proof credential + if: steps.prepare.outputs.required == 'true' + env: + CLOUD_API_URL: ${{ secrets.CLOUD_API_URL }} + CLOUD_API_KEY: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_API_KEY }} + run: | + test -n "$CLOUD_API_URL" + test -n "$CLOUD_API_KEY" + + # Cloud code sync intentionally uploads only paths known to git. Add the + # generated, validated input to this disposable runner's index so it is + # present in the Cloud workspace; this does not create a commit or push. + - name: Include proof input in Cloud code sync + if: steps.prepare.outputs.required == 'true' + run: git add -f -- .relayflow/pr-proof-input.json + + - name: Install the released Agent Relay CLI + if: steps.prepare.outputs.required == 'true' + run: npm install --global "agent-relay@$(node -p "require('./package.json').version")" + + - name: Run isolated Cloud red-green proof + if: steps.prepare.outputs.required == 'true' + id: cloud + env: + CLOUD_API_URL: ${{ secrets.CLOUD_API_URL }} + CLOUD_API_KEY: ${{ secrets.RELAYFLOW_PR_PROOF_CLOUD_API_KEY }} + PR_PROOF_CLOUD_TIMEOUT_MS: '3600000' + run: node scripts/pr-proof/run-cloud.mjs workflows/pr-proof.ts + + - name: Upload Cloud proof logs + if: always() && steps.prepare.outputs.required == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: relayflow-pr-proof-${{ github.event.pull_request.number || inputs.pr_number }} + path: .workflow-artifacts/pr-proof/cloud.log + if-no-files-found: warn + + - name: Finish required status on PR head + # report-status verifies that this run still owns the latest pending + # context before publishing. That makes cancellation terminal without + # allowing a cancelled predecessor to overwrite a replacement run. + if: always() && steps.status.outputs.head_sha != '' + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node scripts/pr-proof/report-status.mjs finish + --sha "${{ steps.status.outputs.head_sha }}" + --job-status "${{ job.status }}" diff --git a/.gitignore b/.gitignore index 60dbfe742..dfa0f28b2 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,7 @@ __pycache__/ .agentworkforce/relay/ .relay/ .agent-relay/ +.relayflow/pr-proof-input.json .msd/ /workflows/* @@ -91,6 +92,7 @@ __pycache__/ !/workflows/cloud-connect/ !/workflows/verify-features.ts !/workflows/audit-feature-manifest.ts +!/workflows/pr-proof.ts # Eval harness JSON reports (generated per run) tests/integration/broker/evals-reports/ diff --git a/packages/cloud/src/api-client.test.ts b/packages/cloud/src/api-client.test.ts index d571ba594..3f51b66de 100644 --- a/packages/cloud/src/api-client.test.ts +++ b/packages/cloud/src/api-client.test.ts @@ -1,8 +1,41 @@ -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; -import { CloudApiClient } from './api-client.js'; +import { CloudApiClient, WorkflowApiKeyClient } from './api-client.js'; import { CloudAuthError } from './types.js'; +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('WorkflowApiKeyClient', () => { + it('uses one API key at the explicit URL and returns a 401 without retrying', async () => { + const fetchSpy = vi.fn(async () => new Response('{}', { status: 401 })); + vi.stubGlobal('fetch', fetchSpy); + + const client = WorkflowApiKeyClient.fromEnv('https://explicit.example/cloud', { + CLOUD_API_URL: 'https://ignored.example/cloud', + CLOUD_API_KEY: 'ci-api-key', + }); + expect(client).not.toBeNull(); + + const response = await client!.fetch('/api/v1/workflows/run', { method: 'POST' }); + + expect(response.status).toBe(401); + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(String(fetchSpy.mock.calls[0][0])).toBe('https://explicit.example/cloud/api/v1/workflows/run'); + const headers = new Headers(fetchSpy.mock.calls[0][1]?.headers); + expect(headers.get('authorization')).toBe('Bearer ci-api-key'); + }); + + it('fails closed when the API key has a malformed Cloud URL', () => { + expect(() => + WorkflowApiKeyClient.fromEnv('not-a-url', { + CLOUD_API_KEY: 'ci-api-key', + }) + ).toThrowError(expect.objectContaining({ code: 'AUTH_ENV_REPROVISION_REQUIRED' })); + }); +}); + describe('CloudApiClient', () => { it('refreshes before an otherwise-valid session reaches refresh-token expiry', async () => { const fetchSpy = vi.fn(async (input: string | URL) => { diff --git a/packages/cloud/src/api-client.ts b/packages/cloud/src/api-client.ts index 3f452c90e..41c59b33a 100644 --- a/packages/cloud/src/api-client.ts +++ b/packages/cloud/src/api-client.ts @@ -42,6 +42,52 @@ export function buildApiUrl(apiUrl: string, p: string): URL { return new URL(trimLeadingSlash(p), withTrailingSlash(apiUrl)); } +function bearerHeaders(headers: HeaderInput | undefined, accessToken: string, defaultJson: boolean): Headers { + const merged = new Headers(headers); + if (defaultJson && !merged.has('content-type')) { + merged.set('content-type', 'application/json'); + } + merged.set('Authorization', `Bearer ${accessToken}`); + return appendAgentRelayTelemetryHeaders(merged); +} + +/** + * A deliberately small, non-refreshing client for workflow automation. It has + * no path to stored sessions, token rotation, browser login, or device login. + */ +export class WorkflowApiKeyClient { + private constructor( + private readonly apiUrl: string, + private readonly apiKey: string + ) {} + + static fromEnv(apiUrl: string, env: NodeJS.ProcessEnv = process.env): WorkflowApiKeyClient | null { + const apiKey = env.CLOUD_API_KEY?.trim(); + if (!apiKey) return null; + + try { + new URL(apiUrl); + } catch (error) { + throw new CloudAuthError( + 'AUTH_ENV_REPROVISION_REQUIRED', + 'CLOUD_API_URL is invalid for CLOUD_API_KEY', + { + cause: error, + } + ); + } + + return new WorkflowApiKeyClient(apiUrl, apiKey); + } + + fetch(p: string, init: RequestInit = {}): Promise { + return fetch(buildApiUrl(this.apiUrl, p), { + ...init, + headers: bearerHeaders(init.headers, this.apiKey, true), + }); + } +} + export class CloudApiClient { private apiUrl: string; private accessToken: string; @@ -242,9 +288,7 @@ export class CloudApiClient { } private buildHeaders(headers: HeaderInput | undefined): Headers { - const merged = new Headers(headers); - merged.set('Authorization', `Bearer ${this.accessToken}`); - return appendAgentRelayTelemetryHeaders(merged); + return bearerHeaders(headers, this.accessToken, false); } private shouldRefresh(): boolean { diff --git a/packages/cloud/src/auth.test.ts b/packages/cloud/src/auth.test.ts index e070775b2..d7e5c6d5f 100644 --- a/packages/cloud/src/auth.test.ts +++ b/packages/cloud/src/auth.test.ts @@ -219,6 +219,25 @@ describe('ensureAuthenticated', () => { return new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(); } + it('does not use workflow API-key auth for general Cloud authentication', async () => { + vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); + fsMocks.readFile.mockResolvedValue( + JSON.stringify({ + ...FILE_AUTH, + accessTokenExpiresAt: farFutureIso(), + }) + ); + + const result = await ensureAuthenticated('https://default.example/cloud'); + + expect(result).toMatchObject({ + apiUrl: FILE_AUTH.apiUrl, + accessToken: FILE_AUTH.accessToken, + refreshToken: FILE_AUTH.refreshToken, + }); + expect(result).not.toHaveProperty('authMode'); + }); + it('returns stored file auth even when apiUrl differs from defaultApiUrl', async () => { // Regression: previously, any host mismatch between the CLI's default // apiUrl and the stored apiUrl forced a browser login on every cloud diff --git a/packages/cloud/src/auth.ts b/packages/cloud/src/auth.ts index 6a052f73b..2338b8021 100644 --- a/packages/cloud/src/auth.ts +++ b/packages/cloud/src/auth.ts @@ -825,14 +825,16 @@ export async function authorizedApiFetch( return { response, auth: activeAuth }; } + let refreshableAuth: StoredAuth = activeAuth; try { - activeAuth = await refreshStoredAuth(activeAuth, { + refreshableAuth = await refreshStoredAuth(refreshableAuth, { force: true, refreshTimeoutMs: options.refreshTimeoutMs, signal: init.signal ?? undefined, }); + activeAuth = refreshableAuth; } catch (error) { - if (isEnvBackedAuth(activeAuth)) { + if (isEnvBackedAuth(refreshableAuth)) { throw toEnvAuthRefreshError(error); } diff --git a/packages/cloud/src/workflows.test.ts b/packages/cloud/src/workflows.test.ts index 4919a8438..57272e3f6 100644 --- a/packages/cloud/src/workflows.test.ts +++ b/packages/cloud/src/workflows.test.ts @@ -9,6 +9,11 @@ const s3SendMock = vi.hoisted(() => vi.fn()); const ensureAuthenticatedMock = vi.hoisted(() => vi.fn()); const authorizedApiFetchMock = vi.hoisted(() => vi.fn()); +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + vi.mock('@aws-sdk/client-s3', () => { class PutObjectCommand { input: unknown; @@ -512,6 +517,37 @@ describe('runWorkflow code sync', () => { expect((runBodies[0] as { paths?: unknown }).paths).toBeUndefined(); }); + it('reports the prepared run id before upload when supervising automation opts in', async () => { + await writeFile('README.md', 'supervised\n'); + const workflowPath = path.join(tmpRoot, 'workflow.yaml'); + await writeFile( + workflowPath, + ['version: "1.0"', 'name: supervised', 'swarm:', ' pattern: dag', 'agents: []', 'workflows: []'].join( + '\n' + ) + ); + const runBodies: unknown[] = []; + mockPrepareAndRun(runBodies); + const errors: string[] = []; + const errorSpy = vi.spyOn(console, 'error').mockImplementation((value) => { + errors.push(String(value)); + }); + process.env.AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID = '1'; + try { + await runWorkflow(workflowPath); + } finally { + delete process.env.AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID; + errorSpy.mockRestore(); + } + + const preparedIndex = errors.indexOf('AGENT_RELAY_CLOUD_PREPARED_RUN_ID=run-1'); + const uploadIndex = errors.indexOf('Uploading to workflow storage...'); + const launchIndex = errors.indexOf('Launching workflow...'); + expect(preparedIndex).toBeGreaterThanOrEqual(0); + expect(preparedIndex).toBeLessThan(uploadIndex); + expect(preparedIndex).toBeLessThan(launchIndex); + }); + it('uploads code through the cloud API when prepare returns cloud-api storage', async () => { await writeFile('README.md', 'cloud-api\n'); const workflowPath = path.join(tmpRoot, 'workflow.yaml'); @@ -761,4 +797,26 @@ describe('workflow schedules', () => { expect(schedules).toHaveLength(1); expect(schedules[0].id).toBe('sched-1'); }); + + it('uses the non-refreshing workflow API-key client without stored authentication', async () => { + vi.stubEnv('CLOUD_API_KEY', 'ci-api-key'); + const fetchSpy = vi.fn( + async () => + new Response(JSON.stringify({ schedules: [scheduleRecord()] }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + vi.stubGlobal('fetch', fetchSpy); + + const schedules = await listWorkflowSchedules({ apiUrl: 'https://ci.example/cloud' }); + + expect(schedules).toHaveLength(1); + expect(ensureAuthenticatedMock).not.toHaveBeenCalled(); + expect(authorizedApiFetchMock).not.toHaveBeenCalled(); + expect(fetchSpy).toHaveBeenCalledOnce(); + const [url, init] = fetchSpy.mock.calls[0]; + expect(String(url)).toBe('https://ci.example/cloud/api/v1/workflows/schedules'); + expect(new Headers(init?.headers).get('authorization')).toBe('Bearer ci-api-key'); + }); }); diff --git a/packages/cloud/src/workflows.ts b/packages/cloud/src/workflows.ts index 1d8792d4c..9c22068ec 100644 --- a/packages/cloud/src/workflows.ts +++ b/packages/cloud/src/workflows.ts @@ -7,6 +7,7 @@ import ignore from 'ignore'; import * as tar from 'tar'; import { ensureAuthenticated, authorizedApiFetch } from './auth.js'; +import { WorkflowApiKeyClient } from './api-client.js'; import { defaultApiUrl, type WorkflowFileType, @@ -58,6 +59,8 @@ type RunWorkflowOptions = { previousRunId?: string; }; +const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; + const CODE_SYNC_EXCLUDES = [ '.git', 'node_modules', @@ -238,7 +241,7 @@ export async function runWorkflow( options: RunWorkflowOptions = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - let auth = await ensureAuthenticated(apiUrl); + const api = await workflowApiClient(apiUrl); const input = await resolveWorkflowInput(workflowArg, options.fileType); if (input.fileType === 'ts') { @@ -269,15 +272,10 @@ export async function runWorkflow( if (syncCode) { const t0 = Date.now(); console.error('Preparing run...'); - const { response: prepResponse, auth: prepAuth } = await authorizedApiFetch( - auth, - '/api/v1/workflows/prepare', - { - method: 'POST', - headers: { Accept: 'application/json' }, - } - ); - auth = prepAuth; + const prepResponse = await api.fetch('/api/v1/workflows/prepare', { + method: 'POST', + headers: { Accept: 'application/json' }, + }); const prepPayload = await readJsonResponse(prepResponse); if (!prepResponse.ok) { @@ -289,24 +287,26 @@ export async function runWorkflow( } const prepared = prepPayload; + // Trusted automation can opt into this machine-readable progress record + // before code upload and final submission. It lets a supervising process + // cancel the prepared run if the CLI is interrupted after the launch POST + // has completed but before the final JSON response is observed. + if (process.env.AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID === '1') { + console.error(`${PREPARED_RUN_ID_MARKER}${prepared.runId}`); + } console.error(` Prepared in ${((Date.now() - t0) / 1000).toFixed(1)}s`); let s3Client: S3Client | null = null; const uploadCodeObject = async (objectKey: string, tarball: Buffer) => { if (isCloudApiWorkflowStorage(prepared)) { - const { response, auth: uploadAuth } = await authorizedApiFetch( - auth, - workflowStorageObjectPath(prepared.runId, objectKey), - { - method: 'PUT', - headers: { - 'content-type': 'application/gzip', - accept: 'application/json', - }, - body: tarball as unknown as BodyInit, - } - ); - auth = uploadAuth; + const response = await api.fetch(workflowStorageObjectPath(prepared.runId, objectKey), { + method: 'PUT', + headers: { + 'content-type': 'application/gzip', + accept: 'application/json', + }, + body: tarball as unknown as BodyInit, + }); const payload = await readJsonResponse(response); if (!response.ok) { throw new Error(`Workflow storage upload failed: ${describeResponseError(response, payload)}`); @@ -410,7 +410,7 @@ export async function runWorkflow( const t3 = Date.now(); console.error('Launching workflow...'); - const { response, auth: updatedAuth } = await authorizedApiFetch(auth, '/api/v1/workflows/run', { + const response = await api.fetch('/api/v1/workflows/run', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -418,7 +418,6 @@ export async function runWorkflow( }, body: JSON.stringify(requestBody), }); - auth = updatedAuth; console.error(` Launched in ${((Date.now() - t3) / 1000).toFixed(1)}s`); @@ -450,7 +449,7 @@ export async function scheduleWorkflow( } const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); + const api = await workflowApiClient(apiUrl); const input = await resolveWorkflowInput(workflowArg, options.fileType); if (input.fileType === 'ts') { @@ -486,7 +485,7 @@ export async function scheduleWorkflow( requestBody.scheduled_at = scheduledAt.toISOString(); } - const { response } = await authorizedApiFetch(auth, '/api/v1/workflows/schedules', { + const response = await api.fetch('/api/v1/workflows/schedules', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -509,8 +508,8 @@ export async function scheduleWorkflow( export async function listWorkflowSchedules(options: { apiUrl?: string } = {}): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); - const { response } = await authorizedApiFetch(auth, '/api/v1/workflows/schedules', { + const api = await workflowApiClient(apiUrl); + const response = await api.fetch('/api/v1/workflows/schedules', { headers: { Accept: 'application/json' }, }); @@ -535,8 +534,8 @@ export async function getRunStatus( options: { apiUrl?: string } = {} ): Promise> { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); - const { response } = await authorizedApiFetch(auth, `/api/v1/workflows/runs/${encodeURIComponent(runId)}`, { + const api = await workflowApiClient(apiUrl); + const response = await api.fetch(`/api/v1/workflows/runs/${encodeURIComponent(runId)}`, { headers: { Accept: 'application/json' }, }); @@ -557,15 +556,11 @@ export async function cancelWorkflow( options: { apiUrl?: string } = {} ): Promise<{ runId: string; status: string }> { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); - const { response } = await authorizedApiFetch( - auth, - `/api/v1/workflows/runs/${encodeURIComponent(runId)}/cancel`, - { - method: 'POST', - headers: { Accept: 'application/json' }, - } - ); + const api = await workflowApiClient(apiUrl); + const response = await api.fetch(`/api/v1/workflows/runs/${encodeURIComponent(runId)}/cancel`, { + method: 'POST', + headers: { Accept: 'application/json' }, + }); const payload = await readJsonResponse(response); if (!response.ok) { @@ -588,7 +583,7 @@ export async function getRunLogs( } = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - const auth = await ensureAuthenticated(apiUrl); + const api = await workflowApiClient(apiUrl); const searchParams = new URLSearchParams(); if (typeof options.offset === 'number') { searchParams.set('offset', String(options.offset)); @@ -599,7 +594,7 @@ export async function getRunLogs( const requestPath = `/api/v1/workflows/runs/${encodeURIComponent(runId)}/logs${searchParams.size ? `?${searchParams.toString()}` : ''}`; - const { response } = await authorizedApiFetch(auth, requestPath, { + const response = await api.fetch(requestPath, { headers: { Accept: 'application/json' }, }); @@ -627,15 +622,12 @@ export async function syncWorkflowPatch( options: { apiUrl?: string } = {} ): Promise { const apiUrl = options.apiUrl ?? defaultApiUrl(); - let auth = await ensureAuthenticated(apiUrl); + const api = await workflowApiClient(apiUrl); // Verify the run is completed - const { response: statusResponse, auth: a1 } = await authorizedApiFetch( - auth, - `/api/v1/workflows/runs/${encodeURIComponent(runId)}`, - { headers: { Accept: 'application/json' } } - ); - auth = a1; + const statusResponse = await api.fetch(`/api/v1/workflows/runs/${encodeURIComponent(runId)}`, { + headers: { Accept: 'application/json' }, + }); if (!statusResponse.ok) { const payload = await readJsonResponse(statusResponse); @@ -648,11 +640,9 @@ export async function syncWorkflowPatch( } // Download the patch - const { response } = await authorizedApiFetch( - auth, - `/api/v1/workflows/runs/${encodeURIComponent(runId)}/patch`, - { headers: { Accept: 'application/json' } } - ); + const response = await api.fetch(`/api/v1/workflows/runs/${encodeURIComponent(runId)}/patch`, { + headers: { Accept: 'application/json' }, + }); const payload = await readJsonResponse(response); if (!response.ok) { @@ -676,6 +666,25 @@ export async function syncWorkflowPatch( // ── Internal helpers ────────────────────────────────────────────────────────── +type WorkflowHttpClient = { + fetch(requestPath: string, init?: RequestInit): Promise; +}; + +async function storedWorkflowClient(apiUrl: string): Promise { + let auth = await ensureAuthenticated(apiUrl); + return { + async fetch(requestPath, init = {}) { + const result = await authorizedApiFetch(auth, requestPath, init); + auth = result.auth; + return result.response; + }, + }; +} + +async function workflowApiClient(apiUrl: string): Promise { + return WorkflowApiKeyClient.fromEnv(apiUrl) ?? storedWorkflowClient(apiUrl); +} + async function readJsonResponse(response: Response): Promise { const rawBody = await response.text(); if (!rawBody) { diff --git a/scripts/pr-proof/cloud-storage.mjs b/scripts/pr-proof/cloud-storage.mjs new file mode 100644 index 000000000..97132806e --- /dev/null +++ b/scripts/pr-proof/cloud-storage.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node + +const ARM_RE = /^(base|head)$/; +const NONCE_RE = /^[0-9a-f]{32}$/; +const DEFAULT_REQUEST_TIMEOUT_MS = 30_000; +const MAX_TIMER_MS = 2_147_483_647; + +function requiredEnvironment(env, name) { + const value = env[name]?.trim(); + if (!value) throw new Error(`${name} is required for Cloud proof evidence storage`); + return value; +} + +function storageUrl(env, input, arm) { + if (!ARM_RE.test(arm)) throw new Error(`Invalid proof arm: ${arm}`); + if (!NONCE_RE.test(input.handoffNonce ?? '')) throw new Error('Invalid proof handoff nonce'); + const apiUrl = requiredEnvironment(env, 'CLOUD_API_URL').replace(/\/$/, '') + '/'; + const orchestratorRunId = env.RUN_ID?.trim(); + const workerRunId = env.AGENT_RELAY_CLOUD_WORKER_RUN_ID?.trim(); + if (orchestratorRunId && workerRunId && orchestratorRunId !== workerRunId) { + throw new Error('Cloud proof runtime exposed conflicting workflow run IDs'); + } + const runId = encodeURIComponent(orchestratorRunId || workerRunId || requiredEnvironment(env, 'RUN_ID')); + const objectKey = ['pr-proof', input.handoffNonce, `${arm}.json`] + .map((segment) => encodeURIComponent(segment)) + .join('/'); + return new URL(`api/v1/workflows/runs/${runId}/storage/${objectKey}`, apiUrl); +} + +function authorization(env) { + return `Bearer ${requiredEnvironment(env, 'CLOUD_API_ACCESS_TOKEN')}`; +} + +export function validateCloudEvidenceEnvironment(input, arm, env = process.env) { + storageUrl(env, input, arm); + authorization(env); +} + +function requestSignal(options) { + if (options.signal) return options.signal; + const requested = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + if (!Number.isFinite(requested) || requested <= 0) { + throw new Error('Cloud evidence request timeout must be a positive finite number'); + } + const timeoutMs = Math.min(Math.floor(requested), MAX_TIMER_MS); + if (timeoutMs < 1) { + throw new Error('Cloud evidence request timeout must be at least one millisecond'); + } + return AbortSignal.timeout(timeoutMs); +} + +export async function uploadCloudEvidence(input, arm, evidence, options = {}) { + const env = options.env ?? process.env; + const fetchImpl = options.fetchImpl ?? fetch; + // The origin is the trusted Cloud runtime URL; run ID, nonce, and arm are + // strictly validated/encoded run-scoped identifiers, not arbitrary hosts. + // codeql[js/file-access-to-http] + const response = await fetchImpl(storageUrl(env, input, arm), { + method: 'PUT', + signal: requestSignal(options), + headers: { + accept: 'application/json', + authorization: authorization(env), + 'content-type': 'application/json', + }, + // Evidence is intentionally sent to this run's authenticated storage + // object after the trusted wrapper has validated its complete contract. + // codeql[js/file-access-to-http] + body: JSON.stringify(evidence), + }); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new Error(`Cloud evidence upload failed (${response.status})${detail ? `: ${detail}` : ''}`); + } +} + +export async function downloadCloudEvidence(input, arm, options = {}) { + const env = options.env ?? process.env; + const fetchImpl = options.fetchImpl ?? fetch; + // The trusted Cloud runtime origin and encoded run-scoped identifiers above + // prevent file-derived data from selecting an arbitrary destination. + // codeql[js/file-access-to-http] + const response = await fetchImpl(storageUrl(env, input, arm), { + signal: requestSignal(options), + headers: { + accept: 'application/json', + authorization: authorization(env), + }, + }); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new Error(`Cloud evidence download failed (${response.status})${detail ? `: ${detail}` : ''}`); + } + try { + return JSON.parse(await response.text()); + } catch (error) { + throw new Error( + `Cloud evidence response was not valid JSON: ${error instanceof Error ? error.message : String(error)}` + ); + } +} diff --git a/scripts/pr-proof/contract.mjs b/scripts/pr-proof/contract.mjs new file mode 100644 index 000000000..b1128947f --- /dev/null +++ b/scripts/pr-proof/contract.mjs @@ -0,0 +1,326 @@ +import path from 'node:path'; + +export const PR_PROOF_VERSION = 1; +export const CASE_ROOT = 'tests/relayflows/cases'; +export const INPUT_PATH = '.relayflow/pr-proof-input.json'; +export const ARTIFACT_ROOT = '.workflow-artifacts/pr-proof'; + +const CASE_ID_RE = /^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$/; +const SHA_RE = /^[0-9a-f]{40}$/; +const REPOSITORY_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const SIGNATURE_RE = /^[a-z0-9](?:[a-z0-9._:-]{0,127})$/; +const HANDOFF_NONCE_RE = /^[0-9a-f]{32}$/; +const MAX_OBSERVATION_DETAILS_LENGTH = 4_000; +const REQUIRED_TITLE_RE = /^(feat|fix)(?:\([^)]*\))?!?:/i; +const TYPE_MARKER_RE = /^\s*-\s*Change type:\s*`([^`]+)`\s*\s*$/im; +const CASE_MARKER_RE = /^\s*-\s*RelayFlow case:\s*`([^`]+)`\s*\s*$/im; + +export class PrProofContractError extends Error { + constructor(message, details = []) { + super(message); + this.name = 'PrProofContractError'; + this.details = details; + } +} + +function nonEmptyString(value) { + return typeof value === 'string' && value.trim() ? value.trim() : null; +} + +function assertObject(value, label, errors) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + errors.push(`${label} must be an object`); + return null; + } + return value; +} + +export function caseManifestPath(caseId) { + if (!CASE_ID_RE.test(caseId)) { + throw new PrProofContractError(`Invalid RelayFlow case id: ${caseId}`); + } + return `${CASE_ROOT}/${caseId}/case.json`; +} + +export function changedRelayFlowCaseIds(files = []) { + const prefix = `${CASE_ROOT}/`; + return [ + ...new Set( + files + .filter((file) => typeof file === 'string' && file.startsWith(prefix)) + .map((file) => file.slice(prefix.length).split('/')) + // Keep malformed directory names in the result. The caller compares + // this set with the one declared valid case, so dropping an invalid + // sibling here would turn an ambiguous PR into a false single-case. + .filter((parts) => parts.length > 1 && parts[0]) + .map((parts) => parts[0]) + ), + ].sort(); +} + +export function parsePrProofMetadata(body = '') { + const valuesFor = (pattern) => + [...body.matchAll(new RegExp(pattern.source, `${pattern.flags}g`))].map((match) => + match[1].trim().toLowerCase() + ); + const changeTypes = valuesFor(TYPE_MARKER_RE); + const caseIds = valuesFor(CASE_MARKER_RE); + return { + changeType: changeTypes[0] ?? null, + caseId: caseIds[0] ?? null, + changeTypeCount: changeTypes.length, + caseIdCount: caseIds.length, + }; +} + +export function classifyPullRequest({ title = '', body = '' }) { + const metadata = parsePrProofMetadata(body); + const conventionalKind = REQUIRED_TITLE_RE.exec(title)?.[1]?.toLowerCase() ?? null; + const titleKind = conventionalKind === 'feat' ? 'feature' : conventionalKind === 'fix' ? 'bugfix' : null; + const errors = []; + + if (metadata.changeTypeCount === 0) { + errors.push('PR body must declare a RelayFlow Proof change type'); + } + if (metadata.changeTypeCount > 1) { + errors.push('PR body must contain exactly one RelayFlow Proof change type marker'); + } + if (metadata.caseIdCount === 0) { + errors.push('PR body must declare a RelayFlow Proof case'); + } + if (metadata.caseIdCount > 1) { + errors.push('PR body must contain exactly one RelayFlow Proof case marker'); + } + + if (!metadata.changeType) { + return { + required: Boolean(titleKind), + kind: titleKind, + caseId: null, + metadata, + errors, + reason: 'missing proof metadata', + }; + } + + if (!['feature', 'bugfix', 'non-functional'].includes(metadata.changeType)) { + errors.push( + `Unsupported RelayFlow Proof change type ${JSON.stringify(metadata.changeType)}; expected feature, bugfix, or non-functional` + ); + } + + const metadataKind = + metadata.changeType === 'feature' || metadata.changeType === 'bugfix' ? metadata.changeType : null; + if (titleKind && metadata.changeType === 'non-functional') { + errors.push(`PR title declares a ${titleKind}, so the change type cannot be non-functional`); + } + if (titleKind && metadataKind && titleKind !== metadataKind) { + errors.push(`PR title declares ${titleKind}, but the PR body declares ${metadataKind}`); + } + + const required = Boolean(metadataKind || titleKind); + const kind = metadataKind ?? titleKind; + if (required) { + if (!metadata.caseId || metadata.caseId === 'n/a') { + errors.push('Feature and bug-fix PRs must declare exactly one RelayFlow case id'); + } else if (!CASE_ID_RE.test(metadata.caseId)) { + errors.push(`Invalid RelayFlow case id: ${metadata.caseId}`); + } + } else if (metadata.caseId !== 'n/a') { + errors.push('Non-functional PRs must declare the RelayFlow case as n/a'); + } + + return { + required, + kind, + caseId: required && metadata.caseId !== 'n/a' ? metadata.caseId : null, + metadata, + errors, + reason: required ? `declared ${kind}` : 'declared non-functional', + }; +} + +function validateExpectation(value, label, allowedOutcome, errors) { + const expectation = assertObject(value, label, errors); + if (!expectation) return null; + const outcome = nonEmptyString(expectation.outcome); + const signature = nonEmptyString(expectation.signature); + if (outcome !== allowedOutcome) { + errors.push(`${label}.outcome must be ${allowedOutcome}`); + } + if (!signature || !SIGNATURE_RE.test(signature)) { + errors.push(`${label}.signature must match ${SIGNATURE_RE}`); + } + return outcome && signature ? { outcome, signature } : null; +} + +export function validateCaseManifest(value, { caseId, kind } = {}) { + const errors = []; + const manifest = assertObject(value, 'case manifest', errors); + if (!manifest) throw new PrProofContractError('Invalid RelayFlow case manifest', errors); + + const id = nonEmptyString(manifest.id); + const manifestKind = nonEmptyString(manifest.kind); + const title = nonEmptyString(manifest.title); + if (manifest.version !== PR_PROOF_VERSION) { + errors.push(`case manifest version must be ${PR_PROOF_VERSION}`); + } + if (!id || !CASE_ID_RE.test(id)) errors.push('case manifest id is invalid'); + if (caseId && id !== caseId) errors.push(`case manifest id ${id} does not match declared case ${caseId}`); + if (!['feature', 'bugfix'].includes(manifestKind)) { + errors.push('case manifest kind must be feature or bugfix'); + } + if (kind && manifestKind !== kind) { + errors.push(`case manifest kind ${manifestKind} does not match PR kind ${kind}`); + } + if (!title || title.length > 160) errors.push('case manifest title must be 1-160 characters'); + + const runner = assertObject(manifest.runner, 'case manifest runner', errors); + const command = Array.isArray(runner?.command) ? runner.command : null; + if (!command || command.length < 2 || command.some((entry) => !nonEmptyString(entry))) { + errors.push('case manifest runner.command must contain an executable and case script path'); + } else { + const executable = command[0]; + if (!['node', 'bash'].includes(executable)) { + errors.push('case manifest runner.command executable must be node or bash'); + } + const expectedRoot = `${CASE_ROOT}/${id}/`; + const scriptPath = command[1].replaceAll('\\', '/'); + if ( + path.posix.isAbsolute(scriptPath) || + scriptPath.includes('..') || + !scriptPath.startsWith(expectedRoot) + ) { + errors.push(`case runner script must stay under ${expectedRoot}`); + } + } + + const timeoutSeconds = Number(manifest.timeoutSeconds); + if (!Number.isInteger(timeoutSeconds) || timeoutSeconds < 30 || timeoutSeconds > 1800) { + errors.push('case manifest timeoutSeconds must be an integer between 30 and 1800'); + } + + const expected = assertObject(manifest.expected, 'case manifest expected', errors); + const baseOutcome = manifestKind === 'feature' ? 'absent' : 'bug'; + const base = validateExpectation(expected?.base, 'expected.base', baseOutcome, errors); + const head = validateExpectation(expected?.head, 'expected.head', 'fixed', errors); + + if (errors.length > 0) { + throw new PrProofContractError('Invalid RelayFlow case manifest', errors); + } + + return { + version: PR_PROOF_VERSION, + id, + kind: manifestKind, + title, + runner: { command: command.map((entry) => entry.trim()) }, + timeoutSeconds, + expected: { base, head }, + }; +} + +export function validateProofInput(value) { + const errors = []; + const input = assertObject(value, 'proof input', errors); + if (!input) throw new PrProofContractError('Invalid PR proof input', errors); + + const repository = nonEmptyString(input.repository); + const baseSha = nonEmptyString(input.baseSha); + const headSha = nonEmptyString(input.headSha); + const caseId = nonEmptyString(input.caseId); + const kind = nonEmptyString(input.kind); + const handoffNonce = nonEmptyString(input.handoffNonce); + const pullRequest = input.pullRequest; + if (input.version !== PR_PROOF_VERSION) errors.push(`proof input version must be ${PR_PROOF_VERSION}`); + if (!repository || !REPOSITORY_RE.test(repository)) errors.push('proof input repository is invalid'); + if (typeof pullRequest !== 'number' || !Number.isInteger(pullRequest) || pullRequest < 1) { + errors.push('proof input pullRequest must be a positive integer'); + } + if (!baseSha || !SHA_RE.test(baseSha)) errors.push('proof input baseSha must be a full lowercase SHA'); + if (!headSha || !SHA_RE.test(headSha)) errors.push('proof input headSha must be a full lowercase SHA'); + if (baseSha && headSha && baseSha === headSha) errors.push('proof input baseSha and headSha must differ'); + if (!caseId || !CASE_ID_RE.test(caseId)) errors.push('proof input caseId is invalid'); + if (!['feature', 'bugfix'].includes(kind)) errors.push('proof input kind must be feature or bugfix'); + if (!handoffNonce || !HANDOFF_NONCE_RE.test(handoffNonce)) { + errors.push('proof input handoffNonce must be 32 lowercase hexadecimal characters'); + } + + let manifest = null; + try { + manifest = validateCaseManifest(input.manifest, { caseId, kind }); + } catch (error) { + if (error instanceof PrProofContractError) errors.push(...error.details); + else throw error; + } + if (errors.length > 0) throw new PrProofContractError('Invalid PR proof input', errors); + + return { + version: PR_PROOF_VERSION, + repository, + pullRequest, + baseSha, + headSha, + caseId, + kind, + handoffNonce, + manifest, + }; +} + +export function validateObservation(value, { caseId, arm, expected }) { + const errors = []; + const observation = assertObject(value, 'case observation', errors); + if (!observation) throw new PrProofContractError('Invalid case observation', errors); + if (observation.version !== PR_PROOF_VERSION) + errors.push(`observation version must be ${PR_PROOF_VERSION}`); + if (observation.caseId !== caseId) errors.push('observation caseId does not match the dispatched case'); + if (observation.arm !== arm) errors.push('observation arm does not match the dispatched arm'); + if (observation.outcome !== expected.outcome) { + errors.push(`observation outcome ${observation.outcome} does not match expected ${expected.outcome}`); + } + if (observation.signature !== expected.signature) { + errors.push( + `observation signature ${observation.signature} does not match expected ${expected.signature}` + ); + } + const details = nonEmptyString(observation.details) ?? ''; + if (details.length > MAX_OBSERVATION_DETAILS_LENGTH) { + errors.push(`observation details must not exceed ${MAX_OBSERVATION_DETAILS_LENGTH} characters`); + } + if (errors.length > 0) throw new PrProofContractError('Invalid case observation', errors); + return { + version: PR_PROOF_VERSION, + caseId, + arm, + outcome: observation.outcome, + signature: observation.signature, + details, + }; +} + +export function validateEvidence(value, input, arm) { + const errors = []; + const evidence = assertObject(value, `${arm} evidence`, errors); + if (!evidence) throw new PrProofContractError(`Invalid ${arm} evidence`, errors); + const expectedSha = arm === 'base' ? input.baseSha : input.headSha; + const expected = input.manifest.expected[arm]; + if (evidence.version !== PR_PROOF_VERSION) errors.push(`${arm} evidence version is invalid`); + if (evidence.caseId !== input.caseId) errors.push(`${arm} evidence caseId is invalid`); + if (evidence.arm !== arm) errors.push(`${arm} evidence arm is invalid`); + if (evidence.repository !== input.repository) errors.push(`${arm} evidence repository is invalid`); + if (evidence.pullRequest !== input.pullRequest) { + errors.push(`${arm} evidence pull request is invalid`); + } + if (evidence.targetSha !== expectedSha) errors.push(`${arm} evidence target SHA is invalid`); + if (evidence.harnessSha !== input.headSha) errors.push(`${arm} evidence harness SHA is invalid`); + if (evidence.handoffNonce !== input.handoffNonce) { + errors.push(`${arm} evidence handoff nonce is invalid`); + } + if (!nonEmptyString(evidence.sandboxId)) errors.push(`${arm} evidence sandboxId is missing`); + if (evidence.runnerExitCode !== 0) errors.push(`${arm} evidence runner exit code is not zero`); + if (evidence.outcome !== expected.outcome) errors.push(`${arm} evidence outcome is invalid`); + if (evidence.signature !== expected.signature) errors.push(`${arm} evidence signature is invalid`); + if (errors.length > 0) throw new PrProofContractError(`Invalid ${arm} evidence`, errors); + return evidence; +} diff --git a/scripts/pr-proof/prepare.mjs b/scripts/pr-proof/prepare.mjs new file mode 100644 index 000000000..416b947d3 --- /dev/null +++ b/scripts/pr-proof/prepare.mjs @@ -0,0 +1,253 @@ +#!/usr/bin/env node + +import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { randomBytes } from 'node:crypto'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + INPUT_PATH, + PR_PROOF_VERSION, + PrProofContractError, + caseManifestPath, + changedRelayFlowCaseIds, + classifyPullRequest, + validateCaseManifest, +} from './contract.mjs'; + +function option(name, fallback = null) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : fallback; +} + +function githubHeaders(token) { + return { + accept: 'application/vnd.github+json', + authorization: `Bearer ${token}`, + 'x-github-api-version': '2022-11-28', + 'user-agent': 'relay-pr-proof-dispatcher', + }; +} + +async function githubJson(url, token) { + const response = await fetch(url, { headers: githubHeaders(token) }); + if (!response.ok) { + throw new Error(`GitHub API ${response.status} for ${url}`); + } + return response.json(); +} + +async function pullRequestFiles(apiUrl, repository, number, token) { + const files = []; + for (let page = 1; page <= 30; page += 1) { + const batch = await githubJson( + `${apiUrl}/repos/${repository}/pulls/${number}/files?per_page=100&page=${page}`, + token + ); + if (!Array.isArray(batch)) throw new Error('GitHub pull request files response was not an array'); + files.push(...batch.map((entry) => entry.filename).filter((entry) => typeof entry === 'string')); + if (batch.length < 100) return files; + } + throw new Error('PR changes more than 3,000 files; RelayFlow proof dispatch refuses ambiguous scope'); +} + +export function pullRequestSnapshot(pullRequest) { + return { + number: pullRequest?.number, + headSha: pullRequest?.head?.sha, + baseSha: pullRequest?.base?.sha, + headRepository: pullRequest?.head?.repo?.full_name, + title: pullRequest?.title, + body: pullRequest?.body ?? '', + }; +} + +export function assertSamePullRequestSnapshot(expected, actual) { + const next = pullRequestSnapshot(actual); + for (const key of Object.keys(expected)) { + if (next[key] !== expected[key]) { + throw new Error( + `Pull request changed during proof preparation (${key}: ${JSON.stringify(expected[key])} -> ${JSON.stringify(next[key])}); dispatch the latest event instead` + ); + } + } +} + +async function readHeadFile(apiUrl, repository, filePath, headSha, token) { + const payload = await githubJson( + `${apiUrl}/repos/${repository}/contents/${filePath}?ref=${encodeURIComponent(headSha)}`, + token + ); + if (payload?.encoding !== 'base64' || typeof payload.content !== 'string') { + throw new Error(`GitHub did not return base64 content for ${filePath}`); + } + return Buffer.from(payload.content.replaceAll('\n', ''), 'base64').toString('utf8'); +} + +function pullRequestFromPayload(payload) { + if (payload.pull_request) return payload.pull_request; + return null; +} + +async function writeGithubOutput(values, outputPath) { + if (!outputPath) return; + await appendFile( + outputPath, + Object.entries(values) + .map(([key, value]) => `${key}=${String(value)}`) + .join('\n') + '\n' + ); +} + +async function writeSummary(lines, summaryPath) { + if (!summaryPath) return; + // GitHub supplies the summary destination; PR fields are rendered only as + // inert Markdown text after classification/snapshot validation. + // codeql[js/http-to-file-access] + await appendFile(summaryPath, `${lines.join('\n')}\n`); +} + +export async function main() { + const eventPath = option('--event', process.env.GITHUB_EVENT_PATH); + const outputPath = option('--output', INPUT_PATH); + const githubOutput = option('--github-output', process.env.GITHUB_OUTPUT); + const summaryPath = option('--summary', process.env.GITHUB_STEP_SUMMARY); + const expectedHeadSha = option('--expected-head-sha'); + const token = process.env.GITHUB_TOKEN?.trim(); + const repository = process.env.GITHUB_REPOSITORY?.trim(); + const apiUrl = (process.env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + if (!eventPath || !token || !repository) { + throw new Error('GITHUB_EVENT_PATH, GITHUB_TOKEN, and GITHUB_REPOSITORY are required'); + } + + const payload = JSON.parse(await readFile(eventPath, 'utf8')); + const eventPullRequest = pullRequestFromPayload(payload); + let pullRequest = eventPullRequest; + const manualNumber = Number(payload.inputs?.pr_number); + if (!pullRequest && Number.isInteger(manualNumber) && manualNumber > 0) { + pullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${manualNumber}`, token); + } + if (!pullRequest) throw new Error('The event does not identify a pull request'); + + const eventNumber = Number(pullRequest.number); + if (!Number.isInteger(eventNumber) || eventNumber < 1) { + throw new Error('Pull request payload is missing a valid number'); + } + // The event can be stale after a synchronize/edited cancellation. Resolve a + // live snapshot before reading the live files endpoint, then compare the + // same fields again after all file and manifest reads. + pullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${eventNumber}`, token); + + const number = pullRequest.number; + const headSha = pullRequest.head?.sha; + const baseSha = pullRequest.base?.sha; + const headRepository = pullRequest.head?.repo?.full_name; + if (!Number.isInteger(number) || !headSha || !baseSha || !headRepository) { + throw new Error('Pull request payload is missing number, repository, or exact SHAs'); + } + if (expectedHeadSha && headSha !== expectedHeadSha) { + throw new Error( + `Pull request head changed during dispatch: status targets ${expectedHeadSha}, preparation resolved ${headSha}` + ); + } + const snapshot = pullRequestSnapshot(pullRequest); + + const classification = classifyPullRequest({ title: pullRequest.title, body: pullRequest.body ?? '' }); + if (classification.errors.length > 0) { + throw new PrProofContractError( + 'Pull request does not satisfy the RelayFlow proof contract', + classification.errors + ); + } + if (!classification.required) { + const finalPullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${number}`, token); + assertSamePullRequestSnapshot(snapshot, finalPullRequest); + await writeGithubOutput({ required: false, case_id: 'n/a' }, githubOutput); + await writeSummary( + ['## RelayFlow PR proof', '', 'Cloud proof is not required for this non-functional change.'], + summaryPath + ); + return; + } + if (headRepository !== repository) { + throw new PrProofContractError('Fork pull requests cannot receive the credential-bearing Cloud proof', [ + `head repository ${headRepository} is not the trusted repository ${repository}`, + 'A maintainer must reproduce the change on a same-repository branch before merge.', + ]); + } + + const caseId = classification.caseId; + const manifestPath = caseManifestPath(caseId); + const changedFiles = await pullRequestFiles(apiUrl, repository, number, token); + const caseRoot = path.posix.dirname(manifestPath) + '/'; + const changedCaseIds = changedRelayFlowCaseIds(changedFiles); + if (!changedFiles.some((file) => file === manifestPath || file.startsWith(caseRoot))) { + throw new PrProofContractError('The declared RelayFlow case is not changed by this PR', [ + `expected a changed file under ${caseRoot}`, + ]); + } + if (changedCaseIds.length !== 1 || changedCaseIds[0] !== caseId) { + throw new PrProofContractError('A PR must change exactly its one declared RelayFlow case', [ + `declared ${caseId}; changed ${changedCaseIds.join(', ') || 'none'}`, + ]); + } + + const manifestSource = await readHeadFile(apiUrl, repository, manifestPath, headSha, token); + let manifestJson; + try { + manifestJson = JSON.parse(manifestSource); + } catch (error) { + throw new PrProofContractError(`RelayFlow case manifest is not valid JSON: ${manifestPath}`, [ + error instanceof Error ? error.message : String(error), + ]); + } + const manifest = validateCaseManifest(manifestJson, { + caseId, + kind: classification.kind, + }); + + const finalPullRequest = await githubJson(`${apiUrl}/repos/${repository}/pulls/${number}`, token); + assertSamePullRequestSnapshot(snapshot, finalPullRequest); + + const proofInput = { + version: PR_PROOF_VERSION, + repository, + pullRequest: number, + baseSha, + headSha, + caseId, + kind: classification.kind, + handoffNonce: randomBytes(16).toString('hex'), + manifestPath, + manifest, + }; + await mkdir(path.dirname(outputPath), { recursive: true }); + // The workflow fixes outputPath, and every network-derived property has + // passed strict type/path/SHA/manifest validation plus a final PR snapshot. + // codeql[js/http-to-file-access] + await writeFile(outputPath, `${JSON.stringify(proofInput, null, 2)}\n`); + await writeGithubOutput({ required: true, case_id: caseId, input_path: outputPath }, githubOutput); + await writeSummary( + [ + '## RelayFlow PR proof', + '', + `- Case: \`${caseId}\``, + `- Base: \`${baseSha}\``, + `- Head: \`${headSha}\``, + '- Order: reproduce on base, then verify on head', + ], + summaryPath + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch(async (error) => { + const details = error instanceof PrProofContractError ? error.details : []; + const lines = ['## RelayFlow PR proof', '', `**Contract failure:** ${error.message}`]; + if (details.length > 0) lines.push('', ...details.map((detail) => `- ${detail}`)); + await writeSummary(lines, option('--summary', process.env.GITHUB_STEP_SUMMARY)).catch(() => {}); + console.error(error.message); + for (const detail of details) console.error(`- ${detail}`); + process.exitCode = 1; + }); +} diff --git a/scripts/pr-proof/process-runner.mjs b/scripts/pr-proof/process-runner.mjs new file mode 100644 index 000000000..44a95e32f --- /dev/null +++ b/scripts/pr-proof/process-runner.mjs @@ -0,0 +1,198 @@ +import { spawn } from 'node:child_process'; +import { StringDecoder } from 'node:string_decoder'; + +const DEFAULT_CAPTURE_BYTES = 128 * 1024; +const DEFAULT_TERMINATION_GRACE_MS = 5_000; +const MAX_TIMER_MS = 2_147_483_647; + +function appendBounded(current, chunk, maximum) { + const bytes = Buffer.from(current + chunk, 'utf8'); + if (bytes.length <= maximum) return bytes.toString('utf8'); + + let start = bytes.length - maximum; + while (start < bytes.length && (bytes[start] & 0xc0) === 0x80) start += 1; + return bytes.subarray(start).toString('utf8'); +} + +function takeUtf8Prefix(value, maximum) { + const bytes = Buffer.from(value, 'utf8'); + if (bytes.length <= maximum) return value; + let end = maximum; + while (end > 0 && (bytes[end] & 0xc0) === 0x80) end -= 1; + return bytes.subarray(0, end).toString('utf8'); +} + +function boundedInteger(value, { fallback, minimum, label }) { + const candidate = value ?? fallback; + if ( + !Number.isFinite(candidate) || + !Number.isInteger(candidate) || + candidate < minimum || + candidate > MAX_TIMER_MS + ) { + throw new Error(`${label} must be an integer between ${minimum} and ${MAX_TIMER_MS}`); + } + return candidate; +} + +function signalProcessTree(child, signal) { + if (process.platform !== 'win32' && child.pid) { + try { + process.kill(-child.pid, signal); + return; + } catch (error) { + if (error?.code !== 'ESRCH') throw error; + } + } + try { + child.kill(signal); + } catch (error) { + if (error?.code !== 'ESRCH') throw error; + } +} + +/** + * Run a subprocess with bounded output and a process-tree timeout. The child + * owns a process group on POSIX so descendants that inherit stdout/stderr are + * terminated with it. After the grace period, the parent-side pipes are also + * closed so an escaped descendant cannot keep this promise pending forever. + */ +export function runBoundedProcess(command, args, options = {}) { + const maximum = boundedInteger(options.maxCaptureBytes, { + fallback: DEFAULT_CAPTURE_BYTES, + minimum: 1, + label: 'maxCaptureBytes', + }); + const maximumLiveOutput = boundedInteger(options.maxLiveOutputBytes, { + fallback: maximum, + minimum: 0, + label: 'maxLiveOutputBytes', + }); + const timeoutMs = + options.timeoutMs === undefined || options.timeoutMs === null + ? null + : boundedInteger(options.timeoutMs, { + fallback: null, + minimum: 1, + label: 'timeoutMs', + }); + const terminationGraceMs = boundedInteger(options.terminationGraceMs, { + fallback: DEFAULT_TERMINATION_GRACE_MS, + minimum: 0, + label: 'terminationGraceMs', + }); + if (options.signal?.aborted) { + return Promise.reject(new Error('Subprocess aborted before launch')); + } + + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: options.cwd, + env: options.env ?? process.env, + detached: process.platform !== 'win32', + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + let timedOut = false; + let aborted = false; + let settled = false; + let hardKill = null; + let forced = false; + let liveOutputBytes = 0; + let liveOutputTruncated = false; + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); + + const forceKill = () => { + if (forced) return; + forced = true; + signalProcessTree(child, 'SIGKILL'); + child.stdout.destroy(); + child.stderr.destroy(); + }; + + const beginTermination = (reason) => { + if (timedOut || aborted || settled) return; + timedOut = reason === 'timeout'; + aborted = reason === 'abort'; + signalProcessTree(child, 'SIGTERM'); + hardKill = setTimeout(forceKill, terminationGraceMs); + }; + + const timeout = timeoutMs ? setTimeout(() => beginTermination('timeout'), timeoutMs) : null; + const abortHandler = () => beginTermination('abort'); + options.signal?.addEventListener('abort', abortHandler, { once: true }); + + const cleanup = () => { + if (timeout) clearTimeout(timeout); + if (hardKill) clearTimeout(hardKill); + options.signal?.removeEventListener('abort', abortHandler); + }; + + const writeLiveOutput = (stream, text) => { + if (options.echo === false || !text) return; + if (liveOutputBytes >= maximumLiveOutput) { + if (!liveOutputTruncated) { + liveOutputTruncated = true; + stream.write('\n[... subprocess live output truncated ...]\n'); + } + return; + } + + const remaining = maximumLiveOutput - liveOutputBytes; + const prefix = takeUtf8Prefix(text, remaining); + const prefixBytes = Buffer.byteLength(prefix, 'utf8'); + if (prefix) stream.write(prefix); + liveOutputBytes += prefixBytes; + if (prefixBytes < Buffer.byteLength(text, 'utf8')) { + liveOutputBytes = maximumLiveOutput; + liveOutputTruncated = true; + stream.write('\n[... subprocess live output truncated ...]\n'); + } + }; + + child.stdout.on('data', (chunk) => { + const text = stdoutDecoder.write(chunk); + if (!text) return; + stdout = appendBounded(stdout, text, maximum); + options.onStdout?.(text); + writeLiveOutput(process.stdout, text); + }); + child.stderr.on('data', (chunk) => { + const text = stderrDecoder.write(chunk); + if (!text) return; + stderr = appendBounded(stderr, text, maximum); + options.onStderr?.(text); + writeLiveOutput(process.stderr, text); + }); + child.on('error', (error) => { + if (settled) return; + settled = true; + cleanup(); + reject(error); + }); + child.on('close', (code, signal) => { + if (settled) return; + settled = true; + // The process-group leader can exit after SIGTERM while a descendant + // with detached stdio remains alive. Force-kill the group before + // clearing the grace timer so that descendant cannot escape cleanup. + if (timedOut || aborted) forceKill(); + cleanup(); + const stdoutTail = stdoutDecoder.end(); + const stderrTail = stderrDecoder.end(); + stdout = appendBounded(stdout, stdoutTail, maximum); + stderr = appendBounded(stderr, stderrTail, maximum); + if (stdoutTail) { + options.onStdout?.(stdoutTail); + writeLiveOutput(process.stdout, stdoutTail); + } + if (stderrTail) { + options.onStderr?.(stderrTail); + writeLiveOutput(process.stderr, stderrTail); + } + resolve({ exitCode: code ?? 1, signal, stdout, stderr, timedOut, aborted }); + }); + }); +} diff --git a/scripts/pr-proof/report-status.mjs b/scripts/pr-proof/report-status.mjs new file mode 100644 index 000000000..b706f5ebe --- /dev/null +++ b/scripts/pr-proof/report-status.mjs @@ -0,0 +1,199 @@ +#!/usr/bin/env node + +import { appendFile, readFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +const SHA_RE = /^[0-9a-f]{40}$/; +const REPOSITORY_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/; +const RUN_ATTEMPT_RE = /^[1-9][0-9]*$/; +const CONTEXT = 'RelayFlow PR proof'; + +function option(name, fallback = null) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : fallback; +} + +function headers(token) { + return { + accept: 'application/vnd.github+json', + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + 'x-github-api-version': '2022-11-28', + 'user-agent': 'relay-pr-proof-status', + }; +} + +async function githubJson(url, token, init = {}, fetchImpl = fetch) { + // Callers construct only GitHub API URLs from the trusted API origin and + // validated repository, PR-number, or full-SHA components. + // codeql[js/file-access-to-http] + const response = await fetchImpl(url, { + ...init, + headers: { ...headers(token), ...(init.headers ?? {}) }, + }); + const payload = await response.json().catch(() => null); + if (!response.ok) { + const detail = payload?.message ? `: ${payload.message}` : ''; + throw new Error(`GitHub API ${response.status} for ${url}${detail}`); + } + return payload; +} + +function validateRepository(value) { + if (!REPOSITORY_RE.test(value ?? '')) throw new Error('GITHUB_REPOSITORY is invalid'); + return value; +} + +export async function resolvePullRequest({ payload, repository, apiUrl, token, fetchImpl }) { + let pullRequest = payload.pull_request ?? null; + const manualNumber = Number(payload.inputs?.pr_number); + if (!pullRequest && Number.isInteger(manualNumber) && manualNumber > 0) { + pullRequest = await githubJson( + `${apiUrl}/repos/${repository}/pulls/${manualNumber}`, + token, + {}, + fetchImpl + ); + } + const number = Number(pullRequest?.number); + const headSha = pullRequest?.head?.sha; + if (!Number.isInteger(number) || number < 1 || !SHA_RE.test(headSha ?? '')) { + throw new Error('The event does not identify a pull request with an exact head SHA'); + } + return { number, headSha }; +} + +function targetUrl(env) { + const server = env.GITHUB_SERVER_URL?.replace(/\/$/, ''); + const repository = env.GITHUB_REPOSITORY; + const runId = env.GITHUB_RUN_ID; + const runAttempt = env.GITHUB_RUN_ATTEMPT?.trim() || '1'; + if (!RUN_ATTEMPT_RE.test(runAttempt)) throw new Error('GITHUB_RUN_ATTEMPT is invalid'); + return server && repository && runId + ? `${server}/${repository}/actions/runs/${runId}/attempts/${runAttempt}` + : undefined; +} + +export async function publishCommitStatus({ sha, state, description, env = process.env, fetchImpl = fetch }) { + const repository = validateRepository(env.GITHUB_REPOSITORY?.trim()); + const token = env.GITHUB_TOKEN?.trim(); + const apiUrl = (env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + if (!token) throw new Error('GITHUB_TOKEN is required to publish the PR proof status'); + if (!SHA_RE.test(sha ?? '')) throw new Error('PR proof status SHA must be a full lowercase SHA'); + if (!['pending', 'success', 'failure', 'error'].includes(state)) { + throw new Error(`Invalid PR proof status state: ${state}`); + } + return githubJson( + `${apiUrl}/repos/${repository}/statuses/${sha}`, + token, + { + method: 'POST', + body: JSON.stringify({ + state, + context: CONTEXT, + description: description.slice(0, 140), + target_url: targetUrl(env), + }), + }, + fetchImpl + ); +} + +export async function publishOwnedCommitStatus({ + sha, + state, + description, + env = process.env, + fetchImpl = fetch, +}) { + const repository = validateRepository(env.GITHUB_REPOSITORY?.trim()); + const token = env.GITHUB_TOKEN?.trim(); + const apiUrl = (env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + const ownerTargetUrl = targetUrl(env); + if (!token || !ownerTargetUrl) { + throw new Error('GITHUB_TOKEN, GITHUB_SERVER_URL, and GITHUB_RUN_ID are required to finish status'); + } + if (!SHA_RE.test(sha ?? '')) throw new Error('PR proof status SHA must be a full lowercase SHA'); + + const combined = await githubJson( + `${apiUrl}/repos/${repository}/commits/${sha}/status`, + token, + {}, + fetchImpl + ); + const latest = Array.isArray(combined?.statuses) + ? combined.statuses.find((status) => status?.context === CONTEXT) + : null; + if (latest?.state !== 'pending' || latest?.target_url !== ownerTargetUrl) { + return { published: false, ownerTargetUrl, latest }; + } + + // The workflow-level concurrency group is the serialization lock for this + // compare-and-write. GitHub permits only one running workflow in the PR's + // group, so a replacement cannot publish its pending status until this + // cancelled predecessor has finished this step and released the group. + await publishCommitStatus({ sha, state, description, env, fetchImpl }); + return { published: true, ownerTargetUrl, latest }; +} + +function conclusionState(jobStatus) { + if (jobStatus === 'success') return 'success'; + if (jobStatus === 'failure') return 'failure'; + return 'error'; +} + +export async function main() { + const command = process.argv[2]; + const env = process.env; + if (command === 'start') { + const eventPath = option('--event', env.GITHUB_EVENT_PATH); + const outputPath = option('--github-output', env.GITHUB_OUTPUT); + const repository = validateRepository(env.GITHUB_REPOSITORY?.trim()); + const token = env.GITHUB_TOKEN?.trim(); + const apiUrl = (env.GITHUB_API_URL ?? 'https://api.github.com').replace(/\/$/, ''); + if (!eventPath || !token) throw new Error('GITHUB_EVENT_PATH and GITHUB_TOKEN are required'); + const payload = JSON.parse(await readFile(eventPath, 'utf8')); + const resolved = await resolvePullRequest({ payload, repository, apiUrl, token, fetchImpl: fetch }); + await publishCommitStatus({ + sha: resolved.headSha, + state: 'pending', + description: `Cloud red/green proof started for PR #${resolved.number}`, + }); + if (outputPath) { + await appendFile(outputPath, `head_sha=${resolved.headSha}\npr_number=${resolved.number}\n`); + } + console.log(`PR_PROOF_STATUS_PENDING pr=${resolved.number} head=${resolved.headSha}`); + return; + } + + if (command === 'finish') { + const sha = option('--sha'); + const jobStatus = option('--job-status'); + const state = conclusionState(jobStatus); + const result = await publishOwnedCommitStatus({ + sha, + state, + description: + state === 'success' + ? 'Declared Cloud red/green proof passed' + : state === 'failure' + ? 'Declared Cloud red/green proof failed' + : `Cloud red/green proof ended with ${jobStatus ?? 'unknown'} status`, + }); + console.log( + result.published + ? `PR_PROOF_STATUS_FINAL head=${sha} state=${state}` + : `PR_PROOF_STATUS_FINAL_SKIPPED head=${sha} reason=status-owned-by-newer-run` + ); + return; + } + + throw new Error('Usage: report-status.mjs '); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/pr-proof/run-arm.mjs b/scripts/pr-proof/run-arm.mjs new file mode 100644 index 000000000..f16d0d2b1 --- /dev/null +++ b/scripts/pr-proof/run-arm.mjs @@ -0,0 +1,210 @@ +#!/usr/bin/env node + +import { mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; +import { constants as fsConstants } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + ARTIFACT_ROOT, + INPUT_PATH, + PR_PROOF_VERSION, + PrProofContractError, + validateCaseManifest, + validateObservation, + validateProofInput, +} from './contract.mjs'; +import { uploadCloudEvidence, validateCloudEvidenceEnvironment } from './cloud-storage.mjs'; +import { runBoundedProcess } from './process-runner.mjs'; + +const MAX_OBSERVATION_FILE_BYTES = 64 * 1024; + +async function readObservationFile(filePath) { + const resultFile = await open( + filePath, + fsConstants.O_RDONLY | fsConstants.O_NONBLOCK | fsConstants.O_NOFOLLOW + ); + try { + const resultStat = await resultFile.stat(); + if (!resultStat.isFile()) throw new Error('observation must be a regular, non-symlink file'); + + const chunks = []; + let totalBytes = 0; + while (totalBytes <= MAX_OBSERVATION_FILE_BYTES) { + const chunk = Buffer.alloc(Math.min(16 * 1024, MAX_OBSERVATION_FILE_BYTES + 1 - totalBytes)); + const { bytesRead } = await resultFile.read(chunk, 0, chunk.length, null); + if (bytesRead === 0) break; + chunks.push(chunk.subarray(0, bytesRead)); + totalBytes += bytesRead; + } + if (totalBytes > MAX_OBSERVATION_FILE_BYTES) { + throw new Error(`observation must be no larger than ${MAX_OBSERVATION_FILE_BYTES} bytes`); + } + return Buffer.concat(chunks, totalBytes).toString('utf8'); + } finally { + await resultFile.close(); + } +} + +export async function runProcess(command, args, options = {}) { + return runBoundedProcess(command, args, options); +} + +async function runChecked(command, args, options = {}) { + const result = await runProcess(command, args, options); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error( + `${command} ${args.join(' ')} ${result.timedOut ? 'timed out' : `failed with exit ${result.exitCode}`}${result.signal ? ` (${result.signal})` : ''}` + ); + } + return result; +} + +async function checkout(repository, sha, destination) { + const remote = `https://github.com/${repository}.git`; + await mkdir(destination, { recursive: true }); + await runChecked('git', ['init', '--quiet', destination]); + await runChecked('git', ['-C', destination, 'remote', 'add', 'origin', remote]); + await runChecked('git', ['-C', destination, 'fetch', '--quiet', '--depth=1', 'origin', sha], { + timeoutMs: 5 * 60_000, + }); + await runChecked('git', ['-C', destination, 'checkout', '--quiet', '--detach', 'FETCH_HEAD']); + const result = await runChecked('git', ['-C', destination, 'rev-parse', 'HEAD']); + const actual = result.stdout.trim(); + if (actual !== sha) throw new Error(`checkout provenance mismatch: expected ${sha}, got ${actual}`); + return actual; +} + +function sanitizedCaseEnvironment({ temporaryHome, targetDir, harnessDir, resultPath, input, arm }) { + const allowed = ['PATH', 'LANG', 'LC_ALL', 'TMPDIR', 'TMP', 'TEMP', 'SYSTEMROOT', 'WINDIR']; + const env = Object.fromEntries( + allowed.map((key) => [key, process.env[key]]).filter((entry) => typeof entry[1] === 'string' && entry[1]) + ); + return { + ...env, + HOME: temporaryHome, + CI: '1', + AGENT_RELAY_TELEMETRY_DISABLED: '1', + RELAY_PR_PROOF_ARM: arm, + RELAY_PR_PROOF_CASE_ID: input.caseId, + RELAY_PR_PROOF_BASE_SHA: input.baseSha, + RELAY_PR_PROOF_HEAD_SHA: input.headSha, + RELAY_PR_PROOF_TARGET_SHA: arm === 'base' ? input.baseSha : input.headSha, + RELAY_PR_PROOF_TARGET_DIR: targetDir, + RELAY_PR_PROOF_HARNESS_DIR: harnessDir, + RELAY_PR_PROOF_RESULT_PATH: resultPath, + }; +} + +function armFromArg() { + const arm = process.argv[2]; + if (arm !== 'base' && arm !== 'head') throw new Error('Usage: run-arm.mjs [input-path]'); + return arm; +} + +export async function main() { + const arm = armFromArg(); + const inputPath = process.argv[3] ?? process.env.RELAY_PR_PROOF_INPUT ?? INPUT_PATH; + const input = validateProofInput(JSON.parse(await readFile(inputPath, 'utf8'))); + const sandboxId = process.env.SANDBOX_ID?.trim(); + if (!sandboxId) throw new Error('SANDBOX_ID is required; this proof arm must run as a Cloud step'); + // Validate the Cloud evidence handoff before checking out or executing any + // PR-authored code, so a misconfigured proof cannot do work it cannot attest. + validateCloudEvidenceEnvironment(input, arm); + + const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), `relay-pr-proof-${arm}-`)); + const harnessDir = path.join(temporaryRoot, 'harness'); + const targetDir = path.join(temporaryRoot, 'target'); + const temporaryHome = path.join(temporaryRoot, 'home'); + const resultPath = path.join(temporaryRoot, 'observation.json'); + const targetSha = arm === 'base' ? input.baseSha : input.headSha; + const evidencePath = path.join(ARTIFACT_ROOT, `${arm}.json`); + + try { + await mkdir(temporaryHome, { recursive: true }); + const harnessSha = await checkout(input.repository, input.headSha, harnessDir); + const actualTargetSha = await checkout(input.repository, targetSha, targetDir); + + const manifestPath = path.join(harnessDir, 'tests', 'relayflows', 'cases', input.caseId, 'case.json'); + const headManifest = validateCaseManifest(JSON.parse(await readFile(manifestPath, 'utf8')), { + caseId: input.caseId, + kind: input.kind, + }); + if (JSON.stringify(headManifest) !== JSON.stringify(input.manifest)) { + throw new Error('Staged case manifest does not match the exact PR head checkout'); + } + + const [command, ...args] = input.manifest.runner.command; + const result = await runProcess(command, args, { + cwd: harnessDir, + env: sanitizedCaseEnvironment({ + temporaryHome, + targetDir, + harnessDir, + resultPath, + input, + arm, + }), + timeoutMs: input.manifest.timeoutSeconds * 1000, + }); + if (result.timedOut) { + throw new Error( + `Case runner exceeded ${input.manifest.timeoutSeconds}s; a timeout cannot count as expected-red evidence` + ); + } + if (result.exitCode !== 0) { + throw new Error( + `Case runner failed with exit ${result.exitCode}; expected-red behavior must be reported as a successful structured observation` + ); + } + + let observationJson; + try { + observationJson = JSON.parse(await readObservationFile(resultPath)); + } catch (error) { + throw new PrProofContractError('Case runner did not write a valid observation JSON file', [ + error instanceof Error ? error.message : String(error), + ]); + } + const observation = validateObservation(observationJson, { + caseId: input.caseId, + arm, + expected: input.manifest.expected[arm], + }); + const evidence = { + version: PR_PROOF_VERSION, + caseId: input.caseId, + arm, + repository: input.repository, + pullRequest: input.pullRequest, + targetSha: actualTargetSha, + harnessSha, + handoffNonce: input.handoffNonce, + sandboxId, + runnerExitCode: result.exitCode, + outcome: observation.outcome, + signature: observation.signature, + details: observation.details, + capturedStdout: result.stdout.slice(-8_000), + capturedStderr: result.stderr.slice(-8_000), + completedAt: new Date().toISOString(), + }; + await mkdir(path.dirname(evidencePath), { recursive: true }); + await writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + await uploadCloudEvidence(input, arm, evidence); + console.log(`PR_PROOF_ARM_COMPLETE arm=${arm} case=${input.caseId} sandbox=${sandboxId}`); + } finally { + await rm(temporaryRoot, { recursive: true, force: true }); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error.message); + if (error instanceof PrProofContractError) { + for (const detail of error.details) console.error(`- ${detail}`); + } + process.exitCode = 1; + }); +} diff --git a/scripts/pr-proof/run-cloud.mjs b/scripts/pr-proof/run-cloud.mjs new file mode 100644 index 000000000..00cd83d22 --- /dev/null +++ b/scripts/pr-proof/run-cloud.mjs @@ -0,0 +1,310 @@ +#!/usr/bin/env node + +import { appendFile, mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { pathToFileURL } from 'node:url'; + +import { runBoundedProcess } from './process-runner.mjs'; + +const TERMINAL_SUCCESS = new Set(['completed', 'succeeded', 'success']); +const TERMINAL_FAILURE = new Set(['failed', 'cancelled', 'canceled', 'timed_out', 'error']); +const LEGACY_REFRESHABLE_AUTH_KEYS = [ + 'CLOUD_API_ACCESS_TOKEN', + 'CLOUD_API_REFRESH_TOKEN', + 'CLOUD_API_ACCESS_TOKEN_EXPIRES_AT', + 'CLOUD_API_REFRESH_TOKEN_EXPIRES_AT', +]; +const MAX_CAPTURE_BYTES = 2 * 1024 * 1024; +const MAX_LIVE_OUTPUT_BYTES = 256 * 1024; +const DEFAULT_COMMAND_TIMEOUT_MS = 2 * 60_000; +const PREPARED_RUN_ID_MARKER = 'AGENT_RELAY_CLOUD_PREPARED_RUN_ID='; +const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; + +function run(command, args, options = {}) { + return runBoundedProcess(command, args, { + env: options.env, + echo: !options.quiet, + maxCaptureBytes: MAX_CAPTURE_BYTES, + maxLiveOutputBytes: MAX_LIVE_OUTPUT_BYTES, + timeoutMs: options.timeoutMs, + signal: options.signal, + onStdout: options.onStdout, + onStderr: options.onStderr, + }); +} + +export function boundedDuration(value, { fallback, minimum, maximum, label }) { + if (value === undefined || value === null || value === '') return fallback; + const parsed = Number(value); + if (!Number.isFinite(parsed) || !Number.isInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new Error(`${label} must be an integer between ${minimum} and ${maximum} milliseconds`); + } + return parsed; +} + +function parseJsonOutput(output, label) { + try { + return JSON.parse(output); + } catch { + const first = output.indexOf('{'); + const last = output.lastIndexOf('}'); + if (first >= 0 && last > first) return JSON.parse(output.slice(first, last + 1)); + throw new Error(`${label} did not return JSON`); + } +} + +function statusFrom(payload) { + for (const candidate of [payload.status, payload.run?.status, payload.workflowRun?.status]) { + if (typeof candidate === 'string') return candidate.toLowerCase(); + } + throw new Error('Cloud status response did not contain a status'); +} + +function requiredCredential(env, name) { + const value = env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +export function preparedRunIdFromOutput(output) { + for (const line of output.split(/\r?\n/)) { + if (!line.startsWith(PREPARED_RUN_ID_MARKER)) continue; + const candidate = line.slice(PREPARED_RUN_ID_MARKER.length).trim(); + if (!RUN_ID_RE.test(candidate)) { + throw new Error('Cloud prepare progress contained an invalid run ID'); + } + return candidate; + } + return null; +} + +export function createPreparedRunProgressParser(onRunId) { + let pending = ''; + + const inspect = (output) => { + for (const line of output.split(/\r?\n/)) { + const runId = preparedRunIdFromOutput(line); + if (runId) onRunId(runId); + } + }; + + return { + write(text) { + pending += text; + const lastNewline = pending.lastIndexOf('\n'); + if (lastNewline < 0) { + // The marker is a short, newline-terminated trusted CLI progress line. + // Bound unrelated unterminated stderr without parsing partial markers. + pending = pending.slice(-8_192); + return; + } + const complete = pending.slice(0, lastNewline + 1); + pending = pending.slice(lastNewline + 1); + inspect(complete); + }, + end() { + if (pending) inspect(pending); + pending = ''; + }, + }; +} + +export function createCliApiKeyEnvironment(env = process.env) { + const apiUrl = requiredCredential(env, 'CLOUD_API_URL'); + const apiKey = requiredCredential(env, 'CLOUD_API_KEY'); + new URL(apiUrl); + + const cliEnv = { ...env, CLOUD_API_URL: apiUrl, CLOUD_API_KEY: apiKey }; + for (const key of LEGACY_REFRESHABLE_AUTH_KEYS) delete cliEnv[key]; + return { cliEnv }; +} + +export async function main() { + const cli = process.env.PR_PROOF_AGENT_RELAY_BIN ?? 'agent-relay'; + const workflowPath = process.argv[2] ?? 'workflows/pr-proof.ts'; + const logsPath = process.env.PR_PROOF_CLOUD_LOG_PATH ?? '.workflow-artifacts/pr-proof/cloud.log'; + const pollMs = boundedDuration(process.env.PR_PROOF_POLL_MS, { + fallback: 15_000, + minimum: 100, + maximum: 60_000, + label: 'PR_PROOF_POLL_MS', + }); + const timeoutMs = boundedDuration(process.env.PR_PROOF_CLOUD_TIMEOUT_MS, { + fallback: 60 * 60_000, + minimum: 60_000, + maximum: 65 * 60_000, + label: 'PR_PROOF_CLOUD_TIMEOUT_MS', + }); + const commandTimeoutMs = boundedDuration(process.env.PR_PROOF_CLOUD_COMMAND_TIMEOUT_MS, { + fallback: DEFAULT_COMMAND_TIMEOUT_MS, + minimum: 1_000, + maximum: 5 * 60_000, + label: 'PR_PROOF_CLOUD_COMMAND_TIMEOUT_MS', + }); + const auth = createCliApiKeyEnvironment(process.env); + let runId = null; + let terminal = false; + let cancelPromise = null; + let shuttingDown = false; + let activeCommandController = null; + let launchProgressError = null; + + const notePreparedRunId = (preparedRunId) => { + try { + if (runId && runId !== preparedRunId) { + throw new Error(`Cloud prepare/run ID mismatch: ${runId} != ${preparedRunId}`); + } + runId = preparedRunId; + } catch (error) { + launchProgressError ??= error; + } + }; + const launchProgress = createPreparedRunProgressParser(notePreparedRunId); + const captureLaunchProgressError = (action) => { + try { + action(); + } catch (error) { + launchProgressError ??= error; + } + }; + + const runTracked = async (command, args, options = {}) => { + const controller = new AbortController(); + activeCommandController = controller; + try { + return await run(command, args, { ...options, signal: controller.signal }); + } finally { + if (activeCommandController === controller) activeCommandController = null; + } + }; + + const cancelRemote = async (reason) => { + if (!runId || terminal) return; + cancelPromise ??= (async () => { + console.warn(`Cancelling Cloud RelayFlow run ${runId} (${reason})`); + const result = await run(cli, ['cloud', 'cancel', runId, '--json'], { + env: auth.cliEnv, + quiet: true, + timeoutMs: commandTimeoutMs, + }); + if (result.exitCode !== 0 || result.timedOut) { + console.warn(`Cloud cancellation failed with exit ${result.exitCode}: ${result.stderr.trim()}`); + } + })(); + await cancelPromise; + }; + + const signalHandler = (signal) => { + if (shuttingDown) return; + shuttingDown = true; + activeCommandController?.abort(); + void (async () => { + await cancelRemote(signal).catch((error) => console.warn(error.message)); + process.exit(signal === 'SIGINT' ? 130 : 143); + })(); + }; + process.once('SIGINT', signalHandler); + process.once('SIGTERM', signalHandler); + + try { + const launch = await runTracked(cli, ['cloud', 'run', workflowPath, '--sync-code', '--json'], { + env: { + ...auth.cliEnv, + AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID: '1', + }, + quiet: true, + timeoutMs: commandTimeoutMs, + onStderr: (text) => captureLaunchProgressError(() => launchProgress.write(text)), + }); + captureLaunchProgressError(() => launchProgress.end()); + if (launchProgressError) throw launchProgressError; + if (launch.aborted) throw new Error('Cloud workflow submission was interrupted'); + if (launch.timedOut) { + await cancelRemote('submission command timed out'); + throw new Error( + 'Cloud workflow submission command timed out and its prepared run was cancelled; it is not retried' + ); + } + if (launch.exitCode !== 0) { + process.stderr.write(launch.stderr); + throw new Error(`Cloud workflow submission failed with exit ${launch.exitCode}`); + } + const launchPayload = parseJsonOutput(launch.stdout, 'Cloud run'); + const launchedRunId = launchPayload.runId; + if (typeof launchedRunId !== 'string' || !RUN_ID_RE.test(launchedRunId)) { + throw new Error('Cloud run response did not contain a valid runId'); + } + if (runId && runId !== launchedRunId) { + throw new Error(`Cloud prepare/run ID mismatch: ${runId} != ${launchedRunId}`); + } + runId = launchedRunId; + console.log(`Cloud RelayFlow run: ${runId}`); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `run_id=${runId}\n`); + + const deadline = Date.now() + timeoutMs; + let terminalStatus = null; + while (Date.now() < deadline) { + await delay(pollMs); + const statusResult = await runTracked(cli, ['cloud', 'status', runId, '--json'], { + env: auth.cliEnv, + quiet: true, + timeoutMs: commandTimeoutMs, + }); + if (statusResult.timedOut) { + throw new Error(`Cloud status command timed out for run ${runId}`); + } + if (statusResult.exitCode !== 0) { + console.warn(`Cloud status poll failed (${statusResult.exitCode}); retrying`); + continue; + } + const status = statusFrom(parseJsonOutput(statusResult.stdout, 'Cloud status')); + console.log(`Cloud RelayFlow status: ${status}`); + if (TERMINAL_SUCCESS.has(status) || TERMINAL_FAILURE.has(status)) { + terminalStatus = status; + terminal = true; + break; + } + } + if (!terminalStatus) { + await cancelRemote('deadline exceeded'); + terminal = true; + throw new Error(`Cloud RelayFlow exceeded ${timeoutMs}ms`); + } + + await mkdir(path.dirname(logsPath), { recursive: true }); + const logs = await runTracked(cli, ['cloud', 'logs', runId], { + env: auth.cliEnv, + quiet: true, + timeoutMs: commandTimeoutMs, + }); + await writeFile(logsPath, logs.stdout + logs.stderr); + if (logs.stdout) process.stdout.write(logs.stdout); + if (logs.stderr) process.stderr.write(logs.stderr); + if (logs.timedOut) throw new Error(`Cloud log retrieval timed out for run ${runId}`); + if (logs.exitCode !== 0) throw new Error(`Cloud log retrieval failed with exit ${logs.exitCode}`); + + if (!TERMINAL_SUCCESS.has(terminalStatus)) { + throw new Error(`Cloud RelayFlow finished with status ${terminalStatus}`); + } + if (process.env.GITHUB_STEP_SUMMARY) { + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `\n- Cloud run: \`${runId}\`\n- Cloud status: **${terminalStatus}**\n` + ); + } + } finally { + activeCommandController?.abort(); + process.removeListener('SIGINT', signalHandler); + process.removeListener('SIGTERM', signalHandler); + if (runId && !terminal) + await cancelRemote('dispatcher exiting').catch((error) => console.warn(error.message)); + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/scripts/pr-proof/verify-evidence.mjs b/scripts/pr-proof/verify-evidence.mjs new file mode 100644 index 000000000..1eb4d9c21 --- /dev/null +++ b/scripts/pr-proof/verify-evidence.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node + +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + ARTIFACT_ROOT, + INPUT_PATH, + PR_PROOF_VERSION, + PrProofContractError, + validateEvidence, + validateProofInput, +} from './contract.mjs'; +import { downloadCloudEvidence } from './cloud-storage.mjs'; + +function option(name, fallback = null) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : fallback; +} + +async function readJson(filePath) { + return JSON.parse(await readFile(filePath, 'utf8')); +} + +export async function verifyEvidenceFiles({ inputPath, arm, artifactRoot = ARTIFACT_ROOT }) { + const input = validateProofInput(await readJson(inputPath)); + const base = await readJson(path.join(artifactRoot, 'base.json')); + const head = arm === 'base' ? null : await readJson(path.join(artifactRoot, 'head.json')); + return verifyEvidenceRecords({ input, arm, base, head }); +} + +export function verifyEvidenceRecords({ input, arm, base: baseRecord, head: headRecord }) { + const base = validateEvidence(baseRecord, input, 'base'); + if (arm === 'base') return { input, base, head: null }; + + const head = validateEvidence(headRecord, input, 'head'); + if (base.sandboxId === head.sandboxId) { + throw new PrProofContractError('Base and head proofs did not run in distinct sandboxes', [ + `both evidence files report sandbox ${base.sandboxId}`, + ]); + } + return { input, base, head }; +} + +export async function verifyCloudEvidence({ inputPath, arm, env = process.env, fetchImpl = fetch }) { + const input = validateProofInput(await readJson(inputPath)); + const base = await downloadCloudEvidence(input, 'base', { env, fetchImpl }); + const head = arm === 'base' ? null : await downloadCloudEvidence(input, 'head', { env, fetchImpl }); + return verifyEvidenceRecords({ input, arm, base, head }); +} + +async function main() { + const arm = option('--arm', 'both'); + if (arm !== 'base' && arm !== 'both') throw new Error('--arm must be base or both'); + const inputPath = option('--input', process.env.RELAY_PR_PROOF_INPUT ?? INPUT_PATH); + const artifactRoot = option('--artifacts', ARTIFACT_ROOT); + const source = option('--source', 'files'); + if (source !== 'files' && source !== 'cloud') throw new Error('--source must be files or cloud'); + const { input, base, head } = + source === 'cloud' + ? await verifyCloudEvidence({ inputPath, arm }) + : await verifyEvidenceFiles({ inputPath, arm, artifactRoot }); + console.log(`PR_PROOF_BASE_VALID case=${input.caseId} outcome=${base.outcome} sandbox=${base.sandboxId}`); + if (!head) return; + + const verdict = { + version: PR_PROOF_VERSION, + verdict: 'PASS', + caseId: input.caseId, + pullRequest: input.pullRequest, + base: { + sha: base.targetSha, + outcome: base.outcome, + signature: base.signature, + sandboxId: base.sandboxId, + }, + head: { + sha: head.targetSha, + outcome: head.outcome, + signature: head.signature, + sandboxId: head.sandboxId, + }, + completedAt: new Date().toISOString(), + }; + await mkdir(artifactRoot, { recursive: true }); + // artifactRoot is trusted workflow configuration; all Cloud-derived verdict + // fields passed the exact nonce/SHA/signature/sandbox evidence contract. + // codeql[js/http-to-file-access] + await writeFile(path.join(artifactRoot, 'verdict.json'), `${JSON.stringify(verdict, null, 2)}\n`); + console.log( + `PR_PROOF_PASS case=${input.caseId} base_sandbox=${base.sandboxId} head_sandbox=${head.sandboxId}` + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error) => { + console.error(error.message); + if (error instanceof PrProofContractError) { + for (const detail of error.details) console.error(`- ${detail}`); + } + process.exitCode = 1; + }); +} diff --git a/tests/fixtures/pr-proof-contract.test.ts b/tests/fixtures/pr-proof-contract.test.ts new file mode 100644 index 000000000..9a1ced05c --- /dev/null +++ b/tests/fixtures/pr-proof-contract.test.ts @@ -0,0 +1,750 @@ +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { describe, expect, it } from 'vitest'; + +// Action-facing code intentionally stays dependency-free ESM so GitHub can run +// it before npm install. Vitest can import it directly for contract coverage. +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { + PrProofContractError, + changedRelayFlowCaseIds, + classifyPullRequest, + validateCaseManifest, + validateEvidence, + validateObservation, + validateProofInput, +} from '../../scripts/pr-proof/contract.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { verifyEvidenceFiles } from '../../scripts/pr-proof/verify-evidence.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { + downloadCloudEvidence, + uploadCloudEvidence, + validateCloudEvidenceEnvironment, +} from '../../scripts/pr-proof/cloud-storage.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { + publishCommitStatus, + publishOwnedCommitStatus, + resolvePullRequest, +} from '../../scripts/pr-proof/report-status.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { assertSamePullRequestSnapshot, pullRequestSnapshot } from '../../scripts/pr-proof/prepare.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { + boundedDuration, + createPreparedRunProgressParser, + createCliApiKeyEnvironment, + preparedRunIdFromOutput, +} from '../../scripts/pr-proof/run-cloud.mjs'; +// @ts-expect-error JavaScript module intentionally has no declaration file. +import { runProcess } from '../../scripts/pr-proof/run-arm.mjs'; + +const BASE_SHA = '1'.repeat(40); +const HEAD_SHA = '2'.repeat(40); +const CASE_ID = '1591-application-ack-reconnect'; +const HANDOFF_NONCE = 'a'.repeat(32); +const PS_PATH = ['/bin/ps', '/usr/bin/ps'].find((candidate) => existsSync(candidate)); + +function proofBody(type = 'bugfix', caseId = CASE_ID) { + return [ + '## RelayFlow Proof', + '', + `- Change type: \`${type}\` `, + `- RelayFlow case: \`${caseId}\` `, + ].join('\n'); +} + +function manifest(kind = 'bugfix') { + return { + version: 1, + id: CASE_ID, + kind, + title: 'Reconnect when application acknowledgements stop', + runner: { + command: ['node', `tests/relayflows/cases/${CASE_ID}/run.mjs`], + }, + timeoutSeconds: 900, + expected: { + base: { + outcome: kind === 'feature' ? 'absent' : 'bug', + signature: 'application_ack_stall_not_detected', + }, + head: { + outcome: 'fixed', + signature: 'application_ack_stall_reconnects', + }, + }, + }; +} + +function input() { + return validateProofInput({ + version: 1, + repository: 'AgentWorkforce/relay', + pullRequest: 1610, + baseSha: BASE_SHA, + headSha: HEAD_SHA, + caseId: CASE_ID, + kind: 'bugfix', + handoffNonce: HANDOFF_NONCE, + manifest: manifest(), + }); +} + +function evidence(arm: 'base' | 'head', sandboxId: string) { + const proofInput = input(); + const expected = proofInput.manifest.expected[arm]; + return { + version: 1, + caseId: CASE_ID, + arm, + repository: 'AgentWorkforce/relay', + pullRequest: 1610, + targetSha: arm === 'base' ? BASE_SHA : HEAD_SHA, + harnessSha: HEAD_SHA, + handoffNonce: HANDOFF_NONCE, + sandboxId, + runnerExitCode: 0, + outcome: expected.outcome, + signature: expected.signature, + }; +} + +describe('RelayFlow PR proof classification', () => { + it('requires proof metadata for a conventional fix PR', () => { + const result = classifyPullRequest({ title: 'fix(broker): reconnect dead links', body: '' }); + expect(result.required).toBe(true); + expect(result.errors).toContainEqual( + expect.stringContaining('must declare a RelayFlow Proof change type') + ); + }); + + it('requires explicit classification even without a conventional title', () => { + const result = classifyPullRequest({ title: 'Update reconnect documentation', body: '' }); + expect(result.required).toBe(false); + expect(result.errors).toContainEqual( + expect.stringContaining('must declare a RelayFlow Proof change type') + ); + expect(result.errors).toContainEqual(expect.stringContaining('must declare a RelayFlow Proof case')); + }); + + it('selects exactly one declared bug-fix case', () => { + const result = classifyPullRequest({ + title: 'fix(broker): reconnect dead links', + body: proofBody(), + }); + expect(result).toMatchObject({ required: true, kind: 'bugfix', caseId: CASE_ID, errors: [] }); + }); + + it('passes a non-functional PR without Cloud work', () => { + const result = classifyPullRequest({ + title: 'docs: explain reconnect behavior', + body: proofBody('non-functional', 'n/a'), + }); + expect(result).toMatchObject({ required: false, caseId: null, errors: [] }); + }); + + it('rejects attempts to mark a fix title non-functional', () => { + const result = classifyPullRequest({ + title: 'fix(broker): reconnect dead links', + body: proofBody('non-functional', 'n/a'), + }); + expect(result.errors).toContainEqual(expect.stringContaining('cannot be non-functional')); + }); + + it('rejects duplicate proof selectors', () => { + const result = classifyPullRequest({ + title: 'fix(broker): reconnect dead links', + body: `${proofBody()}\n${proofBody()}`, + }); + expect(result.errors).toContainEqual(expect.stringContaining('exactly one RelayFlow Proof case')); + expect(result.errors).toContainEqual(expect.stringContaining('exactly one RelayFlow Proof change type')); + }); +}); + +describe('RelayFlow case manifest', () => { + it('identifies case directories without treating shared case docs as cases', () => { + expect( + changedRelayFlowCaseIds([ + 'tests/relayflows/cases/README.md', + `tests/relayflows/cases/${CASE_ID}/case.json`, + `tests/relayflows/cases/${CASE_ID}/run.mjs`, + ]) + ).toEqual([CASE_ID]); + }); + + it('keeps malformed sibling case directories visible so selection fails closed', () => { + expect( + changedRelayFlowCaseIds([ + `tests/relayflows/cases/${CASE_ID}/case.json`, + 'tests/relayflows/cases/INVALID CASE/run.mjs', + ]) + ).toEqual([CASE_ID, 'INVALID CASE']); + }); + + it('accepts a structured external case runner', () => { + expect(validateCaseManifest(manifest(), { caseId: CASE_ID, kind: 'bugfix' })).toEqual(manifest()); + }); + + it('rejects a runner outside its case directory', () => { + const invalid = manifest(); + invalid.runner.command = ['node', 'scripts/untrusted.mjs']; + expect(() => validateCaseManifest(invalid, { caseId: CASE_ID, kind: 'bugfix' })).toThrow( + PrProofContractError + ); + }); + + it('requires an explicit absent outcome for feature bases', () => { + const invalid = manifest('feature'); + invalid.expected.base.outcome = 'bug'; + expect(() => validateCaseManifest(invalid, { caseId: CASE_ID, kind: 'feature' })).toThrow( + /Invalid RelayFlow case manifest/ + ); + }); +}); + +describe('RelayFlow evidence gates', () => { + it('rejects coercible non-numeric pull request provenance', () => { + const invalid = { + ...input(), + pullRequest: true, + }; + try { + validateProofInput(invalid); + throw new Error('expected invalid proof input to fail'); + } catch (error) { + expect(error).toBeInstanceOf(PrProofContractError); + expect((error as { details: string[] }).details).toContain( + 'proof input pullRequest must be a positive integer' + ); + } + }); + + it('accepts exact SHA, outcome, and signature provenance', () => { + const proofInput = input(); + expect(validateEvidence(evidence('base', 'sandbox-base'), proofInput, 'base')).toMatchObject({ + targetSha: BASE_SHA, + outcome: 'bug', + }); + }); + + it('rejects a base crash masquerading as expected red', () => { + const invalid = { ...evidence('base', 'sandbox-base'), runnerExitCode: 1 }; + try { + validateEvidence(invalid, input(), 'base'); + throw new Error('expected invalid evidence to fail'); + } catch (error) { + expect(error).toBeInstanceOf(PrProofContractError); + expect((error as { details: string[] }).details).toContainEqual( + expect.stringContaining('runner exit code') + ); + } + }); + + it('rejects evidence from a stale or different handoff', () => { + const invalid = { ...evidence('base', 'sandbox-base'), handoffNonce: 'b'.repeat(32) }; + expect(() => validateEvidence(invalid, input(), 'base')).toThrow(/Invalid base evidence/); + }); + + it('bounds PR-authored observation details before evidence upload', () => { + expect(() => + validateObservation( + { + version: 1, + caseId: CASE_ID, + arm: 'base', + outcome: 'bug', + signature: manifest().expected.base.signature, + details: 'x'.repeat(4_001), + }, + { caseId: CASE_ID, arm: 'base', expected: manifest().expected.base } + ) + ).toThrow(/Invalid case observation/); + }); + + it('rejects base and head evidence from the same sandbox', async () => { + const root = await mkdtemp(path.join(os.tmpdir(), 'relay-pr-proof-test-')); + const inputPath = path.join(root, 'input.json'); + await writeFile(inputPath, JSON.stringify(input())); + await writeFile(path.join(root, 'base.json'), JSON.stringify(evidence('base', 'same-sandbox'))); + await writeFile(path.join(root, 'head.json'), JSON.stringify(evidence('head', 'same-sandbox'))); + await expect(verifyEvidenceFiles({ inputPath, arm: 'both', artifactRoot: root })).rejects.toThrow( + /distinct sandboxes/ + ); + await rm(root, { recursive: true, force: true }); + }); +}); + +describe('Cloud evidence handoff', () => { + const env = { + CLOUD_API_URL: 'https://cloud.test/cloud', + CLOUD_API_ACCESS_TOKEN: 'sandbox-access-token', + RUN_ID: 'run-123', + }; + + it('uploads evidence to nonce-bound run storage without exposing credentials in the URL', async () => { + let requestUrl = ''; + let requestInit: RequestInit | undefined; + const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { + requestUrl = String(url); + requestInit = init; + return Response.json({ ok: true }); + }; + const record = evidence('base', 'sandbox-base'); + await uploadCloudEvidence(input(), 'base', record, { env, fetchImpl }); + expect(requestUrl).toBe( + `https://cloud.test/cloud/api/v1/workflows/runs/run-123/storage/pr-proof/${HANDOFF_NONCE}/base.json` + ); + expect(requestUrl).not.toContain('sandbox-access-token'); + expect(requestInit?.method).toBe('PUT'); + expect(requestInit?.signal).toBeInstanceOf(AbortSignal); + expect(JSON.parse(String(requestInit?.body))).toMatchObject(record); + }); + + it('downloads structured evidence from the same nonce-bound object', async () => { + const record = evidence('head', 'sandbox-head'); + const fetchImpl = async () => new Response(JSON.stringify(record), { status: 200 }); + await expect(downloadCloudEvidence(input(), 'head', { env, fetchImpl })).resolves.toEqual(record); + }); + + it('accepts the Cloud worker run-id alias and rejects conflicting runtime provenance', async () => { + let requestUrl = ''; + const fetchImpl = async (url: string | URL | Request) => { + requestUrl = String(url); + return Response.json({ ok: true }); + }; + const workerEnv = { + CLOUD_API_URL: env.CLOUD_API_URL, + CLOUD_API_ACCESS_TOKEN: env.CLOUD_API_ACCESS_TOKEN, + AGENT_RELAY_CLOUD_WORKER_RUN_ID: env.RUN_ID, + }; + await uploadCloudEvidence(input(), 'base', evidence('base', 'sandbox-base'), { + env: workerEnv, + fetchImpl, + }); + expect(requestUrl).toContain('/workflows/runs/run-123/storage/'); + expect(() => + validateCloudEvidenceEnvironment(input(), 'base', { + ...workerEnv, + RUN_ID: 'different-run', + }) + ).toThrow(/conflicting workflow run IDs/); + }); + + it('rejects malformed request deadlines instead of disabling or collapsing the timeout', async () => { + await expect( + downloadCloudEvidence(input(), 'base', { + env, + requestTimeoutMs: Number.NaN, + fetchImpl: async () => Response.json({}), + }) + ).rejects.toThrow(/positive finite/); + await expect( + downloadCloudEvidence(input(), 'base', { + env, + requestTimeoutMs: 0.5, + fetchImpl: async () => Response.json({}), + }) + ).rejects.toThrow(/at least one millisecond/); + }); + + it('rejects incomplete Cloud evidence configuration before proof execution', () => { + expect(() => + validateCloudEvidenceEnvironment(input(), 'base', { + CLOUD_API_URL: env.CLOUD_API_URL, + RUN_ID: env.RUN_ID, + }) + ).toThrow(/CLOUD_API_ACCESS_TOKEN/); + }); +}); + +describe('Cloud dispatcher API key lifecycle', () => { + const credentialEnv = { + CLOUD_API_URL: 'https://cloud.test/cloud', + CLOUD_API_KEY: 'ci-api-key', + }; + + it('requires exactly the Cloud URL and API key before dispatch', () => { + expect(() => createCliApiKeyEnvironment({ CLOUD_API_URL: credentialEnv.CLOUD_API_URL })).toThrow( + /CLOUD_API_KEY is required/ + ); + expect(() => createCliApiKeyEnvironment({ CLOUD_API_KEY: credentialEnv.CLOUD_API_KEY })).toThrow( + /CLOUD_API_URL is required/ + ); + }); + + it('rejects unbounded or malformed dispatcher durations', () => { + const options = { fallback: 15_000, minimum: 100, maximum: 60_000, label: 'poll' }; + expect(boundedDuration(undefined, options)).toBe(15_000); + expect(() => boundedDuration('not-a-number', options)).toThrow(/between 100 and 60000/); + expect(() => boundedDuration('60001', options)).toThrow(/between 100 and 60000/); + }); + + it('captures the prepared run before the final Cloud submission response', () => { + expect( + preparedRunIdFromOutput( + `Preparing run...\nAGENT_RELAY_CLOUD_PREPARED_RUN_ID=run-prepared-123\nUploading...\n` + ) + ).toBe('run-prepared-123'); + expect(() => preparedRunIdFromOutput('AGENT_RELAY_CLOUD_PREPARED_RUN_ID=invalid run\n')).toThrow( + /invalid run ID/ + ); + }); + + it('waits for a complete prepared-run progress line split across stderr chunks', () => { + const runIds: string[] = []; + const parser = createPreparedRunProgressParser((runId: string) => runIds.push(runId)); + parser.write('Preparing run...\nAGENT_RELAY_CLOUD_PREPARED_RUN_ID=run-pre'); + expect(runIds).toEqual([]); + parser.write('pared-123\nUploading...\n'); + parser.end(); + expect(runIds).toEqual(['run-prepared-123']); + }); + + it('passes one API key to every CLI subprocess and removes legacy refresh credentials', () => { + const auth = createCliApiKeyEnvironment({ + ...credentialEnv, + CLOUD_API_ACCESS_TOKEN: 'legacy-access', + CLOUD_API_REFRESH_TOKEN: 'legacy-refresh', + CLOUD_API_ACCESS_TOKEN_EXPIRES_AT: '2027-08-25T00:00:00.000Z', + CLOUD_API_REFRESH_TOKEN_EXPIRES_AT: '2027-08-25T00:00:00.000Z', + }); + + expect(auth.cliEnv.CLOUD_API_URL).toBe(credentialEnv.CLOUD_API_URL); + expect(auth.cliEnv.CLOUD_API_KEY).toBe(credentialEnv.CLOUD_API_KEY); + expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN).toBeUndefined(); + expect(auth.cliEnv.CLOUD_API_REFRESH_TOKEN).toBeUndefined(); + expect(auth.cliEnv.CLOUD_API_ACCESS_TOKEN_EXPIRES_AT).toBeUndefined(); + expect(auth.cliEnv.CLOUD_API_REFRESH_TOKEN_EXPIRES_AT).toBeUndefined(); + }); +}); + +describe('process timeout contract', () => { + it('marks a process timed out even when it exits zero after SIGTERM', async () => { + const result = await runProcess( + process.execPath, + ['-e', "process.on('SIGTERM', () => process.exit(0)); setInterval(() => {}, 1000)"], + { timeoutMs: 100 } + ); + expect(result.timedOut).toBe(true); + }); + + it('terminates descendants that inherit output pipes instead of hanging after the parent exits', async () => { + const startedAt = Date.now(); + const script = [ + "const { spawn } = require('node:child_process');", + "spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: ['ignore', 'inherit', 'inherit'] });", + "process.on('SIGTERM', () => process.exit(0));", + 'setInterval(() => {}, 1000);', + ].join(''); + const result = await runProcess(process.execPath, ['-e', script], { timeoutMs: 100 }); + expect(result.timedOut).toBe(true); + expect(Date.now() - startedAt).toBeLessThan(2_000); + }); + + it.skipIf(process.platform === 'win32' || !PS_PATH)( + 'force-kills same-group descendants even when they do not inherit output pipes', + async () => { + const script = [ + "const { spawn } = require('node:child_process');", + "const child = spawn(process.execPath, ['-e', \"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)\"], { stdio: 'ignore' });", + "process.stdout.write(String(child.pid) + '\\n');", + "process.on('SIGTERM', () => process.exit(0));", + 'setInterval(() => {}, 1000);', + ].join(''); + const result = await runProcess(process.execPath, ['-e', script], { + echo: false, + timeoutMs: 100, + terminationGraceMs: 100, + }); + const descendantPid = Number(result.stdout.trim()); + expect(result.timedOut).toBe(true); + expect(descendantPid).toBeGreaterThan(0); + const deadline = Date.now() + 2_000; + let running = true; + while (running && Date.now() < deadline) { + let processState = ''; + try { + processState = execFileSync(PS_PATH, ['-o', 'stat=', '-p', String(descendantPid)], { + encoding: 'utf8', + }).trim(); + } catch (error) { + const status = (error as { status?: number }).status; + if (status !== 1) throw error; + } + running = processState.length > 0 && !processState.startsWith('Z'); + if (running) await new Promise((resolve) => setTimeout(resolve, 20)); + } + if (running) process.kill(descendantPid, 'SIGKILL'); + expect(running).toBe(false); + } + ); + + it('preserves UTF-8 characters split across output chunks', async () => { + const script = [ + 'process.stdout.write(Buffer.from([0xe2]));', + 'setTimeout(() => process.stdout.write(Buffer.from([0x82, 0xac])), 25);', + ].join(''); + const result = await runProcess(process.execPath, ['-e', script], { echo: false }); + expect(result.stdout).toBe('€'); + }); + + it('caps captured and live output by UTF-8 bytes', async () => { + const captured = await runProcess(process.execPath, ['-e', "process.stdout.write('€'.repeat(10))"], { + echo: false, + maxCaptureBytes: 5, + }); + expect(Buffer.byteLength(captured.stdout, 'utf8')).toBeLessThanOrEqual(5); + expect(captured.stdout).not.toContain('�'); + + const writes: string[] = []; + const originalWrite = process.stdout.write; + process.stdout.write = ((chunk: string | Uint8Array) => { + writes.push(String(chunk)); + return true; + }) as typeof process.stdout.write; + try { + await runProcess(process.execPath, ['-e', "process.stdout.write('abcdef')"], { + maxLiveOutputBytes: 4, + }); + } finally { + process.stdout.write = originalWrite; + } + expect(writes.join('')).toContain('abcd'); + expect(writes.join('')).not.toContain('abcdef'); + expect(writes.join('')).toContain('live output truncated'); + }); + + it('rejects malformed process bounds before spawning', async () => { + await expect( + runProcess(process.execPath, ['-e', 'process.exit(0)'], { timeoutMs: Number.NaN }) + ).rejects.toThrow(/timeoutMs must be an integer/); + await expect( + runProcess(process.execPath, ['-e', 'process.exit(0)'], { terminationGraceMs: -1 }) + ).rejects.toThrow(/terminationGraceMs must be an integer/); + }); +}); + +describe('required head status', () => { + const statusEnv = { + GITHUB_REPOSITORY: 'AgentWorkforce/relay', + GITHUB_TOKEN: 'github-token', + GITHUB_API_URL: 'https://api.github.test', + GITHUB_SERVER_URL: 'https://github.test', + GITHUB_RUN_ID: '12345', + GITHUB_RUN_ATTEMPT: '2', + }; + + it('publishes the stable proof context on the exact head SHA', async () => { + let requestUrl = ''; + let requestBody: Record = {}; + const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { + requestUrl = String(url); + requestBody = JSON.parse(String(init?.body)); + return Response.json({ id: 1 }); + }; + await publishCommitStatus({ + sha: HEAD_SHA, + state: 'success', + description: 'proof passed', + env: statusEnv, + fetchImpl, + }); + expect(requestUrl).toBe(`https://api.github.test/repos/AgentWorkforce/relay/statuses/${HEAD_SHA}`); + expect(requestBody).toMatchObject({ context: 'RelayFlow PR proof', state: 'success' }); + }); + + it('publishes status for a fork head while leaving credential rejection to preparation', async () => { + await expect( + resolvePullRequest({ + payload: { + pull_request: { + number: 1612, + head: { sha: HEAD_SHA, repo: { full_name: 'outside/fork' } }, + }, + }, + repository: 'AgentWorkforce/relay', + apiUrl: 'https://api.github.test', + token: 'github-token', + fetchImpl: async () => Response.json({}), + }) + ).resolves.toEqual({ number: 1612, headSha: HEAD_SHA }); + }); + + it('finalizes only the pending status owned by the current workflow run', async () => { + const requests: Array<{ url: string; method: string; body?: Record }> = []; + const ownerTargetUrl = 'https://github.test/AgentWorkforce/relay/actions/runs/12345/attempts/2'; + const fetchImpl = async (url: string | URL | Request, init?: RequestInit) => { + requests.push({ + url: String(url), + method: init?.method ?? 'GET', + ...(init?.body ? { body: JSON.parse(String(init.body)) } : {}), + }); + if (!init?.method) { + return Response.json({ + statuses: [{ context: 'RelayFlow PR proof', state: 'pending', target_url: ownerTargetUrl }], + }); + } + return Response.json({ id: 2 }); + }; + await expect( + publishOwnedCommitStatus({ + sha: HEAD_SHA, + state: 'error', + description: 'cancelled', + env: statusEnv, + fetchImpl, + }) + ).resolves.toMatchObject({ published: true }); + expect(requests).toHaveLength(2); + expect(requests[1]?.body).toMatchObject({ state: 'error' }); + }); + + it('does not let a cancelled predecessor overwrite a newer run', async () => { + let requests = 0; + const fetchImpl = async () => { + requests += 1; + return Response.json({ + statuses: [ + { + context: 'RelayFlow PR proof', + state: 'pending', + target_url: 'https://github.test/AgentWorkforce/relay/actions/runs/newer', + }, + ], + }); + }; + await expect( + publishOwnedCommitStatus({ + sha: HEAD_SHA, + state: 'error', + description: 'cancelled', + env: statusEnv, + fetchImpl, + }) + ).resolves.toMatchObject({ published: false }); + expect(requests).toBe(1); + }); + + it('gives each rerun attempt a distinct status owner marker', async () => { + const targetUrls: string[] = []; + const fetchImpl = async (_url: string | URL | Request, init?: RequestInit) => { + targetUrls.push(JSON.parse(String(init?.body)).target_url); + return Response.json({ id: targetUrls.length }); + }; + await publishCommitStatus({ + sha: HEAD_SHA, + state: 'pending', + description: 'attempt 1', + env: { ...statusEnv, GITHUB_RUN_ATTEMPT: '1' }, + fetchImpl, + }); + await publishCommitStatus({ + sha: HEAD_SHA, + state: 'pending', + description: 'attempt 2', + env: statusEnv, + fetchImpl, + }); + expect(targetUrls).toEqual([ + 'https://github.test/AgentWorkforce/relay/actions/runs/12345/attempts/1', + 'https://github.test/AgentWorkforce/relay/actions/runs/12345/attempts/2', + ]); + }); +}); + +describe('pull request snapshot consistency', () => { + const pullRequest = { + number: 1612, + title: 'feat(ci): prove one case', + body: proofBody('feature'), + head: { sha: HEAD_SHA, repo: { full_name: 'AgentWorkforce/relay' } }, + base: { sha: BASE_SHA }, + }; + + it('accepts the same live metadata and exact base/head pair', () => { + expect(() => + assertSamePullRequestSnapshot(pullRequestSnapshot(pullRequest), structuredClone(pullRequest)) + ).not.toThrow(); + }); + + it('rejects a head or proof-metadata edit during file enumeration', () => { + const snapshot = pullRequestSnapshot(pullRequest); + expect(() => + assertSamePullRequestSnapshot(snapshot, { + ...pullRequest, + head: { ...pullRequest.head, sha: '3'.repeat(40) }, + }) + ).toThrow(/headSha/); + expect(() => + assertSamePullRequestSnapshot(snapshot, { ...pullRequest, body: proofBody('non-functional', 'n/a') }) + ).toThrow(/body/); + }); +}); + +describe('trusted dispatcher source contract', () => { + it('never checks out PR head code on the credential-bearing GitHub runner', async () => { + const source = await readFile('.github/workflows/relayflow-pr-proof.yml', 'utf8'); + expect(source).toContain('pull_request_target:'); + expect(source).toContain('ref: ${{ github.event.pull_request.base.sha || github.sha }}'); + expect(source).toContain('persist-credentials: false'); + expect(source).toContain('git add -f -- .relayflow/pr-proof-input.json'); + expect(source).toContain('statuses: write'); + expect(source).toContain('report-status.mjs start'); + expect(source).toContain('report-status.mjs finish'); + expect(source).toContain('concurrency:'); + expect(source).toContain('cancel-in-progress: true'); + expect(source).toContain("if: always() && steps.status.outputs.head_sha != ''"); + expect(source).not.toContain('always() && !cancelled()'); + expect(source).toContain('--expected-head-sha "${{ steps.status.outputs.head_sha }}"'); + expect(source).toContain('actions/checkout@11d5960a326750d5838078e36cf38b85af677262'); + expect(source).toContain('actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020'); + expect(source).toContain('actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02'); + expect(source).not.toContain('github.event.pull_request.head.sha }}'); + }); + + it('gates head execution on deterministic base evidence', async () => { + const source = await readFile('workflows/pr-proof.ts', 'utf8'); + expect(source).toContain(".onError('fail-fast')"); + expect(source).toContain(".step('gate-base'"); + expect(source).toContain("dependsOn: ['gate-base']"); + expect(source).toContain(".step('gate-red-green'"); + expect(source).toContain('PR_PROOF_ARM_COMPLETE arm=base'); + expect(source).toContain('PR_PROOF_ARM_COMPLETE arm=head'); + expect(source).toContain('--source cloud'); + expect(source).toContain("result.status !== 'completed'"); + }); + + it('records the per-step Cloud sandbox id instead of the orchestrator id', async () => { + const source = await readFile('scripts/pr-proof/run-arm.mjs', 'utf8'); + expect(source).toContain('process.env.SANDBOX_ID'); + expect(source).not.toContain('process.env.DAYTONA_SANDBOX_ID'); + }); + + it('uses one non-refreshing API key and cancels remote work on termination', async () => { + const source = await readFile('scripts/pr-proof/run-cloud.mjs', 'utf8'); + expect(source).toContain("process.once('SIGTERM', signalHandler)"); + expect(source).toContain('activeCommandController?.abort()'); + expect(source).toContain('AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID'); + expect(source).toContain('captureLaunchProgressError(() => launchProgress.write(text))'); + expect(source).toContain("['cloud', 'cancel', runId, '--json']"); + expect(source).toContain("requiredCredential(env, 'CLOUD_API_KEY')"); + expect(source).not.toContain("path.join(authDir, 'cloud-auth.json')"); + expect(source).not.toContain('CLOUD_API_REFRESH_TOKEN='); + }); + + it('emits the prepared Cloud run id before upload and final submission', async () => { + const source = await readFile('packages/cloud/src/workflows.ts', 'utf8'); + const marker = source.indexOf("if (process.env.AGENT_RELAY_CLOUD_REPORT_PREPARED_RUN_ID === '1')"); + const upload = source.indexOf('Creating tarball...'); + const launch = source.indexOf('Launching workflow...'); + expect(marker).toBeGreaterThan(0); + expect(marker).toBeLessThan(upload); + expect(marker).toBeLessThan(launch); + }); +}); diff --git a/tests/relayflows/cases/README.md b/tests/relayflows/cases/README.md new file mode 100644 index 000000000..68c598df4 --- /dev/null +++ b/tests/relayflows/cases/README.md @@ -0,0 +1,147 @@ +# PR-specific RelayFlow cases + +Every Relay feature or bug-fix PR owns exactly one Cloud proof case in this +directory. The case is selected from the PR body and is the only expensive +Cloud scenario the PR check runs. + +Every PR must explicitly classify itself in the PR template. Runtime-neutral +changes declare `non-functional` and `n/a`; missing classification fails the +check instead of being silently treated as runtime-neutral. + +## Required files + +Create `tests/relayflows/cases//case.json` plus the runner named by its +`runner.command`: + +```json +{ + "version": 1, + "id": "1591-application-ack-reconnect", + "kind": "bugfix", + "title": "Reconnect when application acknowledgements stop", + "runner": { + "command": ["node", "tests/relayflows/cases/1591-application-ack-reconnect/run.mjs"] + }, + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "bug", + "signature": "application_ack_stall_not_detected" + }, + "head": { + "outcome": "fixed", + "signature": "application_ack_stall_reconnects" + } + } +} +``` + +Feature cases use `"absent"` for the base outcome and `"fixed"` for head. +Bug-fix cases use `"bug"` and `"fixed"`. + +## Runner contract + +The same runner from the exact PR head is executed against both target SHAs. +The wrapper supplies: + +- `RELAY_PR_PROOF_ARM`: `base` or `head` +- `RELAY_PR_PROOF_TARGET_DIR`: checkout of the target SHA +- `RELAY_PR_PROOF_HARNESS_DIR`: checkout of the exact head SHA +- `RELAY_PR_PROOF_RESULT_PATH`: destination for the observation JSON +- `RELAY_PR_PROOF_BASE_SHA` and `RELAY_PR_PROOF_HEAD_SHA` + +The runner must finish with exit code zero after observing behavior and write: + +```json +{ + "version": 1, + "caseId": "1591-application-ack-reconnect", + "arm": "base", + "outcome": "bug", + "signature": "application_ack_stall_not_detected", + "details": "WebSocket pongs continued but inventory.sync was never acknowledged." +} +``` + +Expected-red is data, not a failing process. A non-zero runner exit, missing +result, timeout, skipped test, missing test name, or build error is an +infrastructure failure and cannot prove the bug. + +Keep the observation file below 64 KiB and `details` at or below 4,000 +characters. Put verbose diagnostics in runner stdout/stderr; the wrapper keeps +bounded tails of both streams in the evidence record. + +Cases should exercise public/production behavior. A unit test added only on the +head cannot prove the base is broken because “test not found” is not a valid +observation. Prefer an external harness that builds the target checkout and +drives its CLI, broker, protocol, or API. + +The wrapper independently attests the exact harness and target checkouts, the +runner process exit, structured observation, handoff nonce, and Cloud sandbox +identity. The case program and its semantic assertion are still PR-authored +test code. As with any new test in a pull request, a reviewer must inspect the +runner and confirm that it actually drives `RELAY_PR_PROOF_TARGET_DIR` and that +its red/green signatures express the claimed behavior. No generic runner can +prove the honesty of arbitrary test code; this check supplies reproducible +review evidence and is not a replacement for required code review. + +## Execution and security + +The `pull_request_target` workflow checks out only the trusted base. It fetches +and validates the case manifest as data, then submits the trusted +`workflows/pr-proof.ts` RelayFlow. PR code runs only in the two Cloud agent +sandboxes. Each prover uploads structured evidence through the run-scoped Cloud +storage API; the case runner never receives Cloud credentials. A cryptographic +nonce generated by the trusted dispatcher binds both handoffs to the current +run. The base proof must pass its deterministic evidence gate before the head +sandbox is launched, and the final gate rejects a stale nonce, identical +sandbox IDs, or incorrect commit provenance. + +The RelayFlow is explicitly fail-fast with zero retries. Automatic repair +agents are disabled: a rejected observation is evidence to report, never an +invitation to edit the harness or artifacts until the gate passes. + +The action temporarily adds the generated `.relayflow/pr-proof-input.json` to +the runner's git index because Cloud code sync uploads git-known paths. It does +not commit or push the generated file. + +Fork PRs do not receive Cloud credentials. A maintainer must reproduce the +change and its case on a same-repository branch before merge. Non-functional +fork PRs can still complete the stable status without entering a +credential-bearing step. + +## Enabling the required check + +The repository needs a dedicated non-refreshing Cloud API key in these GitHub +secrets: + +- `CLOUD_API_URL` +- `RELAYFLOW_PR_PROOF_CLOUD_API_KEY` + +Use a workspace-scoped, least-privilege credential that can prepare, invoke, +read, and cancel workflow runs. Do not copy a human laptop session into CI. +The dispatcher passes one API key to every CLI subprocess and removes any +legacy refreshable-auth environment variables. API-key auth never refreshes or +opens an interactive login; Cloud enforces its workspace binding, scopes, +expiry, and revocation server-side. A `401` fails the proof and requires an +operator to rotate the dedicated key. + +`pull_request_target` itself is attached to the base SHA, so the dispatcher +publishes a separate stable commit-status context named `RelayFlow PR proof` on +the exact PR head SHA. Require that context—not the dispatcher job name—on +`main` branch protection. + +GitHub loads `pull_request_target` workflow code from the default branch. Merge +this infrastructure PR and publish the resulting Agent Relay package before +running the live canary on a subsequent feature or bug-fix PR. The published +CLI emits an opt-in prepared-run marker so the dispatcher can cancel remote +work even if submission is interrupted before its final JSON response. Require +the status context only after that canary proves the credential, Cloud handoff, +submission cancellation, and red/green case end to end. + +The final status step also runs during cancellation. Each run attempt gets a +distinct owner URL. Before publishing a terminal state, it verifies that the +current attempt still owns the latest pending `RelayFlow PR proof` context. +The workflow-level PR concurrency group serializes that compare-and-write with +the replacement's start step, so a cancelled predecessor cannot overwrite a +replacement run. diff --git a/workflows/pr-proof.ts b/workflows/pr-proof.ts new file mode 100644 index 000000000..4664d76af --- /dev/null +++ b/workflows/pr-proof.ts @@ -0,0 +1,88 @@ +/** + * PR-specific Cloud red/green proof. + * + * The trusted GitHub dispatcher runs this file from the PR base branch and + * stages `.relayflow/pr-proof-input.json`. PR head code is never executed on + * the GitHub runner. Each agent step receives a fresh Daytona sandbox from the + * Cloud SandboxedStepExecutor. + * + * Order is deliberate: + * 1. base-prover observes the exact bug/capability absence on the base SHA; + * 2. run-scoped Cloud storage hands nonce-bound evidence to a deterministic + * gate, which rejects crashes, skips, and wrong signatures; + * 3. head-verifier observes the declared fixed behavior on the head SHA; + * 4. a deterministic gate verifies exact SHAs and distinct sandbox IDs. + */ + +import { workflow } from '@relayflows/core'; + +const result = await workflow('relay-pr-proof') + .description( + 'Prove one declared Relay feature or bug-fix case red on the exact PR base SHA and green on the exact head SHA in distinct Cloud sandboxes.' + ) + .pattern('dag') + .channel('relay-pr-proof') + .maxConcurrency(1) + // Proof failures are evidence, not repair assignments. This also opts out + // of RelayFlow's default repair-agent retries so no agent can edit the + // harness or artifacts after a gate rejects them. + .onError('fail-fast') + .timeout(3_600_000) + .agent('base-prover', { + cli: 'codex', + preset: 'worker', + role: 'Run the trusted PR proof base-arm command exactly once without editing repository files.', + interactive: false, + retries: 0, + }) + .agent('head-verifier', { + cli: 'codex', + preset: 'worker', + role: 'Run the trusted PR proof head-arm command exactly once without editing repository files.', + interactive: false, + retries: 0, + }) + .step('prove-base', { + agent: 'base-prover', + task: [ + 'This is a deterministic verification assignment. Do not edit any files.', + 'Run exactly: node scripts/pr-proof/run-arm.mjs base .relayflow/pr-proof-input.json', + 'Wait for it to finish. If it succeeds, print its PR_PROOF_ARM_COMPLETE line and then print DONE.', + 'If it fails, preserve the failure and exit non-zero. Do not reinterpret a crash as bug reproduction.', + ].join('\n'), + verification: { type: 'output_contains', value: 'PR_PROOF_ARM_COMPLETE arm=base' }, + retries: 0, + }) + .step('gate-base', { + type: 'deterministic', + dependsOn: ['prove-base'], + command: + 'node scripts/pr-proof/verify-evidence.mjs --source cloud --arm base --input .relayflow/pr-proof-input.json', + captureOutput: true, + failOnError: true, + }) + .step('verify-head', { + agent: 'head-verifier', + dependsOn: ['gate-base'], + task: [ + 'This is a deterministic verification assignment. Do not edit any files.', + 'Run exactly: node scripts/pr-proof/run-arm.mjs head .relayflow/pr-proof-input.json', + 'Wait for it to finish. If it succeeds, print its PR_PROOF_ARM_COMPLETE line and then print DONE.', + 'If it fails, preserve the failure and exit non-zero. Do not manufacture or alter evidence.', + ].join('\n'), + verification: { type: 'output_contains', value: 'PR_PROOF_ARM_COMPLETE arm=head' }, + retries: 0, + }) + .step('gate-red-green', { + type: 'deterministic', + dependsOn: ['verify-head'], + command: + 'node scripts/pr-proof/verify-evidence.mjs --source cloud --arm both --input .relayflow/pr-proof-input.json', + captureOutput: true, + failOnError: true, + }) + .run({ cwd: process.cwd() }); + +if ('status' in result && result.status !== 'completed') { + process.exitCode = 1; +}