diff --git a/.github/scripts/live-scan-smoke.mjs b/.github/scripts/live-scan-smoke.mjs new file mode 100644 index 00000000..7833df32 --- /dev/null +++ b/.github/scripts/live-scan-smoke.mjs @@ -0,0 +1,486 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { + cp, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, dirname, join, resolve, sep } from "node:path"; + +const PACKAGE = "@openai/codex-security"; +const ARTIFACTS = + "scan-manifest.json findings.json coverage.json report.md".split(" "); +const TOKEN_URL = "https://auth.openai.com/oauth/token"; +const SENSITIVE = + /^(?:ACTIONS_|GITHUB_|GH_|OPENAI_|AZURE_|AWS_|ARM_|GOOGLE_|GCLOUD_|.*(?:TOKEN|SECRET|PASSWORD|PASSWD|API_KEY|PRIVATE_KEY).*)/iu; + +function options(args, env = process.env) { + if (args.length === 1 && args[0] === "--self-test") return { selfTest: true }; + const values = { + archive: env.CODEX_SECURITY_SMOKE_PACKAGE, + model: env.CODEX_SECURITY_SMOKE_MODEL || "gpt-5.6-luna", + effort: env.CODEX_SECURITY_SMOKE_EFFORT || "low", + maxCostUsd: Number(env.CODEX_SECURITY_SMOKE_MAX_COST || "0.25"), + timeoutSeconds: Number(env.CODEX_SECURITY_SMOKE_TIMEOUT_SECONDS || "240"), + artifactsDir: env.CODEX_SECURITY_SMOKE_ARTIFACT_DIR, + expectedGitHead: env.CODEX_SECURITY_EXPECTED_GIT_HEAD, + }; + const names = { + "--package": "archive", + "--model": "model", + "--effort": "effort", + "--max-cost": "maxCostUsd", + "--timeout-seconds": "timeoutSeconds", + "--artifacts-dir": "artifactsDir", + }; + for (let index = 0; index < args.length; index += 2) { + const key = names[args[index]]; + if (!key || !args[index + 1]) + throw new Error(`Invalid smoke argument: ${args[index]}.`); + values[key] = + key === "maxCostUsd" || key === "timeoutSeconds" + ? Number(args[index + 1]) + : args[index + 1]; + } + if (!values.archive) + throw new Error( + "Provide the exact candidate with --package .", + ); + if (!/^[a-z0-9][a-z0-9._/-]*$/iu.test(values.model)) + throw new Error("Invalid model identifier."); + if (!["minimal", "low", "medium", "high", "xhigh"].includes(values.effort)) + throw new Error("Invalid reasoning effort."); + for (const key of ["maxCostUsd", "timeoutSeconds"]) { + if (!Number.isFinite(values[key]) || values[key] <= 0) + throw new Error(`${key} must be positive.`); + } + if ( + values.expectedGitHead && + !/^[0-9a-f]{40}$/u.test(values.expectedGitHead) + ) { + throw new Error( + "CODEX_SECURITY_EXPECTED_GIT_HEAD must be a full commit SHA.", + ); + } + return values; +} + +function cleanEnvironment(env, additions = {}) { + return { + ...Object.fromEntries( + Object.entries(env).filter( + ([key, value]) => value !== undefined && !SENSITIVE.test(key), + ), + ), + ...additions, + }; +} + +function command(executable, args, { cwd, env, timeout = 120_000 }) { + const result = spawnSync(executable, args, { + cwd, + env, + timeout, + encoding: "utf8", + stdio: "pipe", + maxBuffer: 8 * 1024 * 1024, + windowsHide: true, + }); + if (result.error) + throw new Error(`${basename(executable)} failed: ${result.error.message}`); + if (result.status !== 0) { + const detail = (result.stderr || result.stdout || "").trim().slice(-4000); + throw new Error( + `${basename(executable)} exited ${result.status}: ${detail}`, + ); + } + return result.stdout; +} + +async function npm(env) { + const node = dirname(process.execPath); + for (const [index, item] of [ + env.npm_execpath, + "../lib", + ".", + "..", + ].entries()) { + if (typeof item !== "string") continue; + const candidate = index + ? resolve(node, item, "node_modules/npm/bin/npm-cli.js") + : item; + if (basename(candidate) !== "npm-cli.js") continue; + if ((await stat(candidate).catch(() => null))?.isFile()) + return { executable: process.execPath, args: [candidate] }; + } + if (process.platform === "win32") + throw new Error("Node.js does not include npm-cli.js."); + return { executable: "npm", args: [] }; +} + +async function install(archive, consumer, settings) { + await writeFile( + join(consumer, "package.json"), + JSON.stringify({ name: "live-smoke", private: true }), + ); + const runner = await npm(process.env); + const safe = cleanEnvironment(process.env, { + npm_config_audit: "false", + npm_config_fund: "false", + }); + const installArgs = + "install --prefer-offline --include=optional --ignore-scripts --no-audit --no-fund".split( + " ", + ); + command(runner.executable, [...runner.args, ...installArgs, archive], { + cwd: consumer, + env: safe, + timeout: 180_000, + }); + + const root = join(consumer, "node_modules", "@openai", "codex-security"); + const manifest = JSON.parse( + await readFile(join(root, "package.json"), "utf8"), + ); + assert.equal(manifest.name, PACKAGE, "Unexpected candidate package."); + if (settings.expectedGitHead) + assert.equal( + manifest.gitHead, + settings.expectedGitHead, + "Wrong candidate commit.", + ); + const plugin = JSON.parse( + await readFile( + join(root, "_bundled_plugin", ".codex-plugin", "plugin.json"), + "utf8", + ), + ); + assert.equal(plugin.name, "codex-security", "Unexpected candidate plugin."); + assert.equal(typeof plugin.version, "string", "Missing plugin version."); + const relative = manifest.bin?.["codex-security"]; + assert.equal(typeof relative, "string", "Candidate CLI launcher is missing."); + const launcher = resolve(root, relative); + assert.ok( + launcher.startsWith(`${root}${sep}`), + "Candidate launcher escapes.", + ); + assert.ok((await stat(launcher)).isFile(), "Missing candidate launcher."); + assert.equal( + command(process.execPath, [launcher, "--version"], { + cwd: consumer, + env: safe, + }).trim(), + manifest.version, + ); + return { launcher, manifest, plugin }; +} + +async function fixture(directory) { + await mkdir(directory, { recursive: true, mode: 0o700 }); + await writeFile( + join(directory, "server.js"), + [ + 'import { exec } from "node:child_process";', + 'import { createServer } from "node:http";', + "createServer((request, response) => {", + ' const command = new URL(request.url, "http://localhost").searchParams.get("command");', + " exec(`printf ${command}`, (_error, output) => response.end(output));", + "}).listen(3000);\n", + ].join("\n"), + ); + const git = cleanEnvironment(process.env, { + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: process.platform === "win32" ? "NUL" : "/dev/null", + GIT_TERMINAL_PROMPT: "0", + GIT_AUTHOR_NAME: "Codex Security CI", + GIT_AUTHOR_EMAIL: "codex-security-ci@users.noreply.github.com", + GIT_COMMITTER_NAME: "Codex Security CI", + GIT_COMMITTER_EMAIL: "codex-security-ci@users.noreply.github.com", + }); + command("git", ["init", "--quiet"], { cwd: directory, env: git }); + command("git", ["add", "--", "server.js"], { cwd: directory, env: git }); + command("git", ["-c", "commit.gpgsign=false", "commit", "-qm", "fixture"], { + cwd: directory, + env: git, + }); +} + +async function credential(env, request = fetch, endpoint = TOKEN_URL) { + const provider = env.OPENAI_IDENTITY_PROVIDER_ID?.trim(); + const account = env.OPENAI_SERVICE_ACCOUNT_ID?.trim(); + if (!provider && !account) { + const value = env.OPENAI_API_KEY?.trim(); + if (!value) + throw new Error( + "Configure workload identity federation or protected OPENAI_API_KEY.", + ); + return { value, method: "protected_environment", expires: null }; + } + const requestUrl = env.ACTIONS_ID_TOKEN_REQUEST_URL?.trim(); + const requestToken = env.ACTIONS_ID_TOKEN_REQUEST_TOKEN?.trim(); + if (!provider || !account || !requestUrl || !requestToken) { + throw new Error( + "Workload identity federation requires provider, service account, and id-token: write.", + ); + } + const url = new URL(requestUrl); + url.searchParams.set( + "audience", + env.OPENAI_WIF_AUDIENCE || "https://api.openai.com/v1", + ); + const github = await request(url, { + headers: { Authorization: `bearer ${requestToken}` }, + }); + if (!github.ok) + throw new Error(`GitHub identity request failed: ${github.status}.`); + const identity = (await github.json()).value; + if (!identity) throw new Error("GitHub identity response omitted its token."); + const response = await request(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + grant_type: "urn:ietf:params:oauth:grant-type:token-exchange", + subject_token_type: "urn:ietf:params:oauth:token-type:jwt", + subject_token: identity, + identity_provider_id: provider, + service_account_id: account, + }), + }); + if (!response.ok) + throw new Error(`OpenAI token exchange failed: ${response.status}.`); + const exchanged = await response.json(); + if (typeof exchanged.access_token !== "string" || !exchanged.access_token) { + throw new Error("OpenAI token exchange omitted its bearer."); + } + if (env.GITHUB_ACTIONS === "true") + process.stdout.write(`::add-mask::${exchanged.access_token}\n`); + return { + value: exchanged.access_token, + method: "workload_identity_federation", + expires: exchanged.expires_in ?? null, + }; +} + +function scan(installed, repository, output, state, settings, authentication) { + const allowed = + authentication.expires === null + ? settings.timeoutSeconds + : Math.min( + settings.timeoutSeconds, + Math.floor(authentication.expires) - 15, + ); + if (!Number.isFinite(allowed) || allowed < 30) + throw new Error("OpenAI bearer expires too soon to complete a scan."); + const env = cleanEnvironment(process.env, { + OPENAI_API_KEY: authentication.value, + CODEX_SECURITY_STATE_DIR: state, + CI: "true", + NO_COLOR: "1", + }); + const arguments_ = [ + installed.launcher, + "scan", + repository, + "--output-dir", + output, + ]; + arguments_.push("--json", "--headless", "--auth", "api-key"); + arguments_.push("--model", settings.model, "--effort", settings.effort); + arguments_.push("--max-cost", String(settings.maxCostUsd)); + arguments_.push( + "--codex", + "features.multi_agent_v2.max_concurrent_threads_per_session=1", + ); + return JSON.parse( + command(process.execPath, arguments_, { + cwd: repository, + env, + timeout: allowed * 1000, + }), + ); +} + +async function verify(result, output, installed, model) { + const scanResult = result.manifest?.scan; + assert.equal(result.manifest?.documentType, "codex-security.scan-manifest"); + assert.equal(scanResult?.status, "completed", "Scan did not complete."); + assert.equal(scanResult?.producer?.name, "codex-security-plugin"); + assert.equal(scanResult?.producer?.version, installed.plugin.version); + assert.equal( + result.coverage?.completeness, + "complete", + "Incomplete scan coverage.", + ); + assert.equal(result.coverage?.scanId, scanResult.id); + assert.equal(result.findings?.scanId, scanResult.id); + assert.ok(Array.isArray(result.findings?.findings)); + assert.equal(result.turn?.status, "completed"); + assert.equal(result.turn?.model, model); + assert.equal(await realpath(result.scanDir), await realpath(output)); + for (const name of ARTIFACTS) { + const metadata = await stat(join(output, name)); + assert.ok(metadata.isFile() && metadata.size > 0, `Invalid ${name}.`); + } + for (const name of ARTIFACTS.slice(0, 3)) { + const disk = JSON.parse(await readFile(join(output, name), "utf8")); + assert.equal(disk.scan?.id ?? disk.scanId, scanResult.id); + if (name === "coverage.json") assert.equal(disk.completeness, "complete"); + } +} + +async function save(output, destination, summary) { + if (!destination) return; + await mkdir(destination, { recursive: true, mode: 0o700 }); + for (const name of ARTIFACTS) { + await cp(join(output, name), join(destination, name)).catch((error) => { + if (error.code !== "ENOENT") throw error; + }); + } + await writeFile( + join(destination, "smoke-summary.json"), + `${JSON.stringify(summary, null, 2)}\n`, + ); +} + +async function selfTest() { + const secrets = { + PATH: "/safe/bin", + GITHUB_TOKEN: "github-secret", + ACTIONS_ID_TOKEN_REQUEST_TOKEN: "oidc-secret", + OPENAI_API_KEY: "parent-secret", + RANDOM_SERVICE_TOKEN: "other-secret", + }; + const safe = cleanEnvironment(secrets, { OPENAI_API_KEY: "child-only" }); + assert.deepEqual(safe, { PATH: "/safe/bin", OPENAI_API_KEY: "child-only" }); + assert.throws(() => options([]), /--package/u); + assert.throws( + () => options(["--package", "x", "--max-cost", "0"]), + /positive/u, + ); + assert.equal( + (await credential({ OPENAI_API_KEY: "fallback" })).method, + "protected_environment", + ); + await assert.rejects( + credential({ OPENAI_IDENTITY_PROVIDER_ID: "partial" }), + /id-token/u, + ); + + const requests = []; + const fakeFetch = async (url, init) => { + requests.push({ url: String(url), init }); + const result = + requests.length === 1 + ? { value: "github-jwt" } + : { access_token: "openai-bearer", expires_in: 240 }; + return { ok: true, json: async () => result }; + }; + const result = await credential( + { + OPENAI_IDENTITY_PROVIDER_ID: "provider", + OPENAI_SERVICE_ACCOUNT_ID: "account", + OPENAI_WIF_AUDIENCE: "https://api.openai.com/v1", + ACTIONS_ID_TOKEN_REQUEST_URL: + "https://oidc.example.test/token?existing=yes", + ACTIONS_ID_TOKEN_REQUEST_TOKEN: "oidc-capability", + }, + fakeFetch, + ); + assert.equal(result.value, "openai-bearer"); + assert.equal(result.method, "workload_identity_federation"); + assert.equal( + new URL(requests[0].url).searchParams.get("audience"), + "https://api.openai.com/v1", + ); + assert.equal( + requests[0].init.headers.Authorization, + "bearer oidc-capability", + ); + assert.equal(JSON.parse(requests[1].init.body).subject_token, "github-jwt"); + process.stdout.write("Live scan smoke helper self-tests passed.\n"); +} + +async function main() { + const settings = options(process.argv.slice(2)); + if (settings.selfTest) return await selfTest(); + const archive = await realpath(resolve(settings.archive)); + assert.ok( + archive.endsWith(".tgz") && (await stat(archive)).isFile(), + "Candidate must be an npm tarball.", + ); + + const root = await mkdtemp(join(tmpdir(), "codex-security-live-smoke-")); + const consumer = join(root, "consumer"); + const repository = join(root, "fixture"); + const output = join(root, "results"); + const state = join(root, "state"); + let authentication; + try { + await mkdir(consumer, { recursive: true, mode: 0o700 }); + await mkdir(state, { recursive: true, mode: 0o700 }); + const installed = await install(archive, consumer, settings); + await fixture(repository); + // GitHub's identity token is short-lived; finish package installation before minting. + authentication = await credential(process.env); + const startedAt = Date.now(); + const result = scan( + installed, + repository, + output, + state, + settings, + authentication, + ); + await verify(result, output, installed, settings.model); + const summary = { + status: "completed", + packageName: installed.manifest.name, + packageVersion: installed.manifest.version, + packageGitHead: installed.manifest.gitHead ?? null, + pluginVersion: installed.plugin.version, + scanId: result.manifest.scan.id, + model: settings.model, + effort: settings.effort, + maxCostUsd: settings.maxCostUsd, + actualCostUsd: result.cost?.estimatedUsd ?? null, + inputTokens: result.cost?.inputTokens ?? null, + outputTokens: result.cost?.outputTokens ?? null, + authentication: authentication.method, + coverage: result.coverage.completeness, + findings: result.findings.findings.length, + durationMs: Date.now() - startedAt, + platform: process.platform, + artifacts: ARTIFACTS, + }; + await save(output, settings.artifactsDir, summary); + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); + } catch (error) { + let message = error.message ?? String(error); + for (const value of [ + authentication?.value, + process.env.OPENAI_API_KEY, + process.env.ACTIONS_ID_TOKEN_REQUEST_TOKEN, + ]) { + if (value?.length >= 8) message = message.split(value).join("[REDACTED]"); + } + message = message.replace(/Bearer\s+\S+/giu, "Bearer [REDACTED]"); + const failure = { status: "failed", error: message, model: settings.model }; + await save(output, settings.artifactsDir, failure).catch(() => {}); + throw new Error(message); + } finally { + await rm(root, { recursive: true, force: true, maxRetries: 10 }); + } +} + +try { + await main(); +} catch (error) { + process.stderr.write(`codex-security live smoke: ${error.message}\n`); + process.exitCode = 1; +} diff --git a/.github/workflows/live-scan-smoke.yml b/.github/workflows/live-scan-smoke.yml new file mode 100644 index 00000000..16d4856d --- /dev/null +++ b/.github/workflows/live-scan-smoke.yml @@ -0,0 +1,424 @@ +name: live-scan-smoke + +on: + workflow_dispatch: + inputs: + pull_request: + description: Same-repository pull request to smoke-test + required: true + type: string + +permissions: {} + +concurrency: + group: live-scan-smoke-${{ inputs.pull_request }} + cancel-in-progress: false + +jobs: + prepare: + name: Resolve trusted pull request + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + head-sha: ${{ steps.pull-request.outputs.head-sha }} + pull-request: ${{ steps.pull-request.outputs.pull-request }} + + steps: + - name: Require the reviewed workflow on the default branch + if: github.repository != 'openai/codex-security' || github.ref != 'refs/heads/main' + run: | + echo 'Run the reviewed workflow from openai/codex-security main.' >&2 + exit 1 + + - name: Resolve immutable same-repository pull request head + id: pull-request + shell: bash + env: + GH_TOKEN: ${{ github.token }} + INPUT_PULL_REQUEST: ${{ inputs.pull_request }} + EXPECTED_REPOSITORY: ${{ github.repository }} + run: | + set -euo pipefail + + if [[ ! "$INPUT_PULL_REQUEST" =~ ^[1-9][0-9]*$ ]]; then + echo 'pull_request must be a positive pull request number.' >&2 + exit 1 + fi + + gh api "repos/$EXPECTED_REPOSITORY/pulls/$INPUT_PULL_REQUEST" \ + > "$RUNNER_TEMP/pull-request.json" + + node --input-type=module --eval ' + import { appendFileSync, readFileSync } from "node:fs"; + + const pullRequest = JSON.parse( + readFileSync(process.argv[1], "utf8"), + ); + const expectedRepository = process.env.EXPECTED_REPOSITORY; + const allowedAssociations = new Set(["OWNER", "MEMBER", "COLLABORATOR"]); + + if (pullRequest.state !== "open") { + throw new Error("The pull request must be open."); + } + if (pullRequest.base?.repo?.full_name !== expectedRepository) { + throw new Error("The pull request base must be this repository."); + } + if (pullRequest.base?.ref !== "main") { + throw new Error("The pull request must target main."); + } + if (pullRequest.head?.repo?.full_name !== expectedRepository) { + throw new Error("Fork pull requests cannot receive smoke-test credentials."); + } + if (!allowedAssociations.has(pullRequest.author_association)) { + throw new Error("The pull request author must be an organization member or trusted collaborator."); + } + if (!/^[0-9a-f]{40}$/.test(pullRequest.head?.sha ?? "")) { + throw new Error("The pull request head SHA is invalid."); + } + + appendFileSync( + process.env.GITHUB_OUTPUT, + `head-sha=${pullRequest.head.sha}\npull-request=${pullRequest.number}\n`, + ); + console.log(`Testing pull request #${pullRequest.number} at ${pullRequest.head.sha}.`); + ' "$RUNNER_TEMP/pull-request.json" + + mark-pending: + name: Mark candidate smoke checks pending + runs-on: ubuntu-latest + needs: prepare + timeout-minutes: 5 + permissions: + statuses: write + + steps: + - name: Publish pending statuses for this exact candidate + shell: bash + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ needs.prepare.outputs.head-sha }} + TARGET_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + for platform in linux windows; do + gh api \ + --method POST \ + "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f state=pending \ + -f "context=codex-security/live-smoke-$platform" \ + -f "description=Manually dispatched smoke test is running" \ + -f "target_url=$TARGET_URL" \ + --silent + done + + build: + name: Build immutable candidate package + runs-on: ubuntu-latest + needs: [prepare, mark-pending] + timeout-minutes: 10 + permissions: + contents: read + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + + steps: + - name: Checkout exact candidate commit without persisted credentials + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ needs.prepare.outputs.head-sha }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "22.13.0" + package-manager-cache: false + + - name: Set up pnpm + run: npm install --global pnpm@11.9.0 --no-audit --no-fund + + - name: Install frozen candidate dependencies + working-directory: sdk/typescript + run: pnpm install --frozen-lockfile + + - name: Build candidate + working-directory: sdk/typescript + run: pnpm run build + + - name: Embed immutable candidate package provenance + working-directory: sdk/typescript + shell: bash + env: + HEAD_SHA: ${{ needs.prepare.outputs.head-sha }} + run: npm pkg set "gitHead=$HEAD_SHA" + + - name: Pack candidate + working-directory: sdk/typescript + run: pnpm pack --pack-destination ../../dist + + - name: Verify exact package commit and bundled plugin + working-directory: sdk/typescript + shell: bash + env: + HEAD_SHA: ${{ needs.prepare.outputs.head-sha }} + CODEX_SECURITY_EXPECTED_GIT_HEAD: ${{ needs.prepare.outputs.head-sha }} + run: | + set -euo pipefail + shopt -s nullglob + archives=(../../dist/*.tgz) + + if [[ "${#archives[@]}" -ne 1 ]]; then + echo 'Expected exactly one candidate npm package.' >&2 + exit 1 + fi + + pnpm run check:package "${archives[0]}" + package_head="$(tar -xOf "${archives[0]}" package/package.json | node -p 'JSON.parse(require("node:fs").readFileSync(0, "utf8")).gitHead || ""')" + if [[ "$package_head" != "$HEAD_SHA" ]]; then + echo "Packed gitHead must match $HEAD_SHA; received ${package_head:-missing}." >&2 + exit 1 + fi + + - name: Upload immutable candidate package + id: upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-scan-candidate-${{ needs.prepare.outputs.head-sha }} + path: dist/*.tgz + if-no-files-found: error + retention-days: 3 + compression-level: 0 + + smoke: + name: Live scan (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + needs: [prepare, build] + timeout-minutes: 12 + environment: security-live-smoke + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux + runner: ubuntu-latest + - platform: windows + runner: windows-latest + + steps: + - name: Checkout trusted smoke-test helper from reviewed main + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6 + with: + node-version: "22.13.0" + package-manager-cache: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.12" + + - name: Download immutable candidate package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + artifact-ids: ${{ needs.build.outputs.artifact-id }} + path: ${{ runner.temp }}/live-scan-candidate + merge-multiple: true + digest-mismatch: error + + - name: Prepare private Windows scan root + if: runner.os == 'Windows' + id: windows-temp + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $path = Join-Path $env:USERPROFILE '.codex-security-live-smoke' + New-Item -ItemType Directory -Path $path -Force | Out-Null + $sid = (whoami /user /fo csv /nh | ConvertFrom-Csv -Header Name, Sid).Sid + & icacls $path /inheritance:r /grant:r "*${sid}:(OI)(CI)F" '*S-1-5-18:(OI)(CI)F' '*S-1-5-32-544:(OI)(CI)F' | Out-Null + if ($LASTEXITCODE -ne 0) { throw 'Could not secure the Windows live smoke root' } + "path=$path" >> $env:GITHUB_OUTPUT + + - name: Complete real model-backed scan + shell: bash + timeout-minutes: 6 + env: + OPENAI_API_KEY: ${{ secrets.CODEX_SECURITY_SMOKE_OPENAI_API_KEY }} + OPENAI_IDENTITY_PROVIDER_ID: ${{ vars.OPENAI_IDENTITY_PROVIDER_ID }} + OPENAI_SERVICE_ACCOUNT_ID: ${{ vars.OPENAI_SERVICE_ACCOUNT_ID }} + OPENAI_WIF_AUDIENCE: ${{ vars.OPENAI_WIF_AUDIENCE }} + CODEX_SECURITY_EXPECTED_GIT_HEAD: ${{ needs.prepare.outputs.head-sha }} + TEMP: ${{ steps.windows-temp.outputs.path || runner.temp }} + TMP: ${{ steps.windows-temp.outputs.path || runner.temp }} + TMPDIR: ${{ steps.windows-temp.outputs.path || runner.temp }} + run: | + set -euo pipefail + shopt -s nullglob + archives=("$RUNNER_TEMP"/live-scan-candidate/*.tgz) + + if [[ "${#archives[@]}" -ne 1 ]]; then + echo 'Expected exactly one downloaded candidate package.' >&2 + exit 1 + fi + + if [[ -z "${OPENAI_IDENTITY_PROVIDER_ID:-}" && -z "${OPENAI_API_KEY:-}" ]]; then + echo 'Configure security-live-smoke workload identity variables OPENAI_IDENTITY_PROVIDER_ID and OPENAI_SERVICE_ACCOUNT_ID, or configure its protected CODEX_SECURITY_SMOKE_OPENAI_API_KEY secret.' >&2 + exit 1 + fi + + node .github/scripts/live-scan-smoke.mjs \ + --package "${archives[0]}" \ + --model gpt-5.6-luna \ + --effort low \ + --max-cost 0.25 \ + --timeout-seconds 240 \ + --artifacts-dir "$RUNNER_TEMP/live-scan-results" + + - name: Upload synthetic scan results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: live-scan-results-${{ matrix.platform }}-${{ needs.prepare.outputs.head-sha }} + path: ${{ runner.temp }}/live-scan-results + if-no-files-found: ignore + retention-days: 3 + + report: + name: Publish exact-commit smoke results + runs-on: ubuntu-latest + if: ${{ always() && needs.prepare.result == 'success' && needs.mark-pending.result == 'success' }} + needs: [prepare, mark-pending, build, smoke] + timeout-minutes: 5 + permissions: + actions: write + pull-requests: read + statuses: write + outputs: + all-passed: ${{ steps.report.outputs.all-passed }} + + steps: + - name: Report Linux and Windows candidate results + id: report + shell: bash + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ needs.prepare.outputs.head-sha }} + TARGET_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + BUILD_RESULT: ${{ needs.build.result }} + run: | + set -euo pipefail + + if [[ "$BUILD_RESULT" == success ]]; then + gh api --paginate \ + "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?per_page=100" \ + --jq '.jobs[] | {name, conclusion}' \ + > "$RUNNER_TEMP/smoke-jobs.jsonl" + else + : > "$RUNNER_TEMP/smoke-jobs.jsonl" + fi + + all_passed=true + for platform in linux windows; do + platform_result="$(node --input-type=module --eval ' + import { readFileSync } from "node:fs"; + + const jobs = readFileSync(process.argv[1], "utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + const expected = `Live scan (${process.argv[2]})`; + const matches = jobs.filter((job) => job.name === expected); + + if (matches.length > 1) { + throw new Error(`Multiple jobs matched ${expected}.`); + } + process.stdout.write(matches[0]?.conclusion ?? "failure"); + ' "$RUNNER_TEMP/smoke-jobs.jsonl" "$platform")" + + if [[ "$BUILD_RESULT" == success && "$platform_result" == success ]]; then + state=success + description='Candidate completed a real model-backed scan' + else + state=failure + description="Candidate smoke failed: ${platform_result:-$BUILD_RESULT}" + all_passed=false + fi + + gh api \ + --method POST \ + "repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA" \ + -f "state=$state" \ + -f "context=codex-security/live-smoke-$platform" \ + -f "description=$description" \ + -f "target_url=$TARGET_URL" \ + --silent + + printf '%s: %s\n' "$platform" "$state" | tee -a "$GITHUB_STEP_SUMMARY" + done + + printf 'all-passed=%s\n' "$all_passed" >> "$GITHUB_OUTPUT" + + - name: Refresh the pull request required CI gate + if: steps.report.outputs.all-passed == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ needs.prepare.outputs.head-sha }} + PULL_REQUEST: ${{ needs.prepare.outputs.pull-request }} + run: | + set -euo pipefail + + current_head="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PULL_REQUEST" --jq '.head.sha')" + if [[ "$current_head" != "$HEAD_SHA" ]]; then + echo 'Pull request head changed; the new commit requires a fresh manual smoke test.' >&2 + exit 1 + fi + + run_id= + run_conclusion= + for attempt in {1..13}; do + run="$(gh api \ + "repos/$GITHUB_REPOSITORY/actions/workflows/node-ci.yml/runs?event=pull_request&head_sha=$HEAD_SHA&per_page=20" \ + --jq '[.workflow_runs[] | select(.head_sha == env.HEAD_SHA)] | first | if . then [.id, .status, .conclusion // ""] | @tsv else "" end')" + + if [[ -n "$run" ]]; then + IFS=$'\t' read -r candidate_id run_status run_conclusion <<< "$run" + if [[ "$run_status" == completed ]]; then + run_id="$candidate_id" + break + fi + fi + + if [[ "$attempt" -eq 13 ]]; then + echo 'Timed out waiting two minutes for node-ci to complete. Manually rerun its failed jobs after it completes.' >&2 + exit 1 + fi + + echo "Waiting for the node-ci run to complete before refreshing the required gate ($attempt/12)." + sleep 10 + done + + if [[ "$run_conclusion" == success ]]; then + echo 'The existing node-ci run already succeeded after observing both smoke statuses.' \ + | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + gh api \ + --method POST \ + "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/rerun-failed-jobs" \ + --silent + + echo "Re-running failed node-ci checks for exact pull request head $HEAD_SHA." \ + | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index 60892c1e..614bbebc 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -13,6 +13,75 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + bundled-mcp-live-smoke: + name: Require manually initiated bundled MCP smoke tests + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + statuses: read + + steps: + - name: Verify smoke-test results for bundled MCP changes + if: github.event_name == 'pull_request' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { owner, repo } = context.repo; + const pullRequest = context.payload.pull_request; + const bundledMcpPrefix = "sdk/typescript/_bundled_plugin/mcp/"; + const changedFiles = await github.paginate(github.rest.pulls.listFiles, { + owner, + repo, + pull_number: pullRequest.number, + per_page: 100, + }); + + if (!changedFiles.some(({ filename, previous_filename: previousFilename }) => + filename.startsWith(bundledMcpPrefix) || + previousFilename?.startsWith(bundledMcpPrefix) + )) { + core.info("No bundled MCP files changed; a manual smoke test is not required."); + return; + } + + const headSha = pullRequest.head.sha; + const requiredContexts = [ + "codex-security/live-smoke-linux", + "codex-security/live-smoke-windows", + ]; + const statuses = await github.paginate(github.rest.repos.listCommitStatusesForRef, { + owner, + repo, + ref: headSha, + per_page: 100, + }); + const latestStatuses = new Map(); + + for (const status of statuses) { + if (requiredContexts.includes(status.context) && !latestStatuses.has(status.context)) { + latestStatuses.set(status.context, status.state); + } + } + + const missingOrFailed = requiredContexts.filter( + (statusContext) => latestStatuses.get(statusContext) !== "success", + ); + + if (missingOrFailed.length > 0) { + for (const statusContext of missingOrFailed) { + core.error(`${statusContext}: ${latestStatuses.get(statusContext) ?? "not run"}`); + } + core.setFailed( + `Bundled MCP changes require successful manually initiated Linux and Windows smoke scans for ${headSha}. ` + + `Run: gh workflow run live-scan-smoke.yml --repo ${owner}/${repo} --ref main --field pull_request=${pullRequest.number}`, + ); + return; + } + + core.info(`Linux and Windows manual smoke tests passed for ${headSha}.`); + test: name: ${{ matrix.os }} / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} runs-on: ${{ matrix.os }} @@ -250,7 +319,7 @@ jobs: name: windows-latest / node-${{ matrix.node == '22.13.0' && '22' || matrix.node }} runs-on: ubuntu-latest if: always() - needs: [windows-test, windows-verify] + needs: [bundled-mcp-live-smoke, windows-test, windows-verify] strategy: fail-fast: false matrix: @@ -258,5 +327,5 @@ jobs: steps: - name: Require every Windows coverage job - if: needs.windows-test.result != 'success' || needs.windows-verify.result != 'success' + if: needs.bundled-mcp-live-smoke.result != 'success' || needs.windows-test.result != 'success' || needs.windows-verify.result != 'success' run: exit 1 diff --git a/sdk/typescript/tests-ts/live-scan-smoke-workflow.test.ts b/sdk/typescript/tests-ts/live-scan-smoke-workflow.test.ts new file mode 100644 index 00000000..3af2a83d --- /dev/null +++ b/sdk/typescript/tests-ts/live-scan-smoke-workflow.test.ts @@ -0,0 +1,314 @@ +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "bun:test"; + +const workflow = readFileSync( + new URL("../../../.github/workflows/node-ci.yml", import.meta.url), + "utf8", +); +const smokeWorkflow = readFileSync( + new URL("../../../.github/workflows/live-scan-smoke.yml", import.meta.url), + "utf8", +); + +const gateMatch = /^\s{10}script: \|\n([\s\S]*?)(?=^ test:)/mu.exec(workflow); + +if (gateMatch?.[1] === undefined) { + throw new Error("The bundled MCP smoke-test gate is missing."); +} + +const gateScript = gateMatch[1] + .split("\n") + .map((line) => (line.startsWith(" ") ? line.slice(12) : line)) + .join("\n"); + +const runGate = new Function( + "github", + "core", + "context", + `return (async () => {\n${gateScript}\n})();`, +) as ( + github: MockGithub, + core: MockCore, + context: MockContext, +) => Promise; + +interface ChangedFile { + filename: string; + previous_filename?: string; +} + +interface CommitStatus { + context: string; + state: string; +} + +interface MockGithub { + rest: { + pulls: { listFiles: symbol }; + repos: { listCommitStatusesForRef: symbol }; + }; + paginate: ( + endpoint: symbol, + options: Record, + ) => Promise; +} + +interface MockCore { + info: (message: string) => void; + error: (message: string) => void; + setFailed: (message: string) => void; +} + +interface MockContext { + repo: { owner: string; repo: string }; + payload: { + pull_request: { number: number; head: { sha: string } }; + }; +} + +async function inspectGate( + files: ChangedFile[], + statuses: CommitStatus[] = [], +): Promise<{ + calls: { endpoint: symbol; options: Record }[]; + errors: string[]; + failures: string[]; + infos: string[]; +}> { + const calls: { endpoint: symbol; options: Record }[] = []; + const errors: string[] = []; + const failures: string[] = []; + const infos: string[] = []; + const filesEndpoint = Symbol("pull request files"); + const statusesEndpoint = Symbol("commit statuses"); + const github: MockGithub = { + rest: { + pulls: { listFiles: filesEndpoint }, + repos: { listCommitStatusesForRef: statusesEndpoint }, + }, + paginate: async (endpoint, options) => { + calls.push({ endpoint, options }); + return endpoint === filesEndpoint ? files : statuses; + }, + }; + const core: MockCore = { + info: (message) => infos.push(message), + error: (message) => errors.push(message), + setFailed: (message) => failures.push(message), + }; + const context: MockContext = { + repo: { owner: "example", repo: "codex-security" }, + payload: { + pull_request: { + number: 123, + head: { sha: "0123456789abcdef0123456789abcdef01234567" }, + }, + }, + }; + + await runGate(github, core, context); + + return { calls, errors, failures, infos }; +} + +const bundledMcpFile = { + filename: "sdk/typescript/_bundled_plugin/mcp/server.mjs", +}; + +const passingLinux = { + context: "codex-security/live-smoke-linux", + state: "success", +}; + +const passingWindows = { + context: "codex-security/live-smoke-windows", + state: "success", +}; + +describe("manually initiated bundled MCP smoke gate", () => { + test("keeps credential isolation and workload identity exchange covered", () => { + const result = spawnSync( + process.execPath, + [ + fileURLToPath( + new URL( + "../../../.github/scripts/live-scan-smoke.mjs", + import.meta.url, + ), + ), + "--self-test", + ], + { + encoding: "utf8", + timeout: 10_000, + windowsHide: true, + }, + ); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain( + "Live scan smoke helper self-tests passed.", + ); + }); + + test("does not require a live scan for unrelated plugin or SDK changes", async () => { + const result = await inspectGate([ + { filename: "sdk/typescript/src/api.ts" }, + { + filename: + "sdk/typescript/_bundled_plugin/skills/security-scan/SKILL.md", + }, + ]); + + expect(result.failures).toEqual([]); + expect(result.calls).toHaveLength(1); + expect(result.infos[0]).toContain("not required"); + }); + + test("requires both platform results when a bundled MCP file changes", async () => { + const result = await inspectGate([bundledMcpFile]); + + expect(result.calls).toHaveLength(2); + expect(result.calls[0]?.options).toEqual({ + owner: "example", + repo: "codex-security", + pull_number: 123, + per_page: 100, + }); + expect(result.calls[1]?.options).toEqual({ + owner: "example", + repo: "codex-security", + ref: "0123456789abcdef0123456789abcdef01234567", + per_page: 100, + }); + expect(result.errors).toEqual([ + "codex-security/live-smoke-linux: not run", + "codex-security/live-smoke-windows: not run", + ]); + expect(result.failures[0]).toContain( + "gh workflow run live-scan-smoke.yml --repo example/codex-security --ref main --field pull_request=123", + ); + }); + + test("requires a scan when an MCP file is renamed out of its directory", async () => { + const result = await inspectGate([ + { + filename: "sdk/typescript/_bundled_plugin/renamed.mjs", + previous_filename: bundledMcpFile.filename, + }, + ]); + + expect(result.failures).toHaveLength(1); + expect(result.calls).toHaveLength(2); + }); + + test("rejects a missing, failed, or pending platform result", async () => { + for (const statuses of [ + [passingLinux], + [passingLinux, { ...passingWindows, state: "failure" }], + [{ ...passingLinux, state: "pending" }, passingWindows], + ]) { + const result = await inspectGate([bundledMcpFile], statuses); + + expect(result.failures).toHaveLength(1); + } + }); + + test("uses only the newest status for each platform", async () => { + const failedRetry = await inspectGate( + [bundledMcpFile], + [{ ...passingLinux, state: "failure" }, passingLinux, passingWindows], + ); + const successfulRetry = await inspectGate( + [bundledMcpFile], + [passingLinux, { ...passingLinux, state: "failure" }, passingWindows], + ); + + expect(failedRetry.failures).toHaveLength(1); + expect(successfulRetry.failures).toEqual([]); + }); + + test("passes only when Linux and Windows succeeded for the exact PR head", async () => { + const result = await inspectGate( + [bundledMcpFile], + [ + { context: "unrelated/context", state: "failure" }, + passingLinux, + passingWindows, + ], + ); + + expect(result.failures).toEqual([]); + expect(result.infos[0]).toContain( + "0123456789abcdef0123456789abcdef01234567", + ); + }); + + test("makes the existing protected Windows check depend on the gate", () => { + expect(workflow).toContain( + "needs: [bundled-mcp-live-smoke, windows-test, windows-verify]", + ); + expect(workflow).toContain( + "needs.bundled-mcp-live-smoke.result != 'success'", + ); + }); +}); + +describe("manual live scan workflow security boundaries", () => { + test("only runs when a user explicitly dispatches the trusted workflow", () => { + const triggers = /^on:\n([\s\S]*?)(?=^permissions:)/mu.exec( + smokeWorkflow, + )?.[1]; + + expect(triggers).toBeDefined(); + expect(triggers).toContain(" workflow_dispatch:"); + expect(triggers).not.toMatch( + /^ (?:push|pull_request|schedule|workflow_run):/mu, + ); + expect(smokeWorkflow).toContain( + "github.repository != 'openai/codex-security' || github.ref != 'refs/heads/main'", + ); + }); + + test("refuses forks and freezes the exact trusted pull request commit", () => { + expect(smokeWorkflow).toContain( + "pullRequest.head?.repo?.full_name !== expectedRepository", + ); + expect(smokeWorkflow).toContain( + 'new Set(["OWNER", "MEMBER", "COLLABORATOR"])', + ); + expect(smokeWorkflow).toContain( + "ref: ${{ needs.prepare.outputs.head-sha }}", + ); + expect(smokeWorkflow).toContain( + "CODEX_SECURITY_EXPECTED_GIT_HEAD: ${{ needs.prepare.outputs.head-sha }}", + ); + }); + + test("runs a protected real scan on both Linux and Windows", () => { + expect(smokeWorkflow).toContain("environment: security-live-smoke"); + expect(smokeWorkflow).toContain("id-token: write"); + expect(smokeWorkflow).toContain("platform: linux"); + expect(smokeWorkflow).toContain("runner: ubuntu-latest"); + expect(smokeWorkflow).toContain("platform: windows"); + expect(smokeWorkflow).toContain("runner: windows-latest"); + expect(smokeWorkflow).toContain("--effort low"); + expect(smokeWorkflow).toContain("--timeout-seconds 240"); + expect(smokeWorkflow).toContain("--max-cost 0.25"); + }); + + test("reports immutable platform statuses and refreshes the required gate", () => { + expect(smokeWorkflow).toContain( + '"repos/$GITHUB_REPOSITORY/statuses/$HEAD_SHA"', + ); + expect(smokeWorkflow).toContain( + '"context=codex-security/live-smoke-$platform"', + ); + expect(smokeWorkflow).toContain( + '"repos/$GITHUB_REPOSITORY/actions/runs/$run_id/rerun-failed-jobs"', + ); + }); +});