diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml new file mode 100644 index 0000000..97346e7 --- /dev/null +++ b/.github/workflows/pr-risk.yml @@ -0,0 +1,334 @@ +name: PR Risk Grade (reusable) + +# Reusable ADVISORY PR risk grader — the shadow-check rung of the PR risk-grading ladder. +# Grades every PR event into a tier R0 (safest) .. R3 (riskiest) and syncs ONE label +# (`risk:R0` .. `risk:R3`, or `risk:ungraded` when an input was unreadable). That label is +# the entire product: nothing is gated, nothing is blocked, nothing merges, no comment is +# posted. Humans look at the label and agree or disagree; disagreement is recorded by adding +# the `risk-dispute` label (which this workflow never touches) plus a comment saying why. +# +# grade = worst(path_floor, provenance, reversibility) — three deterministic axes; the worst +# tier wins, so no axis can move a PR into a safer lane than another axis put it. No LLM, no +# model call anywhere: `gh` + `jq` over the PR's own API record. See +# scripts/pr-risk/grade-pr-risk.sh for the axes and the unknown contract. +# +# The grader and its default risk map load from THIS repo at the pinned `workflows_ref`, +# never from the graded PR's checkout (no PR code is checked out at all) — a PR cannot edit +# the rules that judge it. A consumer repo sharpens the generic defaults by committing +# `.github/risk.json` (map) / `.github/risk-runbooks.json` (runbook registry), which are read +# from the PR's BASE ref: present-but-invalid fails the run loudly; absent falls back to the +# defaults in scripts/pr-risk/. +# +# CHECKS SETTLE BEFORE THE LABEL DOES: the reversibility axis asks "did tests covering these +# lines actually run", and at event time the rest of the rollup is usually still running (the +# grading job itself is excluded from the rollup it reads — see --self-context in the +# grader). The job re-polls until the other checks settle or `wait_for_checks_minutes` runs +# out, then labels what it has. Pair the caller with a per-PR cancel-in-progress concurrency +# group so a new push supersedes a waiting run instead of stacking behind it. +# +# SECRETS: none. This workflow declares no `secrets:` inputs and callers pass none — the only +# credential in the job is the automatic `GITHUB_TOKEN` (`github.token`), used for the PR read +# and the one label write. There is no `secrets: inherit` to add and nothing to rotate. +# +# The label is applied with the plain GITHUB_TOKEN on purpose: GITHUB_TOKEN-applied labels +# cannot fire `labeled` triggers, so the shadow check is structurally unable to start a +# workflow cascade. Fork PRs under a plain `pull_request` trigger get a read-only token and +# the label write will fail — enroll public repos with `pull_request_target` instead (safe +# here by construction: this workflow never checks out or executes PR code). +# +# ENROLL THIS AS ITS OWN WORKFLOW, not as one job inside an existing CI workflow. The grading +# job is part of the check rollup it reads, so it excludes its own RUN from that rollup; a job +# sharing a run with the rest of CI therefore excludes its siblings too and lands on the honest +# R2 floor instead of grading off a full rollup. (It can never grade a red PR green either way +# — a FAILING check is never excluded.) +# +# Caller pattern (consumer repo, .github/workflows/ci-pr-risk.yml): +# +# name: CI - PR Risk Grade +# on: +# pull_request: +# types: [opened, synchronize, reopened, ready_for_review] +# concurrency: +# group: pr-risk-${{ github.event.pull_request.number }} +# cancel-in-progress: true +# permissions: +# contents: read +# jobs: +# pr-risk: +# permissions: +# contents: read +# issues: write # the risk label rides the issues API +# pull-requests: read +# checks: read # the check rollup the reversibility axis reads +# statuses: read +# uses: Comfy-Org/github-workflows/.github/workflows/pr-risk.yml@ # v1 +# with: +# workflows_ref: + +on: + workflow_call: + inputs: + fleet_logins: + description: >- + GitHub logins whose PRs are supervised-agent output (comma-separated). + Grades provenance `agent-supervised` alongside the `agent-coded` label. + type: string + required: false + default: mattmillerai + bot_logins: + description: >- + Extra logins treated as bots, on top of any `[bot]`-suffixed login + (comma-separated). A bot with no runbook registry entry grades as + human — identity alone never buys trust. + type: string + required: false + default: github-actions,dependabot,renovate,coderabbitai,cursor,comfy-pr-bot,web-flow + label_map: + description: >- + Rename the five grader-owned labels, as `tier=label` pairs + (comma-separated). Default: + `R0=risk:R0,R1=risk:R1,R2=risk:R2,R3=risk:R3,unknown=risk:ungraded`. + Tier KEYS are fixed; only the label text is yours. Missing labels are + created on first use, color-coded green through red. + type: string + required: false + default: '' + wait_for_checks_minutes: + description: >- + How long to wait for the REST of the check rollup to settle before + labeling (the grading run itself is excluded from the rollup it + reads). 0 labels immediately — expect R2 floors from still-pending + checks. CLAMPED to what a 30-minute job can actually spend waiting (25), + so an over-large value degrades to a shorter wait instead of a job + cancelled mid-sleep with the label never applied. + type: number + required: false + default: 10 + repo_map_path: + description: Path of the consumer repo's risk-map override, read from the PR base ref. + type: string + required: false + default: .github/risk.json + repo_runbooks_path: + description: Path of the consumer repo's runbook-registry override, read from the PR base ref. + type: string + required: false + default: .github/risk-runbooks.json + workflows_ref: + description: >- + Ref of Comfy-Org/github-workflows to load the grader + default map + from. REQUIRED, and pin it to the SAME full commit SHA you pin `uses:` + to. There is deliberately no default: a floating default (`main`) let a + caller SHA-pin `uses:` and then download the grader from HEAD of main, + so the two halves of one tool drifted apart and the grading logic + stayed mutable after review. A supply chain with a floating link in it + is not a chain. + type: string + required: true + +permissions: + contents: read + +jobs: + grade: + name: Grade PR risk + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read # read the consumer's .github/risk.json override from the BASE ref + issues: write # THE ONLY WRITE: sync the one risk label (a PR is an issue to the + # labels API). Nothing else is written — no comment, no review, no + # merge — and a GITHUB_TOKEN-applied label cannot fire `labeled` + # triggers, so this write cannot start a workflow cascade. + pull-requests: read # the PR record itself: author, association, fork flag, file list + checks: read # the check rollup the reversibility axis reads (CheckRun contexts) + statuses: read # the same rollup's legacy commit-status contexts + env: + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + GH_TOKEN: ${{ github.token }} + steps: + - name: Load pr-risk tool + # The grader + default map come from THIS workflow's repo (public, pinned + # via workflows_ref) — never from the graded PR. No PR code is checked + # out anywhere in this job. + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: Comfy-Org/github-workflows + ref: ${{ inputs.workflows_ref }} + path: _pr_risk_tool + persist-credentials: false + + - name: Fetch per-repo overrides from the base ref + id: overrides + env: + MAP_PATH: ${{ inputs.repo_map_path }} + RB_PATH: ${{ inputs.repo_runbooks_path }} + run: | + set -uo pipefail + # Read from the BASE ref, so the PR being graded cannot edit the rules + # that judge it. Absent (a genuine 404) falls back to the shipped + # defaults; present-but-invalid is left for the grader's structural + # validation to reject loudly — a repo that commits a corrupt map must + # see red, not silent generic grading. + # + # ONLY A 404 MEANS ABSENT. Treating any non-zero exit as "no override" + # meant a 403 rate-limit, a 5xx or a network blip silently graded the + # PR against the generic default map instead of the repo's sharpened + # one — a LOWER tier computed from an input nobody read, which is the + # confident-answer-from-an-unread-source failure the unknown contract + # forbids everywhere else. So the status code is captured and anything + # that is not 200-or-404 fails the run. + errf="$(mktemp)" + fetch_override() { # -> prints the outfile, or nothing when absent + local p="$1" out="$2" + if gh api "repos/${REPO}/contents/${p}?ref=${BASE_REF}" \ + -H "Accept: application/vnd.github.raw" > "$out" 2>"$errf"; then + echo "using ${p} from ${BASE_REF}" >&2 + printf '%s' "$out" + elif grep -q '(HTTP 404)' "$errf"; then + rm -f "$out" # the redirect already created it empty; nothing must read it + echo "no ${p} on ${BASE_REF} — using the generic default" >&2 + else + echo "::error::could not read ${p} from ${BASE_REF}: $(tr '\n' ' ' < "$errf"). NOT falling back to the generic default: that would grade this PR against rules nobody read." >&2 + return 1 + fi + } + map="$(fetch_override "$MAP_PATH" repo-risk-map.json)" || exit 1 + rb="$(fetch_override "$RB_PATH" repo-risk-runbooks.json)" || exit 1 + rm -f "$errf" + echo "map=$map" >> "$GITHUB_OUTPUT" + echo "runbooks=$rb" >> "$GITHUB_OUTPUT" + + - name: Grade (waits for the rest of the rollup to settle) + id: grade + env: + FLEET_LOGINS: ${{ inputs.fleet_logins }} + BOT_LOGINS: ${{ inputs.bot_logins }} + SELF_CONTEXT: ${{ github.workflow }} + SELF_RUN_ID: ${{ github.run_id }} + WAIT_MINUTES: ${{ inputs.wait_for_checks_minutes }} + MAP_OVERRIDE: ${{ steps.overrides.outputs.map }} + RB_OVERRIDE: ${{ steps.overrides.outputs.runbooks }} + JOB_TIMEOUT_MINUTES: 30 # keep in sync with timeout-minutes on the job + run: | + set -uo pipefail + TOOL="_pr_risk_tool/scripts/pr-risk/grade-pr-risk.sh" + ARGS=( --repo "$REPO" --pr "$PR_NUMBER" + --self-context "$SELF_CONTEXT" --self-run-id "$SELF_RUN_ID" ) + [ -n "$FLEET_LOGINS" ] && ARGS+=( --fleet-logins "$FLEET_LOGINS" ) + [ -n "$BOT_LOGINS" ] && ARGS+=( --bot-logins "$BOT_LOGINS" ) + [ -n "$MAP_OVERRIDE" ] && ARGS+=( --map "$MAP_OVERRIDE" ) + [ -n "$RB_OVERRIDE" ] && ARGS+=( --runbooks "$RB_OVERRIDE" ) + + # CLAMP THE WAIT TO THE JOB'S OWN BUDGET rather than trusting the input + # description. A caller who passes 40 against a 30-minute timeout gets a + # job cancelled mid-sleep: the label step never runs, the PR keeps + # whatever label the previous push left, and the check goes red for a + # reason nobody would guess. 5 minutes of headroom is left for the + # label + summary steps. + max_wait=$(( JOB_TIMEOUT_MINUTES - 5 )) + wait_minutes="${WAIT_MINUTES:-10}" + [ "$wait_minutes" -ge 0 ] 2>/dev/null || wait_minutes=10 + if [ "$wait_minutes" -gt "$max_wait" ]; then + echo "::warning::wait_for_checks_minutes=${wait_minutes} exceeds what a ${JOB_TIMEOUT_MINUTES}-minute job can spend waiting — clamped to ${max_wait}" + wait_minutes="$max_wait" + fi + deadline=$(( $(date +%s) + 60 * wait_minutes )) + empty_deadline=$(( $(date +%s) + 120 )) # grace for checks to REGISTER at all + waited=0 + # Poll backoff: 15s doubling to 120s. The fixed 30s poll idled a runner + # up to the full wait on every `synchronize` event, per PR, across the + # fleet, for a grade that is one API call — increasing backoff cuts both + # runner minutes and API calls without changing the settle behaviour. + delay=15 + settled_once=0 + # rc=3 (the PR could not be read) covers rate limits, secondary rate + # limits and transient 5xx as well as a genuinely missing PR. Labeling + # `ungraded` on the first blip turns a momentary hiccup into a durable + # verdict, so it is retried with backoff. This gets its OWN budget, not + # the settle deadline: a transient API failure has nothing to do with + # how long the caller wants to wait for CI, and sharing the deadline + # would leave `wait_for_checks_minutes: 0` with no retry at all. + unreadable_tries=0 + max_unreadable_tries=4 + read_deadline=$(( $(date +%s) + 150 )) + read_delay=10 + while :; do + rc=0 + bash "$TOOL" "${ARGS[@]}" > record.json || rc=$? + case "$rc" in + 0|1) ;; # graded (1 = graded but unknown — still labeled) + 3) unreadable_tries=$(( unreadable_tries + 1 )) + if [ "$unreadable_tries" -lt "$max_unreadable_tries" ] && [ "$(date +%s)" -lt "$read_deadline" ]; then + echo "PR read failed (attempt ${unreadable_tries}/${max_unreadable_tries}) — retrying in ${read_delay}s" + sleep "$read_delay"; waited=$(( waited + read_delay )) + read_delay=$(( read_delay * 2 )); [ "$read_delay" -le 60 ] || read_delay=60 + continue + fi + echo "PR unreadable via the API after ${unreadable_tries} attempt(s) — labeling ungraded" + printf '{}' > record.json + break ;; + *) echo "grader failed (rc=$rc)"; exit 1 ;; # setup/usage bug: fail loud + esac + pending="$(jq -r '.checks_pending_excl_self // false' record.json)" + state="$(jq -r '.checks_state // "none"' record.json)" + now="$(date +%s)" + [ "$now" -lt "$deadline" ] || break + if [ "$pending" = "true" ]; then + settled_once=0 + elif [ "$state" = "none" ] && [ "$now" -lt "$empty_deadline" ]; then + # An EMPTY rollup is checks that have not registered yet, not CI + # that finished — exiting the poll here would apply a label that a + # check appearing a second later never revises. Bounded by a SHORT + # grace window rather than the whole budget, so a repo with + # genuinely no CI is not made to wait 25 minutes on every PR to + # arrive at the same honest R2. + settled_once=0 + elif [ "$settled_once" -eq 0 ]; then + # First settled reading. Require it to REPEAT before labeling: one + # transiently-quiet poll is not a settled rollup. + settled_once=1 + else + break + fi + sleep "$delay"; waited=$(( waited + delay )) + delay=$(( delay * 2 )); [ "$delay" -le 120 ] || delay=120 + done + tier="$(jq -r '.risk.tier // "unknown"' record.json)" + echo "tier=$tier" >> "$GITHUB_OUTPUT" + echo "waited=$waited" >> "$GITHUB_OUTPUT" + echo "graded tier: $tier (waited ${waited}s for the rollup)" + + - name: Apply risk label + env: + TIER: ${{ steps.grade.outputs.tier }} + LABEL_MAP: ${{ inputs.label_map }} + run: | + set -uo pipefail + bash _pr_risk_tool/scripts/pr-risk/apply-risk-label.sh + + - name: Step summary + if: always() + env: + TIER: ${{ steps.grade.outputs.tier }} + WAITED: ${{ steps.grade.outputs.waited }} + run: | + set -uo pipefail + { + echo "## PR risk grade: ${TIER:-unknown} (advisory)" + echo + if [ -s record.json ] && jq -e '.risk' record.json >/dev/null 2>&1; then + jq -r ' + "Grade = worst(path_floor, provenance, reversibility) — \(.risk.reason)\n", + "| axis | tier | reason |", "|---|---|---|", + (.risk.axes | to_entries[] | "| \(.key) | \(.value.tier // "unknown") | \(.value.reason) |"), + "\nMap `\(.risk.map_version)` · registry `\(.risk.registry_version)` · waited \($waited)s for the rollup · checks `\(.checks_state // "none")`" + ' --arg waited "${WAITED:-0}" record.json + else + echo "The PR could not be read via the API; nothing was graded (labeled ungraded — this is NOT a low-risk verdict)." + fi + echo + echo "This label routes nothing and gates nothing. Disagree with the grade? Add the \`risk-dispute\` label and say why in a comment — the grader never touches that label." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test-pr-risk.yml b/.github/workflows/test-pr-risk.yml new file mode 100644 index 0000000..872b8a2 --- /dev/null +++ b/.github/workflows/test-pr-risk.yml @@ -0,0 +1,60 @@ +# CONTRACT: this is one of THIS repo's own path-filtered script tests, NOT a reusable +# workflow. It is not callable and has no caller pattern: +# TRIGGERS — `pull_request` and `push` to main, both filtered to scripts/pr-risk/** +# and this file. A change elsewhere in the repo does not run it. +# INPUTS — none (no `workflow_call`, no `workflow_dispatch`). +# SECRETS — none. Read-only `contents: read`; the suites are hermetic and make no +# network call (the live-PR path stubs `gh` on PATH with a fixture). +# The reusable workflow these scripts back is .github/workflows/pr-risk.yml — that is +# the file with the inputs/secrets/caller-pattern header. + +name: Test pr-risk scripts + +# Runs the hermetic test suites (+ shellcheck) for the grader and label-sync +# scripts behind pr-risk.yml. The grader stamps an advisory risk tier on every +# consumer repo's PRs, so a grading regression silently mislabels PRs org-wide +# — cheap to guard with a unit run on change. No network: the suites feed the +# grader synthetic records and stub `gh` for the live-PR path. + +on: + pull_request: + paths: + - 'scripts/pr-risk/**' + - '.github/workflows/test-pr-risk.yml' + push: + branches: [main] + paths: + - 'scripts/pr-risk/**' + - '.github/workflows/test-pr-risk.yml' + +permissions: + contents: read + +jobs: + test: + name: shellcheck + suites + runs-on: ubuntu-latest + defaults: + run: + working-directory: scripts/pr-risk + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: shellcheck + run: shellcheck -x grade-pr-risk.sh apply-risk-label.sh tests/test_grade_pr_risk.sh tests/test_apply_risk_label.sh + + - name: default map + registry parse and validate + # The shipped defaults must pass the grader's own structural validation: + # a malformed default would fail every consumer at once. + run: | + bash -c 'source ./grade-pr-risk.sh; read_map risk-map.v0.json map >/dev/null && read_map runbook-registry.v0.json runbooks >/dev/null' + echo "defaults validate" + + - name: grader suite + run: bash tests/test_grade_pr_risk.sh + + - name: label suite + run: bash tests/test_apply_risk_label.sh diff --git a/README.md b/README.md index 17d83d0..b4ff0f7 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ This repo is **public** so any repo — public or private, inside or outside the | [`assign-reviewers.yml`](.github/workflows/assign-reviewers.yml) | Auto-requests expertise-aware, load-balanced PR reviewers with new-folk randomization. Matches changed paths against a caller-repo `.github/reviewers.yml` (path-glob → reviewers, plus a `default_pool`), drops the author + `vars.REVIEWER_EXCLUDE`, ranks candidates by open review load (steering off anyone at/over `vars.REVIEWER_LOAD_CAP`), and may swap a slot for a `vars.REVIEWER_GROWTH_POOL` member. Requests go through the CLOUD_CODE_BOT app token so they work on fork PRs. Requires `vars.APP_ID` + `CLOUD_CODE_BOT_PRIVATE_KEY`. | | [`assign-prs-to-author.yml`](.github/workflows/assign-prs-to-author.yml) | Housekeeping — assigns every open PR with no assignees to its author (bot-authored PRs skipped by default). Run on a schedule from a thin caller; useful when a team tracks PR ownership via assignees. The calling job needs `pull-requests: write` and `issues: write`. | | [`pr-size.yml`](.github/workflows/pr-size.yml) | PR-size cap — fails (or, in `mode: warn`, only reports) when a PR's net diff exceeds `max_lines` non-generated changed lines, keeping diffs reviewable. Excludes dependency lockfiles, `linguist-generated` files (read from the base ref, so a PR can't exempt itself), Go generated-code markers, and per-repo `extra_lockfiles` / `extra_generated_globs`. A `bypass_label` (default `oversized-ok`) waves through a legitimately large change; a sticky bot comment explains overages when `bot_app_id` + `BOT_APP_PRIVATE_KEY` are supplied (degrades to status + step summary without them). Counting logic + tests live in [`scripts/check-pr-size/`](scripts/check-pr-size). | +| [`pr-risk.yml`](.github/workflows/pr-risk.yml) | **Advisory PR risk grading (shadow check)** — grades every PR into a tier `R0` (safest) .. `R3` (riskiest) and syncs one label (`risk:R0`..`risk:R3`, or `risk:ungraded` when an input was unreadable). The label is the entire product: nothing is gated, routed, commented, or merged. Deterministic (`gh` + `jq`, no LLM): `grade = worst(path_floor, provenance, reversibility)` — path-glob map, what-process-produced-the-diff (registered runbooks with identity + diff-shape assertions; forks are R3 with no exceptions), and revertability (persistent-state mutation, deletions under sensitive classes, did green checks cover the lines). Grader + generic defaults live in [`scripts/pr-risk/`](scripts/pr-risk); a consumer sharpens them with `.github/risk.json` / `.github/risk-runbooks.json`, read from the PR's **base ref** so a PR can't edit the rules that judge it. The job excludes its own run from the check rollup and waits (`wait_for_checks_minutes`) for the rest to settle before labeling. Labels ride the plain `GITHUB_TOKEN` (cannot fire `labeled` triggers — no cascade risk); disagreement is recorded with a human-owned `risk-dispute` label. Label text is remappable via `label_map`. `workflows_ref` is **required** — pin it to the same full commit SHA as `uses:`, so the grader cannot be loaded from a floating ref after the caller was reviewed. Enroll it as its own workflow rather than a job inside an existing CI workflow (the rollup exclusion is per-run). The calling job needs `issues: write` + `pull-requests: read` + `checks: read` + `statuses: read`; no secrets. | | [`stale.yml`](.github/workflows/stale.yml) | Stale-PR sweeper (`actions/stale`) plus a Slack digest of what it touched. PRs inactive for N days are labeled `stale`; still-inactive PRs are closed. The digest header names the source repo so batches from different repos posted to the same channel are unambiguous. Thresholds, messages, exempt labels, and the Slack channel are inputs; the caller owns the schedule + dry-run toggle. The calling job needs `pull-requests: write` and `issues: write`. Optional `SLACK_BOT_TOKEN`. | | [`groom.yml`](.github/workflows/groom.yml) | Scheduled/dispatch org-wide **code-cleanup sweep** (finds only — no commits, no PRs, never merges). A read-only FINDER agent scans a clean default-branch checkout (whole-repo, not a diff) for high-value refactors; an INDEPENDENT VERIFIER agent (fresh session) re-checks each as CONFIRM/DOWNGRADE/REJECT with a stable dedup signature; survivors are deduped against a durable GitHub-issue-state ledger and filed as `groom`-labeled GitHub issues (security-adjacent ones get `groom-security` — investigate, don't auto-implement). Mirrors the cursor-review topology: briefs + ledger live in [`.github/groom/`](.github/groom) as the single source of truth. The finder/verifier/builder agent jobs invoke the Claude CLI directly and mint no GitHub token, so they need nothing beyond `contents: read`; filing runs in a separate job as the bot you configure via `bot_app_id` (Comfy: cloud-code-bot). `dry_run` reports what it would file without opening issues. Runs on a **daily base cron** with a runtime cadence gate: set repo Actions variable `GROOM_INTERVAL_DAYS` (default 7 = weekly) to retune how often a real run happens — weekly → every-3-days → daily — with no workflow-file edit; a tick within the interval no-ops before the finder (`workflow_dispatch` bypasses the interval gate, but the volume gate — when the caller leaves it on — still applies). The calling job must grant `contents: read` + `issues: write` + `pull-requests: read` + `actions: read` — the first three are declared by the `file` / `build_select` jobs (needed even with `bot_app_id` set), and the interval gate needs `actions: read` (reads run history for the last real run); GitHub rejects a shorter grant at startup. Requires `ANTHROPIC_API_KEY` (+ `BOT_APP_PRIVATE_KEY` when `bot_app_id` is set). **Opt-in auto-builder** (`builder: true`, BE-4003): the top `max_prs` (default 5) CONFIRMED, non-security findings become **review-gated PRs** (full CI + cursor-review, **never auto-merged**) instead of issues; a credential-free `build` job emits only a patch artifact and a separate `build_pr` job opens the PR as the bot, preserving the security boundary. The ledger's PR-state (open/merged/closed) stops a built finding being re-proposed. Requires `bot_app_id`. `max_prs` is typed **`string`**, not `number`, so a caller can forward its own `workflow_dispatch` input straight through (`max_prs: ${{ github.event.inputs.max_prs \|\| '1' }}`) and let an operator raise the ceiling for one manual run — no `fromJSON()` cast in the caller, and the parse/clamp (empty → default, non-numeric → 0 PRs + warning, never a failed run) happens once inside the reusable. | | [`agents-md-integrity.yml`](.github/workflows/agents-md-integrity.yml) | Enforces the Comfy `AGENTS.md` standard on the caller repo: a top-level `AGENTS.md` must exist and stay under a hard line ceiling (`max_lines`, default 200; warns over `warn_lines`, default 150), a `CLAUDE.md` (if present) must be a thin `@AGENTS.md` shim rather than a divergent copy, no legacy `.cursorrules` (gated `forbid_cursorrules`), every nested monorepo `AGENTS.md` needs a sibling `@AGENTS.md` shim and to be under the ceiling (gated `check_nested`), and `AGENTS.md` should have a CODEOWNERS DRI (`require_codeowners`, warn-only by default). Fails with a non-zero exit + GitHub annotations so it wires in as a required status check. The checker lives in [`.github/agents-md-integrity/`](.github/agents-md-integrity) (pin `workflows_ref` to the same ref as `uses:`); no secrets required. | diff --git a/scripts/pr-risk/README.md b/scripts/pr-risk/README.md new file mode 100644 index 0000000..97386e1 --- /dev/null +++ b/scripts/pr-risk/README.md @@ -0,0 +1,130 @@ +# pr-risk — advisory PR risk grading (the shadow check) + +The scripts behind [`pr-risk.yml`](../../.github/workflows/pr-risk.yml). Every PR +event is graded into a tier and gets ONE label: + +| tier | label (default) | meaning | eventual routing (later phases — nothing routes today) | +|---|---|---|---| +| R0 | `risk:R0` | inert — docs, tests, provably-shaped runbook output | auto-merge candidate | +| R1 | `risk:R1` | contained — bounded, covered, revertable in one click | rubber-stamp | +| R2 | `risk:R2` | standard — ordinary product code | normal review | +| R3 | `risk:R3` | elevated — auth, billing, migrations, IaC, CI, deps, secrets | owner + e2e | +| — | `risk:ungraded` | an input could not be read; deliberately NOT a tier | human review | + +**The label is the entire product.** Nothing is gated, blocked, routed, commented +on, or merged. Humans glance at the label and either agree or disagree. +Disagree by adding the `risk-dispute` label (never touched by the grader) plus a +comment saying why — disputes are the pilot's calibration data. + +## How a grade is computed + +`grade = worst(path_floor, provenance, reversibility)` — three deterministic +axes, worst wins, so no axis can move a PR into a safer lane than another axis +put it. No LLM anywhere; the whole thing is `gh` + `jq` over the PR's API +record. + +1. **Path floor** — [`risk-map.v0.json`](risk-map.v0.json): versioned path-glob + rules. The floor is the worst tier over every rule any changed path matches, + so a docs file can never cancel a migration in the same PR. Matching covers + every path the diff touches, **destination and origin** — a renamed file is + graded under its previous path too, so `git mv auth/x.go misc/x.go` cannot + walk a file out of the rule that guards it. In globs `**` crosses `/` and `*` + does not, and matching is whole-path anchored: a rule without a leading `**/` + matches root-level files ONLY. +2. **Provenance** — what PROCESS produced the diff: `runbook` (a registered + producer in [`runbook-registry.v0.json`](runbook-registry.v0.json) whose + identity AND diff shape both assert), `agent-supervised`, `human`, or + `external` (fork / first-time contributor — R3, no exceptions, even when a + runbook shape matches). Identity is the server-attributed author login, + never the forgeable commit author string. +3. **Reversibility** — mutates persistent state or deletes data → R3; **removes** + a file under a sensitive class → R3 (a delete, or a rename out of that class); + no green check rollup → R2; green but no test file touched → R1; green with + tests touched → R0. "Green" means at least one check actually CONCLUDED + success: a rollup of nothing but skipped/neutral answers "did tests covering + these lines run?" with nothing, so it cannot drop the axis below R2. What + counts as a test file is `reversibility.test_path_patterns` in the map (omit + the key and the grader falls back to a built-in regex that only knows the + Go/TS shapes). + +Anything unreadable grades `unknown` (labeled `risk:ungraded`), never a +confident tier, and never "the axes that did resolve" — a PR whose file list we +could not read is exactly the PR that might touch auth. + +Two CI-specific mechanics worth knowing: + +- **The grading run excludes itself from the check rollup it reads** (its own + check is always in-flight at grade time), and the job re-polls until the rest + of the rollup settles or `wait_for_checks_minutes` runs out — otherwise every + live grade would floor at R2 as an artifact of the measurement. Exclusion is + keyed on `github.run_id` (`--self-run-id`), and a **FAILING check is never + excluded**: self-exclusion may only ever hide our own pending run, never a red + one. Enroll pr-risk as its **own workflow** rather than a job inside an + existing CI workflow — a job sharing a run with the rest of CI excludes its + siblings too, and lands on the honest R2 floor instead of a full rollup. +- **The label is applied with the plain `GITHUB_TOKEN`**, which cannot fire + `labeled` triggers — the shadow check is structurally unable to start a + workflow cascade. Later phases that WANT label-triggered routing switch to an + app token deliberately. + +## Per-repo overrides (read from the base ref) + +The shipped map and registry are deliberately generic. A consumer repo sharpens +them by committing: + +- `.github/risk.json` — the repo's own path→tier map (same schema as + [`risk-map.v0.json`](risk-map.v0.json)) +- `.github/risk-runbooks.json` — the repo's own producer registry (same schema + as [`runbook-registry.v0.json`](runbook-registry.v0.json)) + +Both are read from the PR's **base ref**, so a PR cannot edit the rules that +judge it (editing them — or the grader — at all is R3 by the map's own first +rule). A genuine 404 falls back to the shipped defaults; a present-but-invalid +file fails the run loudly rather than silently grading generic, and so does any +non-404 read failure (a 403 rate-limit or 5xx must not quietly demote the PR to +the generic map, which would be a lower tier computed from an input nobody read). + +A map must MAP every provenance class (`runbook`, `agent-supervised`, `human`, +`external`). Omitting one is refused at load time rather than filled in with a +tier nobody chose — that is how a map that forgot `external` used to grade a fork +the same as a teammate. + +Every graded record carries `map_version` + `registry_version`, so grades made +under different maps stay comparable and a map revision can be replayed against +accumulated records. + +## Relabeling (R0–R3 vs R1–R4 and friends) + +Tier SEMANTICS are fixed (R0 safest .. R3 riskiest, `unknown` separate) +everywhere records are stored. The label TEXT is the caller's, via `label_map`: + +```yaml +with: + label_map: "R0=risk:R1,R1=risk:R2,R2=risk:R3,R3=risk:R4,unknown=risk:ungraded" +``` + +Labels are created on first use, color-coded green → red (gray for ungraded). + +## Files + +- `grade-pr-risk.sh` — the grader. `--pr N --repo o/r` grades a live PR; + `--stdin` grades synthetic records (the no-network test surface). Extracted + from the fleet's offline corpus grader (BE-5507); the identity jq is inlined + from its collector (BE-5030) — keep the two in sync when either changes. The + changed-file list comes from REST `pulls/{n}/files`, not the GraphQL `files` + connection: GraphQL has no previous-path field (so renames are invisible) and + its connection capped the list at 100, which put exactly the PRs a risk grade + helps most in the ungraded lane. A read that comes back short of `changedFiles` + is still `unknown`. +- `apply-risk-label.sh` — the one write. Owns exactly the five mapped labels: + removes stale ones, applies the computed one, touches nothing else. +- `risk-map.v0.json` / `runbook-registry.v0.json` — the generic defaults. +- `tests/` — hermetic suites (synthetic records + a stubbed `gh`); run via + [`test-pr-risk.yml`](../../.github/workflows/test-pr-risk.yml). + +## What is deliberately NOT here + +No auto-merge, no routing, no required check, no PR comment, no LLM judgement, +no linked-ticket requirement. Those are later rungs of the ladder and each one +is its own explicit switch — this workflow exists to accumulate the +agree/disagree evidence that decides whether any of them turn on. diff --git a/scripts/pr-risk/apply-risk-label.sh b/scripts/pr-risk/apply-risk-label.sh new file mode 100755 index 0000000..4295592 --- /dev/null +++ b/scripts/pr-risk/apply-risk-label.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# apply-risk-label.sh — sync a PR's risk label to the computed tier. The one write the +# reusable pr-risk.yml workflow performs. +# +# OWNERSHIP CONTRACT: this script owns EXACTLY the label names in LABEL_MAP's values. It +# removes stale ones and applies the computed one, and never touches any other label — so a +# human who disagrees with a grade records that with their OWN label (the pilot convention is +# `risk-dispute`), which this script will never fight. Editing the grader-owned label by hand +# is futile by design: the next push re-syncs it. +# +# The label is applied with the plain GITHUB_TOKEN on purpose: GITHUB_TOKEN-applied labels do +# not fire `labeled` workflow triggers, which makes the shadow check incapable of starting a +# workflow cascade. When a later phase WANTS the label to trigger routing, that is a deliberate +# switch to an app token (the cursor-review-auto-label.yml pattern), not a default. +# +# Inputs (env): +# REPO owner/name of the repo holding the PR (required) +# PR_NUMBER the PR number (required) +# TIER R0 | R1 | R2 | R3 | unknown ('' and 'null' read as unknown) (required) +# LABEL_MAP tier=label pairs, comma-separated (optional) +# default: R0=risk:R0,R1=risk:R1,R2=risk:R2,R3=risk:R3,unknown=risk:ungraded +# Relabeling (e.g. a 1-indexed R1..R4 scheme) is a caller-side remap of the +# VALUES only; tier keys are fixed R0..R3 + unknown everywhere else. +# DRY_RUN 1 = print the plan, write nothing +# GH_TOKEN token for gh (in CI: the job's GITHUB_TOKEN; needs issues: write) +# +# Missing labels are created on first use (color-coded, described), so enrolling a repo needs +# no manual label setup. +# +# Exit: 0 = label in sync (or dry run). 2 = usage error. 4 = a GitHub write failed — the run +# must go red rather than pretend the label landed. + +set -uo pipefail + +REPO="${REPO:-}" +PR_NUMBER="${PR_NUMBER:-}" +TIER="${TIER:-}" +LABEL_MAP="${LABEL_MAP:-}" +DRY_RUN="${DRY_RUN:-0}" +DEFAULT_MAP="R0=risk:R0,R1=risk:R1,R2=risk:R2,R3=risk:R3,unknown=risk:ungraded" + +log() { printf '[apply-risk-label] %s\n' "$*" >&2; } +die() { printf '[apply-risk-label] ERROR %s\n' "$*" >&2; exit 2; } +fail() { printf '[apply-risk-label] FAIL %s\n' "$*" >&2; exit 4; } + +[ -n "$REPO" ] || die "REPO is required" +[[ "$REPO" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die "bad REPO '$REPO' (want owner/name)" +[[ "$PR_NUMBER" =~ ^[0-9]+$ ]] || die "bad PR_NUMBER '$PR_NUMBER'" +command -v jq >/dev/null 2>&1 || die "jq not found on PATH" +[ "$DRY_RUN" = 1 ] || command -v gh >/dev/null 2>&1 || die "gh not found on PATH" + +# '' and 'null' arrive when the grader refused a confident tier — both are the unknown lane. +case "$TIER" in ""|null) TIER="unknown" ;; esac +case "$TIER" in R0|R1|R2|R3|unknown) ;; *) die "bad TIER '$TIER' (want R0..R3 or unknown)" ;; esac + +[ -n "$LABEL_MAP" ] || LABEL_MAP="$DEFAULT_MAP" + +# Parse "tier=label,tier=label" without eval. All five tiers must resolve: a map that forgets +# `unknown` would leave ungradeable PRs silently unlabeled, which reads as "grader never ran". +label_for() { # -> label on stdout, rc 1 when unmapped + printf '%s' "$LABEL_MAP" | tr ',' '\n' | awk -F= -v t="$1" '$1 == t { print $2; found=1 } END { exit !found }' +} +OWNED=() +for t in R0 R1 R2 R3 unknown; do + l="$(label_for "$t")" || die "LABEL_MAP is missing a label for tier '$t' (got '$LABEL_MAP')" + [ -n "$l" ] || die "LABEL_MAP maps tier '$t' to an empty label" + OWNED+=("$l") +done +TARGET="$(label_for "$TIER")" + +# Colors keyed by TIER (not label text, which callers may remap): green .. red, gray unknown. +color_for() { + case "$1" in + R0) echo "0e8a16" ;; R1) echo "fbca04" ;; R2) echo "d93f0b" ;; R3) echo "b60205" ;; + *) echo "cfd3d7" ;; + esac +} + +if [ "$DRY_RUN" = 1 ]; then + log "DRY RUN — would sync $REPO#$PR_NUMBER to '$TARGET' (owned set: ${OWNED[*]})" + printf '%s\n' "$TARGET" + exit 0 +fi + +# A label name is a PATH SEGMENT in every request below, and GitHub label names legally contain +# spaces, `/`, `#`, `?` and `%`. A caller who remaps `R3=risk high` would otherwise build a +# malformed or misrouted URL: the DELETE fails, `fail` fires, and a rename paints the check red. +# Encoded for the path only — the raw name is what we log, compare and send as a form field. +enc() { jq -rn --arg s "$1" '$s | @uri'; } + +# Current labels on the PR (a PR is an issue to the labels API). --paginate because the endpoint +# returns 30 per page: on a PR with more than 30 labels a stale grader-owned label falls off page +# one, `has()` reports false, the removal loop skips it, and the PR carries two contradictory risk +# labels at once — the exact state the ownership contract above promises cannot happen. +current="$(gh api --paginate "repos/$REPO/issues/$PR_NUMBER/labels?per_page=100" --jq '.[].name' 2>/dev/null \ + | jq -Rsc 'split("\n") | map(select(length > 0))')" \ + || fail "could not read labels on $REPO#$PR_NUMBER" + +has() { jq -e --arg l "$1" 'index($l) != null' >/dev/null 2>&1 <<<"$current"; } + +# Remove stale grader-owned labels (everything in the owned set except the target). +for l in "${OWNED[@]}"; do + [ "$l" = "$TARGET" ] && continue + if has "$l"; then + gh api -X DELETE "repos/$REPO/issues/$PR_NUMBER/labels/$(enc "$l")" >/dev/null 2>&1 \ + || fail "could not remove stale label '$l' from $REPO#$PR_NUMBER" + log "removed stale '$l'" + fi +done + +if has "$TARGET"; then + log "already labeled '$TARGET' — nothing to do" +else + # Ensure the label exists in the repo first, so enrollment needs no manual label setup. + if ! gh api "repos/$REPO/labels/$(enc "$TARGET")" >/dev/null 2>&1; then + gh api -X POST "repos/$REPO/labels" \ + -f name="$TARGET" -f color="$(color_for "$TIER")" \ + -f description="PR risk grade (advisory shadow check; grader-owned)" >/dev/null 2>&1 \ + || log "label '$TARGET' could not be pre-created (may already exist) — trying the add anyway" + fi + gh api -X POST "repos/$REPO/issues/$PR_NUMBER/labels" -f "labels[]=$TARGET" >/dev/null \ + || fail "could not add label '$TARGET' to $REPO#$PR_NUMBER" + log "added '$TARGET'" +fi + +printf '%s\n' "$TARGET" +exit 0 diff --git a/scripts/pr-risk/grade-pr-risk.sh b/scripts/pr-risk/grade-pr-risk.sh new file mode 100755 index 0000000..c7a68eb --- /dev/null +++ b/scripts/pr-risk/grade-pr-risk.sh @@ -0,0 +1,590 @@ +#!/usr/bin/env bash +# grade-pr-risk.sh — deterministic PR risk grader for CI (the reusable pr-risk.yml workflow). +# +# Grades ONE pull request into a risk tier R0 (safest) .. R3 (riskiest), or refuses with +# `unknown` when an input could not be read. The tier is advisory: this script only COMPUTES; +# the workflow around it leaves the label. Nothing here gates, blocks, comments, or merges. +# +# EXTRACTED from the fleet's offline corpus grader (BE-5507), which remains the backfill +# tool. This copy is the CANONICAL grader for the CI path. +# Three pieces were inlined or dropped to make it self-contained: +# * the shared actor-identity jq (logins / classify_login) is inlined from grade-collect.sh +# (BE-5030) — if you change it here, change it there too; the jq is small on purpose +# * gh-lib.sh (fleet API budget accounting) is dropped — CI runs use the job's GITHUB_TOKEN +# * ledger/report modes are dropped — CI grades one open PR; the corpus tooling stays home +# The risk map and runbook registry are the SAME versioned artifacts; every graded record +# carries map_version + registry_version, so fleet-graded and CI-graded records are comparable +# and a map revision can be replayed against either corpus. +# +# DETERMINISTIC BY CONSTRUCTION: `gh` + `jq` only. No model call, no LLM anywhere in the path. +# +# ── grade = worst(path_floor, provenance, reversibility) ─────────────────────────────────── +# Each axis INDEPENDENTLY proposes a tier and the WORST one wins. That is the whole safety +# property: an axis can only ever move a PR into a RISKIER lane. No axis can pull a PR safer +# than another axis put it, so a mis-modelled runbook cannot buy its way past the path map, +# and an unknown on any axis cannot be averaged away. +# +# AXIS 1 — PATH FLOOR. Reads the VERSIONED map (risk-map.v0.json). In the reusable workflow +# the map ships in Comfy-Org/github-workflows and is checked out at the caller's pinned +# ref, never from the graded PR — so a PR cannot edit the rules that judge it. The floor +# is the WORST tier over every rule any changed path matches. +# AXIS 2 — PROVENANCE. runbook / agent-supervised / human / external. A PR whose identity +# matches a runbook but whose DIFF SHAPE does not is not a runbook, and falls back to its +# underlying class. `external` (fork / first-time contributor) is R3 on provenance alone. +# AXIS 3 — REVERSIBILITY. Single clean revert? Mutates persistent state or deletes data? +# Did tests covering the touched lines actually run? Answered from the changed-path list +# + change types + the PR's own check rollup. +# +# ── THE UNKNOWN CONTRACT ──────────────────────────────────────────────────────────────────── +# An unreadable input yields `unknown` and is REPORTED — never a confident tier. Every axis +# returns {tier, status, reason} and a `status: unknown` makes `tier` null. An overall grade +# with ANY unknown axis is `tier: null, status: unknown` — it is NOT silently graded off the +# axes that did resolve, because a PR whose file list we could not read is exactly the PR +# that might touch auth. The workflow labels these `risk:ungraded`, never a tier. +# +# ./grade-pr-risk.sh --repo my-org/my-repo --pr 123 # grade one PR (open or terminal) +# ./grade-pr-risk.sh --stdin # scorecard JSONL in, graded out +# # (the no-network test surface) +# +# Exit: 0 = graded ok. 1 = graded, grade is unknown (reported). 2 = usage/setup error. +# 3 = the PR itself was unreadable — NOTHING was graded (this is NOT "no risk"). +# +# Deliberately bash (shebang), not zsh — CI runners and the test suite both exercise bash. + +set -uo pipefail # no -e: faults are collected and reported, not fatal mid-run + +SKILL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +REPO="" +RISK_MAP="${PR_RISK_MAP:-$SKILL_DIR/risk-map.v0.json}" +RUNBOOKS="${PR_RISK_RUNBOOKS:-$SKILL_DIR/runbook-registry.v0.json}" +FLEET_LOGINS="${PR_RISK_FLEET_LOGINS:-mattmillerai}" +BOT_LOGINS="${PR_RISK_BOT_LOGINS:-github-actions,dependabot,renovate,coderabbitai,cursor,comfy-pr-bot,web-flow}" +SELF_CONTEXT="${PR_RISK_SELF_CONTEXT:-}" +SELF_RUN_ID="${PR_RISK_SELF_RUN_ID:-}" +PR_NUM="" +MODE="" + +log() { printf '[grade-pr-risk] %s\n' "$*" >&2; } +warn() { printf '[grade-pr-risk] WARN %s\n' "$*" >&2; } +die() { printf '[grade-pr-risk] ERROR %s\n' "$*" >&2; exit 2; } + +usage() { + cat >&2 <<'USAGE' +usage: grade-pr-risk.sh (--repo owner/name --pr N | --stdin) [options] + --repo owner/name repo being graded (required with --pr) + --pr N grade ONE pr live (works on an OPEN pr) and print the graded record + --stdin grade scorecard-shaped JSONL records from stdin (no network; tests) + --map FILE risk map (default risk-map.v0.json beside this script) + --runbooks FILE runbook registry (default runbook-registry.v0.json beside this script) + --fleet-logins CSV GitHub logins that are supervised agents (default mattmillerai) + --bot-logins CSV extra logins treated as bots + --self-context NAME when grading FROM CI: the calling workflow's own name. The check + rollup is then computed from individual contexts EXCLUDING that + workflow's runs — the grading job is itself part of the rollup it + reads, so the raw rollup can never be SUCCESS while it runs. Also + emits checks_pending_excl_self so a caller can wait for CI to settle. + --self-run-id ID PREFERRED over --self-context: exclude by workflow-RUN id + (github.run_id) instead of display name, so a same-named workflow + elsewhere is not excluded. A FAILING check is never excluded either + way — self-exclusion may only ever hide our own pending run. +exit: 0 ok | 1 graded but unknown | 2 usage/setup | 3 PR unreadable (nothing graded) +USAGE +} + +# ---- the map --------------------------------------------------------------------------------- +# An unreadable map is fatal — grading against one would grade every PR R0. VALID JSON IS NOT +# ENOUGH: `{}` parses, so a syntactically-valid but STRUCTURALLY EMPTY map used to sail through +# upstream — `default_tier` fell back to R0, no path rule matched anything, and every PR graded +# R0. So the shape is checked too, and every tier STRING in the file is checked against the tier +# enum here rather than being tolerated downstream: `tier_rank` cannot rank a tier it does not +# know, and a fail-safe rank is a worse answer than a refusal at load time. +read_map() { # -> JSON on stdout, rc 0; rc 1 + reason on stderr + local f="$1" kind="${2:-map}" raw shape + [ -f "$f" ] || { echo "$kind $f not found" >&2; return 1; } + raw="$(cat "$f" 2>/dev/null)" || { echo "cannot read $f" >&2; return 1; } + jq -e . >/dev/null 2>&1 <<<"$raw" || { echo "$kind is not valid JSON" >&2; return 1; } + if [ "$kind" = map ]; then + # shellcheck disable=SC2016 # jq program: $vars belong to jq + shape=' + def known: ["R0","R1","R2","R3"]; + if type != "object" then "not a JSON object" + elif (.path_rules | type) != "array" then "path_rules is missing or not an array" + elif (.path_rules | length) == 0 then "path_rules is EMPTY — an empty rule set would grade every PR R0" + elif ([.path_rules[] | select((.class | type) != "string" or (.paths | type) != "array" or (.paths | length) == 0)] | length) > 0 + then "a path rule is missing a class or a non-empty paths list" + elif ([.path_rules[] | .tier | select(IN(known[]) | not)] | length) > 0 + then "a path rule carries a tier outside \(known) — refusing to rank an unknown tier" + elif (.provenance_tiers | type) != "object" then "provenance_tiers is missing or not an object" + elif ([.provenance_tiers | to_entries[] | select(.key | startswith("_") | not) | .value | select(IN(known[]) | not)] | length) > 0 + then "provenance_tiers carries a tier outside \(known)" + # EVERY provenance class must be MAPPED, not just well-typed. Checking only the values let + # a map that OMITS `external` pass, and the lookup then fell back to a tier of its own + # choosing — silently retiring the "external (fork / first-time contributor) is R3, no + # exceptions" invariant that both the default map comment and the README promise. A class + # nobody mapped is a routing decision nobody made, so it is refused at load time. + # (No apostrophes in here: the whole shape program is a single-quoted shell string.) + elif ((["runbook","agent-supervised","human","external"] - [.provenance_tiers | keys[]]) | length) > 0 + then "provenance_tiers is missing a class: \(["runbook","agent-supervised","human","external"] - [.provenance_tiers | keys[]]) — an unmapped class would be graded off a tier nobody chose" + elif ((.reversibility // {}) | has("test_path_patterns")) and (((.reversibility // {}).test_path_patterns | type) != "array") + then "reversibility.test_path_patterns is present but not an array" + elif (.default_tier // "R0") as $d | ($d | IN(known[])) | not then "default_tier is outside \(known)" + elif [(.reversibility // {}) | .no_green_checks_tier, .no_test_touched_tier, .clean_tier | select(. != null and (IN(known[]) | not))] | length > 0 + then "a reversibility tier is outside \(known)" + else empty end' + else + # shellcheck disable=SC2016 # jq program: $vars belong to jq + shape=' + if type != "object" then "not a JSON object" + elif (.runbooks | type) != "array" then "runbooks is missing or not an array" + elif ([.runbooks[] | select((.id | type) != "string" or (.identity | type) != "object" or (.shape | type) != "object")] | length) > 0 + then "a runbook entry is missing an id, an identity or a shape assertion" + elif ([.runbooks[] | select(((.identity.logins // []) | type) != "array" or ((.identity.logins // []) | length) == 0)] | length) > 0 + then "a runbook identity has no logins — identity is the author login, so an entry without one can never assert" + else empty end' + fi + local why; why="$(jq -r "$shape" <<<"$raw" 2>/dev/null)" + [ -z "$why" ] || { echo "$kind is structurally invalid: $why" >&2; return 1; } + printf '%s' "$raw" +} + +# ---- shared actor-identity resolution (jq) --------------------------------------------------- +# INLINED from agent-work's grade-collect.sh (BE-5030), the one definition the fleet's graders +# share. Small on purpose; if you change it here, change it there. NEVER classify on +# commit.author.name/email — that string is `git config user.email` and is forgeable by the +# thing being graded. The PR author login is the resolution GitHub itself made. +# shellcheck disable=SC2016 # jq program: $vars belong to jq +IDENTITY_JQ=' + def logins: ascii_downcase | [splits("[,[:space:]]+")] | map(select(. != "")); + # classify_login: the login STRING -> "unknown" | "bot" | "fleet" | "human". The bot test + # (a `[bot]` suffix plus the caller-supplied bot list) runs BEFORE the fleet test so a bot + # that also appears in $fleetl still reads as a bot. + def classify_login($fleetl; $botl): + if . == null or . == "" then "unknown" + else (ascii_downcase) as $l + | if ($l | endswith("[bot]")) or (($botl | index($l)) != null) then "bot" + elif ($fleetl | index($l)) != null then "fleet" + else "human" end end; +' + +# ---- the grading jq program ------------------------------------------------------------------- +# ONE jq program grades every record. Input: one scorecard-shaped record per line. Output: the +# same record + a `risk` block carrying the tier, the PER-AXIS tiers, the reason, AND the map +# version that produced them — so a later map revision can be REPLAYED against accumulated +# records rather than reconstructed. +grade_program() { +cat <<'JQ' + # --- glob -> anchored regex. `**` crosses separators, `*` does not. ------------------- + # Staged via \u0002 / \u0001 placeholders rather than a lookbehind: the placeholder cannot appear in a path, + # and staging it this way makes "** before *" unambiguous without relying on regex-engine + # lookbehind support. + def glob2re: + gsub("(?[.+?^$(){}|\\[\\]\\\\])"; "\\\(.c)") + | gsub("\\*\\*/"; "\u0002") | gsub("\\*\\*"; "\u0001") | gsub("\\*"; "[^/]*") + | gsub("\u0002"; "(?:.*/)?") | gsub("\u0001"; ".*") + | "^" + . + "$"; + def matches_any($globs): . as $p | any($globs[]?; . as $g | ($p | test($g | glob2re))); + # An UNRECOGNIZED tier ranks as the RISKIEST, never the safest. read_map already refuses a + # map carrying one, so this is defence in depth — but the direction matters: defaulting to + # R0 would let a typo'd or future tier silently DOWNGRADE a PR's grade, which inverts the + # "unknown is never safe" contract in the one place it decides routing. + def tier_rank: {"R0":0,"R1":1,"R2":2,"R3":3}[.] // 3; + def worst($a; $b): if ($a | tier_rank) >= ($b | tier_rank) then $a else $b end; + + $map as $M | $rb as $RB + | ($fleet | logins) as $fleetl | ($bots | logins) as $botl + | ($M.default_tier // "R0") as $DEF + | . as $r + | (.changed_paths) as $paths + | ([$paths[]? | .path]) as $plist + # EVERY path the diff touches, DESTINATION *and* ORIGIN. A RENAMED file is recorded under its + # destination only, so matching `.path` alone let a rename out of a sensitive directory escape + # the floor entirely: move `.github/workflows/deploy.yml` or an `auth/` file to an innocuous + # name and the R3 rule that guards it never matches. The origin path is part of what the PR + # did, so it is graded too. + | ([$paths[]? | .path, (.previous_path // empty)] | unique) as $pall + + # ---- AXIS 1: PATH FLOOR --------------------------------------------------------------- + # WORST over every rule any changed path matches. An R0 rule (docs, tests) can never cancel + # an R3 rule (migrations) in the same PR — that is why this is a max, not a last-match-wins. + | (if $r.changed_paths_status != "ok" or $paths == null + then {tier:null, status:"unknown", + reason:("changed-path list is " + ($r.changed_paths_status // "absent") + " — a PR whose files we cannot read is exactly the PR that might touch auth"), + classes:null} + else + ([$M.path_rules[]? | . as $rule | select($pall | any(. as $p | $p | matches_any($rule.paths)))]) as $hit + | {tier: (reduce $hit[] as $h ($DEF; worst(.; $h.tier))), + status:"ok", + reason: (if ($hit|length) == 0 then "no mapped path touched — floor \($DEF)" + else "matched " + ([$hit[] | "\(.class)=\(.tier)"] | join(", ")) end), + classes: [$hit[] | .class]} + end) as $A1 + + # ---- AXIS 2: PROVENANCE --------------------------------------------------------------- + # Identity first (server-attributed author login, classified by the shared resolver), then + # the runbook shape assertion. A claimed runbook that fails its shape assertion is NOT a + # runbook — provenance alone is never sufficient. + | ($r.author // null) as $author + | ($author | classify_login($fleetl; $botl)) as $cls + | (($r.labels // []) | index("agent-coded") != null) as $agent_coded + # The label list has a STATUS TWIN for the same reason the file list does: `agent-coded` is + # read from it, and a TRUNCATED label list answers "is this agent-coded?" with a confident no + # it has not earned. A consumer map that splits agent-supervised from human would then grade + # off a list nobody confirmed was complete. + | ($r.labels_status // "ok") as $lbst + # `external` is decided from is_fork + author_association, and those arrive with a STATUS + # twin — whether they were actually read has to be asked before they are believed. Reading + # an un-collected `is_fork` as "not a fork" would make `external => R3` — the one provenance + # class never routed unattended — silently unreachable. Unread is `unknown`, and `unknown` + # refuses to grade the axis. + | ($r.provenance_status // (if ($r | has("is_fork")) then "ok" else "absent" end)) as $pvst + | (if $pvst != "ok" or $lbst != "ok" then "unknown" + elif ($r.is_fork // false) or (($r.author_association // "") | IN("FIRST_TIME_CONTRIBUTOR","FIRST_TIMER","NONE")) + then "external" + elif $agent_coded or $cls == "fleet" then "agent-supervised" + elif $cls == "bot" then "runbook-candidate" + elif $cls == "human" then "human" + else "unknown" end) as $base_class + # The shape assertion: identity match AND every changed path inside permitted_paths AND the + # diff-shape bounds AND the title. Anything short of all four is a shape FAILURE, recorded. + # + # IDENTITY IS THE AUTHOR LOGIN, AND THE HEAD REF NARROWS IT — never the other way round. + # This was `login_match or (has_patterns and head_ref_match)`, and because jq binds `and` + # tighter than `or` that made a matching HEAD REF ALONE sufficient: anyone who names a + # branch `.../generated-x` presents sdk-spec-push's identity without being its author. The + # login test is therefore REQUIRED, and head_ref_patterns are an ADDITIONAL condition where + # the producer declares them. The parentheses are load-bearing — do not let this collapse + # back into a bare or/and chain. + | ([$RB.runbooks[]? | . as $bk + | select((($bk.identity.logins // []) | any(. as $l | ($author // "") | ascii_downcase == ($l | ascii_downcase))) + and ((($bk.identity.head_ref_patterns // []) | length) == 0 + or (($r.head_ref // "") | matches_any($bk.identity.head_ref_patterns // [])))) + | {id: $bk.id, lane: $bk.lane, daily_cap: $bk.daily_cap, + paths_ok: (if $r.changed_paths_status != "ok" then null + else ($pall | length) > 0 and all($pall[]; matches_any($bk.permitted_paths // [])) end), + shape_ok: (($r.changed_files // 0) <= ($bk.shape.max_changed_files // 1e9) + and ($r.additions // 0) <= ($bk.shape.max_additions // 1e9) + and ($r.deletions // 0) <= ($bk.shape.max_deletions // 1e9)), + title_ok: (($bk.shape.title_regex // null) as $tr + | if $tr == null then true else (($r.title // "") | test($tr)) end)}]) as $cand + | ([$cand[] | select(.paths_ok == true and .shape_ok and .title_ok)] | first) as $rbk + | ([$cand[] | select(.paths_ok != true or (.shape_ok | not) or (.title_ok | not)) + | "\(.id): paths=\(.paths_ok) shape=\(.shape_ok) title=\(.title_ok)"]) as $shape_failures + # A matched runbook NEVER overrides `external`. An outside diff that also happens to assert + # a runbook's shape is still an outside diff — letting a runbook match downgrade it would + # hand any fork a route past the rule by imitating a known producer's shape. + | (if $base_class == "external" then "external" + elif $rbk != null then "runbook" + elif $base_class == "runbook-candidate" then "human" # a bot we do not have a runbook for is not trusted + else $base_class end) as $prov + | (if $prov == "unknown" or $author == null + then {tier:null, status:"unknown", + reason:(if $pvst != "ok" + then "fork / author-association were not collected (\($pvst)) — the `external` provenance class is un-decidable, and defaulting it to 'not a fork' would silently retire the external => R3 rule" + elif $lbst != "ok" + then "the PR label list is \($lbst) — `agent-coded` cannot be read off a truncated list, and reading it as absent would be a confident answer from a source nobody finished reading" + else "PR author did not resolve to a GitHub account — provenance is unattributable" end), + provenance:null} + # An UNMAPPED class falls back to the RISKIEST tier, never R1 — same direction as + # tier_rank. read_map now REQUIRES all four classes, so this is defence in depth; the + # direction is what matters, because defaulting to R1 is how a map that omitted `external` + # used to grade a fork the same as a teammate. + else {tier: (($M.provenance_tiers // {})[$prov] // "R3"), status:"ok", + provenance: $prov, + runbook: (if $rbk == null then null else $rbk.id end), + runbook_lane: (if $rbk == null then null else $rbk.lane end), + shape_failures: $shape_failures, + reason: (if $rbk != null then "runbook \($rbk.id) — identity, permitted paths, diff shape and title all assert" + elif ($shape_failures | length) > 0 then "\($prov) — claimed a runbook identity but the shape assertion failed (" + ($shape_failures | join("; ")) + ")" + else $prov end)} + end) as $A2 + + # ---- AXIS 3: REVERSIBILITY ------------------------------------------------------------ + # Four questions, answered deterministically and in worsening order: + # mutates persistent state / deletes data? -> R3 (reverting code does not restore state) + # deletes a file under a sensitive class? -> R3 (not a single clean revert) + # did tests covering these lines actually run? no green rollup -> R2; green but no test + # file touched -> R1; green and a test touched -> R0. + # `flag_gated` is RECORDED but never LOWERS a tier — an axis may only move riskier. + | ($M.reversibility // {}) as $RV + # Was the check rollup READ? `checks_status: ok` with a null `checks_state` is GitHub + # genuinely reporting no rollup for this head (a repo with no CI) and IS gradeable — the + # honest R2. A rollup that was never collected is not: reading it as "no green rollup" + # would be a confident answer computed from a source nobody read. + | ($r.checks_status // (if ($r | has("checks_state")) then "ok" else "absent" end)) as $ckst + | (if $r.changed_paths_status != "ok" or $paths == null + then {tier:null, status:"unknown", reason:"changed-path list is \($r.changed_paths_status // "absent") — reversibility is un-answerable without the paths"} + elif $ckst != "ok" + then {tier:null, status:"unknown", reason:"check rollup was not collected (\($ckst)) — 'did tests covering these lines run?' is un-answerable"} + else + ([$A1.classes[]? | select(. as $c | ($RV.irreversible_classes // []) | index($c))]) as $irrev + # A RENAME removes the ORIGIN path as surely as a delete does, so renaming a file OUT of + # a sensitive directory counts here too — otherwise `git mv auth/x.go misc/x.go` reads as + # a clean revert. + | (([$paths[] | select(.change_type == "DELETED") | .path] + + [$paths[] | select(.change_type == "RENAMED") | (.previous_path // empty)]) | unique) as $deleted + # Sensitive-class match over the DELETED paths ONLY. This was computed from $A1.classes — + # the classes matched by ANY changed file — so a PR that merely MODIFIED an auth file + # while deleting an unrelated README reported "deletes N file(s) under a sensitive class" + # and pinned reversibility R3. The two sets have to be the same set for the sentence the + # reason string prints to be true. + | ([$M.path_rules[]? | . as $rule | select($deleted | any(. as $p | $p | matches_any($rule.paths))) | .class] | unique) as $del_classes + | ($del_classes | any(. as $c | ($RV.delete_sensitive_classes // []) | index($c))) as $del_sensitive + | ($pall | any(matches_any($M.flippable_flag_paths // []))) as $flag + # "Did a test file change?" comes from the VERSIONED map when it says, and falls back to + # the built-in regex when it does not. Hardcoding it meant a consumer could not fix it + # with .github/risk.json — the one lever the workflow gives them — and the regex misses + # `test_*.py`, `*_test.py`, `*Test.java` and `*_spec.rb`, so whole ecosystems could never + # reach clean_tier and sat at R1 forever. + # DESTINATION paths only ($plist, not $pall): renaming `x_test.go` to `x.go` REMOVES a + # test, and matching the origin path would read that as "a test file changed" and let it + # reach clean_tier. The path floor uses $pall because widening it there can only ever + # grade RISKIER; widening it here would grade safer, which is the wrong direction. + | (($RV.test_path_patterns // []) as $tp + | if ($tp | length) > 0 then ($plist | any(matches_any($tp))) + else ($plist | any(test("(_test\\.|\\.test\\.|\\.spec\\.|(^|/)tests?/|-test\\.sh$)"))) end) as $touched_test + | ($r.checks_state // null) as $checks + | (if ($irrev | length) > 0 + then {t:"R3", why:("touches " + ($irrev|join(", ")) + " — mutates persistent state or deletes data; reverting the code does not restore it")} + elif (($deleted | length) > 0 and $del_sensitive) + then {t:"R3", why:("removes " + ($deleted|length|tostring) + " file(s) under a sensitive class (" + ($del_classes|join(", ")) + ") — not a single clean revert")} + elif $checks == null or $checks != "SUCCESS" + then {t: ($RV.no_green_checks_tier // "R2"), + why:("no GREEN check rollup (" + ($checks // "absent") + ") — cannot answer whether tests covering these lines actually ran")} + elif ($touched_test | not) + then {t: ($RV.no_test_touched_tier // "R1"), why:"checks green but the diff touches no test file — nothing proves the suite covers THESE lines"} + else {t: ($RV.clean_tier // "R0"), why:"single clean revert, no persistent-state mutation, checks green, tests touched"} end) as $d + | {tier:$d.t, status:"ok", reason:$d.why, flag_gated:$flag, deleted_files:($deleted|length)} + end) as $A3 + + # ---- worst wins ------------------------------------------------------------------------ + # ANY unknown axis makes the OVERALL grade unknown. Grading off the axes that did resolve + # would present a partially-read PR as a confident tier, which is the failure the unknown + # contract exists to forbid. + | ([$A1, $A2, $A3] | map(select(.status != "ok"))) as $unk + | . + {risk: { + map_version: ($M.map_version // "unknown"), + registry_version: ($RB.registry_version // "unknown"), + graded_at: $now, + tier: (if ($unk|length) > 0 then null + else (reduce [$A1.tier, $A2.tier, $A3.tier][] as $t ("R0"; worst(.; $t))) end), + status: (if ($unk|length) > 0 then "unknown" else "ok" end), + reason: (if ($unk|length) > 0 + then "unknown: " + ([$unk[] | .reason] | join(" | ")) + else "worst of path_floor=\($A1.tier), provenance=\($A2.tier), reversibility=\($A3.tier)" end), + axes: {path_floor: $A1, provenance: $A2, reversibility: $A3}}} +JQ +} + +# grade_stream — stdin: scorecard records; stdout: graded records +grade_stream() { + local map="$1" rb="$2" + jq -c --argjson map "$map" --argjson rb "$rb" \ + --arg fleet "$FLEET_LOGINS" --arg bots "$BOT_LOGINS" --arg now "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + "$IDENTITY_JQ$(grade_program)" +} + +# ---- one live PR ------------------------------------------------------------------------------ +# Shaped into the same scorecard-like record the fleet's collector emits, so the SAME grading +# program grades a live PR here and a collected one offline; two graders would drift. +# +# THE GRADING JOB IS PART OF THE ROLLUP IT READS. When this script runs inside a workflow on +# the PR it is grading, its own check run is in progress, so the raw statusCheckRollup.state +# can never be SUCCESS at grade time — every CI-time grade would floor at R2 and the tiers +# would be an artifact of the measurement. With --self-run-id (preferred) or --self-context +# the rollup is therefore recomputed from the individual contexts, EXCLUDING our own check +# runs: any remaining pending => PENDING, any remaining SUCCESS => SUCCESS, remaining contexts +# that are all SKIPPED/NEUTRAL => NEUTRAL, nothing else on the commit => null (the honest +# "no CI" case). `checks_pending_excl_self` is emitted alongside so a CI caller can wait for +# the rest of the rollup to settle instead of labeling a snapshot of half-finished checks. +# More than 100 contexts is `unknown`, never a truncated aggregate. +# +# TWO RULES KEEP SELF-EXCLUSION FROM BECOMING A BLINDFOLD: +# * A FAILURE is scanned over ALL contexts, self INCLUDED. Exclusion may only ever hide our +# own PENDING; it must never be able to hide a red check. Matching on the workflow NAME +# dropped every sibling job of a consumer that put the grading job inside its existing CI +# workflow, so a FAILED test job vanished and the remaining green contexts aggregated to +# SUCCESS — reversibility then graded R0/R1 on a red PR. +# * SUCCESS requires at least one context that actually CONCLUDED SUCCESS. A rollup of +# nothing but SKIPPED / NEUTRAL / null establishes nothing about whether tests ran, which +# is the axis's whole question, so it aggregates to NEUTRAL and floors at R2. +# --self-run-id also makes the match EXACT (github.run_id), so a same-named workflow in the +# consumer repo is no longer excluded. A caller that still embeds the grading job inside a +# multi-job workflow has all of that run's siblings excluded and lands on the honest R2 floor +# rather than a false green — enroll pr-risk as its OWN workflow to grade off a full rollup. +fetch_pr_record() { # -> record JSON on stdout, rc 1 on an unreadable PR + local repo="$1" num="$2" q resp files fstatus + # shellcheck disable=SC2016 # GraphQL: $vars are query variables + q='query($owner:String!,$name:String!,$num:Int!){ + repository(owner:$owner,name:$name){ pullRequest(number:$num){ + number title state isDraft createdAt updatedAt closedAt mergedAt + author{ login } authorAssociation baseRefName headRefName isCrossRepository + additions deletions changedFiles + labels(first:100){ pageInfo{ hasNextPage } nodes{ name } } + commits(last:1){ nodes{ commit{ statusCheckRollup{ state + contexts(first:100){ pageInfo{ hasNextPage } nodes{ __typename + ... on CheckRun{ name status conclusion checkSuite{ workflowRun{ databaseId workflow{ name } } } } + ... on StatusContext{ context state } } } } } } } + } } }' + resp="$(gh api graphql -f query="$q" -F owner="${repo%%/*}" -F name="${repo##*/}" -F num="$num" 2>/dev/null)" || return 1 + jq -e '.data.repository.pullRequest.number != null' >/dev/null 2>&1 <<<"$resp" || return 1 + + # ---- the changed-file list comes from REST, not GraphQL ------------------------------------ + # GraphQL's `files` connection cannot answer this axis. Two reasons, both structural: + # * PullRequestChangedFile has NO previous-path field (its whole field set is additions, + # changeType, deletions, path, viewerViewedState), so a RENAME is only ever visible under + # its DESTINATION — the origin path, which is what the sensitive-path floor needs, is + # simply not on offer. + # * `files(first:100)` capped the list at 100 and graded everything above it `unknown`, + # which put exactly the PRs a risk grade helps most (the 150-file ones) in the ungraded + # lane. + # REST /pulls/{n}/files answers both: `previous_filename` carries the origin, and --paginate + # walks every page. GitHub caps that endpoint at 3000 files; a short read is detected below + # against GraphQL's own changedFiles count and reported `unknown` rather than graded. + fstatus=ok + files="$(gh api --paginate "repos/$repo/pulls/$num/files?per_page=100" \ + --jq '.[] | {path: .filename, + previous_path: (.previous_filename // null), + additions: .additions, deletions: .deletions, + change_type: ((.status // "") | ascii_upcase + | if . == "REMOVED" then "DELETED" else . end)}' 2>/dev/null \ + | jq -sc '.')" || { files=null; fstatus=unreadable; } + [ -n "$files" ] || { files=null; fstatus=unreadable; } + + jq -c --arg repo "$repo" --arg self "$SELF_CONTEXT" --arg selfrun "$SELF_RUN_ID" \ + --argjson files "$files" --arg fstatus "$fstatus" ' + def is_failing: (.__typename == "CheckRun" and ((.conclusion // "") | IN("FAILURE","TIMED_OUT","CANCELLED","ACTION_REQUIRED","STARTUP_FAILURE"))) + or (.__typename == "StatusContext" and ((.state // "") | IN("ERROR","FAILURE"))); + def is_pending: (.__typename == "CheckRun" and ((.status != "COMPLETED") or ((.conclusion // "") == "STALE"))) + or (.__typename == "StatusContext" and ((.state // "") | IN("PENDING","EXPECTED"))); + def is_success: (.__typename == "CheckRun" and ((.conclusion // "") == "SUCCESS")) + or (.__typename == "StatusContext" and ((.state // "") == "SUCCESS")); + # Ours by RUN ID when the caller supplied one (exact), else by workflow display name. + def is_self($self; $selfrun): .__typename == "CheckRun" + and (if $selfrun != "" then ((.checkSuite.workflowRun.databaseId // -1) | tostring) == $selfrun + else (($self != "") and ((.checkSuite.workflowRun.workflow.name // "") == $self)) end); + .data.repository.pullRequest + | ([.labels.nodes[]? | .name] | sort) as $labels + | (.labels.pageInfo.hasNextPage // false) as $labels_trunc + | (.commits.nodes[0].commit.statusCheckRollup) as $ro + # Effective check state. Without a self selector: the raw rollup, the same signal the + # offline corpus grader reads. With one: the self-excluding aggregate described above. + | (if ($self == "" and $selfrun == "") or ($ro == null) + then {state: ($ro.state // null), pending: false, status: "ok"} + elif ($ro.contexts.pageInfo.hasNextPage // false) + then {state: null, pending: false, status: "unknown"} + else + ([$ro.contexts.nodes[]]) as $all + | ([$all[] | select(is_self($self; $selfrun) | not)]) as $ctx + | (if any($all[]; is_failing) then "FAILURE" + elif any($ctx[]; is_pending) then "PENDING" + elif any($ctx[]; is_success) then "SUCCESS" + elif ($ctx | length) > 0 then "NEUTRAL" + else null end) as $st + | {state: $st, pending: ($st == "PENDING"), status: "ok"} + end) as $checks + # The changed-file read, with its status twin. A list SHORTER than changedFiles is a + # truncated read, and a truncated read is `unknown` — never a floor computed from the + # subset of files that happened to fit. + | (if $fstatus != "ok" or $files == null + then {list: null, status: "unreadable", reason: "the changed-file list could not be read from the pulls/{n}/files API"} + elif ($files | length) < (.changedFiles // 0) + then {list: null, status: "unknown", + reason: "changed-file list is short (\($files | length) of \(.changedFiles)) — GitHub caps the files API at 3000 files"} + else {list: $files, status: "ok", reason: null} end) as $fread + | {schema_version:3, repo:$repo, pr:.number, title:.title, author:(.author.login // null), + author_association:.authorAssociation, is_fork:(.isCrossRepository // false), + labels:$labels, agent_coded:($labels | index("agent-coded") != null), + labels_status:(if $labels_trunc then "unknown" else "ok" end), + created_at:.createdAt, updated_at:.updatedAt, closed_at:.closedAt, merged_at:.mergedAt, + base_ref:.baseRefName, head_ref:.headRefName, is_draft:.isDraft, + additions:.additions, deletions:.deletions, changed_files:.changedFiles, + # The status twins the fleet collector emits, so a live grade and a corpus grade of the + # same PR read the same fields. checks_status is `unknown` only when the context list + # was truncated; a null state is an answer from GitHub, not an un-asked question. + checks_state:$checks.state, + checks_status:$checks.status, provenance_status:"ok", + checks_pending_excl_self:$checks.pending, + outcome:(if .mergedAt != null then "merged" elif .state == "CLOSED" then "closed_unmerged" else "open" end), + changed_paths:$fread.list, + changed_paths_status:$fread.status, + changed_paths_reason:$fread.reason}' <<<"$resp" +} + +# ---- main -------------------------------------------------------------------------------------- +main() { + while [ $# -gt 0 ]; do + case "$1" in + --repo) REPO="${2:-}"; shift 2 || die "--repo needs a value" ;; + --pr) PR_NUM="${2:-}"; MODE="pr"; shift 2 || die "--pr needs a value" ;; + --stdin) MODE="stdin"; shift ;; + --map) RISK_MAP="${2:-}"; shift 2 || die "--map needs a value" ;; + --runbooks) RUNBOOKS="${2:-}"; shift 2 || die "--runbooks needs a value" ;; + --fleet-logins) FLEET_LOGINS="${2:-}"; shift 2 || die "--fleet-logins needs a value" ;; + --bot-logins) BOT_LOGINS="${2:-}"; shift 2 || die "--bot-logins needs a value" ;; + --self-context) SELF_CONTEXT="${2:-}"; shift 2 || die "--self-context needs a value" ;; + --self-run-id) SELF_RUN_ID="${2:-}"; shift 2 || die "--self-run-id needs a value" ;; + -h|--help) usage; exit 0 ;; + *) usage; die "unknown argument '$1'" ;; + esac + done + + [ -n "$MODE" ] || { usage; die "one of --pr or --stdin is required"; } + command -v jq >/dev/null 2>&1 || die "jq not found on PATH" + if [ "$MODE" = pr ]; then + [[ "$REPO" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || die "bad --repo '$REPO' (want owner/name)" + [[ "$PR_NUM" =~ ^[0-9]+$ ]] || die "bad --pr '$PR_NUM'" + [ -z "$SELF_RUN_ID" ] || [[ "$SELF_RUN_ID" =~ ^[0-9]+$ ]] || die "bad --self-run-id '$SELF_RUN_ID' (want a numeric github.run_id)" + command -v gh >/dev/null 2>&1 || die "gh not found on PATH" + fi + + # The REASON is captured alongside the value in ONE call each: re-running read_map just to + # collect its stderr would read the file twice, and the two reads could disagree. An unusable + # map is fatal — grading against one would grade every PR R0. + local map rb errf + errf="$(mktemp "${TMPDIR:-/tmp}/grade-pr-risk-err.XXXXXX")" || die "mktemp failed" + map="$(read_map "$RISK_MAP" map 2>"$errf")" \ + || die "risk map unusable ($RISK_MAP): $(tr '\n' ' ' < "$errf")— refusing to grade: an unusable map would grade every PR R0" + rb="$(read_map "$RUNBOOKS" runbooks 2>"$errf")" \ + || die "runbook registry unusable ($RUNBOOKS): $(tr '\n' ' ' < "$errf")— refusing to grade" + rm -f "$errf" + + case "$MODE" in + pr) + local rec + rec="$(fetch_pr_record "$REPO" "$PR_NUM")" \ + || { warn "PR $REPO#$PR_NUM was UNREADABLE — nothing graded (this is NOT 'no risk')"; exit 3; } + local graded; graded="$(printf '%s\n' "$rec" | grade_stream "$map" "$rb")" + [ -n "$graded" ] || die "the grading pass produced nothing for $REPO#$PR_NUM — NOTHING was graded; this is not 'no risk'" + jq . <<<"$graded" + local st; st="$(jq -r '.risk.status' <<<"$graded")" + [ "$st" = ok ] || { warn "grade is UNKNOWN for $REPO#$PR_NUM"; exit 1; } + exit 0 ;; + stdin) + # Per-line tolerant read, same contract as the fleet's corpus path: one corrupt line + # drops exactly that line, never every valid record after it. + local tmp; tmp="$(mktemp -d "${TMPDIR:-/tmp}/grade-pr-risk.XXXXXX")" || die "mktemp failed" + # shellcheck disable=SC2064 + trap "rm -rf '$tmp'" EXIT + jq -R -c 'fromjson? | select(type == "object")' > "$tmp/in.jsonl" 2>/dev/null + local kept; kept="$(wc -l < "$tmp/in.jsonl" | tr -d ' ')" + [ "$kept" -gt 0 ] || { warn "stdin yielded no parseable records — NOTHING graded"; exit 3; } + # A grading pass that FAILED must never read as a graded corpus: both the rc and the + # output count are checked, so a pass that produced nothing can never report a clean run. + if ! grade_stream "$map" "$rb" < "$tmp/in.jsonl" > "$tmp/graded.jsonl"; then + die "the grading pass FAILED (jq returned non-zero) — NOTHING was graded" + fi + local produced; produced="$(wc -l < "$tmp/graded.jsonl" | tr -d ' ')" + [ "$produced" -eq "$kept" ] || die "the grading pass produced $produced record(s) from $kept input(s) — refusing to report a partial pass as a clean one" + cat "$tmp/graded.jsonl" + local unknown; unknown="$(jq -s '[.[] | select(.risk.status != "ok")] | length' "$tmp/graded.jsonl")" + [ "$unknown" -gt 0 ] && { warn "$unknown record(s) graded UNKNOWN — reported, never a confident tier"; exit 1; } + exit 0 ;; + esac +} + +# Sourceable without side effects (the test suite sources this file to exercise the grader +# directly); only a direct invocation runs it. +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + main "$@" +fi diff --git a/scripts/pr-risk/risk-map.v0.json b/scripts/pr-risk/risk-map.v0.json new file mode 100644 index 0000000..5f05718 --- /dev/null +++ b/scripts/pr-risk/risk-map.v0.json @@ -0,0 +1,98 @@ +{ + "map_version": "v0-generic", + "_comment": [ + "The DEFAULT PR risk map for the reusable pr-risk.yml workflow. VERSIONED so a later", + "revision can be REPLAYED against accumulated graded records rather than reconstructed —", + "every graded record carries the map_version that produced it.", + "", + "This default is deliberately GENERIC: it names ecosystem-level risk classes (CI, deps,", + "migrations, IaC, secrets, API contracts), never any one repo's internal layout. A consumer", + "repo sharpens it by committing .github/risk.json; the workflow reads that file from the", + "PR's BASE ref, so a PR cannot edit the rules that judge it — the same defence pr-size.yml", + "uses for linguist-generated.", + "", + "TIERS: R0 safest .. R3 riskiest. grade = worst(path_floor, provenance, reversibility):", + "each axis independently PROPOSES a tier and the worst wins, so no axis can ever move a PR", + "into a SAFER lane than another axis put it.", + "", + "GLOBS: `**` crosses directory separators, `*` does not. Matching is whole-path anchored,", + "so a glob WITHOUT a leading `**/` matches ROOT-LEVEL files only — prefix every rule that", + "should match at any depth, or it silently matches nothing in a real tree.", + "", + "Matching covers every path the diff touches, DESTINATION and ORIGIN: a RENAMED file is", + "graded under its previous path too, so moving a file out of a guarded directory cannot", + "escape that directory's rule." + ], + "tiers": ["R0", "R1", "R2", "R3"], + "default_tier": "R0", + + "path_rules": [ + { "class": "risk-map", "tier": "R3", "why": "touching the map, the registry or the grader itself is automatically R3 — a PR must not be able to lower its own grade, and a PR that edits the judge must not be graded safest by that judge", + "paths": ["**/risk-map.*.json", "**/runbook-registry.*.json", "**/pr-risk/**", + ".github/risk.json", ".github/risk-runbooks.json", + "**/.github/risk.json", "**/.github/risk-runbooks.json"] }, + { "class": "codeowners", "tier": "R3", "why": "CODEOWNERS decides who must review — editing it edits the review gate", + "paths": ["CODEOWNERS", ".github/CODEOWNERS", "docs/CODEOWNERS", "**/CODEOWNERS"] }, + { "class": "ci", "tier": "R3", "why": "any workflow file is automatically R3 — a workflow runs with repo credentials", + "paths": [".github/workflows/**", ".github/actions/**", "**/.github/workflows/**"] }, + { "class": "auth", "tier": "R3", "why": "authentication / trust boundary", + "paths": ["**/auth/**", "**/middleware/auth*", "**/auth_validator*", "**/authvalidator/**", "**/*token_validator*"] }, + { "class": "billing", "tier": "R3", "why": "billing / credits — money moves", + "paths": ["**/billing/**", "**/pricing/**", "**/credits/**"] }, + { "class": "migrations", "tier": "R3", "why": "schema migrations mutate persistent state and do not cleanly revert", + "paths": ["**/migrations/**", "**/migrate/**", "databases/**"] }, + { "class": "iac", "tier": "R3", "why": "infrastructure as code — blast radius is the estate, not the request", + "paths": ["infrastructure/**", "terraform/**", "**/*.tf", "charts/**", "helm/**"] }, + { "class": "deps", "tier": "R3", "why": "dependency manifests pull in code nobody in the PR wrote — every ecosystem, not just one (a package.json `scripts` block is supply-chain-relevant in exactly the same way go.mod is)", + "paths": ["**/go.mod", "**/go.sum", "**/go.work*", + "**/package.json", "**/package-lock.json", "**/pnpm-lock.yaml", "**/yarn.lock", "**/npm-shrinkwrap.json", + "**/requirements*.txt", "**/pyproject.toml", "**/Pipfile", "**/Pipfile.lock", "**/poetry.lock", "**/uv.lock", "**/setup.py", + "**/Cargo.toml", "**/Cargo.lock", + "**/Gemfile", "**/Gemfile.lock", "**/composer.json", "**/composer.lock", "**/*.gemspec"] }, + { "class": "api-contract", "tier": "R3", "why": "cross-repo API contract — consumers break out of band", + "paths": ["openapi.yaml", "openapi.yml", "**/openapi.yaml", "**/openapi.yml", "**/*.proto"] }, + { "class": "secrets", "tier": "R3", "why": "secrets handling", + "paths": ["**/secrets/**", "**/*secret*.go", "**/*secret*.ts", "secrets.env*", "**/kms/**", "**/*credential*"] }, + { "class": "data-deletion", "tier": "R3", "why": "data deletion / compliance — irreversible by construction", + "paths": ["**/deletion/**", "**/gdpr/**"] }, + + { "class": "docs", "tier": "R0", "why": "prose only", + "paths": ["**/*.md", "docs/**", "**/*.txt", "LICENSE", "**/*.mdx"] }, + { "class": "tests", "tier": "R0", "why": "test-only changes cannot break production behaviour", + "paths": ["**/*_test.go", "**/*.test.ts", "**/*.test.tsx", "**/*.spec.ts", "**/test/**", "**/tests/**", "**/*-test.sh"] } + ], + "_path_rule_order": "R3 classes are listed FIRST but ORDER DOES NOT MATTER: the path floor is the WORST tier over every rule any changed path matches, so an R0 doc rule can never cancel an R3 migration rule in the same PR.", + + "provenance_tiers": { + "runbook": "R0", + "agent-supervised": "R1", + "human": "R1", + "external": "R3", + "_why": "external (a fork / first-time contributor) is R3 on provenance alone — an unreviewed outside diff is the one provenance class never routed unattended. runbook is R0 ONLY when the shape assertion also holds (see the runbook registry); a claimed-but-shape-failed runbook falls back to its underlying class." + }, + + "reversibility": { + "irreversible_classes": ["migrations", "data-deletion", "secrets"], + "_why_irreversible": "a PR touching these mutates persistent state or deletes data — reverting the code does not restore the state, so reversibility is pinned R3 regardless of how clean the diff is.", + "delete_sensitive_classes": ["migrations", "data-deletion", "iac", "auth", "billing", "api-contract"], + "_why_delete": "a DELETED file under a persistent-state or trust-boundary class is not a clean single revert — R3.", + "no_green_checks_tier": "R2", + "_why_no_green": "did tests covering the touched lines actually run? A PR with no GREEN check rollup cannot answer yes, so it cannot be graded safer than R2.", + "no_test_touched_tier": "R1", + "_why_no_test": "green checks but the diff touches no test file: the suite ran, but nothing proves it covers THESE lines.", + "clean_tier": "R0", + "test_path_patterns": ["**/*_test.go", + "**/*.test.ts", "**/*.test.tsx", "**/*.test.js", "**/*.test.jsx", + "**/*.spec.ts", "**/*.spec.tsx", "**/*.spec.js", + "**/test_*.py", "**/*_test.py", + "**/*_test.rb", "**/*_spec.rb", + "**/*Test.java", "**/*Tests.java", "**/*Test.kt", "**/*Test.cs", + "**/*_test.rs", "**/*_test.exs", "**/*_test.php", "**/*Test.php", + "**/test/**", "**/tests/**", "**/testdata/**", "**/__tests__/**", + "**/*-test.sh", "**/test_*.sh"], + "_why_test_patterns": "'did a test file change?' is an INPUT ON THE MAP, not a constant in the grader — a consumer sharpens it with .github/risk.json like every other rule. Omit the key entirely and the grader falls back to its built-in regex, which only knows the Go/TS shapes and so can never let a Python, Java or Ruby consumer reach clean_tier." + }, + + "flippable_flag_paths": ["**/feature_flag*/**", "**/featureflags/**", "**/flags/**", "**/*feature_flag*", "**/*featureFlag*"], + "_flippable": "a change gated behind a flippable flag is recoverable without a deploy. Recorded on the reversibility axis as `flag_gated`; in v0 it is REPORTED, never used to LOWER a tier (an axis may only move a PR riskier)." +} diff --git a/scripts/pr-risk/runbook-registry.v0.json b/scripts/pr-risk/runbook-registry.v0.json new file mode 100644 index 0000000..74dc25d --- /dev/null +++ b/scripts/pr-risk/runbook-registry.v0.json @@ -0,0 +1,48 @@ +{ + "registry_version": "v0-generic", + "_comment": [ + "The DEFAULT runbook registry for the reusable pr-risk.yml workflow — the KNOWN automated", + "producers whose output has a fixed diff shape. Read by the grader's provenance axis.", + "", + "PROVENANCE ALONE IS NEVER SUFFICIENT. Every entry carries a `shape` assertion, and the", + "assertion is what separates \"claims to be runbook X\" (an identity anyone with a token", + "can present) from \"actually looks like runbook X's output\". A PR whose identity matches", + "an entry but whose SHAPE does not is NOT a runbook: it falls back to its underlying", + "provenance class and the failure is recorded in `shape_failures`, never silently ignored.", + "", + "IDENTITY is the server-attributed PR author login (and, where the producer is a workflow,", + "the head-ref shape that workflow pushes) — never the forgeable commit author string.", + "", + "This default registers only ecosystem-universal producers. A consumer repo registers its", + "own producers (release bots, spec-sync pushes, caller bumpers) by committing", + ".github/risk-runbooks.json; the workflow reads it from the PR's BASE ref, so a PR cannot", + "register a runbook for itself.", + "", + "`daily_cap` bounds how many PRs a producer may contribute per UTC day before the surplus", + "stops counting as routine runbook output; `lane` is the candidate routing lane a later", + "phase would open for it. Both are RECORDED and gate nothing." + ], + "runbooks": [ + { + "id": "dependabot", + "why": "Dependabot — the highest-volume automated producer, and the one with the best-known diff shape", + "identity": { "logins": ["dependabot[bot]", "dependabot", "dependabot-preview[bot]"], + "head_ref_patterns": ["dependabot/**"] }, + "_permitted_paths_note": "MUST stay a superset of the `deps` class in risk-map.v0.json. Two manifest lists that must agree are one list too many: when this one is SHORT, a Ruby or PHP dependabot PR fails paths_ok, falls back to `human`, and records a shape_failure that says dependabot failed to look like dependabot. The overall tier does not move — the path floor pins those manifests R3 anyway — but the noise trains readers to skip the one field designed to catch real impersonation.", + "permitted_paths": ["go.mod", "go.sum", "**/go.mod", "**/go.sum", "**/go.work*", + "**/package.json", "**/package-lock.json", "**/npm-shrinkwrap.json", + "**/pnpm-lock.yaml", "**/yarn.lock", + "**/requirements*.txt", "**/pyproject.toml", "**/setup.py", + "**/Pipfile", "**/Pipfile.lock", "**/poetry.lock", "**/uv.lock", + "**/Cargo.toml", "**/Cargo.lock", + "**/Gemfile", "**/Gemfile.lock", "**/*.gemspec", + "**/composer.json", "**/composer.lock", + ".github/workflows/**"], + "shape": { "max_changed_files": 4, "max_additions": 2000, "max_deletions": 2000, + "title_regex": "(?i)^((chore|build|fix)\\(deps[^)]*\\):?|bump)[[:space:]]" }, + "daily_cap": 20, + "lane": "dependency-bump", + "_note": "permitted_paths deliberately include the dependency manifests the path map pins R3 — worst-wins keeps the R3 floor. A runbook can never buy its way past the path map; provenance can only PROPOSE a tier." + } + ] +} diff --git a/scripts/pr-risk/tests/test_apply_risk_label.sh b/scripts/pr-risk/tests/test_apply_risk_label.sh new file mode 100755 index 0000000..2c43162 --- /dev/null +++ b/scripts/pr-risk/tests/test_apply_risk_label.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# test_apply_risk_label.sh — hermetic tests for apply-risk-label.sh. No network: DRY_RUN +# covers the mapping/ownership logic, the validation phases exit before any gh call, and the +# write path runs against a `gh` stub on PATH that records the requests it was asked to make. + +set -uo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SELF_DIR/../apply-risk-label.sh" +[ -f "$SCRIPT" ] || { echo "FATAL: $SCRIPT not found" >&2; exit 1; } +command -v jq >/dev/null 2>&1 || { echo "FATAL: jq not found on PATH" >&2; exit 2; } + +SANDBOX="$(mktemp -d "${TMPDIR:-/tmp}/pr-risk-label-test.XXXXXX")" +trap 'rm -rf "$SANDBOX"' EXIT + +PASS=0; FAIL=0 +ok() { PASS=$((PASS+1)); printf 'ok %s\n' "$1"; } +bad() { FAIL=$((FAIL+1)); printf 'FAIL %s\n got: %s\n' "$1" "${2:-}"; } +eq() { if [ "$2" = "$3" ]; then ok "$1"; else bad "$1 (expected '$2')" "$3"; fi; } + +run() { # [label_map] -> stdout (the target label); rc in $? + REPO=test/repo PR_NUMBER=7 TIER="$1" LABEL_MAP="${2:-}" DRY_RUN=1 bash "$SCRIPT" 2>/dev/null +} + +echo "— default map —" +eq "R0 maps to risk:R0" "risk:R0" "$(run R0)" +eq "R3 maps to risk:R3" "risk:R3" "$(run R3)" +eq "unknown maps to risk:ungraded" "risk:ungraded" "$(run unknown)" +eq "empty tier reads as unknown" "risk:ungraded" "$(run '')" +eq "literal null reads as unknown" "risk:ungraded" "$(run null)" + +echo "— caller remap (a 1-indexed R1..R4 scheme is one input) —" +MAP='R0=risk:R1,R1=risk:R2,R2=risk:R3,R3=risk:R4,unknown=risk:ungraded' +eq "R0 remaps to risk:R1" "risk:R1" "$(run R0 "$MAP")" +eq "R3 remaps to risk:R4" "risk:R4" "$(run R3 "$MAP")" + +echo "— validation refuses bad input before any write —" +run R7 >/dev/null 2>&1; eq "bad tier exits 2" 2 "$?" +run R2 'R0=a,R1=b,R2=c,R3=d' >/dev/null 2>&1; eq "map missing unknown exits 2" 2 "$?" +run R2 'R0=,R1=b,R2=c,R3=d,unknown=e' >/dev/null 2>&1; eq "empty label exits 2" 2 "$?" +REPO='bad repo' PR_NUMBER=7 TIER=R1 DRY_RUN=1 bash "$SCRIPT" >/dev/null 2>&1 +eq "bad repo exits 2" 2 "$?" +REPO=test/repo PR_NUMBER=x TIER=R1 DRY_RUN=1 bash "$SCRIPT" >/dev/null 2>&1 +eq "bad pr number exits 2" 2 "$?" + +echo "— the write path: label names are PATH SEGMENTS, and get encoded like it —" +# GitHub label names legally contain spaces, `/`, `#`, `?` and `%`. Interpolated raw, a caller +# remap like `R3=risk high/urgent` built a malformed or misrouted URL: the DELETE failed, `fail` +# fired, and a rename painted the check red. The stub records every request so the test can +# assert on the paths actually built. +mkdir -p "$SANDBOX/bin" +export GH_LOG="$SANDBOX/gh.log" CURRENT_LABELS="$SANDBOX/current.txt" +printf 'risk:R0\nkeep-me\n' > "$CURRENT_LABELS" +cat > "$SANDBOX/bin/gh" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$GH_LOG" +for a in "$@"; do + case "$a" in + *issues/*/labels*) [ "${1:-}" = api ] && [[ " $* " != *" -X POST "* && " $* " != *" -X DELETE "* ]] \ + && cat "$CURRENT_LABELS" + exit 0 ;; + esac +done +exit 0 +STUB +chmod +x "$SANDBOX/bin/gh" + +MAP2='R0=risk:R0,R1=risk:R1,R2=risk:R2,R3=risk high/urgent,unknown=risk:ungraded' +out="$(PATH="$SANDBOX/bin:$PATH" REPO=test/repo PR_NUMBER=7 TIER=R3 LABEL_MAP="$MAP2" \ + bash "$SCRIPT" 2>/dev/null)" +eq "the raw name is what gets returned/logged" "risk high/urgent" "$out" +if grep -q 'risk%20high%2Furgent' "$GH_LOG"; then + ok "the target label is percent-encoded in the request path" +else bad "the target label is percent-encoded in the request path" "$(tr '\n' '|' < "$GH_LOG")"; fi +if grep -q -- '-X POST repos/test/repo/issues/7/labels -f labels\[\]=risk high/urgent' "$GH_LOG"; then + ok "but the FORM FIELD carries the raw name, not the encoding" +else bad "but the FORM FIELD carries the raw name, not the encoding" "$(tr '\n' '|' < "$GH_LOG")"; fi +if grep -q 'DELETE repos/test/repo/issues/7/labels/risk%3AR0' "$GH_LOG"; then + ok "the stale label is removed via an encoded path" +else bad "the stale label is removed via an encoded path" "$(tr '\n' '|' < "$GH_LOG")"; fi +if grep -q -- '--paginate repos/test/repo/issues/7/labels' "$GH_LOG"; then + ok "the label read paginates (a stale label past page 1 must still be found)" +else bad "the label read paginates" "$(tr '\n' '|' < "$GH_LOG")"; fi + +# A label the script does NOT own is never touched, however the grade lands. +if grep -q 'keep-me' "$GH_LOG"; then + bad "an unowned label is never written to" "$(grep keep-me "$GH_LOG" | tr '\n' '|')" +else ok "an unowned label is never written to"; fi + +# Already-correct label: no write at all beyond the read. +: > "$GH_LOG"; printf 'risk:R2\n' > "$CURRENT_LABELS" +PATH="$SANDBOX/bin:$PATH" REPO=test/repo PR_NUMBER=7 TIER=R2 bash "$SCRIPT" >/dev/null 2>&1 +eq "an in-sync label writes nothing" 1 "$(wc -l < "$GH_LOG" | tr -d ' ')" + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ] diff --git a/scripts/pr-risk/tests/test_grade_pr_risk.sh b/scripts/pr-risk/tests/test_grade_pr_risk.sh new file mode 100755 index 0000000..6d1e106 --- /dev/null +++ b/scripts/pr-risk/tests/test_grade_pr_risk.sh @@ -0,0 +1,261 @@ +#!/usr/bin/env bash +# test_grade_pr_risk.sh — hermetic tests for grade-pr-risk.sh. No network: the grading phases +# feed synthetic scorecard records to --stdin, and the live-PR phase stubs `gh` on PATH with a +# fixture GraphQL response. Ported from the fleet's offline grader suite (BE-5507) — the +# safety properties proven there are re-proven here against the extracted script: +# * WORST-WINS: an R0 path rule cannot cancel an R3 one; a runbook (provenance R0) still +# grades R3 when the path floor says R3 — an axis may only ever move a PR RISKIER. +# * PROVENANCE ALONE IS NEVER SUFFICIENT: a runbook IDENTITY whose diff SHAPE does not +# assert is not a runbook, and the failure is recorded. +# * EXTERNAL IS NEVER OVERRIDDEN: a fork imitating a runbook's shape is still external R3. +# * THE UNKNOWN CONTRACT: an unreadable input is tier null + status unknown + exit 1, +# never a confident tier; a structurally empty map is refused outright (exit 2). +# * THE GRADE CARRIES ITS MAP VERSION so a map revision can be replayed later. +# * SELF-EXCLUDING ROLLUP (new here): with --self-context, the grading workflow's own +# in-progress check run does not floor every live grade at R2. +# +# bash tests/test_grade_pr_risk.sh # exit 0 = all green + +set -uo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +GRADER="$SELF_DIR/../grade-pr-risk.sh" +[ -f "$GRADER" ] || { echo "FATAL: $GRADER not found" >&2; exit 1; } +command -v jq >/dev/null 2>&1 || { echo "FATAL: jq not found on PATH" >&2; exit 2; } + +SANDBOX="$(mktemp -d "${TMPDIR:-/tmp}/pr-risk-test.XXXXXX")" +trap 'rm -rf "$SANDBOX"' EXIT + +PASS=0; FAIL=0 +ok() { PASS=$((PASS+1)); printf 'ok %s\n' "$1"; } +bad() { FAIL=$((FAIL+1)); printf 'FAIL %s\n got: %s\n' "$1" "${2:-}"; } +eq() { # + if [ "$2" = "$3" ]; then ok "$1"; else bad "$1 (expected '$2')" "$3"; fi +} + +# rec — one scorecard-shaped record, mirroring what fetch_pr_record emits. The status twins +# default to `ok` (the fully-collected case); phases that want an unread field say so. +rec() { # <paths-json> <paths-status> <checks> [head_ref] [assoc] [fork] + jq -cn --argjson pr "$1" --arg author "$2" --arg title "$3" --argjson paths "$4" \ + --arg pstatus "$5" --arg checks "$6" --arg head "${7:-feature-branch}" \ + --arg assoc "${8:-MEMBER}" --argjson fork "${9:-false}" ' + {repo:"test/repo", pr:$pr, title:$title, author:$author, author_association:$assoc, + is_fork:$fork, labels:[], head_ref:$head, additions:10, deletions:5, changed_files:($paths | if . == null then 99 else length end), + checks_state:(if $checks == "null" then null else $checks end), + checks_status:"ok", provenance_status:"ok", + changed_paths:$paths, changed_paths_status:$pstatus}' +} + +grade() { bash "$GRADER" --stdin 2>/dev/null; } + +echo "— phase 1: worst-wins on the path floor —" +# docs (R0 rule) + migration (R3 rule) in one PR: the R0 rule must not cancel the R3 one. +out="$(rec 1 dev 'fix: tweak' '[{"path":"README.md","additions":1,"deletions":0,"change_type":"MODIFIED"},{"path":"db/migrations/0001_x.sql","additions":9,"deletions":0,"change_type":"ADDED"}]' ok SUCCESS | grade)" +eq "docs cannot cancel migrations" R3 "$(jq -r '.risk.axes.path_floor.tier' <<<"$out")" +eq "overall is R3" R3 "$(jq -r '.risk.tier' <<<"$out")" + +echo "— phase 2: a runbook cannot buy its way past the path floor —" +out="$(rec 2 'dependabot[bot]' 'chore(deps): bump x from 1 to 2' '[{"path":"go.mod","additions":1,"deletions":1,"change_type":"MODIFIED"},{"path":"go.sum","additions":2,"deletions":2,"change_type":"MODIFIED"}]' ok SUCCESS 'dependabot/go_modules/x-2' CONTRIBUTOR | grade)" +eq "provenance is runbook" runbook "$(jq -r '.risk.axes.provenance.provenance' <<<"$out")" +eq "provenance proposes R0" R0 "$(jq -r '.risk.axes.provenance.tier' <<<"$out")" +eq "path floor still decides R3" R3 "$(jq -r '.risk.tier' <<<"$out")" + +echo "— phase 3: provenance alone is never sufficient (shape assertion) —" +# dependabot's identity, but the diff touches a path outside its permitted set. +out="$(rec 3 'dependabot[bot]' 'chore(deps): bump x' '[{"path":"src/evil.go","additions":9,"deletions":0,"change_type":"MODIFIED"}]' ok SUCCESS 'dependabot/go_modules/x-2' CONTRIBUTOR | grade)" +eq "not classified runbook" human "$(jq -r '.risk.axes.provenance.provenance' <<<"$out")" +sf="$(jq -r '.risk.axes.provenance.shape_failures | length' <<<"$out")" +if [ "$sf" -ge 1 ]; then ok "shape failure recorded"; else bad "shape failure recorded" "$sf"; fi + +echo "— phase 4: external is never overridden by a runbook match —" +out="$(rec 4 'dependabot[bot]' 'chore(deps): bump x from 1 to 2' '[{"path":"go.mod","additions":1,"deletions":1,"change_type":"MODIFIED"}]' ok SUCCESS 'dependabot/go_modules/x-2' NONE true | grade)" +eq "fork stays external" external "$(jq -r '.risk.axes.provenance.provenance' <<<"$out")" +eq "external grades R3" R3 "$(jq -r '.risk.tier' <<<"$out")" + +echo "— phase 5: the unknown contract —" +out="$(rec 5 dev 'mystery' null unknown SUCCESS | grade)" +eq "unknown axis nulls the tier" null "$(jq -r '.risk.tier' <<<"$out")" +eq "overall status is unknown" unknown "$(jq -r '.risk.status' <<<"$out")" +rec 5 dev 'mystery' null unknown SUCCESS | bash "$GRADER" --stdin >/dev/null 2>&1 +eq "unknown exits 1" 1 "$?" + +echo "— phase 6: the grade carries its map + registry versions —" +out="$(rec 6 dev 'docs: x' '[{"path":"README.md","additions":1,"deletions":0,"change_type":"MODIFIED"}]' ok SUCCESS | grade)" +eq "map version stamped" v0-generic "$(jq -r '.risk.map_version' <<<"$out")" +eq "registry version stamped" v0-generic "$(jq -r '.risk.registry_version' <<<"$out")" + +echo "— phase 7: a structurally empty map is refused outright —" +printf '{}' > "$SANDBOX/empty-map.json" +rec 7 dev 'docs: x' '[{"path":"README.md","additions":1,"deletions":0,"change_type":"MODIFIED"}]' ok SUCCESS \ + | bash "$GRADER" --stdin --map "$SANDBOX/empty-map.json" >/dev/null 2>&1 +eq "empty map exits 2" 2 "$?" + +echo "— phase 8: reversibility floors and rungs —" +out="$(rec 8 dev 'docs: x' '[{"path":"README.md","additions":1,"deletions":0,"change_type":"MODIFIED"}]' ok PENDING | grade)" +eq "pending checks floor reversibility at R2" R2 "$(jq -r '.risk.axes.reversibility.tier' <<<"$out")" +out="$(rec 9 dev 'test: cover x' '[{"path":"pkg/x_test.go","additions":9,"deletions":0,"change_type":"ADDED"}]' ok SUCCESS | grade)" +eq "green + test touched grades reversibility R0" R0 "$(jq -r '.risk.axes.reversibility.tier' <<<"$out")" +eq "human provenance keeps overall at R1" R1 "$(jq -r '.risk.tier' <<<"$out")" + +echo "— phase 9: --pr with a stubbed gh: the self-excluding rollup —" +# Fixture: our own workflow's check run is IN_PROGRESS (it always is, mid-run); one other +# workflow's run has completed SUCCESS. Raw rollup is PENDING; excluding self it is SUCCESS. +# The changed-file list is a SEPARATE REST call (previous_filename lives only there), so the +# stub dispatches on the request and applies `--jq` the way gh does. +mkdir -p "$SANDBOX/bin" +export FIXTURE_DIR="$SANDBOX" +cat > "$SANDBOX/fixture.json" <<'FIX' +{"data":{"repository":{"pullRequest":{ + "number":42,"title":"docs: tweak readme","state":"OPEN","isDraft":false, + "createdAt":"2026-08-01T00:00:00Z","updatedAt":"2026-08-01T00:10:00Z","closedAt":null,"mergedAt":null, + "author":{"login":"dev"},"authorAssociation":"MEMBER","baseRefName":"main","headRefName":"docs-tweak", + "isCrossRepository":false,"additions":3,"deletions":1,"changedFiles":1, + "labels":{"pageInfo":{"hasNextPage":false},"nodes":[]}, + "commits":{"nodes":[{"commit":{"statusCheckRollup":{"state":"PENDING","contexts":{ + "pageInfo":{"hasNextPage":false}, + "nodes":[ + {"__typename":"CheckRun","name":"Grade PR risk","status":"IN_PROGRESS","conclusion":null, + "checkSuite":{"workflowRun":{"databaseId":999,"workflow":{"name":"CI - PR Risk Grade"}}}}, + {"__typename":"CheckRun","name":"unit tests","status":"COMPLETED","conclusion":"SUCCESS", + "checkSuite":{"workflowRun":{"databaseId":1000,"workflow":{"name":"CI"}}}} + ]}}}}]} +}}}} +FIX +cat > "$SANDBOX/files.json" <<'FIX' +[{"filename":"README.md","additions":3,"deletions":1,"status":"modified"}] +FIX +# `gh api` stub: graphql -> the PR fixture, pulls/{n}/files -> the REST file fixture, and a +# --jq filter is applied to the chosen fixture exactly as gh would. +cat > "$SANDBOX/bin/gh" <<'STUB' +#!/usr/bin/env bash +fixture=""; filter="" +for ((i=1; i<=$#; i++)); do + a="${!i}" + case "$a" in + graphql) fixture="$FIXTURE_DIR/fixture.json" ;; + *pulls/*/files*) fixture="$FIXTURE_DIR/files.json" ;; + --jq) n=$((i+1)); filter="${!n}" ;; + esac +done +[ -n "$fixture" ] || { echo "gh stub: unhandled args: $*" >&2; exit 1; } +if [ -n "$filter" ]; then jq -c "$filter" "$fixture"; else cat "$fixture"; fi +STUB +chmod +x "$SANDBOX/bin/gh" +graded_pr() { PATH="$SANDBOX/bin:$PATH" bash "$GRADER" --repo test/repo --pr 42 "$@" 2>/dev/null; } + +out="$(graded_pr --self-context 'CI - PR Risk Grade')" +eq "self-excluded rollup reads SUCCESS (by name)" SUCCESS "$(jq -r '.checks_state' <<<"$out")" +eq "nothing else pending" false "$(jq -r '.checks_pending_excl_self' <<<"$out")" +eq "live docs PR grades R1" R1 "$(jq -r '.risk.tier' <<<"$out")" +# --self-run-id is the EXACT selector: same result, but keyed on github.run_id, so a +# same-named workflow elsewhere in the consumer repo is no longer swept out of the rollup. +out="$(graded_pr --self-run-id 999)" +eq "self-excluded rollup reads SUCCESS (by run id)" SUCCESS "$(jq -r '.checks_state' <<<"$out")" +# A run id that is NOT ours excludes nothing, so our own in-progress run still reads PENDING. +out="$(graded_pr --self-run-id 12345)" +eq "a foreign run id excludes nothing" PENDING "$(jq -r '.checks_state' <<<"$out")" +# Same fixture with NO self selector: the raw rollup (PENDING) must come through untouched — +# that is the offline/terminal behavior, where no self run is in flight. +out="$(graded_pr)" +eq "raw rollup untouched without a self selector" PENDING "$(jq -r '.checks_state' <<<"$out")" +# Flip the other workflow's run to QUEUED: excluding self must now report pending=true. +jq '.data.repository.pullRequest.commits.nodes[0].commit.statusCheckRollup.contexts.nodes[1].status = "QUEUED" + | .data.repository.pullRequest.commits.nodes[0].commit.statusCheckRollup.contexts.nodes[1].conclusion = null' \ + "$SANDBOX/fixture.json" > "$SANDBOX/f2.json" && cp "$SANDBOX/f2.json" "$SANDBOX/fixture.json" +out="$(graded_pr --self-run-id 999)" +eq "other pending check reports pending" true "$(jq -r '.checks_pending_excl_self' <<<"$out")" + +echo "— phase 10: self-exclusion can never HIDE a red check —" +# The failing check belongs to OUR OWN run (the case a consumer creates by putting the grading +# job inside its existing CI workflow — every sibling job then shares our run id). Excluding it +# from the rollup would aggregate the remaining green contexts to SUCCESS and grade a RED PR +# R0/R1, so the FAILURE scan deliberately covers self too. +jq '.data.repository.pullRequest.commits.nodes[0].commit.statusCheckRollup.contexts.nodes[1] = + {"__typename":"CheckRun","name":"unit tests","status":"COMPLETED","conclusion":"FAILURE", + "checkSuite":{"workflowRun":{"databaseId":999,"workflow":{"name":"CI - PR Risk Grade"}}}}' \ + "$SANDBOX/fixture.json" > "$SANDBOX/f2.json" && cp "$SANDBOX/f2.json" "$SANDBOX/fixture.json" +out="$(graded_pr --self-run-id 999)" +eq "a failing check in our own run still reads FAILURE" FAILURE "$(jq -r '.checks_state' <<<"$out")" +eq "and reversibility cannot go below R2" R2 "$(jq -r '.risk.axes.reversibility.tier' <<<"$out")" + +echo "— phase 11: a rollup of nothing but SKIPPED is not a green rollup —" +# SKIPPED / NEUTRAL / null conclusions establish NOTHING about whether tests covering these +# lines ran, which is the reversibility axis's entire question — so they must not aggregate to +# SUCCESS and let the axis drop to R0/R1. +jq '.data.repository.pullRequest.commits.nodes[0].commit.statusCheckRollup.contexts.nodes = + [{"__typename":"CheckRun","name":"Grade PR risk","status":"IN_PROGRESS","conclusion":null, + "checkSuite":{"workflowRun":{"databaseId":999,"workflow":{"name":"CI - PR Risk Grade"}}}}, + {"__typename":"CheckRun","name":"unit tests","status":"COMPLETED","conclusion":"SKIPPED", + "checkSuite":{"workflowRun":{"databaseId":1000,"workflow":{"name":"CI"}}}}]' \ + "$SANDBOX/fixture.json" > "$SANDBOX/f2.json" && cp "$SANDBOX/f2.json" "$SANDBOX/fixture.json" +out="$(graded_pr --self-run-id 999)" +eq "all-SKIPPED does not aggregate to SUCCESS" NEUTRAL "$(jq -r '.checks_state' <<<"$out")" +eq "no green rollup floors reversibility at R2" R2 "$(jq -r '.risk.axes.reversibility.tier' <<<"$out")" + +echo "— phase 12: a SHORT changed-file read is unknown, never a floor from the files that fit —" +# GraphQL says 5 changed files, the file endpoint returned 1. The old `files(first:100)` path +# graded every PR above the cap `unknown`; the REST read pages past it, but a read that comes +# back SHORT of changedFiles is still an unread input and must refuse to grade. +jq '.data.repository.pullRequest.changedFiles = 5' "$SANDBOX/fixture.json" > "$SANDBOX/f2.json" \ + && cp "$SANDBOX/f2.json" "$SANDBOX/fixture.json" +out="$(graded_pr --self-run-id 999)" +eq "short file read is unknown" unknown "$(jq -r '.changed_paths_status' <<<"$out")" +eq "and the overall grade refuses" null "$(jq -r '.risk.tier' <<<"$out")" + +echo "— phase 13: a RENAME cannot walk a file out of its guarded directory —" +# The origin path is graded too: `git mv src/auth/x.go misc/x.go` used to be recorded under the +# destination ALONE, so the R3 `auth` rule never matched and the move escaped the floor. +out="$(rec 13 dev 'refactor: move things' '[{"path":"misc/x.go","previous_path":"src/auth/x.go","additions":1,"deletions":1,"change_type":"RENAMED"}]' ok SUCCESS | grade)" +eq "the origin path still hits the auth floor" R3 "$(jq -r '.risk.axes.path_floor.tier' <<<"$out")" +eq "renaming a file out of a sensitive class is not a clean revert" R3 "$(jq -r '.risk.axes.reversibility.tier' <<<"$out")" + +echo "— phase 14: 'deletes a sensitive file' means the DELETED files, not any changed file —" +# MODIFIES an auth file and DELETES an unrelated README. The sensitive-class match used to run +# over every changed file, so this reported "deletes N file(s) under a sensitive class" and +# pinned reversibility R3 — a true tier from a false sentence. +out="$(rec 14 dev 'chore: tidy' '[{"path":"src/auth/x.go","additions":2,"deletions":1,"change_type":"MODIFIED"},{"path":"README.md","additions":0,"deletions":9,"change_type":"DELETED"}]' ok SUCCESS | grade)" +eq "deleting a doc is not deleting an auth file" R1 "$(jq -r '.risk.axes.reversibility.tier' <<<"$out")" +eq "the auth path floor still decides the grade" R3 "$(jq -r '.risk.tier' <<<"$out")" +# ...and a genuinely deleted auth file still pins R3. +out="$(rec 15 dev 'chore: drop it' '[{"path":"src/auth/x.go","additions":0,"deletions":9,"change_type":"DELETED"}]' ok SUCCESS | grade)" +eq "deleting an auth file is R3 on reversibility" R3 "$(jq -r '.risk.axes.reversibility.tier' <<<"$out")" + +echo "— phase 15: the default map's globs match at DEPTH, not just at the repo root —" +# `*` does not cross `/`, so an unprefixed glob compiles to a root-only match and the rule +# silently matches nothing in a real tree. The grader + its map are R3: a PR that edits the +# judge must not be graded safest by that judge. +out="$(rec 16 dev 'chore: retune' '[{"path":"scripts/pr-risk/risk-map.v0.json","additions":3,"deletions":1,"change_type":"MODIFIED"}]' ok SUCCESS | grade)" +eq "a nested risk map is R3" R3 "$(jq -r '.risk.axes.path_floor.tier' <<<"$out")" +out="$(rec 17 dev 'chore: retune' '[{"path":"scripts/pr-risk/grade-pr-risk.sh","additions":3,"deletions":1,"change_type":"MODIFIED"}]' ok SUCCESS | grade)" +eq "the grader itself is R3" R3 "$(jq -r '.risk.axes.path_floor.tier' <<<"$out")" +# A shell test OUTSIDE a tests/ directory must still match the R0 tests class. +out="$(rec 18 dev 'test: smoke' '[{"path":"hack/smoke-test.sh","additions":3,"deletions":0,"change_type":"ADDED"}]' ok SUCCESS | grade)" +if jq -e '.risk.axes.path_floor.classes | index("tests")' >/dev/null <<<"$out"; then + ok "a nested *-test.sh matches the tests class" +else bad "a nested *-test.sh matches the tests class" "$(jq -c '.risk.axes.path_floor.classes' <<<"$out")"; fi + +echo "— phase 16: 'did a test file change?' comes from the MAP, so every ecosystem can answer —" +# The built-in regex knows only the Go/TS shapes, so a Python or Java consumer could never +# reach clean_tier and sat at R1 forever. test_path_patterns in the map is the fix. +for p in pkg/test_foo.py pkg/foo_test.py app/FooTest.java spec/foo_spec.rb; do + out="$(rec 19 dev 'test: cover it' "[{\"path\":\"$p\",\"additions\":9,\"deletions\":0,\"change_type\":\"ADDED\"}]" ok SUCCESS | grade)" + eq "$p counts as a touched test" R0 "$(jq -r '.risk.axes.reversibility.tier' <<<"$out")" +done + +echo "— phase 17: a map that forgets a provenance class is refused, not guessed —" +# `{}`-shaped omissions used to pass validation and then grade forks off a fallback tier +# nobody chose, silently retiring "external is R3, no exceptions". +jq 'del(.provenance_tiers.external)' "$SELF_DIR/../risk-map.v0.json" > "$SANDBOX/no-external.json" +rec 20 dev 'docs: x' '[{"path":"README.md","additions":1,"deletions":0,"change_type":"MODIFIED"}]' ok SUCCESS \ + | bash "$GRADER" --stdin --map "$SANDBOX/no-external.json" >/dev/null 2>&1 +eq "a map missing 'external' exits 2" 2 "$?" + +echo "— phase 18: a TRUNCATED label list cannot answer 'is this agent-coded?' —" +out="$(rec 21 dev 'feat: x' '[{"path":"src/x.go","additions":9,"deletions":0,"change_type":"MODIFIED"}]' ok SUCCESS \ + | jq -c '.labels_status = "unknown"' | grade)" +eq "truncated labels make provenance unknown" unknown "$(jq -r '.risk.axes.provenance.status' <<<"$out")" +eq "and the overall grade refuses" null "$(jq -r '.risk.tier' <<<"$out")" + +echo +echo "passed $PASS, failed $FAIL" +[ "$FAIL" -eq 0 ]