From 259738ae6466876e695ea26d41fd99aa728e5a15 Mon Sep 17 00:00:00 2001 From: Vitalii Popov Date: Thu, 5 Nov 2026 13:14:35 +0100 Subject: [PATCH 1/6] Add a reviewrules action and a block review workflow The action reviews the PR diff with reviewrules on the Anthropic API and posts each finding as a review comment on its line. The workflow runs it on block repos as a required workflow of an organization ruleset, when the PR has the ai-review label, so the block repos need no change. --- .github/workflows/block-review.yaml | 67 ++++++++++++ actions/review/reviewrules/action.yaml | 92 +++++++++++++++++ actions/review/reviewrules/post-review.js | 118 ++++++++++++++++++++++ 3 files changed, 277 insertions(+) create mode 100644 .github/workflows/block-review.yaml create mode 100644 actions/review/reviewrules/action.yaml create mode 100644 actions/review/reviewrules/post-review.js diff --git a/.github/workflows/block-review.yaml b/.github/workflows/block-review.yaml new file mode 100644 index 00000000..261d4de2 --- /dev/null +++ b/.github/workflows/block-review.yaml @@ -0,0 +1,67 @@ +name: Block review +# Runs as a required workflow of an organization ruleset, so block repos need no change. +# A ruleset workflow starts only on PR opened, synchronize, and reopened. After you add +# the label, re-run this workflow, or push a new commit. +on: + pull_request: + +permissions: + contents: read + pull-requests: write + +concurrency: + group: block-review-${{ github.repository }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +env: + REVIEW_LABEL: ai-review + +jobs: + review: + runs-on: ubuntu-latest + steps: + # A re-run gets the labels of the first run, so read the labels the PR has now. + - name: Check the review label + id: label + uses: actions/github-script@v7 + with: + script: | + const { data: labels } = await github.rest.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + }); + const requested = labels.some((label) => label.name === process.env.REVIEW_LABEL); + core.setOutput('requested', String(requested)); + if (!requested) core.notice(`Add the label ${process.env.REVIEW_LABEL} and re-run this workflow to get a review.`); + + - uses: actions/checkout@v4 + if: steps.label.outputs.requested == 'true' + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Add the block rule set when the repo has no rule folder + if: steps.label.outputs.requested == 'true' && hashFiles('.reviewrules/config.json') == '' + shell: bash + run: | + mkdir -p .reviewrules + cat > .reviewrules/settings.toml <<'TOML' + [[rule_sets]] + name = "block" + url = "https://github.com/milaboratory/block-review-rules.git" + ref = "v0.1.0" + TOML + cat > .reviewrules/config.json <<'JSON' + { + "ignorePatterns": "**/node_modules/**\n**/dist/**\n**/block-pack/**\n**/.turbo/**\n**/*.tgz\n**/CHANGELOG.md\nsoftware/**/uv.lock\nsoftware/**/src/requirements.txt\nsoftware/**/tests/data/corpus/**\ntest/fixtures/data/**\n**/__pycache__/**", + "rules": [] + } + JSON + + - uses: milaboratory/github-ci/actions/review/reviewrules@v4-beta + if: steps.label.outputs.requested == 'true' + with: + anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} + rule-sets-token: ${{ secrets.REVIEWRULES_RULE_SETS_TOKEN || github.token }} diff --git a/actions/review/reviewrules/action.yaml b/actions/review/reviewrules/action.yaml new file mode 100644 index 00000000..57aa6a63 --- /dev/null +++ b/actions/review/reviewrules/action.yaml @@ -0,0 +1,92 @@ +name: Review a pull request with reviewrules +author: 'MiLaboratories' +description: | + Review the pull request diff with reviewrules (https://github.com/popoffvg/reviewrules) + against the rules in the repo's .reviewrules/ folder and its rule sets. + Post each finding as a review comment on its line. A finding on a line outside + the diff goes to the review body. A finding that is already posted is not posted again. + The step fails only when reviewrules cannot run (exit code 2), not on findings. + + The repo must be checked out at the PR head commit with full history. + +inputs: + anthropic-api-key: + description: Anthropic API key for the review models. + required: true + + github-token: + description: GitHub token with pull-requests:write. The review comments come from its owner. + required: false + default: ${{ github.token }} + + rule-sets-token: + description: GitHub token that can read the rule set repos, for example a private one. + required: false + default: ${{ github.token }} + + version: + description: Commit SHA of reviewrules to install. + required: false + default: '80ff69ce9d16fdd4df659d65add28fe80f437683' + +outputs: + report: + description: Path of the reviewrules JSON report. + value: ${{ steps.review.outputs.report }} + +runs: + using: "composite" + steps: + - uses: actions/setup-go@v5 + with: + go-version: stable + cache: false + + - name: Install reviewrules + shell: bash + env: + VERSION: ${{ inputs.version }} + run: go install "github.com/popoffvg/reviewrules@${VERSION}" + + - name: Review + id: review + shell: bash + env: + CI: 'true' + ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} + RULE_SETS_TOKEN: ${{ inputs.rule-sets-token }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + git config --global url."https://x-access-token:${RULE_SETS_TOKEN}@github.com/".insteadOf "https://github.com/" + + config="${RUNNER_TEMP}/reviewrules.toml" + report="${RUNNER_TEMP}/reviewrules-report.json" + cat > "${config}" <<'TOML' + [ci] + agent = "anthropic" + model = "claude-haiku-5-5" + reasoning_effort = "medium" + + [[ci.profiles]] + action = "hypothesis" + model = "claude-opus-5-5" + reasoning_effort = "medium" + TOML + + code=0 + reviewrules --config "${config}" --base "origin/${BASE_REF}" --json-out "${report}" || code=$? + echo "report=${report}" >> "${GITHUB_OUTPUT}" + if [ "${code}" -ge 2 ]; then + echo "::error::reviewrules could not run (exit code ${code})" + exit "${code}" + fi + + - name: Post findings + uses: actions/github-script@v7 + env: + REPORT: ${{ steps.review.outputs.report }} + with: + github-token: ${{ inputs.github-token }} + script: | + const postReview = require(`${process.env.GITHUB_ACTION_PATH}/post-review.js`); + await postReview({ github, context, core, reportPath: process.env.REPORT }); diff --git a/actions/review/reviewrules/post-review.js b/actions/review/reviewrules/post-review.js new file mode 100644 index 00000000..83e4f810 --- /dev/null +++ b/actions/review/reviewrules/post-review.js @@ -0,0 +1,118 @@ +const fs = require('fs'); + +const MARKER_PREFIX = '`; + +const markerKeys = (body) => + (body || '') + .split(MARKER_PREFIX) + .slice(1) + .map((rest) => rest.slice(0, rest.indexOf(' -->'))); + +const ruleFindings = (report) => + (report.rules || []) + .filter((rule) => rule.verdict === 'FAIL') + .flatMap((rule) => + (rule.findings || []).map((finding) => ({ + path: finding.file, + line: finding.line, + key: `${rule.rule_id}:${finding.file}:${finding.line}`, + text: `**${rule.rule_id}** (${rule.severity})\n\n${finding.message}`, + })), + ); + +const hypothesisText = (check) => + [ + `**Bug confirmed** (${check.hunt})`, + check.question, + check.scenario && `**Input that breaks it:** ${check.scenario}`, + check.edit && `**Fix:** ${check.edit}`, + ...(check.evidence || []).map((line) => `- ${line}`), + ] + .filter(Boolean) + .join('\n\n'); + +const hypothesisFindings = (report) => + ((report.correctness || {}).hypotheses || []) + .filter((check) => check.verdict === 'CONFIRMED') + .map((check) => { + const at = /^([^\s:]+):(\d+)/.exec(check.at || '') || []; + return { + path: at[1], + line: Number(at[2]), + key: `hypothesis:${check.at}:${check.question}`, + text: hypothesisText(check), + }; + }); + +// The right-side lines of each file patch; GitHub accepts a line comment only on these lines. +const commentableLines = (files) => { + const lines = new Map(); + for (const file of files) { + const right = new Set(); + let next = 0; + for (const row of (file.patch || '').split('\n')) { + const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)/.exec(row); + if (hunk) { + next = Number(hunk[1]); + } else if (row.startsWith('+') || row.startsWith(' ')) { + right.add(next); + next += 1; + } + } + lines.set(file.filename, right); + } + return lines; +}; + +const reviewBody = (report, outside) => { + const cost = (report.cost_estimate || {}).total_usd; + const head = `**reviewrules**: ${(report.rules || []).length} rule checks, cost estimate $${Number(cost || 0).toFixed(2)}.`; + if (outside.length === 0) return head; + const rows = outside.map( + (f) => `- \`${f.path || '?'}:${f.line || '?'}\` ${f.text.replace(/\n+/g, ' ')} ${marker(f.key)}`, + ); + return [head, 'Findings outside the diff lines:', ...rows].join('\n\n'); +}; + +module.exports = async ({ github, context, core, reportPath }) => { + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + const pull = context.payload.pull_request; + const target = { owner: context.repo.owner, repo: context.repo.repo, pull_number: pull.number, per_page: 100 }; + + const files = await github.paginate(github.rest.pulls.listFiles, target); + const comments = await github.paginate(github.rest.pulls.listReviewComments, target); + const reviews = await github.paginate(github.rest.pulls.listReviews, target); + const postedKeys = new Set([...comments, ...reviews].flatMap((item) => markerKeys(item.body))); + const lines = commentableLines(files); + + const inline = []; + const outside = []; + for (const finding of [...ruleFindings(report), ...hypothesisFindings(report)]) { + if (postedKeys.has(finding.key)) continue; + const onDiff = finding.path && (lines.get(finding.path) || new Set()).has(finding.line); + (onDiff ? inline : outside).push(finding); + } + + if (inline.length === 0 && outside.length === 0) { + core.info('reviewrules: no new findings'); + return; + } + + await github.rest.pulls.createReview({ + owner: target.owner, + repo: target.repo, + pull_number: target.pull_number, + commit_id: pull.head.sha, + event: 'COMMENT', + body: reviewBody(report, outside), + comments: inline.map((finding) => ({ + path: finding.path, + line: finding.line, + side: 'RIGHT', + body: `${finding.text}\n\n${marker(finding.key)}`, + })), + }); + core.info(`reviewrules: posted ${inline.length} line comments and ${outside.length} findings in the review body`); +}; From 825dc72d6a76aa182b31593c4f809fa4c098240c Mon Sep 17 00:00:00 2001 From: Vitalii Popov Date: Thu, 5 Nov 2026 13:51:10 +0100 Subject: [PATCH 2/6] Read every page of the PR labels in the block review --- .github/workflows/block-review.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/block-review.yaml b/.github/workflows/block-review.yaml index 261d4de2..3bfbe3c9 100644 --- a/.github/workflows/block-review.yaml +++ b/.github/workflows/block-review.yaml @@ -26,10 +26,11 @@ jobs: uses: actions/github-script@v7 with: script: | - const { data: labels } = await github.rest.issues.listLabelsOnIssue({ + const labels = await github.paginate(github.rest.issues.listLabelsOnIssue, { owner: context.repo.owner, repo: context.repo.repo, issue_number: context.payload.pull_request.number, + per_page: 100, }); const requested = labels.some((label) => label.name === process.env.REVIEW_LABEL); core.setOutput('requested', String(requested)); From c25772b09fd66149af70f0996066508b2961ab04 Mon Sep 17 00:00:00 2001 From: Vitalii Popov Date: Thu, 5 Nov 2026 14:01:30 +0100 Subject: [PATCH 3/6] Save the review report and the model sessions as a run artifact The pin moves to the reviewrules commit that adds --session-dir (popoffvg/reviewrules#3). --- actions/review/reviewrules/action.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/actions/review/reviewrules/action.yaml b/actions/review/reviewrules/action.yaml index 57aa6a63..2baf9dc6 100644 --- a/actions/review/reviewrules/action.yaml +++ b/actions/review/reviewrules/action.yaml @@ -6,6 +6,8 @@ description: | Post each finding as a review comment on its line. A finding on a line outside the diff goes to the review body. A finding that is already posted is not posted again. The step fails only when reviewrules cannot run (exit code 2), not on findings. + The artifact reviewrules-attempt- holds report.json and sessions/: parent.json and + one fork-.json per rule or hypothesis fork, with every message of the model calls. The repo must be checked out at the PR head commit with full history. @@ -27,7 +29,7 @@ inputs: version: description: Commit SHA of reviewrules to install. required: false - default: '80ff69ce9d16fdd4df659d65add28fe80f437683' + default: '81894bcb3f637a72bab789fb19dae9882ce3fa27' outputs: report: @@ -59,8 +61,11 @@ runs: run: | git config --global url."https://x-access-token:${RULE_SETS_TOKEN}@github.com/".insteadOf "https://github.com/" + output="${RUNNER_TEMP}/reviewrules" config="${RUNNER_TEMP}/reviewrules.toml" - report="${RUNNER_TEMP}/reviewrules-report.json" + report="${output}/report.json" + mkdir -p "${output}" + echo "output=${output}" >> "${GITHUB_OUTPUT}" cat > "${config}" <<'TOML' [ci] agent = "anthropic" @@ -74,7 +79,7 @@ runs: TOML code=0 - reviewrules --config "${config}" --base "origin/${BASE_REF}" --json-out "${report}" || code=$? + reviewrules --config "${config}" --base "origin/${BASE_REF}" --json-out "${report}" --session-dir "${output}/sessions" || code=$? echo "report=${report}" >> "${GITHUB_OUTPUT}" if [ "${code}" -ge 2 ]; then echo "::error::reviewrules could not run (exit code ${code})" @@ -90,3 +95,11 @@ runs: script: | const postReview = require(`${process.env.GITHUB_ACTION_PATH}/post-review.js`); await postReview({ github, context, core, reportPath: process.env.REPORT }); + + - name: Save the report and the model sessions + if: always() && steps.review.outputs.output != '' + uses: actions/upload-artifact@v4 + with: + name: reviewrules-attempt-${{ github.run_attempt }} + path: ${{ steps.review.outputs.output }} + if-no-files-found: ignore From 2b7632fa060f053e68b5206fdcbdb82e91563b01 Mon Sep 17 00:00:00 2001 From: Vitalii Popov Date: Thu, 5 Nov 2026 14:05:19 +0100 Subject: [PATCH 4/6] Mark a confirmed bug by its hunt and line, not by its question A re-run on AVD PR 7 posted the same bug again because the model worded the question differently. --- actions/review/reviewrules/post-review.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/actions/review/reviewrules/post-review.js b/actions/review/reviewrules/post-review.js index 83e4f810..72c5d644 100644 --- a/actions/review/reviewrules/post-review.js +++ b/actions/review/reviewrules/post-review.js @@ -41,7 +41,8 @@ const hypothesisFindings = (report) => return { path: at[1], line: Number(at[2]), - key: `hypothesis:${check.at}:${check.question}`, + // The model words a question differently on each run, so the key is the hunt and the place. + key: `hypothesis:${check.hunt}:${at[1]}:${at[2]}`, text: hypothesisText(check), }; }); From d71d357a6f2db9634be293d878b627bbd5e960fd Mon Sep 17 00:00:00 2001 From: Vitalii Popov Date: Thu, 5 Nov 2026 14:08:48 +0100 Subject: [PATCH 5/6] Run the block review as a job of node-simple-pnpm A block repo on v4-beta gets the review from the ai-review label alone, with no ruleset. The job checks that secrets.env holds ANTHROPIC_API_KEY, because a reusable workflow sees only the secrets its caller passes. --- .github/workflows/block-review.yaml | 49 +++++++++++++++++-------- .github/workflows/node-simple-pnpm.yaml | 9 +++++ 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/.github/workflows/block-review.yaml b/.github/workflows/block-review.yaml index 3bfbe3c9..8b85d35c 100644 --- a/.github/workflows/block-review.yaml +++ b/.github/workflows/block-review.yaml @@ -1,24 +1,28 @@ name: Block review -# Runs as a required workflow of an organization ruleset, so block repos need no change. -# A ruleset workflow starts only on PR opened, synchronize, and reopened. After you add -# the label, re-run this workflow, or push a new commit. +# node-simple-pnpm.yaml calls it on every PR. It reviews only when the PR has the ai-review label. on: - pull_request: - -permissions: - contents: read - pull-requests: write - -concurrency: - group: block-review-${{ github.repository }}-${{ github.event.pull_request.number }} - cancel-in-progress: true + workflow_call: + secrets: + env: + description: | + The same JSON-encoded map of secret env variables as node-simple-pnpm.yaml gets. + It must hold ANTHROPIC_API_KEY. REVIEWRULES_RULE_SETS_TOKEN is optional: a token + that can read private rule set repos. + required: false env: REVIEW_LABEL: ai-review jobs: review: + name: AI review runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + concurrency: + group: block-review-${{ github.repository }}-${{ github.event.pull_request.number }} + cancel-in-progress: true steps: # A re-run gets the labels of the first run, so read the labels the PR has now. - name: Check the review label @@ -34,7 +38,22 @@ jobs: }); const requested = labels.some((label) => label.name === process.env.REVIEW_LABEL); core.setOutput('requested', String(requested)); - if (!requested) core.notice(`Add the label ${process.env.REVIEW_LABEL} and re-run this workflow to get a review.`); + if (!requested) core.notice(`Add the label ${process.env.REVIEW_LABEL} and push or re-run this job to get a review.`); + + - uses: milaboratory/github-ci/actions/env@v4-beta + if: steps.label.outputs.requested == 'true' + with: + inputs: '{}' + secrets: ${{ secrets.env || '{}' }} + + - name: Check the Anthropic API key + if: steps.label.outputs.requested == 'true' + shell: bash + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "::error::ANTHROPIC_API_KEY is missing. Add it to secrets.env in .github/workflows/build.yaml, the same way as NPMJS_TOKEN." + exit 1 + fi - uses: actions/checkout@v4 if: steps.label.outputs.requested == 'true' @@ -64,5 +83,5 @@ jobs: - uses: milaboratory/github-ci/actions/review/reviewrules@v4-beta if: steps.label.outputs.requested == 'true' with: - anthropic-api-key: ${{ secrets.ANTHROPIC_API_KEY }} - rule-sets-token: ${{ secrets.REVIEWRULES_RULE_SETS_TOKEN || github.token }} + anthropic-api-key: ${{ env.ANTHROPIC_API_KEY }} + rule-sets-token: ${{ env.REVIEWRULES_RULE_SETS_TOKEN || github.token }} diff --git a/.github/workflows/node-simple-pnpm.yaml b/.github/workflows/node-simple-pnpm.yaml index d9d3bd0b..5cfb46b9 100644 --- a/.github/workflows/node-simple-pnpm.yaml +++ b/.github/workflows/node-simple-pnpm.yaml @@ -1274,6 +1274,15 @@ jobs: override-tag: ${{ format('v{0}', env.NPM_PKG_VERSION) }} search-tags: ${{ steps.search-tags.outputs.result }} + ai-review: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: write + uses: milaboratory/github-ci/.github/workflows/block-review.yaml@v4-beta + secrets: + env: ${{ secrets.env }} + monitor-merge-queue-duration: name: monitor merge queue duration runs-on: ${{ inputs.gha-runner-label }} From e43759f61b2f4835842083662b161c845bdf1b5b Mon Sep 17 00:00:00 2001 From: Vitalii Popov Date: Thu, 5 Nov 2026 15:47:42 +0100 Subject: [PATCH 6/6] Pass the block rule set as reviewrules global settings A repo with its own .reviewrules/ now gets the block rule set too. Its own rule_sets replace it. --- .github/workflows/block-review.yaml | 13 ++++++------- actions/review/reviewrules/action.yaml | 9 ++++++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/block-review.yaml b/.github/workflows/block-review.yaml index 8b85d35c..d5bf1f87 100644 --- a/.github/workflows/block-review.yaml +++ b/.github/workflows/block-review.yaml @@ -62,17 +62,11 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Add the block rule set when the repo has no rule folder + - name: Add a rule folder when the repo has none if: steps.label.outputs.requested == 'true' && hashFiles('.reviewrules/config.json') == '' shell: bash run: | mkdir -p .reviewrules - cat > .reviewrules/settings.toml <<'TOML' - [[rule_sets]] - name = "block" - url = "https://github.com/milaboratory/block-review-rules.git" - ref = "v0.1.0" - TOML cat > .reviewrules/config.json <<'JSON' { "ignorePatterns": "**/node_modules/**\n**/dist/**\n**/block-pack/**\n**/.turbo/**\n**/*.tgz\n**/CHANGELOG.md\nsoftware/**/uv.lock\nsoftware/**/src/requirements.txt\nsoftware/**/tests/data/corpus/**\ntest/fixtures/data/**\n**/__pycache__/**", @@ -85,3 +79,8 @@ jobs: with: anthropic-api-key: ${{ env.ANTHROPIC_API_KEY }} rule-sets-token: ${{ env.REVIEWRULES_RULE_SETS_TOKEN || github.token }} + settings: | + [[rule_sets]] + name = "block" + url = "https://github.com/milaboratory/block-review-rules.git" + ref = "v0.1.0" diff --git a/actions/review/reviewrules/action.yaml b/actions/review/reviewrules/action.yaml index 2baf9dc6..65ea0fe2 100644 --- a/actions/review/reviewrules/action.yaml +++ b/actions/review/reviewrules/action.yaml @@ -26,10 +26,15 @@ inputs: required: false default: ${{ github.token }} + settings: + description: Global reviewrules settings, TOML. A key in the repo .reviewrules/settings.toml replaces the same key here. + required: false + default: '' + version: description: Commit SHA of reviewrules to install. required: false - default: '81894bcb3f637a72bab789fb19dae9882ce3fa27' + default: 'f5d723d9563aae80ce104d912a8d5cf06058bca0' outputs: report: @@ -58,6 +63,7 @@ runs: ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} RULE_SETS_TOKEN: ${{ inputs.rule-sets-token }} BASE_REF: ${{ github.event.pull_request.base.ref }} + SETTINGS: ${{ inputs.settings }} run: | git config --global url."https://x-access-token:${RULE_SETS_TOKEN}@github.com/".insteadOf "https://github.com/" @@ -66,6 +72,7 @@ runs: report="${output}/report.json" mkdir -p "${output}" echo "output=${output}" >> "${GITHUB_OUTPUT}" + printf '%s\n' "${SETTINGS}" > "${RUNNER_TEMP}/settings.toml" cat > "${config}" <<'TOML' [ci] agent = "anthropic"