diff --git a/.github/workflows/block-review.yaml b/.github/workflows/block-review.yaml new file mode 100644 index 00000000..d5bf1f87 --- /dev/null +++ b/.github/workflows/block-review.yaml @@ -0,0 +1,86 @@ +name: Block review +# node-simple-pnpm.yaml calls it on every PR. It reviews only when the PR has the ai-review label. +on: + workflow_call: + secrets: + env: + description: | + The same JSON-encoded map of secret env variables as node-simple-pnpm.yaml gets. + It must hold ANTHROPIC_API_KEY. REVIEWRULES_RULE_SETS_TOKEN is optional: a token + that can read private rule set repos. + required: false + +env: + REVIEW_LABEL: ai-review + +jobs: + review: + name: AI review + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + concurrency: + group: block-review-${{ github.repository }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + steps: + # A re-run gets the labels of the first run, so read the labels the PR has now. + - name: Check the review label + id: label + uses: actions/github-script@v7 + with: + script: | + const labels = await github.paginate(github.rest.issues.listLabelsOnIssue, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + per_page: 100, + }); + const requested = labels.some((label) => label.name === process.env.REVIEW_LABEL); + core.setOutput('requested', String(requested)); + if (!requested) core.notice(`Add the label ${process.env.REVIEW_LABEL} and push or re-run this job to get a review.`); + + - uses: milaboratory/github-ci/actions/env@v4-beta + if: steps.label.outputs.requested == 'true' + with: + inputs: '{}' + secrets: ${{ secrets.env || '{}' }} + + - name: Check the Anthropic API key + if: steps.label.outputs.requested == 'true' + shell: bash + run: | + if [ -z "${ANTHROPIC_API_KEY:-}" ]; then + echo "::error::ANTHROPIC_API_KEY is missing. Add it to secrets.env in .github/workflows/build.yaml, the same way as NPMJS_TOKEN." + exit 1 + fi + + - uses: actions/checkout@v4 + if: steps.label.outputs.requested == 'true' + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Add a rule folder when the repo has none + if: steps.label.outputs.requested == 'true' && hashFiles('.reviewrules/config.json') == '' + shell: bash + run: | + mkdir -p .reviewrules + cat > .reviewrules/config.json <<'JSON' + { + "ignorePatterns": "**/node_modules/**\n**/dist/**\n**/block-pack/**\n**/.turbo/**\n**/*.tgz\n**/CHANGELOG.md\nsoftware/**/uv.lock\nsoftware/**/src/requirements.txt\nsoftware/**/tests/data/corpus/**\ntest/fixtures/data/**\n**/__pycache__/**", + "rules": [] + } + JSON + + - uses: milaboratory/github-ci/actions/review/reviewrules@v4-beta + if: steps.label.outputs.requested == 'true' + with: + anthropic-api-key: ${{ env.ANTHROPIC_API_KEY }} + rule-sets-token: ${{ env.REVIEWRULES_RULE_SETS_TOKEN || github.token }} + settings: | + [[rule_sets]] + name = "block" + url = "https://github.com/milaboratory/block-review-rules.git" + ref = "v0.1.0" diff --git a/.github/workflows/node-simple-pnpm.yaml b/.github/workflows/node-simple-pnpm.yaml index d9d3bd0b..5cfb46b9 100644 --- a/.github/workflows/node-simple-pnpm.yaml +++ b/.github/workflows/node-simple-pnpm.yaml @@ -1274,6 +1274,15 @@ jobs: override-tag: ${{ format('v{0}', env.NPM_PKG_VERSION) }} search-tags: ${{ steps.search-tags.outputs.result }} + ai-review: + if: github.event_name == 'pull_request' + permissions: + contents: read + pull-requests: write + uses: milaboratory/github-ci/.github/workflows/block-review.yaml@v4-beta + secrets: + env: ${{ secrets.env }} + monitor-merge-queue-duration: name: monitor merge queue duration runs-on: ${{ inputs.gha-runner-label }} diff --git a/actions/review/reviewrules/action.yaml b/actions/review/reviewrules/action.yaml new file mode 100644 index 00000000..65ea0fe2 --- /dev/null +++ b/actions/review/reviewrules/action.yaml @@ -0,0 +1,112 @@ +name: Review a pull request with reviewrules +author: 'MiLaboratories' +description: | + Review the pull request diff with reviewrules (https://github.com/popoffvg/reviewrules) + against the rules in the repo's .reviewrules/ folder and its rule sets. + Post each finding as a review comment on its line. A finding on a line outside + the diff goes to the review body. A finding that is already posted is not posted again. + The step fails only when reviewrules cannot run (exit code 2), not on findings. + The artifact reviewrules-attempt- holds report.json and sessions/: parent.json and + one fork-.json per rule or hypothesis fork, with every message of the model calls. + + The repo must be checked out at the PR head commit with full history. + +inputs: + anthropic-api-key: + description: Anthropic API key for the review models. + required: true + + github-token: + description: GitHub token with pull-requests:write. The review comments come from its owner. + required: false + default: ${{ github.token }} + + rule-sets-token: + description: GitHub token that can read the rule set repos, for example a private one. + required: false + default: ${{ github.token }} + + settings: + description: Global reviewrules settings, TOML. A key in the repo .reviewrules/settings.toml replaces the same key here. + required: false + default: '' + + version: + description: Commit SHA of reviewrules to install. + required: false + default: 'f5d723d9563aae80ce104d912a8d5cf06058bca0' + +outputs: + report: + description: Path of the reviewrules JSON report. + value: ${{ steps.review.outputs.report }} + +runs: + using: "composite" + steps: + - uses: actions/setup-go@v5 + with: + go-version: stable + cache: false + + - name: Install reviewrules + shell: bash + env: + VERSION: ${{ inputs.version }} + run: go install "github.com/popoffvg/reviewrules@${VERSION}" + + - name: Review + id: review + shell: bash + env: + CI: 'true' + ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }} + RULE_SETS_TOKEN: ${{ inputs.rule-sets-token }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + SETTINGS: ${{ inputs.settings }} + run: | + git config --global url."https://x-access-token:${RULE_SETS_TOKEN}@github.com/".insteadOf "https://github.com/" + + output="${RUNNER_TEMP}/reviewrules" + config="${RUNNER_TEMP}/reviewrules.toml" + report="${output}/report.json" + mkdir -p "${output}" + echo "output=${output}" >> "${GITHUB_OUTPUT}" + printf '%s\n' "${SETTINGS}" > "${RUNNER_TEMP}/settings.toml" + cat > "${config}" <<'TOML' + [ci] + agent = "anthropic" + model = "claude-haiku-5-5" + reasoning_effort = "medium" + + [[ci.profiles]] + action = "hypothesis" + model = "claude-opus-5-5" + reasoning_effort = "medium" + TOML + + code=0 + reviewrules --config "${config}" --base "origin/${BASE_REF}" --json-out "${report}" --session-dir "${output}/sessions" || code=$? + echo "report=${report}" >> "${GITHUB_OUTPUT}" + if [ "${code}" -ge 2 ]; then + echo "::error::reviewrules could not run (exit code ${code})" + exit "${code}" + fi + + - name: Post findings + uses: actions/github-script@v7 + env: + REPORT: ${{ steps.review.outputs.report }} + with: + github-token: ${{ inputs.github-token }} + script: | + const postReview = require(`${process.env.GITHUB_ACTION_PATH}/post-review.js`); + await postReview({ github, context, core, reportPath: process.env.REPORT }); + + - name: Save the report and the model sessions + if: always() && steps.review.outputs.output != '' + uses: actions/upload-artifact@v4 + with: + name: reviewrules-attempt-${{ github.run_attempt }} + path: ${{ steps.review.outputs.output }} + if-no-files-found: ignore diff --git a/actions/review/reviewrules/post-review.js b/actions/review/reviewrules/post-review.js new file mode 100644 index 00000000..72c5d644 --- /dev/null +++ b/actions/review/reviewrules/post-review.js @@ -0,0 +1,119 @@ +const fs = require('fs'); + +const MARKER_PREFIX = '`; + +const markerKeys = (body) => + (body || '') + .split(MARKER_PREFIX) + .slice(1) + .map((rest) => rest.slice(0, rest.indexOf(' -->'))); + +const ruleFindings = (report) => + (report.rules || []) + .filter((rule) => rule.verdict === 'FAIL') + .flatMap((rule) => + (rule.findings || []).map((finding) => ({ + path: finding.file, + line: finding.line, + key: `${rule.rule_id}:${finding.file}:${finding.line}`, + text: `**${rule.rule_id}** (${rule.severity})\n\n${finding.message}`, + })), + ); + +const hypothesisText = (check) => + [ + `**Bug confirmed** (${check.hunt})`, + check.question, + check.scenario && `**Input that breaks it:** ${check.scenario}`, + check.edit && `**Fix:** ${check.edit}`, + ...(check.evidence || []).map((line) => `- ${line}`), + ] + .filter(Boolean) + .join('\n\n'); + +const hypothesisFindings = (report) => + ((report.correctness || {}).hypotheses || []) + .filter((check) => check.verdict === 'CONFIRMED') + .map((check) => { + const at = /^([^\s:]+):(\d+)/.exec(check.at || '') || []; + return { + path: at[1], + line: Number(at[2]), + // The model words a question differently on each run, so the key is the hunt and the place. + key: `hypothesis:${check.hunt}:${at[1]}:${at[2]}`, + text: hypothesisText(check), + }; + }); + +// The right-side lines of each file patch; GitHub accepts a line comment only on these lines. +const commentableLines = (files) => { + const lines = new Map(); + for (const file of files) { + const right = new Set(); + let next = 0; + for (const row of (file.patch || '').split('\n')) { + const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)/.exec(row); + if (hunk) { + next = Number(hunk[1]); + } else if (row.startsWith('+') || row.startsWith(' ')) { + right.add(next); + next += 1; + } + } + lines.set(file.filename, right); + } + return lines; +}; + +const reviewBody = (report, outside) => { + const cost = (report.cost_estimate || {}).total_usd; + const head = `**reviewrules**: ${(report.rules || []).length} rule checks, cost estimate $${Number(cost || 0).toFixed(2)}.`; + if (outside.length === 0) return head; + const rows = outside.map( + (f) => `- \`${f.path || '?'}:${f.line || '?'}\` ${f.text.replace(/\n+/g, ' ')} ${marker(f.key)}`, + ); + return [head, 'Findings outside the diff lines:', ...rows].join('\n\n'); +}; + +module.exports = async ({ github, context, core, reportPath }) => { + const report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + const pull = context.payload.pull_request; + const target = { owner: context.repo.owner, repo: context.repo.repo, pull_number: pull.number, per_page: 100 }; + + const files = await github.paginate(github.rest.pulls.listFiles, target); + const comments = await github.paginate(github.rest.pulls.listReviewComments, target); + const reviews = await github.paginate(github.rest.pulls.listReviews, target); + const postedKeys = new Set([...comments, ...reviews].flatMap((item) => markerKeys(item.body))); + const lines = commentableLines(files); + + const inline = []; + const outside = []; + for (const finding of [...ruleFindings(report), ...hypothesisFindings(report)]) { + if (postedKeys.has(finding.key)) continue; + const onDiff = finding.path && (lines.get(finding.path) || new Set()).has(finding.line); + (onDiff ? inline : outside).push(finding); + } + + if (inline.length === 0 && outside.length === 0) { + core.info('reviewrules: no new findings'); + return; + } + + await github.rest.pulls.createReview({ + owner: target.owner, + repo: target.repo, + pull_number: target.pull_number, + commit_id: pull.head.sha, + event: 'COMMENT', + body: reviewBody(report, outside), + comments: inline.map((finding) => ({ + path: finding.path, + line: finding.line, + side: 'RIGHT', + body: `${finding.text}\n\n${marker(finding.key)}`, + })), + }); + core.info(`reviewrules: posted ${inline.length} line comments and ${outside.length} findings in the review body`); +};