From 8bdb25977d6793e168be0b51891314ffe6983600 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 10 Jul 2026 20:02:14 +0100 Subject: [PATCH] Make the required CI gate fail closed Classify behavior, documentation, and unknown paths explicitly so source, deployment, dependency, and unrecognized changes cannot satisfy branch protection without meaningful checks. Align extended validation and bootstrap metadata with the pnpm and Node 24 project baseline. Closes #148 --- .github/workflows/extended-validation.yml | 78 ++++++++---- .github/workflows/pr-fast-ci.yml | 140 +++++++++++++++++----- project.bootstrap.yaml | 11 +- test/workflow.test.ts | 121 +++++++++++++++++++ 4 files changed, 296 insertions(+), 54 deletions(-) diff --git a/.github/workflows/extended-validation.yml b/.github/workflows/extended-validation.yml index a13d0b5..12b7c73 100644 --- a/.github/workflows/extended-validation.yml +++ b/.github/workflows/extended-validation.yml @@ -15,7 +15,7 @@ permissions: contents: read env: - NODE_VERSION: '20' + NODE_VERSION: '24.14.1' PYTHON_VERSION: '3.12' defaults: @@ -27,8 +27,9 @@ jobs: name: Detect Extended Validation Scope runs-on: ['self-hosted', 'synology', 'shell-only', 'public'] outputs: - app: ${{ steps.preset.outputs.app || steps.filter.outputs.app || 'false' }} - ci: ${{ steps.preset.outputs.ci || steps.filter.outputs.ci || 'false' }} + behavior: ${{ steps.preset.outputs.behavior || steps.filter.outputs.behavior || 'false' }} + docs: ${{ steps.preset.outputs.docs || steps.filter.outputs.docs || 'false' }} + unknown: ${{ steps.preset.outputs.unknown || steps.filter.outputs.unknown || 'false' }} extended: ${{ steps.preset.outputs.extended || steps.filter.outputs.extended || 'false' }} steps: - uses: actions/checkout@v7 @@ -41,8 +42,9 @@ jobs: if: github.event_name != 'push' run: | cat >>"$GITHUB_OUTPUT" <<'EOF' - app=true - ci=true + behavior=true + docs=false + unknown=false extended=true EOF @@ -51,28 +53,58 @@ jobs: if: github.event_name == 'push' with: filters: | - app: + behavior: + - 'src/**' + - 'test/**' + - 'config/**' + - 'docker/**' + - 'actions/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'tsconfig*.json' + - 'vitest*.ts' + - 'stryker.config.json' - 'project.bootstrap.yaml' - - 'AGENTS.md' - - 'CLAUDE.md' - '.devcontainer/**' - '.githooks/**' - '.github/workflows/**' - 'scripts/**' - - 'docs/bootstrap/**' - - 'README.md' - - 'docs/**' - ci: - - 'project.bootstrap.yaml' - - 'AGENTS.md' - - 'CLAUDE.md' - - '.devcontainer/**' - - '.githooks/**' - - '.github/workflows/**' - - 'scripts/**' - - 'docs/bootstrap/**' - '.env.example' - 'CODEOWNERS' + - 'infra/**' + - 'ops/**' + docs: + - 'AGENTS.md' + - 'CLAUDE.md' + - 'README.md' + - 'docs/**' + - 'LICENSE*' + unknown: + - '**' + - '!src/**' + - '!test/**' + - '!config/**' + - '!docker/**' + - '!actions/**' + - '!package.json' + - '!pnpm-lock.yaml' + - '!tsconfig*.json' + - '!vitest*.ts' + - '!stryker.config.json' + - '!project.bootstrap.yaml' + - '!.devcontainer/**' + - '!.githooks/**' + - '!.github/workflows/**' + - '!scripts/**' + - '!.env.example' + - '!CODEOWNERS' + - '!infra/**' + - '!ops/**' + - '!AGENTS.md' + - '!CLAUDE.md' + - '!README.md' + - '!docs/**' + - '!LICENSE*' extended: - 'infra/**' - 'ops/**' @@ -82,7 +114,7 @@ jobs: runs-on: ['self-hosted', 'synology', 'shell-only', 'public'] timeout-minutes: 15 needs: changes - if: needs.changes.outputs.app == 'true' || needs.changes.outputs.ci == 'true' + if: needs.changes.outputs.behavior == 'true' || needs.changes.outputs.unknown == 'true' steps: - uses: actions/checkout@v7 @@ -98,7 +130,7 @@ jobs: runs-on: ['self-hosted', 'synology', 'shell-only', 'public'] timeout-minutes: 20 needs: changes - if: needs.changes.outputs.extended == 'true' || needs.changes.outputs.app == 'true' + if: needs.changes.outputs.extended == 'true' || needs.changes.outputs.behavior == 'true' || needs.changes.outputs.unknown == 'true' steps: - uses: actions/checkout@v7 @@ -147,7 +179,7 @@ jobs: runs-on: ['self-hosted', 'synology', 'shell-only', 'public'] timeout-minutes: 30 needs: changes - if: needs.changes.outputs.extended == 'true' || needs.changes.outputs.app == 'true' + if: needs.changes.outputs.extended == 'true' || needs.changes.outputs.behavior == 'true' || needs.changes.outputs.unknown == 'true' steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/pr-fast-ci.yml b/.github/workflows/pr-fast-ci.yml index 0ef1659..570be7a 100644 --- a/.github/workflows/pr-fast-ci.yml +++ b/.github/workflows/pr-fast-ci.yml @@ -13,7 +13,7 @@ permissions: pull-requests: read env: - NODE_VERSION: '20' + NODE_VERSION: '24.14.1' PYTHON_VERSION: '3.12' SHELL: /bin/bash @@ -30,35 +30,66 @@ jobs: - shell-only - public outputs: - app: ${{ steps.filter.outputs.app }} - ci: ${{ steps.filter.outputs.ci }} + behavior: ${{ steps.filter.outputs.behavior }} + docs: ${{ steps.filter.outputs.docs }} + unknown: ${{ steps.filter.outputs.unknown }} steps: - uses: dorny/paths-filter@v4 id: filter with: filters: | - app: + behavior: + - 'src/**' + - 'test/**' + - 'config/**' + - 'docker/**' + - 'actions/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'tsconfig*.json' + - 'vitest*.ts' + - 'stryker.config.json' - 'project.bootstrap.yaml' - - 'AGENTS.md' - - 'CLAUDE.md' - - '.devcontainer/**' - - '.githooks/**' - - '.github/workflows/**' - - 'scripts/**' - - 'docs/bootstrap/**' - - 'README.md' - - 'docs/**' - ci: - - 'project.bootstrap.yaml' - - 'AGENTS.md' - - 'CLAUDE.md' - '.devcontainer/**' - '.githooks/**' - '.github/workflows/**' - 'scripts/**' - - 'docs/bootstrap/**' - '.env.example' - 'CODEOWNERS' + - 'infra/**' + - 'ops/**' + docs: + - 'AGENTS.md' + - 'CLAUDE.md' + - 'README.md' + - 'docs/**' + - 'LICENSE*' + unknown: + - '**' + - '!src/**' + - '!test/**' + - '!config/**' + - '!docker/**' + - '!actions/**' + - '!package.json' + - '!pnpm-lock.yaml' + - '!tsconfig*.json' + - '!vitest*.ts' + - '!stryker.config.json' + - '!project.bootstrap.yaml' + - '!.devcontainer/**' + - '!.githooks/**' + - '!.github/workflows/**' + - '!scripts/**' + - '!.env.example' + - '!CODEOWNERS' + - '!infra/**' + - '!ops/**' + - '!AGENTS.md' + - '!CLAUDE.md' + - '!README.md' + - '!docs/**' + - '!LICENSE*' fast-checks: name: Fast Checks @@ -72,7 +103,7 @@ jobs: if: >- github.event.pull_request.draft == false && github.event.pull_request.head.repo.full_name == github.repository && - (needs.changes.outputs.app == 'true' || needs.changes.outputs.ci == 'true') + (needs.changes.outputs.behavior == 'true' || needs.changes.outputs.unknown == 'true') steps: - uses: actions/checkout@v7 with: @@ -115,7 +146,7 @@ jobs: if: >- github.event.pull_request.draft == false && github.event.pull_request.head.repo.full_name != github.repository && - (needs.changes.outputs.app == 'true' || needs.changes.outputs.ci == 'true') + (needs.changes.outputs.behavior == 'true' || needs.changes.outputs.unknown == 'true') steps: - uses: actions/checkout@v7 with: @@ -167,6 +198,11 @@ jobs: steps: - name: Check required PR jobs env: + BEHAVIOR_CHANGED: ${{ needs.changes.outputs.behavior }} + DOCS_CHANGED: ${{ needs.changes.outputs.docs }} + DRAFT_PULL_REQUEST: ${{ github.event.pull_request.draft }} + UNKNOWN_CHANGED: ${{ needs.changes.outputs.unknown }} + SAME_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name == github.repository }} RESULTS: >- changes=${{ needs.changes.result }} fast-checks=${{ needs.fast-checks.result }} @@ -174,15 +210,63 @@ jobs: hosted-fork-fast-checks=${{ needs.hosted-fork-fast-checks.result }} hosted-fork-validate-secrets=${{ needs.hosted-fork-validate-secrets.result }} run: | - failed=0 + declare -A results for entry in $RESULTS; do - job="${entry%%=*}" - status="${entry##*=}" - if [[ "$status" == "success" || "$status" == "skipped" ]]; then - echo "OK $job => $status" - else - echo "FAIL $job => $status" + results["${entry%%=*}"]="${entry##*=}" + done + + expect_status() { + local job="$1" + local expected="$2" + local actual="${results[$job]:-missing}" + if [[ "$actual" != "$expected" ]]; then + echo "FAIL $job => $actual (expected $expected)" failed=1 + else + echo "OK $job => $actual" fi - done + } + + failed=0 + expect_status changes success + + if [[ "$DRAFT_PULL_REQUEST" == "true" ]]; then + echo "INFO draft pull request; validation jobs are intentionally skipped" + expect_status fast-checks skipped + expect_status validate-secrets skipped + expect_status hosted-fork-fast-checks skipped + expect_status hosted-fork-validate-secrets skipped + exit "$failed" + fi + + checks_required=false + if [[ "$BEHAVIOR_CHANGED" == "true" || "$UNKNOWN_CHANGED" == "true" ]]; then + checks_required=true + elif [[ "$DOCS_CHANGED" == "true" ]]; then + echo "INFO docs-only change; behavior checks are intentionally skipped" + else + echo "FAIL change classification produced no matching scope" + failed=1 + fi + + if [[ "$SAME_REPOSITORY" == "true" ]]; then + expect_status validate-secrets success + expect_status hosted-fork-validate-secrets skipped + if [[ "$checks_required" == "true" ]]; then + expect_status fast-checks success + else + expect_status fast-checks skipped + fi + expect_status hosted-fork-fast-checks skipped + else + expect_status validate-secrets skipped + expect_status hosted-fork-validate-secrets success + expect_status fast-checks skipped + if [[ "$checks_required" == "true" ]]; then + expect_status hosted-fork-fast-checks success + else + expect_status hosted-fork-fast-checks skipped + fi + fi + exit "$failed" diff --git a/project.bootstrap.yaml b/project.bootstrap.yaml index aa98567..c7d05c6 100644 --- a/project.bootstrap.yaml +++ b/project.bootstrap.yaml @@ -23,7 +23,7 @@ repo: - docs/bootstrap/** archetype: kind: generic-empty - packageManager: npm + packageManager: pnpm moduleName: github_runner_fleet github: createRepo: false @@ -52,12 +52,17 @@ github: hasDiscussions: false ci: runnerPolicy: hybrid-safe - nodeVersion: "20" + nodeVersion: "24.14.1" pythonVersion: "3.12" fastChecks: + - lint + - test + - build - secrets extendedChecks: - - template-review + - extended-validation + - mutation-test + - drift-detect nightlyCron: 0 7 * * * agents: manageCodexHome: true diff --git a/test/workflow.test.ts b/test/workflow.test.ts index 2e6c2f7..3cf7fcf 100644 --- a/test/workflow.test.ts +++ b/test/workflow.test.ts @@ -276,6 +276,127 @@ describe("CI workflow", () => { expect(workflow.jobs["ci-gate"]["runs-on"]).toEqual(shellSafePublicRunner); }); + test("classifies behavior, docs-only, and unknown PR changes fail-closed", () => { + const workflow = YAML.parse( + fs.readFileSync(path.resolve(".github/workflows/pr-fast-ci.yml"), "utf8") + ) as { + jobs: Record>; + }; + + const changes = workflow.jobs.changes; + const filterStep = (changes.steps as Array>).find( + (step) => step.uses === "dorny/paths-filter@v4" + ); + const filters = String((filterStep?.with as Record).filters); + const fastChecks = workflow.jobs["fast-checks"]; + const forkChecks = workflow.jobs["hosted-fork-fast-checks"]; + const gateScript = String( + ( + workflow.jobs["ci-gate"].steps as Array> + ).find((step) => step.name === "Check required PR jobs")?.run + ); + + for (const behaviorPath of [ + "src/**", + "test/**", + "config/**", + "docker/**", + "actions/**", + "package.json", + "pnpm-lock.yaml" + ]) { + expect(filters).toContain(`- '${behaviorPath}'`); + expect(filters).toContain(`- '!${behaviorPath}'`); + } + expect(filters).toContain("docs:"); + expect(filters).toContain("- 'docs/**'"); + expect(filters).toContain("unknown:"); + expect(filters).toContain("- '**'"); + expect(changes.outputs).toMatchObject({ + behavior: "${{ steps.filter.outputs.behavior }}", + docs: "${{ steps.filter.outputs.docs }}", + unknown: "${{ steps.filter.outputs.unknown }}" + }); + for (const job of [fastChecks, forkChecks]) { + expect(String(job.if)).toContain( + "needs.changes.outputs.behavior == 'true'" + ); + expect(String(job.if)).toContain( + "needs.changes.outputs.unknown == 'true'" + ); + } + expect(gateScript).toContain( + 'if [[ "$BEHAVIOR_CHANGED" == "true" || "$UNKNOWN_CHANGED" == "true" ]]' + ); + expect(gateScript).toContain( + "INFO docs-only change; behavior checks are intentionally skipped" + ); + expect(gateScript).toContain( + "FAIL change classification produced no matching scope" + ); + expect(gateScript).toContain( + "INFO draft pull request; validation jobs are intentionally skipped" + ); + expect(gateScript).toContain("expect_status fast-checks success"); + expect(gateScript).toContain("expect_status hosted-fork-fast-checks success"); + }); + + test("keeps extended validation classification aligned with the PR gate", () => { + const workflow = YAML.parse( + fs.readFileSync( + path.resolve(".github/workflows/extended-validation.yml"), + "utf8" + ) + ) as { + jobs: Record>; + }; + + const changes = workflow.jobs.changes; + const filterStep = (changes.steps as Array>).find( + (step) => step.uses === "dorny/paths-filter@v4" + ); + const filters = String((filterStep?.with as Record).filters); + + expect(filters).toContain("behavior:"); + expect(filters).toContain("docs:"); + expect(filters).toContain("unknown:"); + expect(changes.outputs).toHaveProperty("behavior"); + expect(changes.outputs).toHaveProperty("docs"); + expect(changes.outputs).toHaveProperty("unknown"); + for (const jobName of ["fast-checks", "extended-checks", "mutation-tests"]) { + const condition = String(workflow.jobs[jobName].if); + expect(condition).toContain("needs.changes.outputs.behavior == 'true'"); + expect(condition).toContain("needs.changes.outputs.unknown == 'true'"); + } + }); + + test("records the package manager and CI commands actually used by the repository", () => { + const bootstrap = YAML.parse( + fs.readFileSync(path.resolve("project.bootstrap.yaml"), "utf8") + ) as { + archetype: { packageManager: string }; + ci: { + nodeVersion: string; + fastChecks: string[]; + extendedChecks: string[]; + }; + }; + + expect(bootstrap.archetype.packageManager).toBe("pnpm"); + expect(bootstrap.ci.nodeVersion).toBe("24.14.1"); + expect(bootstrap.ci.fastChecks).toEqual([ + "lint", + "test", + "build", + "secrets" + ]); + expect(bootstrap.ci.extendedChecks).toEqual([ + "extended-validation", + "mutation-test", + "drift-detect" + ]); + }); + test("restricts Claude comment triggers to trusted repository actors", () => { const workflow = YAML.parse( fs.readFileSync(path.resolve(".github/workflows/claude.yml"), "utf8")