From 71ba4ae796292423fa330a9baeaf6352c0b3fb81 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:46:26 -0400 Subject: [PATCH 01/18] docs(standards): add organization health defaults Adds organization-wide community health defaults, validation, ownership, contribution guidance, security policy, and hardened workflow checks. --- .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/bug_report.yml | 77 ++++++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.yml | 42 ++++ .github/ISSUE_TEMPLATE/question.yml | 36 +++ .github/PULL_REQUEST_TEMPLATE.md | 35 +++ .github/markdownlint-profile.yaml | 3 + .../tests/community_health_contract_test.py | 223 ++++++++++++++++++ .github/workflows/standards-validation.yml | 97 ++++++++ .github/workflows/update-stats.yml | 9 +- CONTRIBUTING.md | 56 +++++ SECURITY.md | 49 ++++ 12 files changed, 631 insertions(+), 2 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/ISSUE_TEMPLATE/question.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/markdownlint-profile.yaml create mode 100644 .github/tests/community_health_contract_test.py create mode 100644 .github/workflows/standards-validation.yml create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..787cb14 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @scttbnsn @ALARGECOMPANY @biggest-littlest diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..0782aa8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,77 @@ +name: Bug report +description: Report a reproducible problem +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for reporting a bug. Do not include secrets or private data, and redact logs and configuration. Email security@codeswhat.com instead of filing a public issue for a suspected vulnerability. + + - type: input + id: version + attributes: + label: Version or commit + description: Provide the release version, image digest, or commit SHA. + placeholder: "e.g. 1.2.3 or abc1234" + validations: + required: true + + - type: textarea + id: description + attributes: + label: What happened? + description: Describe the observed behavior and its impact. + validations: + required: true + + - type: textarea + id: expected + attributes: + label: What did you expect? + description: Describe what should have happened instead. + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Provide the smallest complete reproduction you can. + placeholder: | + 1. Configure ... + 2. Run ... + 3. Observe ... + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: Include relevant OS, architecture, runtime, and install method. + placeholder: | + - OS and architecture: + - Runtime or toolchain: + - Install method: + validations: + required: true + + - type: textarea + id: configuration + attributes: + label: Redacted configuration + description: Include only configuration needed to reproduce the problem. + render: yaml + + - type: textarea + id: logs + attributes: + label: Redacted logs + description: Remove credentials, private paths, hostnames, and user data. + render: text + + - type: textarea + id: additional + attributes: + label: Additional context + description: Add screenshots, links, or other useful context. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..632633e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Report a security vulnerability + url: mailto:security@codeswhat.com + about: Report suspected vulnerabilities privately, not in a public issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..ff53175 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,42 @@ +name: Feature request +description: Propose a focused improvement +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Search existing issues and pull requests before proposing a feature. Describe the need before the implementation. + + - type: textarea + id: problem + attributes: + label: Problem + description: What use case or limitation should this change address? + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed change + description: Describe the behavior you want and who benefits from it. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Describe current workarounds or other approaches you considered. + + - type: textarea + id: compatibility + attributes: + label: Compatibility and security + description: Note migration, breaking-change, privacy, or security effects. + + - type: textarea + id: additional + attributes: + label: Additional context + description: Add examples, screenshots, or links that clarify the request. diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 0000000..2d697bf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,36 @@ +name: Question or help +description: Ask about setup, usage, or project behavior +labels: ["question"] +body: + - type: markdown + attributes: + value: | + Check the README, documentation, and existing issues first. Email security@codeswhat.com instead of asking publicly about a suspected vulnerability. + + - type: textarea + id: question + attributes: + label: Question + description: What are you trying to understand or accomplish? + validations: + required: true + + - type: textarea + id: attempted + attributes: + label: What have you tried? + description: Include the relevant documentation and commands you used. + validations: + required: true + + - type: input + id: version + attributes: + label: Version or commit + description: Provide the release version or commit SHA when relevant. + + - type: textarea + id: context + attributes: + label: Additional context + description: Add redacted configuration, logs, screenshots, or links. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..96f9712 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,35 @@ +# Pull request + +## Summary + + + +Related issue: + +## Changes + + + +## Verification + + + +## Security and compatibility + + + +## Checklist + +- [ ] This pull request targets the repository's documented target branch. +- [ ] The change is focused and follows the repository's contributor + instructions. +- [ ] Tests cover non-trivial behavior, or the verification section explains + why they do not apply. +- [ ] Relevant documentation and changelog entries are updated, or they do not + apply. +- [ ] Required local checks pass. +- [ ] The change contains no secrets, credentials, or private data. +- [ ] All required checks and reviews are complete before merge. diff --git a/.github/markdownlint-profile.yaml b/.github/markdownlint-profile.yaml new file mode 100644 index 0000000..b6a8dcf --- /dev/null +++ b/.github/markdownlint-profile.yaml @@ -0,0 +1,3 @@ +config: + MD033: false + MD041: false diff --git a/.github/tests/community_health_contract_test.py b/.github/tests/community_health_contract_test.py new file mode 100644 index 0000000..9439282 --- /dev/null +++ b/.github/tests/community_health_contract_test.py @@ -0,0 +1,223 @@ +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class CommunityHealthContractTest(unittest.TestCase): + def test_required_default_files_exist(self): + required = ( + "SECURITY.md", + "CONTRIBUTING.md", + ".github/CODEOWNERS", + ".github/markdownlint-profile.yaml", + ".github/ISSUE_TEMPLATE/bug_report.yml", + ".github/ISSUE_TEMPLATE/feature_request.yml", + ".github/ISSUE_TEMPLATE/question.yml", + ".github/ISSUE_TEMPLATE/config.yml", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/workflows/standards-validation.yml", + ) + + missing = [path for path in required if not (ROOT / path).is_file()] + + self.assertEqual([], missing, f"missing community health files: {missing}") + + def test_codeowners_has_org_catch_all(self): + codeowners = self.read_text(".github/CODEOWNERS") + + self.assertEqual( + "* @scttbnsn @ALARGECOMPANY @biggest-littlest\n", + codeowners, + ) + + def test_security_policy_has_org_baseline(self): + policy = self.read_text("SECURITY.md") + + for expected in ( + "## Supported versions", + "## Reporting a vulnerability", + "## Security scope", + "Do not open a public GitHub issue", + "security@codeswhat.com", + "private vulnerability reporting", + "48 hours", + "7 days", + ): + self.assertIn(expected, policy) + + def test_contributor_guide_matches_org_workflow(self): + guide = self.read_text("CONTRIBUTING.md") + + for expected in ( + "AGENTS.md", + "active development or integration branch", + "Conventional Commits", + "feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert", + "without emoji", + "tests", + "documentation", + "required checks and reviews", + "SECURITY.md", + ): + self.assertIn(expected, guide) + + def test_issue_forms_have_unique_ids_and_required_reproduction_fields(self): + bug_report = self.read_text(".github/ISSUE_TEMPLATE/bug_report.yml") + feature_request = self.read_text( + ".github/ISSUE_TEMPLATE/feature_request.yml" + ) + question = self.read_text(".github/ISSUE_TEMPLATE/question.yml") + + self.assertIn('labels: ["bug"]', bug_report) + self.assertIn('labels: ["enhancement"]', feature_request) + self.assertIn('labels: ["question"]', question) + self.assert_form_ids_are_unique(bug_report) + self.assert_form_ids_are_unique(feature_request) + self.assert_form_ids_are_unique(question) + + required_bug_fields = {"version", "description", "expected", "reproduce"} + actual_required = self.required_form_ids(bug_report) + self.assertTrue(required_bug_fields <= actual_required) + + self.assertIn("security@codeswhat.com", bug_report) + self.assertIn("redact", bug_report.lower()) + + def test_issue_template_config_routes_security_reports_privately(self): + config = self.read_text(".github/ISSUE_TEMPLATE/config.yml") + + self.assertIn("blank_issues_enabled: false", config) + self.assertIn("url: mailto:security@codeswhat.com", config) + self.assertIn("privately", config.lower()) + + def test_pull_request_template_requires_verification_and_safe_content(self): + template = self.read_text(".github/PULL_REQUEST_TEMPLATE.md") + + for expected in ( + "## Summary", + "## Changes", + "## Verification", + "## Security and compatibility", + "## Checklist", + "target branch", + "tests", + "documentation", + "required checks and reviews", + "secrets, credentials, or private data", + ): + self.assertIn(expected, template) + + def test_standards_workflow_has_unconditional_stable_jobs(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + + self.assertEqual(2, workflow.count("name: Standards Validation")) + self.assertIn(" name: CodeQL\n", workflow) + self.assertIn(" pull_request:\n", workflow) + self.assertIn(" - main\n", workflow) + self.assertIn(" - dev/repository-standards\n", workflow) + self.assertNotIn("\n push:", workflow) + self.assertIn("permissions: {}", workflow) + self.assertIn("runs-on: ubuntu-24.04", workflow) + self.assertNotIn("matrix:", workflow) + self.assertNotIn("continue-on-error:", workflow) + self.assertNotIn("\n if:", workflow) + + validation_job = workflow.split(" validation:\n", 1)[1].split( + "\n codeql:\n", 1 + )[0] + self.assertIn(" actions: read", validation_job) + self.assertIn(" contents: read", validation_job) + self.assertNotIn("write", validation_job) + + codeql_job = workflow.split("\n codeql:\n", 1)[1] + self.assertIn(" actions: read", codeql_job) + self.assertIn(" contents: read", codeql_job) + self.assertIn(" security-events: write", codeql_job) + + jobs = re.findall( + r"^ ([a-z][a-z0-9-]+):$", + workflow.split("\njobs:\n", 1)[1], + re.MULTILINE, + ) + self.assertEqual(["validation", "codeql"], jobs) + + def test_standards_workflow_pins_actions_and_runs_every_contract(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + for expected in ( + "persist-credentials: false", + "community_health_contract_test.py", + "python3 -m compileall -q .", + "YAML.parse_file", + "json.load", + "markdownlint-cli2@0.23.2", + "markdownlint-profile.yaml", + "actionlint", + "zizmorcore/zizmor-action@", + "github/codeql-action/init@", + "languages: python", + "github/codeql-action/analyze@", + ): + self.assertIn(expected, workflow) + + def test_zizmor_action_owns_its_supported_cli_version(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + zizmor_step = workflow.split( + " - name: Validate workflows with zizmor\n", 1 + )[1].split("\n codeql:\n", 1)[0] + + self.assertNotIn("\n version:", zizmor_step) + + def test_linter_configs_do_not_suppress_workflow_findings(self): + markdown_config = self.read_text(".github/markdownlint-profile.yaml") + + self.assertIn("MD033: false", markdown_config) + self.assertIn("MD041: false", markdown_config) + self.assertNotIn("ignores:", markdown_config) + self.assertEqual(2, markdown_config.count(": false")) + self.assertFalse((ROOT / ".github/zizmor.yml").exists()) + + def test_profile_workflow_pins_actions_and_pushes_without_persisted_credentials(self): + workflow = self.read_text(".github/workflows/update-stats.yml") + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertEqual(2, len(actions)) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + checkout_step = workflow.split(" - name: Checkout\n", 1)[1].split( + "\n - name:", 1 + )[0] + push_step = workflow.split(" - name: Commit and push\n", 1)[1] + + self.assertIn("persist-credentials: false", checkout_step) + self.assertIn("GH_TOKEN: ${{ github.token }}", push_step) + self.assertIn("gh auth setup-git", push_step) + + def read_text(self, relative_path): + path = ROOT / relative_path + self.assertTrue(path.is_file(), f"missing community health file: {relative_path}") + return path.read_text() + + def assert_form_ids_are_unique(self, form): + ids = re.findall(r"^ id: ([a-z][a-z0-9_-]*)$", form, re.MULTILINE) + self.assertEqual(len(ids), len(set(ids))) + + def required_form_ids(self, form): + required = set() + for field in form.split("\n - type:"): + field_id = re.search(r"^ id: ([a-z][a-z0-9_-]*)$", field, re.MULTILINE) + if field_id and " required: true" in field: + required.add(field_id.group(1)) + return required + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml new file mode 100644 index 0000000..edfbdfd --- /dev/null +++ b/.github/workflows/standards-validation.yml @@ -0,0 +1,97 @@ +name: Standards Validation + +on: + pull_request: + branches: + - main + - dev/repository-standards + +permissions: {} + +jobs: + validation: + name: Standards Validation + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + + - name: Validate community health contract + run: python3 .github/tests/community_health_contract_test.py + + - name: Compile Python + run: python3 -m compileall -q . + + - name: Parse YAML and JSON + run: | + set -euo pipefail + ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' + python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' + + - name: Lint Markdown + run: | + set -euo pipefail + npx --yes markdownlint-cli2@0.23.2 "**/*.md" "#profile/README.md" + npx --yes markdownlint-cli2@0.23.2 "profile/README.md" --config .github/markdownlint-profile.yaml + + - name: Validate workflows with actionlint + run: | + set -euo pipefail + bash <(curl -fsSL https://raw.githubusercontent.com/rhysd/actionlint/914e7df21a07ef503a81201c76d2b11c789d3fca/scripts/download-actionlint.bash) 1.7.12 "$RUNNER_TEMP" + "$RUNNER_TEMP/actionlint" -color + + - name: Validate workflows with zizmor + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + with: + advanced-security: false + annotations: true + collect: workflows + inputs: .github/workflows/ + online-audits: false + + codeql: + name: CodeQL + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + languages: python + + - name: Analyze + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + category: /language:python diff --git a/.github/workflows/update-stats.yml b/.github/workflows/update-stats.yml index ba295c7..cb79f7c 100644 --- a/.github/workflows/update-stats.yml +++ b/.github/workflows/update-stats.yml @@ -17,10 +17,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -31,7 +33,10 @@ jobs: python generate_svg.py - name: Commit and push + env: + GH_TOKEN: ${{ github.token }} run: | + gh auth setup-git git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git add profile/dark_mode.svg profile/light_mode.svg cache/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1d31cae --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contributing to CodesWhat projects + +Bug fixes, focused features, tests, and documentation improvements are welcome. +Repository-specific instructions take precedence over this organization +default. + +## Before starting + +1. Read the repository's `README.md`, `AGENTS.md`, and development or release + documentation. +2. Search existing issues and pull requests for the same problem. +3. Open an issue before starting a large feature, breaking change, architecture + change, or release-process change. +4. From the documented active development or integration branch, create a + focused branch in your fork. + +Do not commit credentials, private data, local planning files, editor state, or +generated artifacts that the repository does not track. + +## Making a change + +- Keep the change focused on one concern. +- Follow the repository's existing language, formatting, and architecture + conventions. +- Add regression tests for bug fixes and coverage for non-trivial behavior. +- Update documentation and changelog entries when public behavior changes. +- Run the exact formatter, lint, test, build, and local hook commands documented + by the repository. + +## Commits + +Use plain Conventional Commits without emoji: + +```text +(): +``` + +Allowed types are +`feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert`. Keep the subject +concise and describe the change in the imperative mood. + +## Pull requests + +- Select the repository's documented target branch. Do not assume the GitHub + default branch is the correct target. +- Explain the problem, the change, and the verification performed. +- Call out security, compatibility, migration, or release effects. +- Include tests and documentation, or explain why they do not apply. +- Wait for all required checks and reviews before merge. + +Maintainers may ask for changes or decline work that does not fit the project's +scope. Contributions are licensed under the license of the repository receiving +them. + +Report suspected vulnerabilities privately as described in +[`SECURITY.md`](SECURITY.md), never in a public issue or pull request. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5cf164d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,49 @@ +# Security policy + +## Supported versions + +CodesWhat ships security fixes for the latest maintained release or branch of +each project. A repository-specific policy, release notes, or README may define +a narrower support window and takes precedence over this organization default. + +| Release or branch | Supported | +| --- | --- | +| Latest maintained release or branch | Yes | +| Older or unmaintained releases and branches | No | + +For a project that does not publish versioned releases, support applies to the +latest commit on its documented maintained branch. + +## Reporting a vulnerability + +Do not open a public GitHub issue for a suspected security vulnerability. + +Open this repository's **Security** tab and select **Report a vulnerability** +to use GitHub private vulnerability reporting when it is enabled. Otherwise, +email ****. Include the affected version or commit, +minimal reproduction steps, observed and expected behavior, and your assessment +of the impact. Redact credentials, private data, and identifying environment +details. + +You can expect: + +- acknowledgement within 48 hours; +- a status update within 7 days; and +- a fix or mitigation as soon as feasible, depending on severity and release + safety. + +CodesWhat coordinates disclosure with the reporter and credits reporters in +release notes unless they prefer to remain anonymous. + +## Security scope + +Unless a repository-specific policy says otherwise, the following are in +scope: + +- source code and configuration maintained in the repository; +- artifacts published by CodesWhat from that repository; and +- repository-owned build, test, and release automation. + +Third-party services, dependencies, and deployments are out of scope unless the +repository introduces or amplifies the reported impact. If the boundary is +unclear, report the issue privately and let the maintainers triage it. From 22feaaa1cd49740a453e9cd611745edbb13c6e8d Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:15:14 -0400 Subject: [PATCH 02/18] ci(greptile): require manual review requests (#11) --- .github/tests/greptile_config_contract_test.py | 18 ++++++++++++++++++ .github/workflows/standards-validation.yml | 1 + greptile.json | 3 +++ 3 files changed, 22 insertions(+) create mode 100644 .github/tests/greptile_config_contract_test.py create mode 100644 greptile.json diff --git a/.github/tests/greptile_config_contract_test.py b/.github/tests/greptile_config_contract_test.py new file mode 100644 index 0000000..33f7fb8 --- /dev/null +++ b/.github/tests/greptile_config_contract_test.py @@ -0,0 +1,18 @@ +import json +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class GreptileConfigContractTest(unittest.TestCase): + def test_reviews_are_manual_only(self): + with (ROOT / "greptile.json").open() as config_file: + config = json.load(config_file) + + self.assertEqual({"skipReview": "AUTOMATIC"}, config) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index edfbdfd..d9079a1 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -44,6 +44,7 @@ jobs: set -euo pipefail ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' + python3 .github/tests/greptile_config_contract_test.py - name: Lint Markdown run: | diff --git a/greptile.json b/greptile.json new file mode 100644 index 0000000..34c493d --- /dev/null +++ b/greptile.json @@ -0,0 +1,3 @@ +{ + "skipReview": "AUTOMATIC" +} From bc5ab596a2c715cd8eb7b87230abb5e568c33021 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:54:26 -0400 Subject: [PATCH 03/18] ci(workflows): add reusable CI foundation (#13) * ci(workflows): add reusable CI foundation * fix(workflows): harden reusable release contracts --- .github/tests/reusable_ci_contract_test.py | 527 +++++++++++++++++++++ .github/workflows/go-ci.yml | 460 ++++++++++++++++++ .github/workflows/node-ci.yml | 186 ++++++++ .github/workflows/release-gate.yml | 172 +++++++ .github/workflows/standards-validation.yml | 1 + 5 files changed, 1346 insertions(+) create mode 100644 .github/tests/reusable_ci_contract_test.py create mode 100644 .github/workflows/go-ci.yml create mode 100644 .github/workflows/node-ci.yml create mode 100644 .github/workflows/release-gate.yml diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py new file mode 100644 index 0000000..a6ec3f5 --- /dev/null +++ b/.github/tests/reusable_ci_contract_test.py @@ -0,0 +1,527 @@ +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = { + "go": ROOT / ".github/workflows/go-ci.yml", + "node": ROOT / ".github/workflows/node-ci.yml", + "release": ROOT / ".github/workflows/release-gate.yml", +} +TARGET_SHA = "a" * 40 + + +class ReusableCIContractTest(unittest.TestCase): + def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self): + go_inputs = [ + "module-directory", + "go-version-file", + "go-cache-dependency-path", + "lint-check-name", + "test-check-name", + "fuzzers-json", + "run-govulncheck", + "run-qlty", + "run-goreleaser", + "run-workflow-security", + "run-commit-message", + "run-codeql", + ] + self.egress_inputs( + ( + "test", + "lint", + "govulncheck", + "workflow-security", + "commit-message", + "goreleaser", + "codeql", + "qlty", + "fuzz", + ) + ) + node_inputs = [ + "node-version", + "lockfile-path", + "lint-check-name", + "test-check-name", + "build-check-name", + "run-lint", + "run-test", + "run-build", + ] + self.egress_inputs(("lint", "test", "build")) + release_inputs = [ + "target-sha", + "workflow-files-json", + "max-attempts", + "sleep-seconds", + ] + + expected_inputs = { + "go": go_inputs, + "node": node_inputs, + "release": release_inputs, + } + expected_jobs = { + "go": { + "test", + "lint", + "govulncheck", + "workflow-security", + "commit-message", + "goreleaser", + "codeql", + "qlty", + "fuzz", + }, + "node": {"lint", "test", "build"}, + "release": {"gate"}, + } + + for name, path in WORKFLOWS.items(): + workflow = self.read_workflow(path) + header = workflow.split("\npermissions:", 1)[0] + inputs = re.findall(r"^ ([a-z][a-z0-9-]+):$", header, re.MULTILINE) + self.assertEqual(expected_inputs[name], inputs) + self.assertRegex(workflow, r"(?m)^on:\n workflow_call:\s+inputs:") + self.assertNotRegex( + workflow, + r"(?m)^ (push|pull_request|schedule|workflow_dispatch):", + ) + self.assertNotIn("concurrency:", workflow) + self.assertNotIn("outputs:", header) + self.assertNotIn("secrets:", header) + self.assertNotIn("secrets: inherit", workflow) + self.assertIn("permissions: {}", workflow) + + jobs = set( + re.findall( + r"^ ([a-z][a-z0-9-]+):$", + workflow.split("\njobs:\n", 1)[1], + re.MULTILINE, + ) + ) + self.assertEqual(expected_jobs[name], jobs) + + self.assert_input("go", "module-directory", "string", default=".") + self.assert_input("go", "go-version-file", "string", default="go.mod") + self.assert_input("go", "go-cache-dependency-path", "string", default="go.sum") + self.assert_input("go", "lint-check-name", "string", default="Go Lint") + self.assert_input("go", "test-check-name", "string", default="Go Test") + self.assert_input("go", "fuzzers-json", "string", default="[]") + for input_name in ( + "run-govulncheck", + "run-qlty", + "run-goreleaser", + "run-workflow-security", + "run-commit-message", + "run-codeql", + ): + self.assert_input("go", input_name, "boolean", default="false") + + self.assert_input("node", "node-version", "string", default="24") + self.assert_input("node", "lockfile-path", "string", default="package-lock.json") + self.assert_input("node", "lint-check-name", "string", default="Node Lint") + self.assert_input("node", "test-check-name", "string", default="Node Test") + self.assert_input("node", "build-check-name", "string", default="Node Build") + for input_name in ("run-lint", "run-test", "run-build"): + self.assert_input("node", input_name, "boolean", default="false") + + for workflow_name, job_names in ( + ("go", ("test", "lint", "govulncheck", "workflow-security", "commit-message", "goreleaser", "codeql", "qlty", "fuzz")), + ("node", ("lint", "test", "build")), + ): + for job_name in job_names: + self.assert_input(workflow_name, f"{job_name}-egress-policy", "string", default="audit") + self.assert_input(workflow_name, f"{job_name}-allowed-endpoints", "string", default="") + + self.assert_input("release", "target-sha", "string", required=True) + self.assert_input("release", "workflow-files-json", "string", required=True) + self.assert_input("release", "max-attempts", "number", default="12") + self.assert_input("release", "sleep-seconds", "number", default="300") + + def test_central_jobs_have_exact_names_fixed_commands_and_per_job_egress(self): + go = self.read_workflow(WORKFLOWS["go"]) + expected_go_names = { + "test": "${{ inputs.test-check-name }}", + "lint": "${{ inputs.lint-check-name }}", + "govulncheck": "Govulncheck", + "workflow-security": "Workflow Security", + "commit-message": "Commit Message", + "goreleaser": "GoReleaser Config", + "codeql": "CodeQL Analysis", + "qlty": "Qlty Check", + "fuzz": 'Go Fuzz (${{ matrix.fuzzer.name }})', + } + for job, display_name in expected_go_names.items(): + self.assertEqual(display_name, self.job_name(go, job)) + self.assert_job_egress(go, job) + + expected_go_scripts = [ + "./scripts/ci/go-test.sh", + "./scripts/ci/go-lint.sh", + "./scripts/ci/go-govulncheck.sh", + "./scripts/ci/commit-message.sh", + "./scripts/ci/go-release-check.sh", + "./scripts/ci/go-codeql-build.sh", + "./scripts/ci/go-qlty.sh", + "./scripts/ci/go-fuzz.sh", + ] + self.assertEqual(expected_go_scripts, self.fixed_scripts(go)) + self.assertIn("fuzzer: ${{ fromJSON(inputs.fuzzers-json) }}", go) + self.assertIn( + "if: inputs.fuzzers-json != '' && fromJSON(inputs.fuzzers-json)[0] != null", + go, + ) + self.assertIn("go-version-file: ${{ inputs.go-version-file }}", go) + self.assertIn("cache-dependency-path: ${{ inputs.go-cache-dependency-path }}", go) + self.assertIn("MODULE_DIRECTORY: ${{ inputs.module-directory }}", go) + + node = self.read_workflow(WORKFLOWS["node"]) + expected_node_names = { + "lint": "${{ inputs.lint-check-name }}", + "test": "${{ inputs.test-check-name }}", + "build": "${{ inputs.build-check-name }}", + } + for job, display_name in expected_node_names.items(): + self.assertEqual(display_name, self.job_name(node, job)) + self.assert_job_egress(node, job) + self.assertEqual( + [ + "./scripts/ci/node-lint.sh", + "./scripts/ci/node-test.sh", + "./scripts/ci/node-build.sh", + ], + self.fixed_scripts(node), + ) + self.assertIn("node-version: ${{ inputs.node-version }}", node) + self.assertIn("cache-dependency-path: ${{ inputs.lockfile-path }}", node) + + for workflow in (go, node, self.read_workflow(WORKFLOWS["release"])): + self.assertIn("runs-on: ubuntu-24.04", workflow) + for action in re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE): + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + for run_block in re.findall(r"run: \|\n((?: {10}.*\n|\n)+)", workflow): + self.assertNotIn("${{ inputs.", run_block) + + def test_artifact_uploads_are_central_and_fixed(self): + go = self.read_workflow(WORKFLOWS["go"]) + node = self.read_workflow(WORKFLOWS["node"]) + + self.assertEqual( + ["artifacts/go-test/", "artifacts/go-fuzz/"], + re.findall(r"^\s+path: (artifacts/[^\s]+)$", go, re.MULTILINE), + ) + self.assertEqual( + ["artifacts/node-test/", "artifacts/node-build/"], + re.findall(r"^\s+path: (artifacts/[^\s]+)$", node, re.MULTILINE), + ) + for workflow in (go, node): + upload_count = workflow.count("actions/upload-artifact@") + self.assertEqual(upload_count, workflow.count("if-no-files-found: ignore")) + self.assertEqual(upload_count, workflow.count("retention-days: 14")) + self.assertEqual(upload_count, workflow.count("if: always()")) + + def test_release_gate_checks_every_workflow_by_exact_sha_push_and_nonempty_branch(self): + exact_success = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "dev/v1.7", + "status": "completed", + "conclusion": "success", + } + wrong_runs = [ + {**exact_success, "event": "pull_request"}, + {**exact_success, "head_branch": ""}, + {**exact_success, "head_sha": "b" * 40}, + ] + result = self.run_release_gate( + { + "ci-verify.yml": wrong_runs + [exact_success], + "e2e-playwright.yml": [ + {**exact_success, "head_branch": "main"}, + ], + } + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn(f"Found successful CI Verify push run for {TARGET_SHA}.", result.stdout) + self.assertIn(f"Found successful E2E Playwright push run for {TARGET_SHA}.", result.stdout) + + def test_release_gate_keeps_authoritative_client_side_run_filtering(self): + release = self.read_workflow(WORKFLOWS["release"]) + + self.assertIn("/runs?per_page=100\"", release) + self.assertNotRegex(release, r"runs\?[^\"\n]*(?:event|head_sha)=") + self.assertIn("spuriously returning 0 results since 2026-04-27", release) + + def test_release_gate_uses_one_shared_poll_budget_for_every_workflow(self): + in_progress = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "dev/v1.7", + "status": "in_progress", + "conclusion": None, + } + success = {**in_progress, "status": "completed", "conclusion": "success"} + result = self.run_release_gate( + {}, + runs_sequences_by_workflow={ + "ci-verify.yml": [[in_progress], [success]], + "e2e-playwright.yml": [[in_progress], [success]], + }, + max_attempts="2", + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(1, result.stdout.count("FAKE_SLEEP"), result.stdout) + + def test_release_gate_waits_when_an_exact_run_is_still_in_progress(self): + exact_run = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + } + result = self.run_release_gate( + { + "ci-verify.yml": [ + {**exact_run, "status": "completed", "conclusion": "failure"}, + {**exact_run, "status": "in_progress", "conclusion": None}, + ], + } + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("Attempt 1/1: waiting for CI Verify", result.stdout) + self.assertIn( + f"Timed out waiting for successful CI Verify push run for {TARGET_SHA}.", + result.stdout, + ) + + def test_release_gate_fails_closed_when_any_workflow_lacks_exact_success(self): + result = self.run_release_gate( + { + "ci-verify.yml": [ + { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ], + "e2e-playwright.yml": [ + { + "head_sha": "b" * 40, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ], + } + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn( + f"Timed out waiting for successful E2E Playwright push run for {TARGET_SHA}.", + result.stdout, + ) + + def test_release_gate_rejects_unsafe_inputs_before_api_calls(self): + bad_sha = self.run_release_gate({"ci-verify.yml": []}, target_sha="main") + bad_files = self.run_release_gate( + {"ci-verify.yml": []}, + workflow_files_json='["../../release.yml"]', + ) + bad_attempts = self.run_release_gate( + {"ci-verify.yml": []}, + max_attempts="0", + ) + + self.assertNotEqual(0, bad_sha.returncode) + self.assertIn("target-sha must be a full lowercase commit SHA", bad_sha.stdout) + self.assertNotEqual(0, bad_files.returncode) + self.assertIn("workflow-files-json must be a nonempty JSON array", bad_files.stdout) + self.assertNotEqual(0, bad_attempts.returncode) + self.assertIn("max-attempts must be a positive integer", bad_attempts.stdout) + + def test_release_gate_rejects_non_github_api_hosts_before_api_calls(self): + result = self.run_release_gate( + { + "ci-verify.yml": [ + { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ] + }, + api_url="https://github.example/api/v3", + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("release-gate supports api.github.com only", result.stdout) + + def test_standards_validation_runs_this_contract(self): + workflow = self.read_workflow(ROOT / ".github/workflows/standards-validation.yml") + self.assertEqual(1, workflow.count("python3 .github/tests/reusable_ci_contract_test.py")) + + def egress_inputs(self, job_names): + return [item for job in job_names for item in (f"{job}-egress-policy", f"{job}-allowed-endpoints")] + + def read_workflow(self, path): + self.assertTrue(path.is_file(), f"missing reusable workflow: {path.name}") + return path.read_text() + + def input_header(self, workflow_name): + workflow = self.read_workflow(WORKFLOWS[workflow_name]) + return workflow.split("\npermissions:", 1)[0] + + def assert_input(self, workflow_name, input_name, input_type, default=None, required=False): + header = self.input_header(workflow_name) + block = header.split(f" {input_name}:\n", 1)[1] + next_input = re.search(r"^ [a-z][a-z0-9-]+:$", block, re.MULTILINE) + if next_input: + block = block[: next_input.start()] + self.assertIn(f" type: {input_type}\n", block) + if default is not None: + expected = ( + f' default: "{default}"\n' + if default in {"", "[]"} + else f" default: {default}\n" + ) + self.assertIn(expected, block) + if required: + self.assertIn(" required: true\n", block) + + def job_section(self, workflow, job_name): + jobs = workflow.split("\njobs:\n", 1)[1] + section = jobs.split(f" {job_name}:\n", 1)[1] + next_job = re.search(r"^ [a-z][a-z0-9-]+:$", section, re.MULTILINE) + return section[: next_job.start()] if next_job else section + + def job_name(self, workflow, job_name): + section = self.job_section(workflow, job_name) + return re.search(r"^ name: (.+)$", section, re.MULTILINE).group(1).strip('"') + + def assert_job_egress(self, workflow, job_name): + section = self.job_section(workflow, job_name) + self.assertIn(f"egress-policy: ${{{{ inputs.{job_name}-egress-policy }}}}", section) + self.assertIn(f"allowed-endpoints: ${{{{ inputs.{job_name}-allowed-endpoints }}}}", section) + + def fixed_scripts(self, workflow): + return re.findall(r"^\s+run: (\./scripts/ci/[^\s]+)$", workflow, re.MULTILINE) + + def release_gate_script(self): + workflow = self.read_workflow(WORKFLOWS["release"]) + marker = " - name: Verify required CI workflows\n" + self.assertIn(marker, workflow) + step = workflow.split(marker, 1)[1] + block = step.split(" run: |\n", 1)[1] + lines = [] + for line in block.splitlines(): + if line.startswith(" "): + lines.append(line[10:]) + elif not line: + lines.append("") + else: + break + self.assertTrue(lines, "release gate shell step is empty") + return "\n".join(lines) + + def run_release_gate( + self, + runs_by_workflow, + target_sha=TARGET_SHA, + workflow_files_json=None, + max_attempts="1", + api_url="https://api.github.com", + runs_sequences_by_workflow=None, + ): + with tempfile.TemporaryDirectory() as temp_dir: + temp = Path(temp_dir) + fake_bin = temp / "bin" + fake_bin.mkdir() + fixtures = temp / "fixtures" + fixtures.mkdir() + + metadata = { + "ci-verify.yml": (123, "CI Verify"), + "e2e-playwright.yml": (456, "E2E Playwright"), + } + sequences = { + workflow_file: [runs] + for workflow_file, runs in runs_by_workflow.items() + } + sequences.update(runs_sequences_by_workflow or {}) + for workflow_file, runs_sequence in sequences.items(): + workflow_id = metadata[workflow_file][0] + for index, runs in enumerate(runs_sequence, start=1): + (fixtures / f"{workflow_id}.{index}.json").write_text( + json.dumps({"workflow_runs": runs}) + ) + (fixtures / f"{workflow_id}.last").write_text(str(len(runs_sequence))) + + curl = fake_bin / "curl" + curl.write_text( + "#!/usr/bin/env bash\n" + "for argument in \"$@\"; do url=\"$argument\"; done\n" + "case \"$url\" in\n" + " */actions/workflows/ci-verify.yml) printf '%s' '{\"id\":123,\"name\":\"CI Verify\",\"path\":\".github/workflows/ci-verify.yml\"}' ;;\n" + " */actions/workflows/e2e-playwright.yml) printf '%s' '{\"id\":456,\"name\":\"E2E Playwright\",\"path\":\".github/workflows/e2e-playwright.yml\"}' ;;\n" + " */actions/workflows/123/runs?per_page=100) workflow_id=123 ;;\n" + " */actions/workflows/456/runs?per_page=100) workflow_id=456 ;;\n" + " *) printf 'unexpected URL: %s\\n' \"$url\" >&2; exit 22 ;;\n" + "esac\n" + "if [ -n \"${workflow_id:-}\" ]; then\n" + " count_file=\"$RUNS_FIXTURES/$workflow_id.count\"\n" + " count=0\n" + " [ ! -f \"$count_file\" ] || count=$(cat \"$count_file\")\n" + " count=$((count + 1))\n" + " printf '%s' \"$count\" > \"$count_file\"\n" + " last=$(cat \"$RUNS_FIXTURES/$workflow_id.last\")\n" + " [ \"$count\" -le \"$last\" ] || count=\"$last\"\n" + " cat \"$RUNS_FIXTURES/$workflow_id.$count.json\"\n" + "fi\n" + ) + curl.chmod(0o755) + + sleep = fake_bin / "sleep" + sleep.write_text("#!/usr/bin/env bash\nprintf 'FAKE_SLEEP %s\\n' \"$1\"\n") + sleep.chmod(0o755) + + if workflow_files_json is None: + workflow_files_json = json.dumps(list(sequences)) + env = os.environ.copy() + env.update( + { + "GH_TOKEN": "test-token", + "GITHUB_API_URL": api_url, + "GITHUB_REPOSITORY": "CodesWhat/example", + "MAX_ATTEMPTS": max_attempts, + "PATH": f"{fake_bin}:{env['PATH']}", + "RUNS_FIXTURES": str(fixtures), + "SLEEP_SECONDS": "0", + "TARGET_SHA": target_sha, + "WORKFLOW_FILES_JSON": workflow_files_json, + } + ) + return subprocess.run( + ["bash", "-c", self.release_gate_script()], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml new file mode 100644 index 0000000..46601d4 --- /dev/null +++ b/.github/workflows/go-ci.yml @@ -0,0 +1,460 @@ +name: Reusable Go CI + +on: + workflow_call: + inputs: + module-directory: + description: Repository-relative Go module directory + required: false + type: string + default: . + go-version-file: + description: Repository-relative Go version file + required: false + type: string + default: go.mod + go-cache-dependency-path: + description: Repository-relative Go cache dependency path + required: false + type: string + default: go.sum + lint-check-name: + description: Exact display name for the Go lint job + required: false + type: string + default: Go Lint + test-check-name: + description: Exact display name for the Go test job + required: false + type: string + default: Go Test + fuzzers-json: + description: JSON array of caller-owned fuzzer name and pkg objects + required: false + type: string + default: "[]" + run-govulncheck: + description: Run the fixed repository govulncheck gate + required: false + type: boolean + default: false + run-qlty: + description: Run the fixed repository Qlty gate + required: false + type: boolean + default: false + run-goreleaser: + description: Run the fixed repository GoReleaser gate + required: false + type: boolean + default: false + run-workflow-security: + description: Run the central workflow security gate + required: false + type: boolean + default: false + run-commit-message: + description: Run the fixed repository commit-message gate on pull requests + required: false + type: boolean + default: false + run-codeql: + description: Run CodeQL for Actions and Go + required: false + type: boolean + default: false + test-egress-policy: + description: Harden Runner egress policy for Go test + required: false + type: string + default: audit + test-allowed-endpoints: + description: Harden Runner allowlist for Go test + required: false + type: string + default: "" + lint-egress-policy: + description: Harden Runner egress policy for Go lint + required: false + type: string + default: audit + lint-allowed-endpoints: + description: Harden Runner allowlist for Go lint + required: false + type: string + default: "" + govulncheck-egress-policy: + description: Harden Runner egress policy for govulncheck + required: false + type: string + default: audit + govulncheck-allowed-endpoints: + description: Harden Runner allowlist for govulncheck + required: false + type: string + default: "" + workflow-security-egress-policy: + description: Harden Runner egress policy for workflow security + required: false + type: string + default: audit + workflow-security-allowed-endpoints: + description: Harden Runner allowlist for workflow security + required: false + type: string + default: "" + commit-message-egress-policy: + description: Harden Runner egress policy for commit-message validation + required: false + type: string + default: audit + commit-message-allowed-endpoints: + description: Harden Runner allowlist for commit-message validation + required: false + type: string + default: "" + goreleaser-egress-policy: + description: Harden Runner egress policy for GoReleaser + required: false + type: string + default: audit + goreleaser-allowed-endpoints: + description: Harden Runner allowlist for GoReleaser + required: false + type: string + default: "" + codeql-egress-policy: + description: Harden Runner egress policy for CodeQL + required: false + type: string + default: audit + codeql-allowed-endpoints: + description: Harden Runner allowlist for CodeQL + required: false + type: string + default: "" + qlty-egress-policy: + description: Harden Runner egress policy for Qlty + required: false + type: string + default: audit + qlty-allowed-endpoints: + description: Harden Runner allowlist for Qlty + required: false + type: string + default: "" + fuzz-egress-policy: + description: Harden Runner egress policy for Go fuzzing + required: false + type: string + default: audit + fuzz-allowed-endpoints: + description: Harden Runner allowlist for Go fuzzing + required: false + type: string + default: "" + +permissions: {} + +jobs: + test: + name: ${{ inputs.test-check-name }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.test-egress-policy }} + allowed-endpoints: ${{ inputs.test-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go test gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-test.sh + + - name: Upload Go test artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: go-test-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/go-test/ + if-no-files-found: ignore + retention-days: 14 + + lint: + name: ${{ inputs.lint-check-name }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.lint-egress-policy }} + allowed-endpoints: ${{ inputs.lint-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go lint gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-lint.sh + + govulncheck: + name: Govulncheck + if: inputs.run-govulncheck + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.govulncheck-egress-policy }} + allowed-endpoints: ${{ inputs.govulncheck-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository govulncheck gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-govulncheck.sh + + workflow-security: + name: Workflow Security + if: inputs.run-workflow-security + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.workflow-security-egress-policy }} + allowed-endpoints: ${{ inputs.workflow-security-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + with: + token: ${{ github.token }} + advanced-security: false + online-audits: false + + commit-message: + name: Commit Message + if: inputs.run-commit-message && github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.commit-message-egress-policy }} + allowed-endpoints: ${{ inputs.commit-message-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Run repository commit-message gate + env: + BASE_REF: ${{ github.base_ref }} + HEAD_REF: ${{ github.head_ref }} + run: ./scripts/ci/commit-message.sh + + goreleaser: + name: GoReleaser Config + if: inputs.run-goreleaser + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.goreleaser-egress-policy }} + allowed-endpoints: ${{ inputs.goreleaser-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository GoReleaser gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-release-check.sh + + codeql: + name: CodeQL Analysis + if: inputs.run-codeql + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.codeql-egress-policy }} + allowed-endpoints: ${{ inputs.codeql-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + languages: actions, go + queries: security-and-quality + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Build for CodeQL + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-codeql-build.sh + + - name: Analyze with CodeQL + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + + qlty: + name: Qlty Check + if: inputs.run-qlty + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.qlty-egress-policy }} + allowed-endpoints: ${{ inputs.qlty-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Qlty + uses: qltysh/qlty-action/install@08a0a862c159eae9b9003081da6663d96efef637 # v2.3.0 + + - name: Run repository Qlty gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-qlty.sh + + fuzz: + name: "Go Fuzz (${{ matrix.fuzzer.name }})" + if: inputs.fuzzers-json != '' && fromJSON(inputs.fuzzers-json)[0] != null + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + fuzzer: ${{ fromJSON(inputs.fuzzers-json) }} + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.fuzz-egress-policy }} + allowed-endpoints: ${{ inputs.fuzz-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go fuzzer + env: + FUZZER: ${{ matrix.fuzzer.name }} + MODULE_DIRECTORY: ${{ inputs.module-directory }} + PKG: ${{ matrix.fuzzer.pkg }} + run: ./scripts/ci/go-fuzz.sh + + - name: Upload Go fuzz artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: go-fuzz-${{ matrix.fuzzer.name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/go-fuzz/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml new file mode 100644 index 0000000..e699a9b --- /dev/null +++ b/.github/workflows/node-ci.yml @@ -0,0 +1,186 @@ +name: Reusable Node CI + +on: + workflow_call: + inputs: + node-version: + description: Node.js version used by enabled jobs + required: false + type: string + default: 24 + lockfile-path: + description: Repository-relative npm lockfile path or multiline paths + required: false + type: string + default: package-lock.json + lint-check-name: + description: Exact display name for the Node lint job + required: false + type: string + default: Node Lint + test-check-name: + description: Exact display name for the Node test job + required: false + type: string + default: Node Test + build-check-name: + description: Exact display name for the Node build job + required: false + type: string + default: Node Build + run-lint: + description: Run the fixed repository Node lint gate + required: false + type: boolean + default: false + run-test: + description: Run the fixed repository Node test gate + required: false + type: boolean + default: false + run-build: + description: Run the fixed repository Node build gate + required: false + type: boolean + default: false + lint-egress-policy: + description: Harden Runner egress policy for Node lint + required: false + type: string + default: audit + lint-allowed-endpoints: + description: Harden Runner allowlist for Node lint + required: false + type: string + default: "" + test-egress-policy: + description: Harden Runner egress policy for Node test + required: false + type: string + default: audit + test-allowed-endpoints: + description: Harden Runner allowlist for Node test + required: false + type: string + default: "" + build-egress-policy: + description: Harden Runner egress policy for Node build + required: false + type: string + default: audit + build-allowed-endpoints: + description: Harden Runner allowlist for Node build + required: false + type: string + default: "" + +permissions: {} + +jobs: + lint: + name: ${{ inputs.lint-check-name }} + if: inputs.run-lint + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.lint-egress-policy }} + allowed-endpoints: ${{ inputs.lint-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node lint gate + run: ./scripts/ci/node-lint.sh + + test: + name: ${{ inputs.test-check-name }} + if: inputs.run-test + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.test-egress-policy }} + allowed-endpoints: ${{ inputs.test-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node test gate + run: ./scripts/ci/node-test.sh + + - name: Upload Node test artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-test-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/node-test/ + if-no-files-found: ignore + retention-days: 14 + + build: + name: ${{ inputs.build-check-name }} + if: inputs.run-build + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.build-egress-policy }} + allowed-endpoints: ${{ inputs.build-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node build gate + run: ./scripts/ci/node-build.sh + + - name: Upload Node build artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-build-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/node-build/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 0000000..f86ff64 --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,172 @@ +name: Reusable Release Gate + +on: + workflow_call: + inputs: + target-sha: + description: Full commit SHA whose branch push runs must have succeeded + required: true + type: string + workflow-files-json: + description: Nonempty JSON array of workflow filenames to verify + required: true + type: string + max-attempts: + description: Maximum polls for each required workflow + required: false + type: number + default: 12 + sleep-seconds: + description: Seconds between polls + required: false + type: number + default: 300 + +permissions: {} + +jobs: + gate: + name: Release Gate + runs-on: ubuntu-24.04 + timeout-minutes: 360 + permissions: + actions: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: api.github.com:443 + + - name: Verify required CI workflows + env: + GH_TOKEN: ${{ github.token }} + MAX_ATTEMPTS: ${{ inputs.max-attempts }} + SLEEP_SECONDS: ${{ inputs.sleep-seconds }} + TARGET_SHA: ${{ inputs.target-sha }} + WORKFLOW_FILES_JSON: ${{ inputs.workflow-files-json }} + run: | + set -euo pipefail + + if [ "${GITHUB_API_URL}" != "https://api.github.com" ]; then + echo "::error::release-gate supports api.github.com only so its blocked egress allowlist matches the API host." + exit 1 + fi + if ! [[ "${TARGET_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::target-sha must be a full lowercase commit SHA." + exit 1 + fi + if ! [[ "${MAX_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::max-attempts must be a positive integer." + exit 1 + fi + if ! [[ "${SLEEP_SECONDS}" =~ ^[0-9]+$ ]]; then + echo "::error::sleep-seconds must be a nonnegative integer." + exit 1 + fi + if ! jq -e ' + type == "array" + and length > 0 + and all(.[]; + type == "string" + and test("^[A-Za-z0-9][A-Za-z0-9._-]*[.]ya?ml$") + ) + ' <<< "${WORKFLOW_FILES_JSON}" >/dev/null; then + echo "::error::workflow-files-json must be a nonempty JSON array of workflow filenames." + exit 1 + fi + + workflow_files=() + while IFS= read -r workflow_file; do + workflow_files+=("${workflow_file}") + done < <(jq -r '.[]' <<< "${WORKFLOW_FILES_JSON}") + + workflow_ids=() + workflow_names=() + for workflow_file in "${workflow_files[@]}"; do + workflow_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow_file}" + workflow_json="$(curl --proto '=https' --tlsv1.2 --retry 3 --fail --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${workflow_url}")" + workflow_id="$(echo "${workflow_json}" | jq -r '.id // empty')" + workflow_name="$(echo "${workflow_json}" | jq -r '.name // empty')" + workflow_path="$(echo "${workflow_json}" | jq -r '.path // empty')" + expected_path=".github/workflows/${workflow_file}" + if [ -z "${workflow_id}" ] || [ -z "${workflow_name}" ] || [ "${workflow_path}" != "${expected_path}" ]; then + echo "::error::Failed to resolve workflow metadata for ${workflow_file}." + exit 1 + fi + workflow_ids+=("${workflow_id}") + workflow_names+=("${workflow_name}") + done + + workflow_succeeded=() + for index in "${!workflow_ids[@]}"; do + workflow_succeeded["${index}"]=false + done + + for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do + waiting=false + for index in "${!workflow_ids[@]}"; do + if [ "${workflow_succeeded[${index}]}" = "true" ]; then + continue + fi + + workflow_id="${workflow_ids[${index}]}" + workflow_name="${workflow_names[${index}]}" + # GitHub's `?head_sha=` and `?event=` query filters have been spuriously returning 0 results since 2026-04-27. + # Filter the latest 100 runs client-side instead. + runs_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow_id}/runs?per_page=100" + runs_json="$(curl --proto '=https' --tlsv1.2 --retry 3 --fail --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${runs_url}")" + filtered="$(echo "${runs_json}" | jq --arg sha "${TARGET_SHA}" '{ + workflow_runs: [.workflow_runs[] + | select( + .head_sha == $sha + and .event == "push" + and ((.head_branch // "") | length > 0) + )] + }')" + + success_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.conclusion == "success")] | length')" + in_progress_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.status != "completed")] | length')" + completed_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.status == "completed")] | length')" + + if [ "${success_count}" -gt 0 ]; then + echo "Found successful ${workflow_name} push run for ${TARGET_SHA}." + workflow_succeeded["${index}"]=true + continue + fi + + if [ "${completed_count}" -gt 0 ] && [ "${in_progress_count}" -eq 0 ]; then + echo "::error::${workflow_name} push runs for ${TARGET_SHA} completed without success." + echo "${filtered}" | jq '.workflow_runs[] | {id, status, conclusion, head_branch, html_url}' + exit 1 + fi + + echo "Attempt ${attempt}/${MAX_ATTEMPTS}: waiting for ${workflow_name} push run for ${TARGET_SHA}." + waiting=true + done + + if [ "${waiting}" != "true" ]; then + break + fi + if [ "${attempt}" -lt "${MAX_ATTEMPTS}" ]; then + sleep "${SLEEP_SECONDS}" + fi + done + + timed_out=false + for index in "${!workflow_ids[@]}"; do + if [ "${workflow_succeeded[${index}]}" != "true" ]; then + echo "::error::Timed out waiting for successful ${workflow_names[${index}]} push run for ${TARGET_SHA}." + timed_out=true + fi + done + if [ "${timed_out}" = "true" ]; then + exit 1 + fi diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index d9079a1..192e5bd 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -45,6 +45,7 @@ jobs: ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' python3 .github/tests/greptile_config_contract_test.py + python3 .github/tests/reusable_ci_contract_test.py - name: Lint Markdown run: | From 16a6680ee0bdeb32f5a768557413503adc4c36c8 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:35:36 -0400 Subject: [PATCH 04/18] feat(quality): standardize long-run reporting (#15) * feat(quality): add normalized reporting foundation * test(quality): run reporting contracts in standards validation * fix(quality): align report validator with schema * test(quality): verify GitHub integration outputs --- .../quality-result-config/target-result.json | 22 + .../quality-result-filter/target-result.json | 22 + .../quality-result-stats/target-result.json | 16 + .../quality-result-verify/target-result.json | 16 + .../quality-result-stats/target-result.json | 15 + .../quality-result-verify/target-result.json | 16 + .../quality-result-ops-1/target-result.json | 22 + .../quality-result-ops-1/target-result.json | 22 + .../quality-result-ops-2/target-result.json | 22 + .../quality-result-config/target-result.json | 22 + .../target-result.invalid | 21 + .github/tests/quality_report_contract_test.py | 447 ++++++++++ .../workflows/quality-report-aggregate.yml | 157 ++++ .github/workflows/standards-validation.yml | 1 + decisions/quality-reporting.md | 162 ++++ quality-report/v1/quality_report.py | 765 ++++++++++++++++++ quality-report/v1/schema.json | 496 ++++++++++++ 17 files changed, 2244 insertions(+) create mode 100644 .github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid create mode 100644 .github/tests/quality_report_contract_test.py create mode 100644 .github/workflows/quality-report-aggregate.yml create mode 100644 decisions/quality-reporting.md create mode 100644 quality-report/v1/quality_report.py create mode 100644 quality-report/v1/schema.json diff --git a/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json new file mode 100644 index 0000000..d4f7466 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/config", + "outcome": "passed", + "metrics": { + "killed": 40, + "timeout": 10, + "survived": 5, + "no_coverage": 5, + "invalid": 2, + "ignored": 3, + "tool_score_numerator": 40, + "tool_score_denominator": 45, + "tool_score_pct": 88.89, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json new file mode 100644 index 0000000..a4b3164 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/filter", + "outcome": "passed", + "metrics": { + "killed": 30, + "timeout": 0, + "survived": 0, + "no_coverage": 10, + "invalid": 1, + "ignored": 2, + "tool_score_numerator": 30, + "tool_score_denominator": 30, + "tool_score_pct": 100.0, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json new file mode 100644 index 0000000..627a14e --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/docker/FuzzDecodeStats", + "outcome": "passed", + "metrics": { + "budget_seconds": 120, + "elapsed_seconds": 111.25, + "executions": 650000, + "new_interesting_inputs": 2 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json new file mode 100644 index 0000000..472e276 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 288.25, + "executions": 1250000, + "new_interesting_inputs": 3 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json new file mode 100644 index 0000000..0db4a95 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json @@ -0,0 +1,15 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/docker/FuzzDecodeStats", + "outcome": "crashed", + "metrics": null, + "diagnostic": "tool process exited 2", + "reproduction": { + "seed": "-test.fuzzcachedir=/tmp/fuzz-cache" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json new file mode 100644 index 0000000..c2f7e89 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 300.0, + "executions": 990000, + "new_interesting_inputs": 0 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json b/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json new file mode 100644 index 0000000..4a5d472 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-1", + "outcome": "passed", + "metrics": { + "killed": 20, + "timeout": 2, + "survived": 3, + "no_coverage": 5, + "invalid": 1, + "ignored": 2, + "tool_score_numerator": 22, + "tool_score_denominator": 25, + "tool_score_pct": 88.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json new file mode 100644 index 0000000..f79465e --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-1", + "outcome": "passed", + "metrics": { + "killed": 3, + "timeout": 1, + "survived": 1, + "no_coverage": 1, + "invalid": 0, + "ignored": 0, + "tool_score_numerator": 4, + "tool_score_denominator": 5, + "tool_score_pct": 80.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json new file mode 100644 index 0000000..403a77f --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-2", + "outcome": "passed", + "metrics": { + "killed": 2, + "timeout": 0, + "survived": 2, + "no_coverage": 0, + "invalid": 1, + "ignored": 1, + "tool_score_numerator": 2, + "tool_score_denominator": 4, + "tool_score_pct": 50.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json new file mode 100644 index 0000000..d1f3655 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/config", + "outcome": "passed", + "metrics": { + "killed": 12, + "timeout": 1, + "survived": 2, + "no_coverage": 1, + "invalid": 0, + "ignored": 1, + "tool_score_numerator": 12, + "tool_score_denominator": 14, + "tool_score_pct": 85.71, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid new file mode 100644 index 0000000..783dac7 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid @@ -0,0 +1,21 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/filter", + "outcome": "passed", + "metrics": { + "killed": 31, + "timeout": 4, + "survived": 2, + "no_coverage": 3, + "invalid": 1, + "ignored": 0, + "tool_score_numerator": 31, + "tool_score_denominator": 33, + "tool_score_pct": 93.94, + "tool_score_definition": "killed / (killed + survived)" + } + } diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py new file mode 100644 index 0000000..b456ce7 --- /dev/null +++ b/.github/tests/quality_report_contract_test.py @@ -0,0 +1,447 @@ +import json +from pathlib import Path +import runpy +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +CONTRACT_ROOT = ROOT / "quality-report" / "v1" +FIXTURES = ROOT / ".github" / "tests" / "fixtures" / "quality-report" / "v1" +SCRIPT = CONTRACT_ROOT / "quality_report.py" +SCHEMA = CONTRACT_ROOT / "schema.json" +WORKFLOW = ROOT / ".github" / "workflows" / "quality-report-aggregate.yml" +DECISION = ROOT / "decisions" / "quality-reporting.md" +SHA = "0123456789abcdef0123456789abcdef01234567" +QUALITY_REPORT = runpy.run_path(str(SCRIPT)) + + +class QualityReportContractTest(unittest.TestCase): + def test_fixture_inventory_covers_failure_and_score_boundaries(self): + self.assertEqual( + { + "canonical-score", + "complete", + "crash", + "incomplete", + "native-score", + "parse-error", + }, + {path.name for path in FIXTURES.iterdir() if path.is_dir()}, + ) + + def test_complete_fuzz_fixture_emits_a_valid_report_and_summary(self): + result, report, summary = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("quality-report/v1", report["schema_version"]) + self.assertEqual("report", report["kind"]) + self.assertEqual( + {"expected": 2, "reported": 2, "complete": True}, + report["completeness"], + ) + self.assertEqual("passed", report["outcome"]) + self.assertEqual(420, report["metrics"]["declared_budget_seconds"]) + self.assertEqual(399.5, report["metrics"]["elapsed_seconds"]) + self.assertEqual(1900000, report["metrics"]["executions"]) + self.assertEqual(5, report["metrics"]["new_interesting_inputs"]) + self.assertNotIn("canonical_score_pct", report["metrics"]) + self.assertIn("complete 2/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_incomplete_fixture_fails_closed_without_a_subset_score(self): + result, report, summary = self.aggregate( + "incomplete", + track="mutation", + tool="stryker", + expected=["apps-api-ops-1", "apps-api-ops-2"], + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual( + {"expected": 2, "reported": 1, "complete": False}, + report["completeness"], + ) + self.assertEqual("error", report["outcome"]) + self.assertIsNone(report["metrics"]) + self.assertIn("missing target: apps-api-ops-2", report["errors"]) + self.assertIn("partial 1/2", summary) + self.assertNotIn("Canonical score", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_crash_fixture_fails_even_when_every_target_reports(self): + result, report, summary = self.aggregate( + "crash", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + + self.assertNotEqual(0, result.returncode) + self.assertTrue(report["completeness"]["complete"]) + self.assertEqual("crashed", report["outcome"]) + self.assertIsNone(report["metrics"]) + self.assertIn("tool process exited 2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_parse_error_fixture_still_emits_an_error_report(self): + result, report, summary = self.aggregate( + "parse-error", + track="mutation", + tool="gremlins", + expected=["./internal/config", "./internal/filter"], + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual("error", report["outcome"]) + self.assertFalse(report["completeness"]["complete"]) + self.assertIsNone(report["metrics"]) + self.assertTrue( + any("invalid JSON" in error for error in report["errors"]), + report["errors"], + ) + self.assertIn("partial 1/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_canonical_score_counts_timeout_and_no_coverage(self): + result, report, summary = self.aggregate( + "canonical-score", + track="mutation", + tool="gremlins", + expected=["./internal/config", "./internal/filter"], + ) + + self.assertEqual(0, result.returncode, result.stderr) + metrics = report["metrics"] + self.assertEqual(70, metrics["killed"]) + self.assertEqual(10, metrics["timeout"]) + self.assertEqual(5, metrics["survived"]) + self.assertEqual(15, metrics["no_coverage"]) + self.assertEqual(80, metrics["detected"]) + self.assertEqual(20, metrics["missed"]) + self.assertEqual(80.0, metrics["canonical_score_pct"]) + self.assertEqual(93.33, metrics["tool_score_pct"]) + self.assertIn("Canonical score: **80.00%**", summary) + + def test_decision_defines_canonical_score_as_a_percentage(self): + decision = DECISION.read_text() + + self.assertIn( + "canonical_score_pct = 100 * detected / (detected + missed)", + decision, + ) + + def test_decision_documents_track_and_tool_pairings(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn("`mutation` permits `gremlins` and `stryker`", decision) + self.assertIn("`fuzz` permits `fast-check` and `go-fuzz`", decision) + + def test_native_score_is_weighted_and_keeps_its_definition(self): + result, report, summary = self.aggregate( + "native-score", + track="mutation", + tool="stryker", + expected=["apps-api-ops-1", "apps-api-ops-2"], + ) + + self.assertEqual(0, result.returncode, result.stderr) + metrics = report["metrics"] + self.assertEqual(60.0, metrics["canonical_score_pct"]) + self.assertEqual(66.67, metrics["tool_score_pct"]) + self.assertEqual(6, metrics["tool_score_numerator"]) + self.assertEqual(9, metrics["tool_score_denominator"]) + self.assertEqual( + "detected / mutants with test coverage", + metrics["tool_score_definition"], + ) + self.assertIn("Native score: **66.67%**", summary) + + def test_validator_rejects_unknown_report_fields(self): + result, report, _ = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + self.assertEqual(0, result.returncode, result.stderr) + report["unreviewed"] = True + + validation = self.validate_report(report) + + self.assertNotEqual(0, validation.returncode) + self.assertIn("unknown field", validation.stderr) + + def test_validator_rejects_missing_start_time_from_a_success_report(self): + result, report, _ = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + self.assertEqual(0, result.returncode, result.stderr) + report["run"]["started_at"] = None + + validation = self.validate_report(report) + + self.assertNotEqual(0, validation.returncode) + self.assertIn("started_at", validation.stderr) + + def test_validator_rejects_target_names_beyond_the_schema_limit(self): + target = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + target["target"]["name"] = "x" * 257 + + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], "256"): + QUALITY_REPORT["validate_target_document"](target) + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], "256"): + QUALITY_REPORT["parse_expected_targets"]( + json.dumps([target["target"]["name"]]) + ) + + def test_validator_requires_a_positive_declared_fuzz_budget(self): + metrics = { + "declared_budget_seconds": 0, + "elapsed_seconds": 0, + "executions": 0, + "new_interesting_inputs": 0, + } + + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], ">= 1"): + QUALITY_REPORT["validate_fuzz_report_metrics"](metrics, [], "$.metrics") + + def test_validator_requires_time_and_offset_in_timestamps(self): + for timestamp in ("2026-08-14", "2026-08-14T15:00:00"): + with self.subTest(timestamp=timestamp): + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], "RFC 3339" + ): + QUALITY_REPORT["require_timestamp"](timestamp, "$.run.started_at") + + def test_schema_is_versioned_and_fail_closed(self): + with SCHEMA.open() as schema_file: + schema = json.load(schema_file) + + self.assertEqual( + "https://json-schema.org/draft/2020-12/schema", + schema["$schema"], + ) + self.assertEqual( + "https://codeswhat.com/schemas/quality-report/v1/schema.json", + schema["$id"], + ) + self.assertEqual("quality-report/v1", schema["$defs"]["version"]["const"]) + self.assertNotIn("additionalProperties", schema) + self.assertTrue(self.all_objects_are_closed(schema)) + + def test_reusable_workflow_matches_the_public_contract(self): + workflow = WORKFLOW.read_text() + + self.assertIn(" workflow_call:", workflow) + for input_name in ( + "track", + "tool", + "policy", + "expected_targets", + "result_artifact_pattern", + "report_name", + "retention_days", + "fail_on_incomplete", + ): + self.assertIn(f" {input_name}:\n", workflow) + for output_name in ( + "report_artifact_name", + "completeness", + "expected_targets", + "reported_targets", + "outcome", + "canonical_score_pct", + ): + self.assertIn(f" {output_name}:\n", workflow) + + self.assertIn(" default: quality-result-*", workflow) + self.assertIn(" default: 90", workflow) + self.assertIn(" default: true", workflow) + self.assertIn(" if: always()", workflow) + self.assertIn( + "repository: ${{ fromJSON(toJSON(job)).workflow_repository }}", workflow + ) + self.assertIn("ref: ${{ fromJSON(toJSON(job)).workflow_sha }}", workflow) + self.assertIn( + "name: quality-report-${{ inputs.track }}-${{ github.run_id }}-${{ github.run_attempt }}", + workflow, + ) + self.assertIn("retention-days: ${{ inputs.retention_days }}", workflow) + self.assertIn("if: always()", workflow) + + actions = [ + line.split("uses: ", 1)[1].split()[0] + for line in workflow.splitlines() + if "uses: " in line + ] + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + for forbidden in ("git add", "git commit", "git push", "badges/"): + self.assertNotIn(forbidden, workflow) + + def test_public_decision_records_fail_closed_and_no_source_badges(self): + decision = DECISION.read_text() + + for expected in ( + "quality-report/v1", + "target-result.json", + "if: always()", + "quality-report---", + "90 days", + "partial N/M", + "Do not commit badge JSON to `main`", + "Native percentages are not comparable", + ): + self.assertIn(expected, decision) + + def test_standards_validation_runs_this_contract(self): + workflow = (ROOT / ".github/workflows/standards-validation.yml").read_text() + self.assertEqual( + 1, + workflow.count("python3 .github/tests/quality_report_contract_test.py"), + ) + + def aggregate(self, fixture, *, track, tool, expected): + self.assertTrue(SCRIPT.is_file(), f"missing aggregator: {SCRIPT}") + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) / "input" + shutil.copytree(FIXTURES / fixture, input_dir) + for invalid_fixture in input_dir.rglob("target-result.invalid"): + invalid_fixture.rename(invalid_fixture.with_name("target-result.json")) + output_dir = Path(temp_dir) / "output" + github_output = Path(temp_dir) / "github-output" + summary_path = Path(temp_dir) / "step-summary" + command = [ + "python3", + str(SCRIPT), + "aggregate", + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + "--repository", + "CodesWhat/example", + "--ref", + "refs/heads/dev/example", + "--sha", + SHA, + "--run-id", + "1234567890", + "--run-attempt", + "2", + "--event", + "workflow_dispatch", + "--run-url", + "https://github.com/CodesWhat/example/actions/runs/1234567890", + "--started-at", + "2026-08-13T12:34:56Z", + "--track", + track, + "--tool", + tool, + "--policy", + "advisory", + "--expected-targets", + json.dumps(expected), + "--report-name", + "Fixture quality report", + "--fail-on-incomplete", + "true", + "--github-output", + str(github_output), + "--github-step-summary", + str(summary_path), + ] + result = subprocess.run(command, capture_output=True, text=True) + report_path = output_dir / "report.json" + summary_file = output_dir / "summary.md" + self.assertTrue(report_path.is_file(), result.stderr) + self.assertTrue(summary_file.is_file(), result.stderr) + report = json.loads(report_path.read_text()) + summary = summary_file.read_text() + self.assertTrue(github_output.is_file(), result.stderr) + self.assertTrue(summary_path.is_file(), result.stderr) + github_outputs = dict( + line.split("=", 1) for line in github_output.read_text().splitlines() + ) + canonical_score = "" + if ( + report["track"] == "mutation" + and report["metrics"] is not None + and report["metrics"]["canonical_score_pct"] is not None + ): + canonical_score = "{0:.2f}".format( + report["metrics"]["canonical_score_pct"] + ) + self.assertEqual( + { + "report_artifact_name": "quality-report-{0}-1234567890-2".format( + track + ), + "completeness": str(report["completeness"]["complete"]).lower(), + "expected_targets": str(report["completeness"]["expected"]), + "reported_targets": str(report["completeness"]["reported"]), + "outcome": report["outcome"], + "canonical_score_pct": canonical_score, + }, + github_outputs, + ) + self.assertEqual(summary, summary_path.read_text()) + return result, report, summary + + def validate_report(self, report): + with tempfile.TemporaryDirectory() as temp_dir: + report_path = Path(temp_dir) / "report.json" + report_path.write_text(json.dumps(report)) + return subprocess.run( + ["python3", str(SCRIPT), "validate", str(report_path)], + capture_output=True, + text=True, + ) + + def all_objects_are_closed(self, value): + if isinstance(value, dict): + if ( + value.get("type") == "object" + and value.get("additionalProperties") is not False + ): + return False + return all(self.all_objects_are_closed(item) for item in value.values()) + if isinstance(value, list): + return all(self.all_objects_are_closed(item) for item in value) + return True + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/quality-report-aggregate.yml b/.github/workflows/quality-report-aggregate.yml new file mode 100644 index 0000000..86ef420 --- /dev/null +++ b/.github/workflows/quality-report-aggregate.yml @@ -0,0 +1,157 @@ +name: Quality Report Aggregate + +on: + workflow_call: + inputs: + track: + description: Quality track, mutation or fuzz + required: true + type: string + tool: + description: Native tool that produced the normalized target results + required: true + type: string + policy: + description: Reporting policy, advisory or signal + required: true + type: string + expected_targets: + description: Non-empty JSON array of unique target names + required: true + type: string + result_artifact_pattern: + description: Artifact-name pattern containing target-result.json files + required: false + type: string + default: quality-result-* + report_name: + description: Human-readable report name + required: true + type: string + retention_days: + description: Aggregate artifact retention in days + required: false + type: number + default: 90 + fail_on_incomplete: + description: Must remain true for CodesWhat telemetry + required: false + type: boolean + default: true + outputs: + report_artifact_name: + description: Fixed aggregate artifact name + value: ${{ jobs.aggregate.outputs.report_artifact_name }} + completeness: + description: Whether every expected target reported valid JSON + value: ${{ jobs.aggregate.outputs.completeness }} + expected_targets: + description: Number of expected targets + value: ${{ jobs.aggregate.outputs.expected_targets }} + reported_targets: + description: Number of valid expected target results + value: ${{ jobs.aggregate.outputs.reported_targets }} + outcome: + description: Aggregate outcome + value: ${{ jobs.aggregate.outputs.outcome }} + canonical_score_pct: + description: Canonical mutation score, empty for fuzz or partial reports + value: ${{ jobs.aggregate.outputs.canonical_score_pct }} + +permissions: {} + +jobs: + aggregate: + name: Quality Report Aggregate + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + outputs: + report_artifact_name: ${{ steps.aggregate.outputs.report_artifact_name }} + completeness: ${{ steps.aggregate.outputs.completeness }} + expected_targets: ${{ steps.aggregate.outputs.expected_targets }} + reported_targets: ${{ steps.aggregate.outputs.reported_targets }} + outcome: ${{ steps.aggregate.outputs.outcome }} + canonical_score_pct: ${{ steps.aggregate.outputs.canonical_score_pct }} + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out the called workflow contract + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # actionlint 1.7.12 does not model these current reusable-workflow fields. + repository: ${{ fromJSON(toJSON(job)).workflow_repository }} + ref: ${{ fromJSON(toJSON(job)).workflow_sha }} + path: quality-report-contract + persist-credentials: false + + - name: Read workflow run metadata + id: metadata + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + started_at="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq .run_started_at)" + test -n "$started_at" + echo "started_at=${started_at}" >> "$GITHUB_OUTPUT" + + - name: Download normalized target results + id: download + if: always() + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ inputs.result_artifact_pattern }} + path: quality-result-artifacts + + - name: Validate and aggregate target results + id: aggregate + if: always() + env: + EXPECTED_TARGETS: ${{ inputs.expected_targets }} + FAIL_ON_INCOMPLETE: ${{ inputs.fail_on_incomplete }} + POLICY: ${{ inputs.policy }} + REPORT_NAME: ${{ inputs.report_name }} + RUN_STARTED_AT: ${{ steps.metadata.outputs.started_at }} + TOOL: ${{ inputs.tool }} + TRACK: ${{ inputs.track }} + run: | + set -euo pipefail + python3 quality-report-contract/quality-report/v1/quality_report.py aggregate \ + --input-dir quality-result-artifacts \ + --output-dir quality-report-output \ + --repository "$GITHUB_REPOSITORY" \ + --ref "$GITHUB_REF" \ + --sha "$GITHUB_SHA" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --event "$GITHUB_EVENT_NAME" \ + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --started-at "$RUN_STARTED_AT" \ + --track "$TRACK" \ + --tool "$TOOL" \ + --policy "$POLICY" \ + --expected-targets "$EXPECTED_TARGETS" \ + --report-name "$REPORT_NAME" \ + --fail-on-incomplete "$FAIL_ON_INCOMPLETE" \ + --github-output "$GITHUB_OUTPUT" \ + --github-step-summary "$GITHUB_STEP_SUMMARY" + + - name: Upload normalized quality report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: quality-report-${{ inputs.track }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + quality-report-output/report.json + quality-report-output/summary.md + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index 192e5bd..ceb4606 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -45,6 +45,7 @@ jobs: ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' python3 .github/tests/greptile_config_contract_test.py + python3 .github/tests/quality_report_contract_test.py python3 .github/tests/reusable_ci_contract_test.py - name: Lint Markdown diff --git a/decisions/quality-reporting.md b/decisions/quality-reporting.md new file mode 100644 index 0000000..6182c97 --- /dev/null +++ b/decisions/quality-reporting.md @@ -0,0 +1,162 @@ +# Quality Reporting + +Last updated: 2026-08-13 + +## Decision + +Keep each repository's native test tool. Normalize the result envelope, not the +runner: + +- Drydock keeps Stryker and fast-check. +- Sockguard keeps Gremlins and Go fuzzing. +- Portwing keeps Gremlins and Go fuzzing. + +Mutation stays advisory and absent from required PR checks. A scheduled or +manual mutation workflow must still fail when its telemetry is incomplete or a +tool crashes. Non-blocking means it does not gate a PR. It does not mean the +workflow may report success after losing results. + +Long fuzzing succeeds only when every declared target completes its budget. A +crash, repeated boundary flake, setup failure, or missing target result makes +the workflow red. + +## Evidence behind the decision + +The 2026-08-01 Drydock mutation run completed at the workflow level while 13 of +27 Stryker shards failed. Its aggregate used `--allow-missing` and published an +81.79% score from the remaining 14 shards. The workflow status and numeric +badge therefore described a partial result as success. + +Sockguard's existing 97.8% badge uses `killed / (killed + lived)`. Counting +timeout as detected and no-coverage as missed gives 93.91% for the same run. +Portwing has no aggregate badge; applying the same canonical definition to its +latest complete Gremlins run gives 77.31%, versus 81.98% under Gremlins' +efficacy definition. Native percentages are not comparable until their +denominators are explicit. + +Sockguard inventories all 48 in-tree fuzzers and covers every target in at +least one tier. Portwing's `FuzzVerifyRequest` runs in primary CI and scheduled +long tiers. Drydock's five fast-check fuzz tests run as ordinary Vitest tests +with default run counts and have no scheduled long-fuzz workflow. + +## Normalized envelope + +Every matrix leg writes one `target-result.json`. An always-running aggregate +job validates the set and publishes `quality-report/v1` with: + +- `repository`, `ref`, `sha` +- `run`: `id`, `attempt`, `event`, `url`, `started_at` +- `track`: `mutation` or `fuzz` +- `tool`: `mutation` permits `gremlins` and `stryker`; `fuzz` permits + `fast-check` and `go-fuzz` +- `policy`: `advisory` or `signal` +- `completeness`: `expected`, `reported`, `complete` +- `outcome`: `passed`, `failed`, `crashed`, `flaked`, `cancelled`, `error` +- `targets`: the declared target names and their individual results +- track-specific `metrics` + +The schema and dependency-free validator live under [`quality-report/v1`](../quality-report/v1). +Both target-result fragments and aggregate reports carry the exact +`quality-report/v1` version. Objects reject unknown fields, JSON parsing rejects +duplicate fields and non-finite numbers, and semantic validation recomputes +completeness, outcomes, and scores. Callers pass `expected_targets` as a +non-empty JSON array of unique target names. + +Mutation metrics are: + +- `killed`, `timeout`, `survived`, `no_coverage`, `invalid`, `ignored` +- `detected = killed + timeout` +- `missed = survived + no_coverage` +- `canonical_score_pct = 100 * detected / (detected + missed)`, rounded + half-up to two decimals, or `null` when the denominator is zero +- `tool_score_pct` and `tool_score_definition` + +Each target also supplies the native numerator and denominator. The aggregate +weights those counts instead of averaging percentages, and rejects mixed native +score definitions. The native score remains available with its definition. +Only the canonical score is comparable across mutation tools. + +Fuzz metrics include the declared budget, elapsed time, and executions or new +interesting inputs when the runner exposes them. Do not compare execution +counts between runners or tools. A failed, crashed, or flaked fuzz result must +retain a seed, path, or corpus reference for reproduction. + +## Reusable workflow contract + +Inputs: + +- `track` +- `tool` +- `policy` +- `expected_targets` +- `result_artifact_pattern`, default `quality-result-*` +- `report_name` +- `retention_days`, default `90` +- `fail_on_incomplete`, always `true` for CodesWhat telemetry + +Outputs: + +- `report_artifact_name` +- `completeness` +- `expected_targets` +- `reported_targets` +- `outcome` +- `canonical_score_pct` for mutation reports + +The aggregate artifact is named +`quality-report---` and contains `report.json` plus +`summary.md`. Native reports and crash corpora remain separate artifacts. +Target result JSON is always uploaded; crash forensics remain failure-only. +Normalized aggregate artifacts use at least 90 days of retention. + +The reusable workflow checks out its own contract at the exact called-workflow +commit via `job.workflow_repository` and `job.workflow_sha`. Product workflows +remain responsible for native tool adapters and uploading their target-result +artifacts. + +## Status and badge semantics + +- The aggregate job runs under `if: always()` and fails on an incomplete set, + tool crash, parse error, invalid contract, or policy failure. +- Numeric mutation badges publish only a complete canonical score. An + incomplete run displays `partial N/M`, never a subset score or stale score. +- Do not commit badge JSON to `main`. Bot commits bypass the dev-to-main flow + and make telemetry part of source history. +- Use the native workflow-status badge as the stable shared badge. Put numeric + scores and native dashboard links in the run summary and retained artifact + until there is a non-source telemetry endpoint. +- Long-fuzz failures must retain the runner seed, path, or corpus needed to + reproduce the failure. + +## Migration order + +1. Add and test the schema, validator, and reusable aggregate in + `CodesWhat/.github`. +2. Canary Portwing. It has the smallest Gremlins and fuzz matrices. Keep + `FuzzVerifyRequest` in a long tier and preserve its source-to-tier inventory + contract. +3. Migrate Sockguard's Gremlins aggregation, retire its direct badge commit, + and preserve its existing fuzz inventory contract. +4. Migrate Drydock's Stryker aggregate, remove partial score publication, then + add a scheduled fast-check workflow with explicit run count and seed/path + reproduction. +5. Manually dispatch each workflow on the exact active dev head before syncing + it to `main`. Scheduled workflows exercise `main`, not the active dev branch. + +Native runner adapters and product callers are intentionally outside this +foundation change. Each migration supplies and tests its own mapping from +native output into the target-result contract. + +## Risks and controls + +- The three first-of-month mutation runs compete for organization runner + capacity. Stagger their schedules. +- Gremlins text parsing is version-sensitive. Pin the tool and test fixtures + from real output. +- Stryker reports compile/runtime-invalid mutants and may lose shards. Preserve + native categories and fail closed on a missing shard. +- Fast-check failures are not reproducible without seed and path. Make both + mandatory failure fields. +- Artifacts expire. Ninety days is the minimum retention for normalized + reports; durable trend storage is a separate decision. +- `continue-on-error` on a matrix leg must not make the final aggregate green. diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py new file mode 100644 index 0000000..8b8c93a --- /dev/null +++ b/quality-report/v1/quality_report.py @@ -0,0 +1,765 @@ +#!/usr/bin/env python3 +"""Validate and aggregate CodesWhat quality-report/v1 result envelopes.""" + +import argparse +from datetime import datetime +from decimal import Decimal, ROUND_HALF_UP +import json +import math +from pathlib import Path +import re +import sys +from urllib.parse import urlparse + + +VERSION = "quality-report/v1" +KINDS = {"target-result", "report"} +TRACK_TOOLS = { + "mutation": {"gremlins", "stryker"}, + "fuzz": {"fast-check", "go-fuzz"}, +} +POLICIES = {"advisory", "signal"} +OUTCOMES = {"passed", "failed", "crashed", "flaked", "cancelled", "error"} +OUTCOME_PRIORITY = ("error", "crashed", "cancelled", "flaked", "failed") +MUTATION_COUNT_FIELDS = ( + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", +) +TARGET_MUTATION_FIELDS = set(MUTATION_COUNT_FIELDS) | { + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition", +} +REPORT_MUTATION_FIELDS = TARGET_MUTATION_FIELDS | { + "detected", + "missed", + "canonical_score_pct", +} +TARGET_FUZZ_REQUIRED = {"budget_seconds", "elapsed_seconds"} +TARGET_FUZZ_OPTIONAL = {"executions", "new_interesting_inputs"} +REPORT_FUZZ_FIELDS = { + "declared_budget_seconds", + "elapsed_seconds", + "executions", + "new_interesting_inputs", +} +SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +RFC3339_PATTERN = re.compile( + r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})$" +) + + +class ContractError(ValueError): + pass + + +def reject_constant(value): + raise ContractError("non-finite JSON number: {0}".format(value)) + + +def reject_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ContractError("duplicate JSON field: {0}".format(key)) + result[key] = value + return result + + +def load_json_text(text, source): + try: + return json.loads( + text, + object_pairs_hook=reject_duplicate_keys, + parse_constant=reject_constant, + ) + except (json.JSONDecodeError, ContractError) as error: + raise ContractError("{0}: invalid JSON: {1}".format(source, error)) + + +def load_json_file(path): + try: + return load_json_text(path.read_text(), str(path)) + except OSError as error: + raise ContractError("{0}: cannot read JSON: {1}".format(path, error)) + + +def require_object(value, path, required, optional=()): + if not isinstance(value, dict): + raise ContractError("{0}: expected object".format(path)) + required = set(required) + allowed = required | set(optional) + missing = sorted(required - set(value)) + unknown = sorted(set(value) - allowed) + if missing: + raise ContractError( + "{0}: missing field(s): {1}".format(path, ", ".join(missing)) + ) + if unknown: + raise ContractError( + "{0}: unknown field(s): {1}".format(path, ", ".join(unknown)) + ) + return value + + +def require_string(value, path, *, pattern=None, max_length=None): + if not isinstance(value, str) or not value or "\n" in value or "\r" in value: + raise ContractError("{0}: expected non-empty single-line string".format(path)) + if max_length is not None and len(value) > max_length: + raise ContractError( + "{0}: expected at most {1} characters".format(path, max_length) + ) + if pattern is not None and pattern.fullmatch(value) is None: + raise ContractError("{0}: invalid value".format(path)) + return value + + +def require_enum(value, path, choices): + require_string(value, path) + if value not in choices: + raise ContractError( + "{0}: expected one of {1}".format(path, ", ".join(sorted(choices))) + ) + return value + + +def require_integer(value, path, *, minimum=0): + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ContractError("{0}: expected integer >= {1}".format(path, minimum)) + return value + + +def require_number(value, path, *, minimum=0): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ContractError("{0}: expected number >= {1}".format(path, minimum)) + if not math.isfinite(value) or value < minimum: + raise ContractError("{0}: expected finite number >= {1}".format(path, minimum)) + return value + + +def require_nullable_integer(value, path): + if value is not None: + require_integer(value, path) + + +def require_percentage(value, path, *, nullable=False): + if value is None and nullable: + return + require_number(value, path) + if value > 100: + raise ContractError("{0}: expected percentage <= 100".format(path)) + + +def require_url(value, path): + require_string(value, path) + parsed = urlparse(value) + if parsed.scheme != "https" or not parsed.netloc: + raise ContractError("{0}: expected HTTPS URL".format(path)) + + +def require_timestamp(value, path): + require_string(value, path) + try: + if RFC3339_PATTERN.fullmatch(value) is None: + raise ValueError + datetime.fromisoformat(value.replace("Z", "+00:00").replace("z", "+00:00")) + except ValueError: + raise ContractError("{0}: expected RFC 3339 timestamp".format(path)) + + +def percentage(numerator, denominator): + if denominator == 0: + return None + value = Decimal(numerator) * Decimal(100) / Decimal(denominator) + return float(value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)) + + +def validate_mutation_target_metrics(metrics, path): + require_object(metrics, path, TARGET_MUTATION_FIELDS) + for field in MUTATION_COUNT_FIELDS: + require_integer(metrics[field], "{0}.{1}".format(path, field)) + numerator = require_integer( + metrics["tool_score_numerator"], path + ".tool_score_numerator" + ) + denominator = require_integer( + metrics["tool_score_denominator"], path + ".tool_score_denominator" + ) + require_percentage( + metrics["tool_score_pct"], path + ".tool_score_pct", nullable=True + ) + require_string(metrics["tool_score_definition"], path + ".tool_score_definition") + expected = percentage(numerator, denominator) + if metrics["tool_score_pct"] != expected: + raise ContractError( + "{0}.tool_score_pct: expected {1} from numerator/denominator".format( + path, expected + ) + ) + + +def validate_fuzz_target_metrics(metrics, path): + require_object(metrics, path, TARGET_FUZZ_REQUIRED, TARGET_FUZZ_OPTIONAL) + require_integer(metrics["budget_seconds"], path + ".budget_seconds", minimum=1) + require_number(metrics["elapsed_seconds"], path + ".elapsed_seconds") + for field in TARGET_FUZZ_OPTIONAL: + if field in metrics: + require_integer(metrics[field], "{0}.{1}".format(path, field)) + + +def validate_reproduction(value, path): + require_object(value, path, (), {"seed", "path", "corpus"}) + if not value: + raise ContractError("{0}: expected seed, path, or corpus".format(path)) + for field, item in value.items(): + require_string(item, "{0}.{1}".format(path, field)) + + +def validate_target(target, track, path): + require_object( + target, + path, + {"name", "outcome", "metrics"}, + {"diagnostic", "reproduction"}, + ) + require_string(target["name"], path + ".name", max_length=256) + outcome = require_enum(target["outcome"], path + ".outcome", OUTCOMES) + metrics = target["metrics"] + if metrics is None: + if outcome == "passed": + raise ContractError( + "{0}.metrics: passed target requires metrics".format(path) + ) + elif track == "mutation": + validate_mutation_target_metrics(metrics, path + ".metrics") + else: + validate_fuzz_target_metrics(metrics, path + ".metrics") + + if outcome == "passed": + if "diagnostic" in target or "reproduction" in target: + raise ContractError( + "{0}: passed target cannot contain diagnostic or reproduction".format( + path + ) + ) + else: + if "diagnostic" not in target: + raise ContractError( + "{0}: non-passed target requires diagnostic".format(path) + ) + require_string(target["diagnostic"], path + ".diagnostic") + if track == "fuzz" and outcome in {"failed", "crashed", "flaked"}: + if "reproduction" not in target: + raise ContractError( + "{0}: failed fuzz target requires reproduction".format(path) + ) + if "reproduction" in target: + validate_reproduction(target["reproduction"], path + ".reproduction") + + +def validate_target_document(document, path="$"): + require_object( + document, + path, + {"schema_version", "kind", "track", "tool", "target"}, + ) + if document["schema_version"] != VERSION: + raise ContractError("{0}.schema_version: expected {1}".format(path, VERSION)) + if document["kind"] != "target-result": + raise ContractError("{0}.kind: expected target-result".format(path)) + track = require_enum(document["track"], path + ".track", set(TRACK_TOOLS)) + tool = require_enum( + document["tool"], path + ".tool", set().union(*TRACK_TOOLS.values()) + ) + if tool not in TRACK_TOOLS[track]: + raise ContractError("{0}.tool: {1} is not a {2} tool".format(path, tool, track)) + validate_target(document["target"], track, path + ".target") + return document + + +def aggregate_mutation_metrics(targets): + metrics = {field: 0 for field in MUTATION_COUNT_FIELDS} + definitions = set() + for target in targets: + target_metrics = target["metrics"] + for field in MUTATION_COUNT_FIELDS: + metrics[field] += target_metrics[field] + metrics.setdefault("tool_score_numerator", 0) + metrics.setdefault("tool_score_denominator", 0) + metrics["tool_score_numerator"] += target_metrics["tool_score_numerator"] + metrics["tool_score_denominator"] += target_metrics["tool_score_denominator"] + definitions.add(target_metrics["tool_score_definition"]) + if len(definitions) != 1: + raise ContractError("target results use inconsistent native score definitions") + metrics["detected"] = metrics["killed"] + metrics["timeout"] + metrics["missed"] = metrics["survived"] + metrics["no_coverage"] + metrics["canonical_score_pct"] = percentage( + metrics["detected"], metrics["detected"] + metrics["missed"] + ) + metrics["tool_score_pct"] = percentage( + metrics["tool_score_numerator"], metrics["tool_score_denominator"] + ) + metrics["tool_score_definition"] = definitions.pop() + return metrics + + +def aggregate_fuzz_metrics(targets): + metrics = { + "declared_budget_seconds": sum( + target["metrics"]["budget_seconds"] for target in targets + ), + "elapsed_seconds": sum( + target["metrics"]["elapsed_seconds"] for target in targets + ), + } + for field in ("executions", "new_interesting_inputs"): + if all(field in target["metrics"] for target in targets): + metrics[field] = sum(target["metrics"][field] for target in targets) + else: + metrics[field] = None + return metrics + + +def derive_outcome(targets, errors): + if errors: + return "error" + outcomes = {target["outcome"] for target in targets} + for outcome in OUTCOME_PRIORITY: + if outcome in outcomes: + return outcome + return "passed" + + +def validate_mutation_report_metrics(metrics, targets, path): + require_object(metrics, path, REPORT_MUTATION_FIELDS) + for field in MUTATION_COUNT_FIELDS + ( + "detected", + "missed", + "tool_score_numerator", + "tool_score_denominator", + ): + require_integer(metrics[field], "{0}.{1}".format(path, field)) + require_percentage( + metrics["canonical_score_pct"], path + ".canonical_score_pct", nullable=True + ) + require_percentage( + metrics["tool_score_pct"], path + ".tool_score_pct", nullable=True + ) + require_string(metrics["tool_score_definition"], path + ".tool_score_definition") + expected = aggregate_mutation_metrics(targets) + if metrics != expected: + raise ContractError( + "{0}: aggregate mutation metrics do not match targets".format(path) + ) + + +def validate_fuzz_report_metrics(metrics, targets, path): + require_object(metrics, path, REPORT_FUZZ_FIELDS) + require_integer( + metrics["declared_budget_seconds"], + path + ".declared_budget_seconds", + minimum=1, + ) + require_number(metrics["elapsed_seconds"], path + ".elapsed_seconds") + require_nullable_integer(metrics["executions"], path + ".executions") + require_nullable_integer( + metrics["new_interesting_inputs"], path + ".new_interesting_inputs" + ) + expected = aggregate_fuzz_metrics(targets) + if metrics != expected: + raise ContractError( + "{0}: aggregate fuzz metrics do not match targets".format(path) + ) + + +def validate_report(document, path="$"): + require_object( + document, + path, + { + "schema_version", + "kind", + "repository", + "ref", + "sha", + "run", + "report_name", + "track", + "tool", + "policy", + "completeness", + "outcome", + "targets", + "metrics", + "errors", + }, + ) + if document["schema_version"] != VERSION: + raise ContractError("{0}.schema_version: expected {1}".format(path, VERSION)) + if document["kind"] != "report": + raise ContractError("{0}.kind: expected report".format(path)) + require_string(document["repository"], path + ".repository") + require_string(document["ref"], path + ".ref") + require_string(document["sha"], path + ".sha", pattern=SHA_PATTERN) + require_string(document["report_name"], path + ".report_name") + track = require_enum(document["track"], path + ".track", set(TRACK_TOOLS)) + tool = require_enum( + document["tool"], path + ".tool", set().union(*TRACK_TOOLS.values()) + ) + if tool not in TRACK_TOOLS[track]: + raise ContractError("{0}.tool: {1} is not a {2} tool".format(path, tool, track)) + require_enum(document["policy"], path + ".policy", POLICIES) + + run = require_object( + document["run"], + path + ".run", + {"id", "attempt", "event", "url", "started_at"}, + ) + require_string(run["id"], path + ".run.id") + require_integer(run["attempt"], path + ".run.attempt", minimum=1) + require_string(run["event"], path + ".run.event") + require_url(run["url"], path + ".run.url") + if run["started_at"] is not None: + require_timestamp(run["started_at"], path + ".run.started_at") + + completeness = require_object( + document["completeness"], + path + ".completeness", + {"expected", "reported", "complete"}, + ) + require_integer( + completeness["expected"], path + ".completeness.expected", minimum=1 + ) + require_integer(completeness["reported"], path + ".completeness.reported") + if not isinstance(completeness["complete"], bool): + raise ContractError(path + ".completeness.complete: expected boolean") + + targets = document["targets"] + if not isinstance(targets, list): + raise ContractError(path + ".targets: expected array") + names = [] + for index, target in enumerate(targets): + validate_target(target, track, "{0}.targets[{1}]".format(path, index)) + names.append(target["name"]) + if len(names) != len(set(names)): + raise ContractError(path + ".targets: duplicate target name") + if names != sorted(names): + raise ContractError(path + ".targets: targets must be sorted by name") + if completeness["reported"] != len(targets): + raise ContractError(path + ".completeness.reported: does not match targets") + + errors = document["errors"] + if not isinstance(errors, list) or any( + not isinstance(error, str) or not error for error in errors + ): + raise ContractError(path + ".errors: expected non-empty strings") + if errors != sorted(set(errors)): + raise ContractError(path + ".errors: errors must be sorted and unique") + if run["started_at"] is None and not errors: + raise ContractError(path + ".run.started_at: required for an error-free report") + expected_complete = ( + completeness["expected"] == completeness["reported"] and not errors + ) + if completeness["complete"] != expected_complete: + raise ContractError(path + ".completeness.complete: inconsistent value") + + outcome = require_enum(document["outcome"], path + ".outcome", OUTCOMES) + if outcome != derive_outcome(targets, errors): + raise ContractError(path + ".outcome: does not match target results") + + metrics = document["metrics"] + metrics_available = ( + completeness["complete"] + and bool(targets) + and all(target["metrics"] is not None for target in targets) + ) + if not metrics_available: + if metrics is not None: + raise ContractError(path + ".metrics: must be null for partial results") + elif track == "mutation": + validate_mutation_report_metrics(metrics, targets, path + ".metrics") + else: + validate_fuzz_report_metrics(metrics, targets, path + ".metrics") + return document + + +def validate_document(document): + if not isinstance(document, dict): + raise ContractError("$: expected object") + kind = document.get("kind") + if kind not in KINDS: + raise ContractError("$.kind: expected one of report, target-result") + if kind == "target-result": + return validate_target_document(document) + return validate_report(document) + + +def parse_expected_targets(raw): + value = load_json_text(raw, "expected_targets") + if not isinstance(value, list) or not value: + raise ContractError("expected_targets: expected a non-empty JSON array") + targets = [] + for index, target in enumerate(value): + targets.append( + require_string( + target, + "expected_targets[{0}]".format(index), + max_length=256, + ) + ) + if len(targets) != len(set(targets)): + raise ContractError("expected_targets: duplicate target") + return targets + + +def relative_label(path, root): + try: + return str(path.relative_to(root)) + except ValueError: + return path.name + + +def collect_targets(input_dir, expected_targets, track, tool): + errors = [] + reported = {} + paths = sorted(input_dir.rglob("target-result.json")) if input_dir.is_dir() else [] + if not paths: + errors.append("no target-result.json files found") + for path in paths: + label = relative_label(path, input_dir) + try: + document = load_json_file(path) + validate_target_document(document, label) + if document["track"] != track: + raise ContractError( + "{0}.track: expected {1}, got {2}".format( + label, track, document["track"] + ) + ) + if document["tool"] != tool: + raise ContractError( + "{0}.tool: expected {1}, got {2}".format( + label, tool, document["tool"] + ) + ) + target = document["target"] + name = target["name"] + if name not in expected_targets: + raise ContractError("{0}: unexpected target: {1}".format(label, name)) + if name in reported: + raise ContractError("{0}: duplicate target: {1}".format(label, name)) + reported[name] = target + except ContractError as error: + errors.append(str(error)) + for target in expected_targets: + if target not in reported: + errors.append("missing target: {0}".format(target)) + return [reported[name] for name in sorted(reported)], sorted(set(errors)) + + +def build_summary(report): + completeness = report["completeness"] + if completeness["complete"]: + completeness_text = "complete {0}/{1}".format( + completeness["reported"], completeness["expected"] + ) + else: + completeness_text = "partial {0}/{1}".format( + completeness["reported"], completeness["expected"] + ) + lines = [ + "# {0}".format(report["report_name"]), + "", + "- Track: `{0}` via `{1}`".format(report["track"], report["tool"]), + "- Policy: `{0}`".format(report["policy"]), + "- Completeness: **{0}**".format(completeness_text), + "- Outcome: **{0}**".format(report["outcome"]), + ] + metrics = report["metrics"] + if report["track"] == "mutation" and metrics is not None: + canonical = metrics["canonical_score_pct"] + native = metrics["tool_score_pct"] + if canonical is not None: + lines.append("- Canonical score: **{0:.2f}%**".format(canonical)) + if native is not None: + lines.append( + "- Native score: **{0:.2f}%** ({1})".format( + native, metrics["tool_score_definition"] + ) + ) + lines.extend(("", "## Targets", "")) + for target in report["targets"]: + detail = "" + if "diagnostic" in target: + detail = ": {0}".format(target["diagnostic"]) + lines.append( + "- `{0}`: {1}{2}".format(target["name"], target["outcome"], detail) + ) + if report["errors"]: + lines.extend(("", "## Contract errors", "")) + lines.extend("- {0}".format(error) for error in report["errors"]) + return "\n".join(lines) + "\n" + + +def append_lines(path, values): + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as output: + for key, value in values.items(): + output.write("{0}={1}\n".format(key, value)) + + +def command_aggregate(args): + errors = [] + try: + expected_targets = parse_expected_targets(args.expected_targets) + except ContractError as error: + expected_targets = ["invalid-expected-targets"] + errors.append(str(error)) + if args.fail_on_incomplete != "true": + errors.append("fail_on_incomplete must be true for CodesWhat telemetry") + try: + require_timestamp(args.started_at, "started_at") + started_at = args.started_at + except ContractError as error: + started_at = None + errors.append(str(error)) + + targets, collection_errors = collect_targets( + args.input_dir, expected_targets, args.track, args.tool + ) + errors.extend(collection_errors) + errors = sorted(set(errors)) + completeness = { + "expected": len(expected_targets), + "reported": len(targets), + "complete": len(targets) == len(expected_targets) and not errors, + } + metrics = None + if completeness["complete"] and all( + target["metrics"] is not None for target in targets + ): + try: + if args.track == "mutation": + metrics = aggregate_mutation_metrics(targets) + else: + metrics = aggregate_fuzz_metrics(targets) + except ContractError as error: + errors.append(str(error)) + errors = sorted(set(errors)) + completeness["complete"] = False + + report = { + "schema_version": VERSION, + "kind": "report", + "repository": args.repository, + "ref": args.ref, + "sha": args.sha, + "run": { + "id": args.run_id, + "attempt": args.run_attempt, + "event": args.event, + "url": args.run_url, + "started_at": started_at, + }, + "report_name": args.report_name, + "track": args.track, + "tool": args.tool, + "policy": args.policy, + "completeness": completeness, + "outcome": derive_outcome(targets, errors), + "targets": targets, + "metrics": metrics, + "errors": errors, + } + validate_report(report) + + args.output_dir.mkdir(parents=True, exist_ok=True) + summary = build_summary(report) + (args.output_dir / "report.json").write_text( + json.dumps(report, indent=2, allow_nan=False) + "\n" + ) + (args.output_dir / "summary.md").write_text(summary) + append_lines( + args.github_output, + { + "report_artifact_name": "quality-report-{0}-{1}-{2}".format( + args.track, args.run_id, args.run_attempt + ), + "completeness": str(completeness["complete"]).lower(), + "expected_targets": completeness["expected"], + "reported_targets": completeness["reported"], + "outcome": report["outcome"], + "canonical_score_pct": ( + "" + if metrics is None + or args.track != "mutation" + or metrics["canonical_score_pct"] is None + else "{0:.2f}".format(metrics["canonical_score_pct"]) + ), + }, + ) + if args.github_step_summary is not None: + args.github_step_summary.parent.mkdir(parents=True, exist_ok=True) + with args.github_step_summary.open("a") as step_summary: + step_summary.write(summary) + return 0 if completeness["complete"] and report["outcome"] == "passed" else 1 + + +def command_validate(args): + document = load_json_file(args.document) + validate_document(document) + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate_parser = subparsers.add_parser("validate") + validate_parser.add_argument("document", type=Path) + validate_parser.set_defaults(handler=command_validate) + + aggregate_parser = subparsers.add_parser("aggregate") + aggregate_parser.add_argument("--input-dir", type=Path, required=True) + aggregate_parser.add_argument("--output-dir", type=Path, required=True) + aggregate_parser.add_argument("--repository", required=True) + aggregate_parser.add_argument("--ref", required=True) + aggregate_parser.add_argument("--sha", required=True) + aggregate_parser.add_argument("--run-id", required=True) + aggregate_parser.add_argument("--run-attempt", type=int, required=True) + aggregate_parser.add_argument("--event", required=True) + aggregate_parser.add_argument("--run-url", required=True) + aggregate_parser.add_argument("--started-at", required=True) + aggregate_parser.add_argument("--track", choices=sorted(TRACK_TOOLS), required=True) + aggregate_parser.add_argument( + "--tool", choices=sorted(set().union(*TRACK_TOOLS.values())), required=True + ) + aggregate_parser.add_argument("--policy", choices=sorted(POLICIES), required=True) + aggregate_parser.add_argument("--expected-targets", required=True) + aggregate_parser.add_argument("--report-name", required=True) + aggregate_parser.add_argument( + "--fail-on-incomplete", choices=("true", "false"), required=True + ) + aggregate_parser.add_argument("--github-output", type=Path) + aggregate_parser.add_argument("--github-step-summary", type=Path) + aggregate_parser.set_defaults(handler=command_aggregate) + return parser + + +def main(): + args = build_parser().parse_args() + try: + return args.handler(args) + except ContractError as error: + print("quality-report: {0}".format(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/quality-report/v1/schema.json b/quality-report/v1/schema.json new file mode 100644 index 0000000..46c38f1 --- /dev/null +++ b/quality-report/v1/schema.json @@ -0,0 +1,496 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeswhat.com/schemas/quality-report/v1/schema.json", + "title": "CodesWhat quality-report/v1", + "description": "Normalized target results and aggregate quality reports.", + "oneOf": [ + { + "$ref": "#/$defs/targetResult" + }, + { + "$ref": "#/$defs/report" + } + ], + "$defs": { + "version": { + "type": "string", + "const": "quality-report/v1" + }, + "outcome": { + "type": "string", + "enum": [ + "passed", + "failed", + "crashed", + "flaked", + "cancelled", + "error" + ] + }, + "reproduction": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "seed": { + "type": "string", + "minLength": 1 + }, + "path": { + "type": "string", + "minLength": 1 + }, + "corpus": { + "type": "string", + "minLength": 1 + } + } + }, + "mutationTargetMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition" + ], + "properties": { + "killed": { + "type": "integer", + "minimum": 0 + }, + "timeout": { + "type": "integer", + "minimum": 0 + }, + "survived": { + "type": "integer", + "minimum": 0 + }, + "no_coverage": { + "type": "integer", + "minimum": 0 + }, + "invalid": { + "type": "integer", + "minimum": 0 + }, + "ignored": { + "type": "integer", + "minimum": 0 + }, + "tool_score_numerator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_denominator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_definition": { + "type": "string", + "minLength": 1 + } + } + }, + "fuzzTargetMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "budget_seconds", + "elapsed_seconds" + ], + "properties": { + "budget_seconds": { + "type": "integer", + "minimum": 1 + }, + "elapsed_seconds": { + "type": "number", + "minimum": 0 + }, + "executions": { + "type": "integer", + "minimum": 0 + }, + "new_interesting_inputs": { + "type": "integer", + "minimum": 0 + } + } + }, + "target": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "outcome", + "metrics" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\r\\n]+$" + }, + "outcome": { + "$ref": "#/$defs/outcome" + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationTargetMetrics" + }, + { + "$ref": "#/$defs/fuzzTargetMetrics" + }, + { + "type": "null" + } + ] + }, + "diagnostic": { + "type": "string", + "minLength": 1 + }, + "reproduction": { + "$ref": "#/$defs/reproduction" + } + } + }, + "targetResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "track", + "tool", + "target" + ], + "properties": { + "schema_version": { + "$ref": "#/$defs/version" + }, + "kind": { + "type": "string", + "const": "target-result" + }, + "track": { + "type": "string", + "enum": [ + "mutation", + "fuzz" + ] + }, + "tool": { + "type": "string", + "enum": [ + "stryker", + "gremlins", + "go-fuzz", + "fast-check" + ] + }, + "target": { + "$ref": "#/$defs/target" + } + } + }, + "mutationReportMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", + "detected", + "missed", + "canonical_score_pct", + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition" + ], + "properties": { + "killed": { + "type": "integer", + "minimum": 0 + }, + "timeout": { + "type": "integer", + "minimum": 0 + }, + "survived": { + "type": "integer", + "minimum": 0 + }, + "no_coverage": { + "type": "integer", + "minimum": 0 + }, + "invalid": { + "type": "integer", + "minimum": 0 + }, + "ignored": { + "type": "integer", + "minimum": 0 + }, + "detected": { + "type": "integer", + "minimum": 0 + }, + "missed": { + "type": "integer", + "minimum": 0 + }, + "canonical_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_numerator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_denominator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_definition": { + "type": "string", + "minLength": 1 + } + } + }, + "fuzzReportMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "declared_budget_seconds", + "elapsed_seconds", + "executions", + "new_interesting_inputs" + ], + "properties": { + "declared_budget_seconds": { + "type": "integer", + "minimum": 1 + }, + "elapsed_seconds": { + "type": "number", + "minimum": 0 + }, + "executions": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "new_interesting_inputs": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + } + } + }, + "run": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "attempt", + "event", + "url", + "started_at" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "attempt": { + "type": "integer", + "minimum": 1 + }, + "event": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https://" + }, + "started_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } + }, + "completeness": { + "type": "object", + "additionalProperties": false, + "required": [ + "expected", + "reported", + "complete" + ], + "properties": { + "expected": { + "type": "integer", + "minimum": 1 + }, + "reported": { + "type": "integer", + "minimum": 0 + }, + "complete": { + "type": "boolean" + } + } + }, + "report": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "repository", + "ref", + "sha", + "run", + "report_name", + "track", + "tool", + "policy", + "completeness", + "outcome", + "targets", + "metrics", + "errors" + ], + "properties": { + "schema_version": { + "$ref": "#/$defs/version" + }, + "kind": { + "type": "string", + "const": "report" + }, + "repository": { + "type": "string", + "minLength": 1 + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "run": { + "$ref": "#/$defs/run" + }, + "report_name": { + "type": "string", + "minLength": 1 + }, + "track": { + "type": "string", + "enum": [ + "mutation", + "fuzz" + ] + }, + "tool": { + "type": "string", + "enum": [ + "stryker", + "gremlins", + "go-fuzz", + "fast-check" + ] + }, + "policy": { + "type": "string", + "enum": [ + "advisory", + "signal" + ] + }, + "completeness": { + "$ref": "#/$defs/completeness" + }, + "outcome": { + "$ref": "#/$defs/outcome" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/$defs/target" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationReportMetrics" + }, + { + "$ref": "#/$defs/fuzzReportMetrics" + }, + { + "type": "null" + } + ] + }, + "errors": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + } + } + } + } +} From 99a7a997425bec33f51328d10a784f4612a4d3d3 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:50:56 -0400 Subject: [PATCH 05/18] fix(quality): enforce report contract boundaries --- .../quality-result-stats/target-result.json | 2 +- .../quality-result-verify/target-result.json | 2 +- .../quality-result-stats/target-result.json | 2 +- .github/tests/quality_report_contract_test.py | 259 +++++++++++++++++- decisions/quality-reporting.md | 7 + quality-report/v1/quality_report.py | 23 +- quality-report/v1/schema.json | 216 ++++++++++++++- 7 files changed, 495 insertions(+), 16 deletions(-) diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json index 627a14e..1f79ab0 100644 --- a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json @@ -8,7 +8,7 @@ "outcome": "passed", "metrics": { "budget_seconds": 120, - "elapsed_seconds": 111.25, + "elapsed_seconds": 120.0, "executions": 650000, "new_interesting_inputs": 2 } diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json index 472e276..3e0c633 100644 --- a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json @@ -8,7 +8,7 @@ "outcome": "passed", "metrics": { "budget_seconds": 300, - "elapsed_seconds": 288.25, + "elapsed_seconds": 300.0, "executions": 1250000, "new_interesting_inputs": 3 } diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json index 0db4a95..4530c33 100644 --- a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json @@ -9,7 +9,7 @@ "metrics": null, "diagnostic": "tool process exited 2", "reproduction": { - "seed": "-test.fuzzcachedir=/tmp/fuzz-cache" + "path": "internal/docker/testdata/fuzz/FuzzDecodeStats/6f2c1b8d9a4e5c37" } } } diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index b456ce7..6c65a24 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -3,6 +3,7 @@ import runpy import shutil import subprocess +import sys import tempfile import unittest @@ -52,7 +53,7 @@ def test_complete_fuzz_fixture_emits_a_valid_report_and_summary(self): ) self.assertEqual("passed", report["outcome"]) self.assertEqual(420, report["metrics"]["declared_budget_seconds"]) - self.assertEqual(399.5, report["metrics"]["elapsed_seconds"]) + self.assertEqual(420.0, report["metrics"]["elapsed_seconds"]) self.assertEqual(1900000, report["metrics"]["executions"]) self.assertEqual(5, report["metrics"]["new_interesting_inputs"]) self.assertNotIn("canonical_score_pct", report["metrics"]) @@ -97,6 +98,23 @@ def test_crash_fixture_fails_even_when_every_target_reports(self): self.assertIn("tool process exited 2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_crash_fixture_uses_a_native_go_crasher_path(self): + result = json.loads( + ( + FIXTURES + / "crash" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + + self.assertEqual( + { + "path": "internal/docker/testdata/fuzz/FuzzDecodeStats/6f2c1b8d9a4e5c37" + }, + result["target"]["reproduction"], + ) + def test_parse_error_fixture_still_emits_an_error_report(self): result, report, summary = self.aggregate( "parse-error", @@ -116,6 +134,33 @@ def test_parse_error_fixture_still_emits_an_error_report(self): self.assertIn("partial 1/2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_non_utf8_target_still_emits_an_error_report(self): + def corrupt_target(input_dir): + target = next(input_dir.rglob("target-result.json")) + target.write_bytes(b"\xff") + + result, report, summary = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + input_mutator=corrupt_target, + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual("error", report["outcome"]) + self.assertFalse(report["completeness"]["complete"]) + self.assertIsNone(report["metrics"]) + self.assertTrue( + any("cannot read JSON" in error for error in report["errors"]), + report["errors"], + ) + self.assertIn("partial 1/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + def test_canonical_score_counts_timeout_and_no_coverage(self): result, report, summary = self.aggregate( "canonical-score", @@ -150,6 +195,14 @@ def test_decision_documents_track_and_tool_pairings(self): self.assertIn("`mutation` permits `gremlins` and `stryker`", decision) self.assertIn("`fuzz` permits `fast-check` and `go-fuzz`", decision) + def test_decision_defines_go_fuzz_as_the_native_go_engine(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn( + "`go-fuzz` identifies Go's native `go test -fuzz` engine, not the legacy third-party go-fuzz tool.", + decision, + ) + def test_native_score_is_weighted_and_keeps_its_definition(self): result, report, summary = self.aggregate( "native-score", @@ -235,6 +288,21 @@ def test_validator_requires_a_positive_declared_fuzz_budget(self): with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], ">= 1"): QUALITY_REPORT["validate_fuzz_report_metrics"](metrics, [], "$.metrics") + def test_passed_fuzz_target_requires_its_full_declared_budget(self): + target = { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 299.99, + }, + } + + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], "completed fuzz budget" + ): + QUALITY_REPORT["validate_target"](target, "fuzz", "$.target") + def test_validator_requires_time_and_offset_in_timestamps(self): for timestamp in ("2026-08-14", "2026-08-14T15:00:00"): with self.subTest(timestamp=timestamp): @@ -243,6 +311,91 @@ def test_validator_requires_time_and_offset_in_timestamps(self): ): QUALITY_REPORT["require_timestamp"](timestamp, "$.run.started_at") + def test_invalid_json_contract_error_retains_its_cause(self): + with self.assertRaises(QUALITY_REPORT["ContractError"]) as raised: + QUALITY_REPORT["load_json_text"]("{", "target-result.json") + + self.assertIsInstance(raised.exception.__cause__, json.JSONDecodeError) + + def test_invalid_timestamp_contract_error_retains_its_cause(self): + with self.assertRaises(QUALITY_REPORT["ContractError"]) as raised: + QUALITY_REPORT["require_timestamp"]("2026-08-14", "$.run.started_at") + + self.assertIsInstance(raised.exception.__cause__, ValueError) + + def test_json_parser_rejects_duplicate_fields_and_non_finite_numbers(self): + cases = ( + ('{"value": 1, "value": 2}', "duplicate JSON field"), + ('{"value": NaN}', "non-finite JSON number"), + ('{"value": Infinity}', "non-finite JSON number"), + ) + + for document, message in cases: + with self.subTest(document=document): + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], message): + QUALITY_REPORT["load_json_text"](document, "target-result.json") + + def test_mutation_aggregate_rejects_mixed_native_score_definitions(self): + targets = [] + for definition in ("detected / covered", "killed / lived"): + targets.append( + { + "metrics": { + "killed": 1, + "timeout": 0, + "survived": 0, + "no_coverage": 0, + "invalid": 0, + "ignored": 0, + "tool_score_numerator": 1, + "tool_score_denominator": 1, + "tool_score_pct": 100.0, + "tool_score_definition": definition, + } + } + ) + + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], + "inconsistent native score definitions", + ): + QUALITY_REPORT["aggregate_mutation_metrics"](targets) + + def test_target_collection_rejects_unexpected_and_duplicate_names(self): + document = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + target_name = document["target"]["name"] + + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) + (input_dir / "unexpected").mkdir() + (input_dir / "unexpected" / "target-result.json").write_text( + json.dumps(document) + ) + targets, errors = QUALITY_REPORT["collect_targets"]( + input_dir, ["expected-target"], "fuzz", "go-fuzz" + ) + self.assertEqual([], targets) + self.assertTrue(any("unexpected target" in error for error in errors)) + + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) + for directory in ("first", "second"): + path = input_dir / directory + path.mkdir() + (path / "target-result.json").write_text(json.dumps(document)) + targets, errors = QUALITY_REPORT["collect_targets"]( + input_dir, [target_name], "fuzz", "go-fuzz" + ) + self.assertEqual([target_name], [target["name"] for target in targets]) + self.assertTrue(any("duplicate target" in error for error in errors)) + def test_schema_is_versioned_and_fail_closed(self): with SCHEMA.open() as schema_file: schema = json.load(schema_file) @@ -259,6 +412,97 @@ def test_schema_is_versioned_and_fail_closed(self): self.assertNotIn("additionalProperties", schema) self.assertTrue(self.all_objects_are_closed(schema)) + def test_schema_encodes_python_target_and_track_guards(self): + schema = json.loads(SCHEMA.read_text()) + definitions = schema["$defs"] + + self.assertEqual( + [ + { + "if": { + "properties": {"outcome": {"const": "passed"}}, + "required": ["outcome"], + }, + "then": { + "properties": {"metrics": {"not": {"type": "null"}}}, + "not": { + "anyOf": [ + {"required": ["diagnostic"]}, + {"required": ["reproduction"]}, + ] + }, + }, + "else": {"required": ["diagnostic"]}, + } + ], + definitions["target"]["allOf"], + ) + self.assertEqual( + {"gremlins", "stryker"}, + set( + definitions["targetResult"]["allOf"][0]["then"]["properties"][ + "tool" + ]["enum"] + ), + ) + self.assertEqual( + {"fast-check", "go-fuzz"}, + set( + definitions["targetResult"]["allOf"][0]["else"]["properties"][ + "tool" + ]["enum"] + ), + ) + self.assertEqual( + "#/$defs/mutationTargetMetrics", + definitions["mutationTarget"]["allOf"][1]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzTargetMetrics", + definitions["fuzzTarget"]["allOf"][1]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + ["reproduction"], + definitions["fuzzTarget"]["allOf"][2]["then"]["required"], + ) + self.assertEqual( + "#/$defs/mutationTarget", + definitions["report"]["allOf"][0]["then"]["properties"]["targets"][ + "items" + ]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzTarget", + definitions["report"]["allOf"][0]["else"]["properties"]["targets"][ + "items" + ]["$ref"], + ) + self.assertEqual( + "#/$defs/mutationReportMetrics", + definitions["report"]["allOf"][0]["then"]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzReportMetrics", + definitions["report"]["allOf"][0]["else"]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertTrue(definitions["report"]["properties"]["targets"]["uniqueItems"]) + + def test_decision_names_python_as_the_cross_field_semantic_authority(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn( + "The Python validator is authoritative for cross-field semantics that JSON Schema cannot express, including unique and sorted target names, recomputed completeness, outcomes, and aggregate metrics.", + decision, + ) + def test_reusable_workflow_matches_the_public_contract(self): workflow = WORKFLOW.read_text() @@ -332,18 +576,20 @@ def test_standards_validation_runs_this_contract(self): workflow.count("python3 .github/tests/quality_report_contract_test.py"), ) - def aggregate(self, fixture, *, track, tool, expected): + def aggregate(self, fixture, *, track, tool, expected, input_mutator=None): self.assertTrue(SCRIPT.is_file(), f"missing aggregator: {SCRIPT}") with tempfile.TemporaryDirectory() as temp_dir: input_dir = Path(temp_dir) / "input" shutil.copytree(FIXTURES / fixture, input_dir) + if input_mutator is not None: + input_mutator(input_dir) for invalid_fixture in input_dir.rglob("target-result.invalid"): invalid_fixture.rename(invalid_fixture.with_name("target-result.json")) output_dir = Path(temp_dir) / "output" github_output = Path(temp_dir) / "github-output" summary_path = Path(temp_dir) / "step-summary" command = [ - "python3", + sys.executable, str(SCRIPT), "aggregate", "--input-dir", @@ -383,7 +629,9 @@ def aggregate(self, fixture, *, track, tool, expected): "--github-step-summary", str(summary_path), ] - result = subprocess.run(command, capture_output=True, text=True) + result = subprocess.run( + command, capture_output=True, text=True, timeout=30 + ) report_path = output_dir / "report.json" summary_file = output_dir / "summary.md" self.assertTrue(report_path.is_file(), result.stderr) @@ -425,9 +673,10 @@ def validate_report(self, report): report_path = Path(temp_dir) / "report.json" report_path.write_text(json.dumps(report)) return subprocess.run( - ["python3", str(SCRIPT), "validate", str(report_path)], + [sys.executable, str(SCRIPT), "validate", str(report_path)], capture_output=True, text=True, + timeout=30, ) def all_objects_are_closed(self, value): diff --git a/decisions/quality-reporting.md b/decisions/quality-reporting.md index 6182c97..b826c22 100644 --- a/decisions/quality-reporting.md +++ b/decisions/quality-reporting.md @@ -55,6 +55,9 @@ job validates the set and publishes `quality-report/v1` with: - `targets`: the declared target names and their individual results - track-specific `metrics` +`go-fuzz` identifies Go's native `go test -fuzz` engine, not the legacy +third-party go-fuzz tool. + The schema and dependency-free validator live under [`quality-report/v1`](../quality-report/v1). Both target-result fragments and aggregate reports carry the exact `quality-report/v1` version. Objects reject unknown fields, JSON parsing rejects @@ -62,6 +65,10 @@ duplicate fields and non-finite numbers, and semantic validation recomputes completeness, outcomes, and scores. Callers pass `expected_targets` as a non-empty JSON array of unique target names. +The Python validator is authoritative for cross-field semantics that JSON +Schema cannot express, including unique and sorted target names, recomputed +completeness, outcomes, and aggregate metrics. + Mutation metrics are: - `killed`, `timeout`, `survived`, `no_coverage`, `invalid`, `ignored` diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py index 8b8c93a..274308f 100644 --- a/quality-report/v1/quality_report.py +++ b/quality-report/v1/quality_report.py @@ -79,14 +79,16 @@ def load_json_text(text, source): parse_constant=reject_constant, ) except (json.JSONDecodeError, ContractError) as error: - raise ContractError("{0}: invalid JSON: {1}".format(source, error)) + raise ContractError("{0}: invalid JSON: {1}".format(source, error)) from error def load_json_file(path): try: return load_json_text(path.read_text(), str(path)) - except OSError as error: - raise ContractError("{0}: cannot read JSON: {1}".format(path, error)) + except (OSError, UnicodeDecodeError) as error: + raise ContractError( + "{0}: cannot read JSON: {1}".format(path, error) + ) from error def require_object(value, path, required, optional=()): @@ -168,8 +170,10 @@ def require_timestamp(value, path): if RFC3339_PATTERN.fullmatch(value) is None: raise ValueError datetime.fromisoformat(value.replace("Z", "+00:00").replace("z", "+00:00")) - except ValueError: - raise ContractError("{0}: expected RFC 3339 timestamp".format(path)) + except ValueError as error: + raise ContractError( + "{0}: expected RFC 3339 timestamp".format(path) + ) from error def percentage(numerator, denominator): @@ -238,6 +242,15 @@ def validate_target(target, track, path): validate_mutation_target_metrics(metrics, path + ".metrics") else: validate_fuzz_target_metrics(metrics, path + ".metrics") + if ( + outcome == "passed" + and metrics["elapsed_seconds"] < metrics["budget_seconds"] + ): + raise ContractError( + "{0}.metrics.elapsed_seconds: passed target has not completed fuzz budget".format( + path + ) + ) if outcome == "passed": if "diagnostic" in target or "reproduction" in target: diff --git a/quality-report/v1/schema.json b/quality-report/v1/schema.json index 46c38f1..2ce2acc 100644 --- a/quality-report/v1/schema.json +++ b/quality-report/v1/schema.json @@ -172,7 +172,112 @@ "reproduction": { "$ref": "#/$defs/reproduction" } - } + }, + "allOf": [ + { + "if": { + "properties": { + "outcome": { + "const": "passed" + } + }, + "required": [ + "outcome" + ] + }, + "then": { + "properties": { + "metrics": { + "not": { + "type": "null" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "diagnostic" + ] + }, + { + "required": [ + "reproduction" + ] + } + ] + } + }, + "else": { + "required": [ + "diagnostic" + ] + } + } + ] + }, + "mutationTarget": { + "allOf": [ + { + "$ref": "#/$defs/target" + }, + { + "properties": { + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationTargetMetrics" + }, + { + "type": "null" + } + ] + } + } + } + ] + }, + "fuzzTarget": { + "allOf": [ + { + "$ref": "#/$defs/target" + }, + { + "properties": { + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/fuzzTargetMetrics" + }, + { + "type": "null" + } + ] + } + } + }, + { + "if": { + "properties": { + "outcome": { + "enum": [ + "failed", + "crashed", + "flaked" + ] + } + }, + "required": [ + "outcome" + ] + }, + "then": { + "required": [ + "reproduction" + ] + } + } + ] }, "targetResult": { "type": "object", @@ -211,7 +316,47 @@ "target": { "$ref": "#/$defs/target" } - } + }, + "allOf": [ + { + "if": { + "properties": { + "track": { + "const": "mutation" + } + }, + "required": [ + "track" + ] + }, + "then": { + "properties": { + "tool": { + "enum": [ + "gremlins", + "stryker" + ] + }, + "target": { + "$ref": "#/$defs/mutationTarget" + } + } + }, + "else": { + "properties": { + "tool": { + "enum": [ + "fast-check", + "go-fuzz" + ] + }, + "target": { + "$ref": "#/$defs/fuzzTarget" + } + } + } + } + ] }, "mutationReportMetrics": { "type": "object", @@ -465,6 +610,7 @@ }, "targets": { "type": "array", + "uniqueItems": true, "items": { "$ref": "#/$defs/target" } @@ -490,7 +636,71 @@ "minLength": 1 } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "track": { + "const": "mutation" + } + }, + "required": [ + "track" + ] + }, + "then": { + "properties": { + "tool": { + "enum": [ + "gremlins", + "stryker" + ] + }, + "targets": { + "items": { + "$ref": "#/$defs/mutationTarget" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationReportMetrics" + }, + { + "type": "null" + } + ] + } + } + }, + "else": { + "properties": { + "tool": { + "enum": [ + "fast-check", + "go-fuzz" + ] + }, + "targets": { + "items": { + "$ref": "#/$defs/fuzzTarget" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/fuzzReportMetrics" + }, + { + "type": "null" + } + ] + } + } + } + } + ] } } } From 9a5362a6666d6ee42a3c5e47bf78b88eb21021f6 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:08:46 -0400 Subject: [PATCH 06/18] fix(quality): decode reports as utf-8 --- .github/tests/quality_report_contract_test.py | 30 +++++++++++++++++++ quality-report/v1/quality_report.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index 6c65a24..858507c 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -6,6 +6,7 @@ import sys import tempfile import unittest +from unittest import mock ROOT = Path(__file__).resolve().parents[2] @@ -161,6 +162,35 @@ def corrupt_target(input_dir): self.assertIn("partial 1/2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_utf8_target_load_does_not_depend_on_the_locale_encoding(self): + document = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + document["target"]["name"] = "internal/docker/FuzzDécodage" + + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "target-result.json" + path.write_bytes(json.dumps(document, ensure_ascii=False).encode("utf-8")) + + def read_with_ascii_default(*, encoding=None, errors=None): + return path.read_bytes().decode( + encoding or "ascii", errors or "strict" + ) + + with mock.patch.object( + Path, "read_text", side_effect=read_with_ascii_default + ): + loaded = QUALITY_REPORT["load_json_file"](path) + + self.assertEqual( + "internal/docker/FuzzDécodage", loaded["target"]["name"] + ) + def test_canonical_score_counts_timeout_and_no_coverage(self): result, report, summary = self.aggregate( "canonical-score", diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py index 274308f..1c4a4c2 100644 --- a/quality-report/v1/quality_report.py +++ b/quality-report/v1/quality_report.py @@ -84,7 +84,7 @@ def load_json_text(text, source): def load_json_file(path): try: - return load_json_text(path.read_text(), str(path)) + return load_json_text(path.read_text(encoding="utf-8"), str(path)) except (OSError, UnicodeDecodeError) as error: raise ContractError( "{0}: cannot read JSON: {1}".format(path, error) From a39c5bd4697a7e513694c4a227dc8b15099f12a8 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:38:16 -0400 Subject: [PATCH 07/18] test(quality): pin fixture encoding --- .github/tests/quality_report_contract_test.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index 858507c..6fabc70 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -163,13 +163,19 @@ def corrupt_target(input_dir): self.assertEqual(0, self.validate_report(report).returncode) def test_utf8_target_load_does_not_depend_on_the_locale_encoding(self): - document = json.loads( - ( - FIXTURES - / "complete" - / "quality-result-stats" - / "target-result.json" - ).read_text() + fixture = ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ) + original_read_text = Path.read_text + with mock.patch.object( + Path, "read_text", autospec=True, side_effect=original_read_text + ) as fixture_read: + document = json.loads(fixture.read_text(encoding="utf-8")) + self.assertEqual( + mock.call(fixture, encoding="utf-8"), fixture_read.call_args ) document["target"]["name"] = "internal/docker/FuzzDécodage" From 7be30f375a4303edd1f135e919d02b9ac3cfe875 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:23:49 -0400 Subject: [PATCH 08/18] ci(profile): make asset generation read-only (#10) * ci(profile): make asset generation read-only * fix(profile): restrict asset validation egress --- .../tests/community_health_contract_test.py | 78 ++++++++++++++++--- .github/workflows/profile-assets.yml | 60 ++++++++++++++ .github/workflows/standards-validation.yml | 14 ++-- .github/workflows/update-stats.yml | 44 ----------- .gitignore | 1 + cache/.gitignore | 2 - cache/stats.json | 22 ------ .../generate_profile_svg.py | 6 +- 8 files changed, 140 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/profile-assets.yml delete mode 100644 .github/workflows/update-stats.yml delete mode 100644 cache/.gitignore delete mode 100644 cache/stats.json rename generate_svg.py => scripts/generate_profile_svg.py (98%) diff --git a/.github/tests/community_health_contract_test.py b/.github/tests/community_health_contract_test.py index 9439282..7d36a25 100644 --- a/.github/tests/community_health_contract_test.py +++ b/.github/tests/community_health_contract_test.py @@ -1,5 +1,7 @@ from pathlib import Path import re +import subprocess +import sys import unittest @@ -119,6 +121,9 @@ def test_standards_workflow_has_unconditional_stable_jobs(self): self.assertIn(" - dev/repository-standards\n", workflow) self.assertNotIn("\n push:", workflow) self.assertIn("permissions: {}", workflow) + self.assertIn("concurrency:", workflow) + self.assertIn("group: standards-validation-", workflow) + self.assertIn("cancel-in-progress: true", workflow) self.assertIn("runs-on: ubuntu-24.04", workflow) self.assertNotIn("matrix:", workflow) self.assertNotIn("continue-on-error:", workflow) @@ -127,14 +132,14 @@ def test_standards_workflow_has_unconditional_stable_jobs(self): validation_job = workflow.split(" validation:\n", 1)[1].split( "\n codeql:\n", 1 )[0] - self.assertIn(" actions: read", validation_job) - self.assertIn(" contents: read", validation_job) + self.assertIn(" actions: read # Read workflow metadata", validation_job) + self.assertIn(" contents: read # Check out and validate repository files", validation_job) self.assertNotIn("write", validation_job) codeql_job = workflow.split("\n codeql:\n", 1)[1] - self.assertIn(" actions: read", codeql_job) - self.assertIn(" contents: read", codeql_job) - self.assertIn(" security-events: write", codeql_job) + self.assertIn(" actions: read # Read workflow metadata", codeql_job) + self.assertIn(" contents: read # Check out Python sources", codeql_job) + self.assertIn(" security-events: write # Upload CodeQL analysis", codeql_job) jobs = re.findall( r"^ ([a-z][a-z0-9-]+):$", @@ -184,22 +189,73 @@ def test_linter_configs_do_not_suppress_workflow_findings(self): self.assertEqual(2, markdown_config.count(": false")) self.assertFalse((ROOT / ".github/zizmor.yml").exists()) - def test_profile_workflow_pins_actions_and_pushes_without_persisted_credentials(self): - workflow = self.read_text(".github/workflows/update-stats.yml") + def test_profile_generator_has_one_canonical_location(self): + self.assertFalse((ROOT / "generate_svg.py").exists()) + self.assertTrue((ROOT / "scripts/generate_profile_svg.py").is_file()) + + def test_dead_profile_cache_is_removed(self): + self.assertFalse((ROOT / "cache").exists()) + + def test_font_reference_is_local_only(self): + gitignore = self.read_text(".gitignore") + self.assertIn("profile/font_reference.svg", gitignore.splitlines()) + + def test_profile_generator_reproduces_committed_assets(self): + generator = ROOT / "scripts/generate_profile_svg.py" + self.assertTrue(generator.is_file(), f"missing generator: {generator}") + outputs = ( + ROOT / "profile/dark_mode.svg", + ROOT / "profile/light_mode.svg", + ) + before = {path: path.read_bytes() for path in outputs} + + subprocess.run( + [sys.executable, generator], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + after = {path: path.read_bytes() for path in outputs} + self.assertEqual(before, after) + + def test_profile_workflow_is_read_only_bounded_and_pinned(self): + workflow = self.read_text(".github/workflows/profile-assets.yml") actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) self.assertEqual(2, len(actions)) for action in actions: self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") - checkout_step = workflow.split(" - name: Checkout\n", 1)[1].split( + harden_marker = " - name: Harden runner\n" + self.assertIn(harden_marker, workflow) + harden_step = workflow.split(harden_marker, 1)[1].split("\n - name:", 1)[0] + + self.assertIn("egress-policy: block", harden_step) + self.assertIn("allowed-endpoints: github.com:443", harden_step) + + checkout_marker = " - name: Check out repository\n" + self.assertIn(checkout_marker, workflow) + checkout_step = workflow.split(checkout_marker, 1)[1].split( "\n - name:", 1 )[0] - push_step = workflow.split(" - name: Commit and push\n", 1)[1] self.assertIn("persist-credentials: false", checkout_step) - self.assertIn("GH_TOKEN: ${{ github.token }}", push_step) - self.assertIn("gh auth setup-git", push_step) + self.assertIn("permissions: {}", workflow) + self.assertIn("name: Profile Assets", workflow) + self.assertIn("contents: read", workflow) + self.assertIn("timeout-minutes:", workflow) + self.assertIn("concurrency:", workflow) + self.assertIn("step-security/harden-runner@", workflow) + self.assertIn("python3 scripts/generate_profile_svg.py", workflow) + self.assertIn("git diff --exit-code -- profile/", workflow) + self.assertIn("git status --porcelain --untracked-files=all -- profile/", workflow) + self.assertNotIn("contents: write", workflow) + self.assertNotIn("git push", workflow) + self.assertNotIn("GH_TOKEN", workflow) + self.assertNotIn("schedule:", workflow) + self.assertFalse((ROOT / ".github/workflows/update-stats.yml").exists()) def read_text(self, relative_path): path = ROOT / relative_path diff --git a/.github/workflows/profile-assets.yml b/.github/workflows/profile-assets.yml new file mode 100644 index 0000000..0fd6154 --- /dev/null +++ b/.github/workflows/profile-assets.yml @@ -0,0 +1,60 @@ +name: Profile Assets + +on: + pull_request: + branches: + - main + - dev/repository-standards + paths: + - .github/tests/community_health_contract_test.py + - .github/workflows/profile-assets.yml + - .gitignore + - profile/*.svg + - scripts/generate_profile_svg.py + push: + branches: + - main + paths: + - .github/tests/community_health_contract_test.py + - .github/workflows/profile-assets.yml + - .gitignore + - profile/*.svg + - scripts/generate_profile_svg.py + workflow_dispatch: + +permissions: {} + +concurrency: + group: profile-assets-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + validate: + name: Profile Assets + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: github.com:443 + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Verify generated profile assets + run: | + set -euo pipefail + python3 scripts/generate_profile_svg.py + git diff --exit-code -- profile/ + changes="$(git status --porcelain --untracked-files=all -- profile/)" + if [[ -n "$changes" ]]; then + printf '%s\n' "$changes" + exit 1 + fi diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index ceb4606..30ea7f4 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -8,14 +8,18 @@ on: permissions: {} +concurrency: + group: standards-validation-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: validation: name: Standards Validation runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: - actions: read - contents: read + actions: read # Read workflow metadata + contents: read # Check out and validate repository files steps: - name: Harden runner @@ -74,9 +78,9 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: - actions: read - contents: read - security-events: write + actions: read # Read workflow metadata + contents: read # Check out Python sources + security-events: write # Upload CodeQL analysis steps: - name: Harden runner diff --git a/.github/workflows/update-stats.yml b/.github/workflows/update-stats.yml deleted file mode 100644 index cb79f7c..0000000 --- a/.github/workflows/update-stats.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Update Org Profile Stats - -on: - push: - branches: - - main - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -permissions: - contents: write - -jobs: - update: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.11' - - - name: Generate SVGs - env: - GH_TOKEN: ${{ secrets.ACCESS_TOKEN }} - run: | - python generate_svg.py - - - name: Commit and push - env: - GH_TOKEN: ${{ github.token }} - run: | - gh auth setup-git - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - git add profile/dark_mode.svg profile/light_mode.svg cache/ - git diff --staged --quiet || git commit -m "Update stats" - git push diff --git a/.gitignore b/.gitignore index c54ade8..d66d040 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ # Local planning/scratch workspace — not for commit .planning/ +profile/font_reference.svg diff --git a/cache/.gitignore b/cache/.gitignore deleted file mode 100644 index 688f1fa..0000000 --- a/cache/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Track cache files (needed for CI) -!*.json diff --git a/cache/stats.json b/cache/stats.json deleted file mode 100644 index 42f10e4..0000000 --- a/cache/stats.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "timestamp": "2026-02-09T23:00:00.795596", - "org": { - "public_repos": 2, - "private_repos": 3, - "total_repos": 5, - "total_stars": 0, - "total_forks": 0, - "total_commits": 398, - "total_prs": 11, - "total_issues": 1, - "members": 0, - "loc_added": 354732, - "loc_deleted": 283673, - "loc_total": 71059, - "languages": [ - "Python", - "Swift", - "TypeScript" - ] - } -} \ No newline at end of file diff --git a/generate_svg.py b/scripts/generate_profile_svg.py similarity index 98% rename from generate_svg.py rename to scripts/generate_profile_svg.py index 02f2b1b..436b7c6 100644 --- a/generate_svg.py +++ b/scripts/generate_profile_svg.py @@ -211,16 +211,16 @@ def art_color(ch): def main(): """Generate both dark and light mode SVGs.""" - script_dir = Path(__file__).parent + repo_root = Path(__file__).resolve().parents[1] dark_svg = generate_svg("dark") - dark_path = script_dir / "profile" / "dark_mode.svg" + dark_path = repo_root / "profile" / "dark_mode.svg" with open(dark_path, 'w', encoding='utf-8') as f: f.write(dark_svg) print(f"Generated: {dark_path}") light_svg = generate_svg("light") - light_path = script_dir / "profile" / "light_mode.svg" + light_path = repo_root / "profile" / "light_mode.svg" with open(light_path, 'w', encoding='utf-8') as f: f.write(light_svg) print(f"Generated: {light_path}") From 55c58e856c5f2a680b8e616158b76328831fad76 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:31:52 -0400 Subject: [PATCH 09/18] ci(review): add deduplicated Greptile summon (#9) * ci(review): add deduplicated Greptile summon * fix(review): serialize exact-head Greptile summons * test(review): lock Greptile security controls --- .../tests/greptile_summon_contract_test.py | 92 ++++++++++++++ .github/workflows/greptile-summon.yml | 120 ++++++++++++++++++ .github/workflows/standards-validation.yml | 4 +- 3 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 .github/tests/greptile_summon_contract_test.py create mode 100644 .github/workflows/greptile-summon.yml diff --git a/.github/tests/greptile_summon_contract_test.py b/.github/tests/greptile_summon_contract_test.py new file mode 100644 index 0000000..7584077 --- /dev/null +++ b/.github/tests/greptile_summon_contract_test.py @@ -0,0 +1,92 @@ +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/greptile-summon.yml" + + +class GreptileSummonContractTest(unittest.TestCase): + def test_reusable_workflow_has_exact_head_inputs_and_narrow_permissions(self): + workflow = self.read_workflow() + + for expected in ( + " workflow_call:\n", + " pr_number:\n", + " type: number\n", + " head_sha:\n", + " type: string\n", + " focus:\n", + "permissions: {}", + " runs-on: ubuntu-24.04", + " timeout-minutes: 5", + " pull-requests: write", + "uses: step-security/harden-runner@", + "egress-policy: block", + "api.github.com:443", + "github-token: ${{ github.token }}", + ): + self.assertIn(expected, workflow) + + self.assertNotIn("actions/checkout", workflow) + + def test_workflow_verifies_head_and_deduplicates_before_commenting(self): + workflow = self.read_workflow() + + for expected in ( + "github.rest.pulls.get", + 'pull.state !== "open"', + "pull.head.sha !== headSha", + "github.paginate", + "github.rest.issues.listComments", + "greptile-summon:${headSha}", + 'comment.user?.login === "github-actions[bot]"', + "github.rest.issues.createComment", + "@greptileai", + ): + self.assertIn(expected, workflow) + + self.assertIn( + "greptile-summon-${{ github.repository }}-${{ inputs.pr_number }}-${{ inputs.head_sha }}", + workflow, + ) + self.assertIn(" cancel-in-progress: false", workflow) + self.assertGreaterEqual(workflow.count("github.rest.pulls.get"), 2) + self.assertIn("currentPull.head.sha !== headSha", workflow) + self.assertIn("HEAD_SHA: ${{ inputs.head_sha }}", workflow) + self.assertIn("REVIEW_FOCUS: ${{ inputs.focus }}", workflow) + self.assertNotIn("const headSha = '${{ inputs.head_sha }}'", workflow) + self.assertNotIn("const focus = '${{ inputs.focus }}'", workflow) + + def test_workflow_rejects_partial_pull_request_numbers(self): + workflow = self.read_workflow() + + self.assertIn("const prNumberText = process.env.PR_NUMBER", workflow) + self.assertIn("const prNumber = Number(prNumberText)", workflow) + self.assertIn("!/^[1-9]\\d*$/.test(prNumberText)", workflow) + self.assertIn("PR_NUMBER: ${{ inputs.pr_number }}", workflow) + self.assertIn("if (!/^[0-9a-f]{40}$/.test(headSha))", workflow) + self.assertIn("if (focus.length === 0 || focus.length > 500)", workflow) + self.assertNotIn("Number.parseInt(process.env.PR_NUMBER", workflow) + + def test_workflow_pins_actions_and_is_run_by_standards_validation(self): + workflow = self.read_workflow() + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + validation = ( + ROOT / ".github/workflows/standards-validation.yml" + ).read_text() + self.assertIn("greptile_summon_contract_test.py", validation) + + def read_workflow(self): + self.assertTrue(WORKFLOW.is_file(), f"missing workflow: {WORKFLOW}") + return WORKFLOW.read_text() + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/greptile-summon.yml b/.github/workflows/greptile-summon.yml new file mode 100644 index 0000000..3484f9b --- /dev/null +++ b/.github/workflows/greptile-summon.yml @@ -0,0 +1,120 @@ +name: Summon Greptile + +on: + workflow_call: + inputs: + pr_number: + description: Pull request number in the caller repository + required: true + type: number + head_sha: + description: Exact pull request head to review + required: true + type: string + focus: + description: Short review focus based on the risk in the diff + required: false + default: Review for correctness, security issues, and cross-file regressions. + type: string + +permissions: {} + +jobs: + summon: + name: Summon Greptile + runs-on: ubuntu-24.04 + timeout-minutes: 5 + concurrency: + group: greptile-summon-${{ github.repository }}-${{ inputs.pr_number }}-${{ inputs.head_sha }} + cancel-in-progress: false + permissions: + pull-requests: write # Create the intentional Greptile summon comment. + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + + - name: Comment once on the exact pull request head + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PR_NUMBER: ${{ inputs.pr_number }} + HEAD_SHA: ${{ inputs.head_sha }} + REVIEW_FOCUS: ${{ inputs.focus }} + with: + github-token: ${{ github.token }} + script: | + const prNumberText = process.env.PR_NUMBER; + const prNumber = Number(prNumberText); + const headSha = process.env.HEAD_SHA; + const focus = process.env.REVIEW_FOCUS.trim(); + const { owner, repo } = context.repo; + + if (!/^[1-9]\d*$/.test(prNumberText) || !Number.isSafeInteger(prNumber)) { + core.setFailed(`Invalid pull request number: ${prNumberText}`); + return; + } + if (!/^[0-9a-f]{40}$/.test(headSha)) { + core.setFailed(`Invalid full head SHA: ${headSha}`); + return; + } + if (focus.length === 0 || focus.length > 500) { + core.setFailed("Review focus must contain 1 to 500 characters"); + return; + } + + const { data: pull } = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber, + }); + if (pull.state !== "open") { + core.setFailed(`Pull request #${prNumber} is ${pull.state}`); + return; + } + if (pull.head.sha !== headSha) { + core.setFailed( + `Pull request head changed: expected ${headSha}, got ${pull.head.sha}`, + ); + return; + } + + const marker = ``; + const comments = await github.paginate( + github.rest.issues.listComments, + { owner, repo, issue_number: prNumber, per_page: 100 }, + ); + const alreadySummoned = comments.some( + (comment) => comment.user?.login === "github-actions[bot]" + && comment.body?.includes(marker), + ); + if (alreadySummoned) { + core.notice(`Greptile already summoned for ${headSha}`); + return; + } + + const { data: currentPull } = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber, + }); + if (currentPull.state !== "open") { + core.setFailed(`Pull request #${prNumber} is ${currentPull.state}`); + return; + } + if (currentPull.head.sha !== headSha) { + core.setFailed( + `Pull request head changed: expected ${headSha}, got ${currentPull.head.sha}`, + ); + return; + } + + await github.rest.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body: `@greptileai Review exact head \`${headSha}\`. ${focus}\n\n${marker}`, + }); diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index 30ea7f4..0ea5ad0 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -38,7 +38,9 @@ jobs: node-version: 24 - name: Validate community health contract - run: python3 .github/tests/community_health_contract_test.py + run: | + python3 .github/tests/community_health_contract_test.py + python3 .github/tests/greptile_summon_contract_test.py - name: Compile Python run: python3 -m compileall -q . From 6e7a78dc13d099cb27669c0a0f8a75836de97f03 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 08:24:16 -0400 Subject: [PATCH 10/18] ci(workflows): add run-test and run-lint toggles to go-ci (#19) go-ci.yml's test and lint jobs ran unconditionally, so a Go-less repo that only wants the language-agnostic workflow-security (zizmor) job couldn't call it. Add run-test/run-lint boolean inputs, mirroring the existing run-govulncheck/run-workflow-security/etc. toggle pattern, defaulting to true so existing callers see no behavior change. Fixes: #18 --- .github/tests/reusable_ci_contract_test.py | 4 ++++ .github/workflows/go-ci.yml | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py index a6ec3f5..b958eb6 100644 --- a/.github/tests/reusable_ci_contract_test.py +++ b/.github/tests/reusable_ci_contract_test.py @@ -25,6 +25,8 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) "lint-check-name", "test-check-name", "fuzzers-json", + "run-test", + "run-lint", "run-govulncheck", "run-qlty", "run-goreleaser", @@ -113,6 +115,8 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) self.assert_input("go", "lint-check-name", "string", default="Go Lint") self.assert_input("go", "test-check-name", "string", default="Go Test") self.assert_input("go", "fuzzers-json", "string", default="[]") + for input_name in ("run-test", "run-lint"): + self.assert_input("go", input_name, "boolean", default="true") for input_name in ( "run-govulncheck", "run-qlty", diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 46601d4..8e14b1c 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -33,6 +33,16 @@ on: required: false type: string default: "[]" + run-test: + description: Run the fixed repository Go test gate + required: false + type: boolean + default: true + run-lint: + description: Run the fixed repository Go lint gate + required: false + type: boolean + default: true run-govulncheck: description: Run the fixed repository govulncheck gate required: false @@ -159,6 +169,7 @@ permissions: {} jobs: test: name: ${{ inputs.test-check-name }} + if: inputs.run-test runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: @@ -198,6 +209,7 @@ jobs: lint: name: ${{ inputs.lint-check-name }} + if: inputs.run-lint runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: From 04ad251abff43346cd46b836f323421a0f99bd00 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:05:50 -0400 Subject: [PATCH 11/18] ci(workflows): add module-directory input to node-ci (#22) * ci(workflows): add module-directory input to node-ci Mirrors go-ci's module-directory idiom: a string input defaulting to "." threaded into each fixed script's env as MODULE_DIRECTORY, so a repo with several independently-gated Node projects can call node-ci once per project. The default preserves current behavior for existing callers. Extends the reusable CI contract test to assert the new input and its threading, matching how run-test/run-lint were added for go-ci in #19. * test(workflows): assert module-directory threads into all three node jobs --- .github/tests/reusable_ci_contract_test.py | 5 +++++ .github/workflows/node-ci.yml | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py index b958eb6..b4fc0fa 100644 --- a/.github/tests/reusable_ci_contract_test.py +++ b/.github/tests/reusable_ci_contract_test.py @@ -47,6 +47,7 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) ) ) node_inputs = [ + "module-directory", "node-version", "lockfile-path", "lint-check-name", @@ -127,6 +128,7 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) ): self.assert_input("go", input_name, "boolean", default="false") + self.assert_input("node", "module-directory", "string", default=".") self.assert_input("node", "node-version", "string", default="24") self.assert_input("node", "lockfile-path", "string", default="package-lock.json") self.assert_input("node", "lint-check-name", "string", default="Node Lint") @@ -204,6 +206,9 @@ def test_central_jobs_have_exact_names_fixed_commands_and_per_job_egress(self): ) self.assertIn("node-version: ${{ inputs.node-version }}", node) self.assertIn("cache-dependency-path: ${{ inputs.lockfile-path }}", node) + self.assertEqual( + 3, node.count("MODULE_DIRECTORY: ${{ inputs.module-directory }}") + ) for workflow in (go, node, self.read_workflow(WORKFLOWS["release"])): self.assertIn("runs-on: ubuntu-24.04", workflow) diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index e699a9b..c953151 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -3,6 +3,11 @@ name: Reusable Node CI on: workflow_call: inputs: + module-directory: + description: Repository-relative Node module directory + required: false + type: string + default: . node-version: description: Node.js version used by enabled jobs required: false @@ -105,6 +110,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node lint gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-lint.sh test: @@ -135,6 +142,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node test gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-test.sh - name: Upload Node test artifacts @@ -174,6 +183,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node build gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-build.sh - name: Upload Node build artifacts From 7f9be8b854dd345814537fd0d7f24b2509760f96 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:30:27 -0400 Subject: [PATCH 12/18] docs(onboarding): record the qlty alignment baseline (#24) --- REPOSITORY_ONBOARDING.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index a06bb7a..7510744 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -115,10 +115,8 @@ repositories. Language-specific tests do not. - [ ] Add language formatters to `pre-commit` only when they can safely operate on staged files. Verify the hook does not rewrite unrelated work. -Qlty Cloud enrollment and usage data are not an organization-wide prerequisite. -Do not make a Qlty upload or check required until the repository is enrolled and -the check has proved stable. Local Qlty checks may still be stricter than this -baseline. +Qlty spans local and CI gates; the full baseline, including why Qlty Cloud +checks stay non-required, is in the Qlty subsection of section 4. ## 3. Protect `main` @@ -194,6 +192,37 @@ Add these only when the behavior exists: - [ ] Translation synchronization only for a repository with a translation source of truth and configured provider credentials. +### Qlty + +Qlty appears in three places. The committed configuration and the repository-run +gates are the alignment surface; the Qlty Cloud GitHub App is not. + +- [ ] Commit `.qlty/qlty.toml` (`config_version = "0"`) with the organization + exclude baseline for generated, minified, and vendored paths. Drydock's file + is the reference posture and Portwing's is its Go-repo mirror; start from the + closer of the two. The same file drives the local CLI and Qlty Cloud, so + changes to it are quality-gate changes, not formatting. +- [ ] Add the repository's Qlty gate script (`scripts/qlty-check-gate.sh all`) + to `pre-push`, fail-fast like every other local gate. An advisory smells + gate may run alongside it (Drydock pattern) but must not mask the gating + script's failure. +- [ ] Gate Qlty in CI with the repository-run check, which needs no Qlty Cloud + account. Go repositories call `go-ci.yml` with `run-qlty` enabled, + `qlty-egress-policy: block`, and the proven endpoint allowlist — copy the + allowlist from Portwing's `ci-verify.yml` rather than re-deriving it, and do + not widen it. Node repositories run the SHA-pinned + `qltysh/qlty-action/install` plus the same gate script inside their own + workflow (Drydock pattern). + +The Qlty Cloud GitHub App's statuses (`qlty check`, `qlty coverage`, +`qlty coverage diff`) currently error organization-wide with out-of-minutes +billing failures. Treat them as non-gating: never add them to required status +checks, and do not chase these failures on PRs — the repository-run gate above +is the enforced check. Qlty Cloud enrollment and usage data are not an +organization-wide prerequisite; do not make a Cloud upload or check required +until the repository is enrolled and the check has proved stable. Local Qlty +checks may still be stricter than this baseline. + ## 5. Add delivery controls only for shipped artifacts - [ ] Keep release construction separate from ordinary CI. A release-cut From f4c1d509fe98551d6e6d6c819d2bb618711e460f Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 18:34:58 -0400 Subject: [PATCH 13/18] docs(onboarding): align with the codified standards registry (#26) * docs(onboarding): align with the codified standards registry - docs(onboarding): name Codecov as the coverage cloud; Qlty Cloud App and maintainability badge stay, checks stay non-required - docs(onboarding): trivy deprecated in favor of Grype, including the qlty plugin blocks in the two reference configs (drydock#753, portwing#135) - docs(onboarding): CodeRabbit free Pro is public-only; private repos use cross-account human review - docs(onboarding): add the greptile.json contract and the label-gated second-opinion caller * docs(onboarding): reword the CodeRabbit private-repo claim as org policy - docs(onboarding): free-plan private-repo reviews exist but are rate-limited and never fired here; the skip is policy, not a plan fact - docs(onboarding): pair the Greptile caller with auto-applied CodeRabbit labeling so the second-opinion label is criteria-driven --- REPOSITORY_ONBOARDING.md | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index 7510744..8929480 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -100,6 +100,20 @@ repositories. Language-specific tests do not. request an explicit CodeRabbit review on the first PR to prove it works. - [ ] Confirm CodeRabbit can access the repository and posts an actual review comment. A green check without review output is not enough for this test. + Automated Pro-level reviews are free on public repositories only; on + private repositories the free plan rate-limits automated reviews, and they + have not fired for this organization in practice. Organization policy: on a + private repository skip both CodeRabbit items, use cross-account human + review, and add the config when the repository goes public. +- [ ] Keep `greptile.json` at exactly `{"skipReview": "AUTOMATIC"}` + (contract-tested in this repository) so Greptile never reviews unbidden. A + repository that wants opt-in second opinions adds a label-gated caller + workflow (`.github/workflows/greptile.yml`, firing on the `second-opinion` + label) that calls this repository's `greptile-summon.yml` at a pinned full + commit SHA. Pair the caller with a CodeRabbit `labeling_instructions` entry + for `second-opinion` and `auto_apply_labels` enabled, so applying the label + is criteria-driven rather than left to memory. The label is Greptile's only + trigger; never wire it to review every PR directly. ### Local gates @@ -156,7 +170,8 @@ must push, and use `step-security/harden-runner` with the smallest practical egress allowlist. Every repository with executable code needs the applicable format or lint and -test gates. Require coverage when the language and test tooling measure it, a +test gates. Require coverage when the language and test tooling measure it +(uploaded to Codecov, not Qlty Cloud), a production build when the repository ships a buildable artifact, dependency review when it has dependencies, workflow validation when it has GitHub Actions, and CodeQL when its language is supported. Add the matching language or artifact @@ -174,8 +189,9 @@ checks below: - **GitHub Actions:** actionlint and zizmor on every PR, plus tests for non-trivial embedded shell and workflow contracts. - **Container image:** build the real Dockerfile, smoke-test the image, and scan - the built image and locked dependencies with Grype or the current organization - scanner. + the built image and locked dependencies with Grype. Trivy is deprecated + organization-wide in favor of Grype; do not adopt it, including as a Qlty + plugin. - **Public repository:** OpenSSF Scorecard on its supported triggers and dependency review on PRs. @@ -200,8 +216,11 @@ gates are the alignment surface; the Qlty Cloud GitHub App is not. - [ ] Commit `.qlty/qlty.toml` (`config_version = "0"`) with the organization exclude baseline for generated, minified, and vendored paths. Drydock's file is the reference posture and Portwing's is its Go-repo mirror; start from the - closer of the two. The same file drives the local CLI and Qlty Cloud, so - changes to it are quality-gate changes, not formatting. + closer of the two. Do not copy the trivy plugin block from either reference: + trivy is deprecated in favor of Grype and its removal from both files is + tracked (drydock#753, portwing#135). The trufflehog plugin stays. The same + file drives the local CLI and Qlty Cloud, so changes to it are quality-gate + changes, not formatting. - [ ] Add the repository's Qlty gate script (`scripts/qlty-check-gate.sh all`) to `pre-push`, fail-fast like every other local gate. An advisory smells gate may run alongside it (Drydock pattern) but must not mask the gating @@ -218,8 +237,12 @@ The Qlty Cloud GitHub App's statuses (`qlty check`, `qlty coverage`, `qlty coverage diff`) currently error organization-wide with out-of-minutes billing failures. Treat them as non-gating: never add them to required status checks, and do not chase these failures on PRs — the repository-run gate above -is the enforced check. Qlty Cloud enrollment and usage data are not an -organization-wide prerequisite; do not make a Cloud upload or check required +is the enforced check. Coverage reporting is Codecov's job (decided +2026-08-16): upload coverage to Codecov and carry its badge in the README. +The Qlty Cloud App and its maintainability badge stay installed, but its +coverage upload is not the coverage system and its checks stay non-required. +Qlty Cloud enrollment and usage data are not an organization-wide +prerequisite; do not make a Cloud upload or check required until the repository is enrolled and the check has proved stable. Local Qlty checks may still be stricter than this baseline. From 5ac2e3fa208ef8724e163230adf8ed18ff607424 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 19:41:50 -0400 Subject: [PATCH 14/18] chore(repo): meet our own onboarding checklist (#28) * chore(repo): meet our own onboarding checklist - chore(repo): MIT LICENSE (infrastructure repos are MIT; products AGPL) - docs(repo): root AGENTS.md with repo-specific rules and validation - build(hooks): lefthook with commit-msg + pre-push mirroring CI via scripts/validate.sh * fix(hooks): tighten the commit-msg exemptions and mirror zizmor's CI flags - fix(hooks): merge/revert exemptions match git's generated subjects only, so a hand-typed 'Merge ...' subject no longer bypasses the check - fix(hooks): require a non-whitespace character after the colon - fix(hooks): zizmor runs --no-online-audits locally, matching CI's online-audits: false for local/CI parity * fix(hooks): exempt only git-generated merge and revert subjects --- AGENTS.md | 50 +++++++++++++++++++++++++++++++++++++ LICENSE | 21 ++++++++++++++++ lefthook.yml | 12 +++++++++ scripts/check-commit-msg.sh | 25 +++++++++++++++++++ scripts/validate.sh | 45 +++++++++++++++++++++++++++++++++ 5 files changed, 153 insertions(+) create mode 100644 AGENTS.md create mode 100644 LICENSE create mode 100644 lefthook.yml create mode 100755 scripts/check-commit-msg.sh create mode 100755 scripts/validate.sh diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..0d27337 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,50 @@ +# Agent instructions for CodesWhat/.github + +This repository is the organization's meta layer: community health defaults, +the repository onboarding checklist (`REPOSITORY_ONBOARDING.md`), reusable CI +workflows, the organization profile README and its generator, and the +contract tests that pin the shapes other repositories depend on. + +## What lives here + +- `.github/workflows/go-ci.yml`, `node-ci.yml`, `release-gate.yml`, + `greptile-summon.yml`, `quality-report-aggregate.yml` — reusable workflows + consumed by other CodesWhat repositories. +- `.github/tests/` — contract tests. These are the public API of the org's + shared configs (the `greptile.json` shape, the quality-report v1 contract, + the reusable CI inputs). Change a contract test and its consumers together, + never one side alone. +- `profile/` — the organization profile README plus generated SVG assets + from `scripts/generate_profile_svg.py`. The untracked + `profile/font_reference.svg` is user-owned; leave it alone. + +## Rules that are specific to this repository + +- Consumers pin the reusable workflows to frozen full commit SHAs. Treat + every workflow change as breaking for pinned consumers: add new inputs + with safe defaults, never repurpose an existing input, and let consumers + adopt by rolling their pin deliberately. +- PRs target `dev/repository-standards`. `main` advances only through + promotion PRs. Reconcile before promoting: + `git merge -s ours origin/main -m "chore(sync): reconcile main before promotion"`. + Verify promotions with tree equality + (`git diff --quiet origin/main origin/dev/repository-standards`), never + commit ancestry. +- CodeRabbit auto-reviews PRs against `dev/*` branches only (scoped in + `.coderabbit.yaml`); summon it explicitly anywhere else and read its + inline comments before merging. +- Commits are plain Conventional Commits, `(scope): `, + no emoji, no AI attribution trailers. +- Never weaken branch protection or rulesets to land a change. + +## Validation + +Run before pushing (the lefthook pre-push hook runs the same set): + +```bash +bash scripts/validate.sh +``` + +That script mirrors the Standards Validation CI job: contract tests, Python +compile, YAML/JSON parse, markdownlint (with the separate profile config for +`profile/README.md`), and actionlint/zizmor when installed locally. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..af599c9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 CodesWhat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..3872369 --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,12 @@ +# Local git hooks. Install once with: npx --yes lefthook install +# pre-push mirrors the Standards Validation CI job via scripts/validate.sh. + +commit-msg: + commands: + conventional-commits: + run: bash scripts/check-commit-msg.sh {1} + +pre-push: + commands: + validate: + run: bash scripts/validate.sh diff --git a/scripts/check-commit-msg.sh b/scripts/check-commit-msg.sh new file mode 100755 index 0000000..3b3320e --- /dev/null +++ b/scripts/check-commit-msg.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# commit-msg hook: enforce plain Conventional Commits on the subject line. +set -euo pipefail + +msg_file="$1" +subject="$(head -n 1 "$msg_file")" + +case "$subject" in + "Merge branch '"*|"Merge pull request #"*|"Merge remote-tracking branch '"*|fixup!\ *|squash!\ *) exit 0 ;; +esac + +if [[ "$subject" =~ ^Revert\ \".+\"$ ]]; then + exit 0 +fi + +pattern='^(feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)(\([a-z0-9./-]+\))?!?: [^[:space:]]' +if ! [[ "$subject" =~ $pattern ]]; then + { + echo "commit subject must be plain Conventional Commits:" + echo " (scope): " + echo "allowed types: feat fix docs style refactor perf test build ci chore revert" + echo "got: $subject" + } >&2 + exit 1 +fi diff --git a/scripts/validate.sh b/scripts/validate.sh new file mode 100755 index 0000000..5d02c10 --- /dev/null +++ b/scripts/validate.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Local mirror of the Standards Validation CI job. Run before pushing; +# lefthook pre-push runs this same script. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +echo "==> clean tree" +if ! git diff --quiet || ! git diff --cached --quiet; then + echo "commit or stash local changes before pushing" >&2 + exit 1 +fi + +echo "==> contract tests" +python3 .github/tests/community_health_contract_test.py +python3 .github/tests/greptile_summon_contract_test.py +python3 .github/tests/greptile_config_contract_test.py +python3 .github/tests/quality_report_contract_test.py +python3 .github/tests/reusable_ci_contract_test.py + +echo "==> compile python" +python3 -m compileall -q . + +echo "==> parse yaml and json" +ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' +python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' + +echo "==> markdownlint" +npx --yes markdownlint-cli2@0.23.2 "**/*.md" "#profile/README.md" +npx --yes markdownlint-cli2@0.23.2 "profile/README.md" --config .github/markdownlint-profile.yaml + +echo "==> actionlint" +if command -v actionlint >/dev/null 2>&1; then + actionlint -color +else + echo "actionlint not installed locally; CI will run it" +fi + +echo "==> zizmor" +if command -v zizmor >/dev/null 2>&1; then + zizmor --no-online-audits .github/workflows/ +else + echo "zizmor not installed locally; CI will run it" +fi + +echo "==> ok" From 30d6b13d3340a67297ca9c2a0370715fc317bfde Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 21:48:43 -0400 Subject: [PATCH 15/18] docs(community): org-default code of conduct + community checklist (#30) * docs(community): add org-default code of conduct and community checklist items CODE_OF_CONDUCT.md is Contributor Covenant 2.0 (drydock's tuned copy) with the org contact security@codeswhat.com, cascading to every repo without a local one. Onboarding checklist gains the cascade-first rule and the Discussions on/off split for product vs meta repos. * test(community): assert the code of conduct in the community-health contract --- .../tests/community_health_contract_test.py | 12 ++ CODE_OF_CONDUCT.md | 128 ++++++++++++++++++ REPOSITORY_ONBOARDING.md | 8 ++ 3 files changed, 148 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/.github/tests/community_health_contract_test.py b/.github/tests/community_health_contract_test.py index 7d36a25..69c2020 100644 --- a/.github/tests/community_health_contract_test.py +++ b/.github/tests/community_health_contract_test.py @@ -13,6 +13,7 @@ def test_required_default_files_exist(self): required = ( "SECURITY.md", "CONTRIBUTING.md", + "CODE_OF_CONDUCT.md", ".github/CODEOWNERS", ".github/markdownlint-profile.yaml", ".github/ISSUE_TEMPLATE/bug_report.yml", @@ -66,6 +67,17 @@ def test_contributor_guide_matches_org_workflow(self): ): self.assertIn(expected, guide) + def test_code_of_conduct_has_org_contact_and_attribution(self): + conduct = self.read_text("CODE_OF_CONDUCT.md") + + for expected in ( + "Contributor Covenant", + "security@codeswhat.com", + "monitored for both security and conduct reports", + "## Enforcement", + ): + self.assertIn(expected, conduct) + def test_issue_forms_have_unique_ids_and_required_reproduction_fields(self): bug_report = self.read_text(".github/ISSUE_TEMPLATE/bug_report.yml") feature_request = self.read_text( diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..8d99e9f --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +**** (monitored for both security and conduct reports). +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +. Translations are available at +. diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index 8929480..f0bfd3b 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -48,6 +48,14 @@ repositories. Language-specific tests do not. planning details in it. - [ ] Add `CONTRIBUTING.md` when the repository accepts contributions. It must agree with `AGENTS.md` about commands, branches, reviews, tests, and commits. +- [ ] Rely on this repository's org-default community-health files + (CONTRIBUTING baseline, `SECURITY.md`, issue forms, PR template, + `CODE_OF_CONDUCT.md`). Add a local override only when the repository needs + repo-specific content, never as a reflex copy. +- [ ] Public product repositories: enable GitHub Discussions and state the + routing in the README's Community section (durable bugs and feature + requests go to Issues, open-ended Q&A and ideas to Discussions, chat to + Discord). Meta and infrastructure repositories keep Discussions off. - [ ] Add tool-specific contributor instructions only when the tool is used. For example, `CLAUDE.md` can supplement `AGENTS.md`, but it does not replace the required `AGENTS.md`. From dd74a99506bd364b882b8e064744f1f1e1a3d93d Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Thu, 20 Aug 2026 18:58:45 -0400 Subject: [PATCH 16/18] feat(workflows): add the shared star-chart refresh reusable workflow (#32) * feat(workflows): add the shared star-chart refresh reusable workflow Replaces both retired star-chart engines org-wide. The chart becomes a first-party SVG generated from GitHub's own stargazer timestamps and committed into the consuming repository, so it needs no secret and makes no request at render time. That property is the point. A live route that loses its credential serves a plausible placeholder at HTTP 200 forever with nothing reporting red, which is exactly how drydock's chart sat broken. A committed artifact fails visibly or not at all. The generator is embedded in the workflow rather than checked out from a second repository, so a caller's SHA pin covers every line of behaviour with nothing resolved at run time. Verified against live data before committing: byte-identical output to the reference implementation for drydock at 238 stars and 3 API calls, and a clean no-op exit on a repo with a single star. - feat(workflows): starchart-refresh.yml, egress-blocked to api.github.com and github.com, contents: write as its only elevated scope - test(workflows): contract test covering the embedded generator, env-var input handling, the self-contained SVG, and the conditional commit-back - ci(validation): run the new contract test in standards validation - docs(onboarding): document the caller shape and why the artifact is committed rather than served * test(workflows): syntax-check the embedded star-chart generator This workflow never runs in this repository, so a syntax error inside the heredoc would first surface in a consumer's scheduled job, days later and in someone else's lane. The test recovers the generator the way the shell will actually see it, stripping the run block's base indentation rather than reading the file as written, since a heredoc body that looks correct in YAML can still reach node malformed. Then node --check parses it. Verified with a negative control rather than assumed: injecting a syntax error into the generator fails the test, and reverting passes it. * fix(workflows): reject the inputs that would publish a wrong star chart All three from CodeRabbit on #32, and the max-pages one was a real bug of exactly the kind this workflow exists to prevent. max-pages: 0 made pages 0, which fetched nothing, which hit the "too few stars" clean exit. A repository with 238 stars would have reported a green no-op. A cap below the needed page count was worse than that: it drew a chart from the first N pages and published a partial history as a whole one behind a ::warning:: nobody reads. Both now fail loudly, and the cap must be a positive integer. branch had no runtime guard. Omitting a default only prevents omission, so a caller could still pass main and, on a repository whose ruleset let the push through, commit straight to the default branch. Rejected before checkout rather than at the push, where the error would be confusing. output-path was read through the environment, which stops script injection but not traversal. An absolute or ../ path reached writeFileSync outside the checkout, and the commit step then found nothing staged and reported success. Writes now use the resolved and validated path rather than the raw input, since a check that doesn't govern the write is decoration. Verified behaviourally, not by reading: each rejected input throws, the one-star exit still no-ops, nothing lands outside the workspace, and the happy path is still byte-identical to the reference output for drydock. --- .../tests/starchart_refresh_contract_test.py | 203 +++++++++++++++ .github/workflows/standards-validation.yml | 1 + .github/workflows/starchart-refresh.yml | 232 ++++++++++++++++++ REPOSITORY_ONBOARDING.md | 13 + 4 files changed, 449 insertions(+) create mode 100644 .github/tests/starchart_refresh_contract_test.py create mode 100644 .github/workflows/starchart-refresh.yml diff --git a/.github/tests/starchart_refresh_contract_test.py b/.github/tests/starchart_refresh_contract_test.py new file mode 100644 index 0000000..1454c19 --- /dev/null +++ b/.github/tests/starchart_refresh_contract_test.py @@ -0,0 +1,203 @@ +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/starchart-refresh.yml" + + +class StarchartRefreshContractTest(unittest.TestCase): + def test_reusable_workflow_shape_and_narrow_permissions(self): + workflow = self.read_workflow() + + for expected in ( + " workflow_call:\n", + " branch:\n", + " required: true\n", + " output-path:\n", + " default: docs/assets/star-history.svg\n", + " max-pages:\n", + " type: number\n", + "permissions: {}", + " runs-on: ubuntu-24.04", + " timeout-minutes: 10", + "uses: step-security/harden-runner@", + "egress-policy: block", + "api.github.com:443", + "github.com:443", + ): + self.assertIn(expected, workflow) + + # contents: write is the whole point of this workflow, but it must be + # the ONLY elevated scope. A second write scope here would be a + # commit-back job that can also move issues, releases, or packages. + job_scopes = re.findall(r"^ (\w[\w-]*): write", workflow, re.MULTILINE) + self.assertEqual(job_scopes, ["contents"]) + + def test_generator_is_embedded_rather_than_fetched_at_run_time(self): + """The caller pins this file by SHA. Anything resolved at run time + escapes that pin, so the generator lives inline and the only network + reads are GitHub's own API.""" + workflow = self.read_workflow() + + self.assertIn("node --input-type=module - <<'GENERATOR'", workflow) + self.assertIn("application/vnd.github.star+json", workflow) + self.assertIn("https://api.github.com/", workflow) + + # No second repository checkout, and no curl/wget/npm pulling code in. + self.assertEqual(workflow.count("actions/checkout@"), 1) + self.assertNotIn("repository: CodesWhat/.github", workflow) + for forbidden in ("curl ", "wget ", "npx ", "npm install", "pip install"): + self.assertNotIn(forbidden, workflow) + + def test_untrusted_input_is_read_from_the_environment(self): + """Caller-controlled values reach the script as env vars, never as + ${{ }} interpolated into a shell or JavaScript body.""" + workflow = self.read_workflow() + + for expected in ( + "TARGET_REPO: ${{ github.repository }}", + "OUTPUT_PATH: ${{ inputs.output-path }}", + "MAX_PAGES: ${{ inputs.max-pages }}", + "TARGET_BRANCH: ${{ inputs.branch }}", + "const repo = process.env.TARGET_REPO", + "const out = process.env.OUTPUT_PATH", + ): + self.assertIn(expected, workflow) + + generator = workflow.split("<<'GENERATOR'", 1)[1].split("GENERATOR", 1)[0] + self.assertNotIn("${{", generator) + + def test_chart_is_self_contained_with_no_external_references(self): + """A committed artifact that reaches out at render time would + reintroduce exactly the silent failure this replaced.""" + workflow = self.read_workflow() + + self.assertIn(" maxPages)", workflow) + self.assertNotIn("Math.min(Math.ceil(total / 100), maxPages)", workflow) + self.assertNotIn("::warning::capping", workflow) + + def test_too_few_stars_is_a_clean_exit_not_a_failure(self): + """A young repo having one star is a real state, not a broken build. + Reporting red there trains people to ignore the signal.""" + workflow = self.read_workflow() + + self.assertIn("if (stars.length < 2)", workflow) + self.assertIn("process.exit(0)", workflow) + self.assertNotIn("process.exit(1)", workflow) + self.assertNotIn("process.exit(2)", workflow) + + def test_embedded_generator_is_valid_javascript(self): + """This workflow never runs in this repository, so a syntax error in + the heredoc would first surface in a consumer's scheduled job. Parse + it here instead.""" + node = shutil.which("node") + if node is None: + self.skipTest("node is not available") + + source = self.read_generator() + with tempfile.TemporaryDirectory() as tmp: + script = Path(tmp) / "generator.mjs" + script.write_text(source) + result = subprocess.run( + [node, "--check", str(script)], + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_workflow_pins_actions_and_is_run_by_standards_validation(self): + workflow = self.read_workflow() + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + validation = (ROOT / ".github/workflows/standards-validation.yml").read_text() + self.assertIn("starchart_refresh_contract_test.py", validation) + + def read_workflow(self): + self.assertTrue(WORKFLOW.is_file(), f"missing workflow: {WORKFLOW}") + return WORKFLOW.read_text() + + def read_generator(self): + """Recover the generator exactly as the shell will see it: YAML strips + the run block's base indentation, so a heredoc body that only looks + right in the file can still reach node malformed.""" + workflow = self.read_workflow() + opener = "node --input-type=module - <<'GENERATOR'\n" + self.assertIn(opener, workflow) + + indent = " " * (len(workflow.split(opener)[0].rsplit("\n", 1)[-1])) + self.assertTrue(indent, "expected the run block to be indented") + + body = workflow.split(opener, 1)[1].split(f"\n{indent}GENERATOR", 1)[0] + return "\n".join( + line[len(indent):] if line.startswith(indent) else line + for line in body.split("\n") + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index 0ea5ad0..3966ebe 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -53,6 +53,7 @@ jobs: python3 .github/tests/greptile_config_contract_test.py python3 .github/tests/quality_report_contract_test.py python3 .github/tests/reusable_ci_contract_test.py + python3 .github/tests/starchart_refresh_contract_test.py - name: Lint Markdown run: | diff --git a/.github/workflows/starchart-refresh.yml b/.github/workflows/starchart-refresh.yml new file mode 100644 index 0000000..8517969 --- /dev/null +++ b/.github/workflows/starchart-refresh.yml @@ -0,0 +1,232 @@ +name: Star Chart Refresh + +# Regenerates a repository's star-history chart as a first-party SVG and +# commits it back. The chart is a committed artifact on purpose: it needs no +# secret and no request at render time, so it cannot fail silently the way a +# live route or a third-party embed can (standards/readme-shape.md). +# +# Callers declare their own triggers and pin this file by full commit SHA: +# +# on: +# schedule: [{cron: "17 6 * * 1"}] +# workflow_dispatch: +# permissions: {} +# jobs: +# starchart: +# permissions: +# contents: write +# uses: CodesWhat/.github/.github/workflows/starchart-refresh.yml@ +# with: +# branch: dev/v1.7 +# +# The generator is embedded rather than checked out from a second repository +# so that the caller's SHA pin covers every line of behaviour, with nothing +# resolved at run time. + +on: + workflow_call: + inputs: + branch: + description: Branch to read and commit the refreshed chart to. Never main under the strict release flow. + required: true + type: string + output-path: + description: Path the SVG is written to, relative to the repository root. + required: false + default: docs/assets/star-history.svg + type: string + max-pages: + description: Safety cap on stargazer pages fetched (100 stars per page). + required: false + default: 100 + type: number + +permissions: {} + +jobs: + refresh: + name: Star Chart Refresh + runs-on: ubuntu-24.04 + timeout-minutes: 10 + concurrency: + group: starchart-refresh-${{ github.repository }}-${{ inputs.branch }} + cancel-in-progress: false + permissions: + contents: write # Commit the regenerated SVG back to the caller's branch. + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + + - name: Reject a protected branch as the commit-back target + env: + TARGET_BRANCH: ${{ inputs.branch }} + run: | + set -euo pipefail + # Omitting a default only prevents omission. Under the strict + # release flow nothing commits back to a default branch, and a + # named rejection here beats a confusing push rejection at the end + # of the job, or none at all on a repo that forgot its ruleset. + case "${TARGET_BRANCH#refs/heads/}" in + main | master | HEAD | "") + echo "::error::refusing to commit the chart to '$TARGET_BRANCH'; pass an integration branch such as dev/v1.7" >&2 + exit 1 + ;; + esac + + - name: Check out the caller at the target branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.branch }} + # Kept, unlike everywhere else in this org: the whole job is a + # commit-back, so the pushing credential has to survive checkout. + persist-credentials: true # zizmor: ignore[artipacked] + + - name: Generate the star-history SVG + env: + # Read via the environment, never interpolated into the script body. + TARGET_REPO: ${{ github.repository }} + OUTPUT_PATH: ${{ inputs.output-path }} + MAX_PAGES: ${{ inputs.max-pages }} + GITHUB_API_TOKEN: ${{ github.token }} + run: | + node --input-type=module - <<'GENERATOR' + import { mkdirSync, writeFileSync } from 'node:fs' + import { dirname, isAbsolute, relative, resolve } from 'node:path' + + const repo = process.env.TARGET_REPO + const out = process.env.OUTPUT_PATH + const maxPages = Number(process.env.MAX_PAGES) + const auth = process.env.GITHUB_API_TOKEN + const workspace = process.env.GITHUB_WORKSPACE + + // Reading through the environment stops script injection but not + // path traversal. A write outside the checkout is worse than an + // error: the commit step finds nothing staged and reports success. + const target = resolve(workspace, out) + if (isAbsolute(out) || relative(workspace, target).startsWith('..')) { + throw new Error(`output-path must stay inside the repository: ${out}`) + } + if (!Number.isInteger(maxPages) || maxPages < 1) { + throw new Error(`max-pages must be a positive integer, got: ${process.env.MAX_PAGES}`) + } + + const api = async (path, accept) => { + const res = await fetch(`https://api.github.com/${path}`, { + headers: { + accept, + authorization: `Bearer ${auth}`, + 'user-agent': 'codeswhat-starchart', + 'x-github-api-version': '2022-11-28', + }, + }) + if (!res.ok) throw new Error(`GET ${path} -> ${res.status} ${res.statusText}`) + return res.json() + } + + const total = (await api(`repos/${repo}`, 'application/vnd.github+json')).stargazers_count + const pages = Math.ceil(total / 100) + // Truncating is a hard failure, not a warning. A chart drawn from + // the first N pages is a partial history rendered as a whole one, + // and a log line nobody reads is how that ships unnoticed. + if (pages > maxPages) { + throw new Error( + `${repo} has ${total} stars needing ${pages} pages but max-pages is ${maxPages}; ` + + 'raise the cap rather than publishing a truncated history', + ) + } + + const stars = [] + for (let p = 1; p <= pages; p++) { + const page = await api( + `repos/${repo}/stargazers?per_page=100&page=${p}`, + 'application/vnd.github.star+json', + ) + for (const s of page) if (s.starred_at) stars.push(new Date(s.starred_at).getTime()) + } + stars.sort((a, b) => a - b) + + // Too few points to plot is a real state for a young repo, not a + // failure. Leaving the previous SVG in place beats committing an + // empty chart or reporting red on nothing being wrong. + if (stars.length < 2) { + console.log(`::notice::${repo} has ${stars.length} star(s); leaving the chart untouched`) + process.exit(0) + } + + const W = 800, H = 400, P = { t: 30, r: 30, b: 45, l: 60 } + const [t0, t1] = [stars[0], stars.at(-1)] + const x = (t) => P.l + ((t - t0) / (t1 - t0 || 1)) * (W - P.l - P.r) + const y = (n) => H - P.b - (n / stars.length) * (H - P.t - P.b) + + const pts = stars.map((t, i) => `${x(t).toFixed(1)},${y(i + 1).toFixed(1)}`) + const line = `M ${x(t0).toFixed(1)},${y(0).toFixed(1)} L ${pts.join(' L ')}` + const area = `${line} L ${x(t1).toFixed(1)},${(H - P.b).toFixed(1)} L ${x(t0).toFixed(1)},${(H - P.b).toFixed(1)} Z` + + const fmt = (ms) => new Date(ms).toISOString().slice(0, 7) + const ticks = 4 + const xLabels = Array.from({ length: ticks + 1 }, (_, i) => { + const t = t0 + ((t1 - t0) * i) / ticks + // Anchor the end labels inward so they can't clip the viewBox edges. + const anchor = i === 0 ? 'start' : i === ticks ? 'end' : 'middle' + return `${fmt(t)}` + }).join('\n ') + const yLabels = Array.from({ length: ticks + 1 }, (_, i) => { + const n = Math.round((stars.length * i) / ticks) + return ` + ${n}` + }).join('\n ') + + // Write the path that was validated, not the raw input. In Actions + // the working directory is the workspace so they agree, but a + // check that doesn't govern the write is decoration. + mkdirSync(dirname(target), { recursive: true }) + writeFileSync(target, ` + Star history for ${repo} + + + ${repo} · ${stars.length} stars + + ${yLabels} + + + + + ${xLabels} + + + `) + console.log(`${repo}: ${stars.length} stars, ${pages} API call(s) -> ${out}`) + GENERATOR + + - name: Commit the chart only when it actually changed + env: + OUTPUT_PATH: ${{ inputs.output-path }} + TARGET_BRANCH: ${{ inputs.branch }} + run: | + set -euo pipefail + # --porcelain rather than `git diff`, which reports clean for a + # path that is new and therefore still untracked on first run. + if [ -z "$(git status --porcelain -- "$OUTPUT_PATH")" ]; then + echo "::notice::chart unchanged; nothing to commit" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -- "$OUTPUT_PATH" + git commit -m "chore(docs): refresh the star-history chart" + git push origin "HEAD:$TARGET_BRANCH" diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index f0bfd3b..896dda6 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -215,6 +215,19 @@ Add these only when the behavior exists: with a clear enforced or advisory threshold. - [ ] Translation synchronization only for a repository with a translation source of truth and configured provider credentials. +- [ ] Star-history chart refresh for a public repository whose README carries a + Star History section. Call this repository's `starchart-refresh.yml` at a + pinned full commit SHA from a thin caller on `schedule` plus + `workflow_dispatch`, granting the job `contents: write` and passing the + active integration branch as `branch`. It regenerates a first-party SVG from + GitHub's stargazer timestamps and commits it only when the chart actually + changed. The chart is a committed artifact rather than a live route or a + third-party embed on purpose: it needs no secret and makes no request at + render time, so a stale one is visible and a missing one is a visibly broken + image, where a route that loses its credential serves a plausible placeholder + at HTTP 200 indefinitely. Do not embed `star-history.com` or `warpchart.dev`; + both are retired organization-wide, and adopting the chart means removing + what it replaced in the same change. ### Qlty From 79801afde32380fa5797d6daedbffc65dfca2a92 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:28:55 -0400 Subject: [PATCH 17/18] Main-is-released check, and the codified star-chart shape (#34) * ci(standards): assert main points at a release tag Reusable workflow for the invariant behind "main is the released version, not the newest work": every commit on main is a tagged release, so an untagged main head is itself the alarm. Callers pin it by SHA and run it on a schedule plus push to main. It separates three states that all look like "not tagged" from the outside. A repository with zero tags cannot be evaluated at all and says so rather than reporting drift. A drifted main reports the newest reachable tag and how many commits it is behind. A prerelease on main is its own failure by default, since a release candidate on the default branch is the exact drift this exists to catch. Read-only: contents: read, egress blocked to github.com, and no credentials persisted through checkout. fetch-depth: 0 because tags only travel with full history and a shallow clone would fail for the wrong reason and read as real drift. * ci(starchart): render the codified chart shape in both themes Scott drew the target and it is now the renderer. The chart reads as native GitHub UI rather than as a third-party embed: a 900x460 card on GitHub's own border colour, sans for the words and mono for every number, a 2px accent line over a faint gradient, interior gridlines and a solid baseline. The accent is the repository's logo colour, passed as a new required input, and an accent that is not a colour now fails instead of drawing a chart with no line. Three behaviours the renderer decides rather than hard-codes, each because the naive version produced something wrong on a real repository. The y-axis searches step-and-tick-count pairs, since rounding the step alone put drydock's 239 stars on a 0-400 axis with the curve in the bottom 60% of the plot. The curve is a monotone cubic, since a cardinal spline overshoots on a curve this flat and an overshoot on a cumulative count draws a dip that never happened. X labels drop to day precision when month names collide, which is the actual condition rather than a guessed span threshold. Two files ship now, not one. GitHub's theme toggle does not reach a media query inside an -embedded SVG, so a self-theming file shows a white card to anyone reading GitHub dark with a light OS. It does drive a element in the README, so the pair is generated from one fetch and the markup chooses. They commit together or not at all: a with a fresh light chart and a stale dark one shows two different histories depending on who is looking, and nothing reports it. The documented trigger moves from a cron to the release cut. A committed artifact refreshed on a schedule mutates underneath a tag, which is what the main-is-released rule forbids. The renderer block is generated from ops render-chart.mjs by splice-into-workflow.mjs rather than hand-copied, and byte parity with that module was verified against live drydock data before this landed. Also fixes an assertion in the main-is-released test that sliced the whole if-block as the decisive expression and so could never pass. * docs(onboarding): add the main-is-released caller to the section 4 checklist --- .../tests/main_is_released_contract_test.py | 93 +++++ .../tests/starchart_refresh_contract_test.py | 64 ++- .github/workflows/main-is-released.yml | 92 +++++ .github/workflows/standards-validation.yml | 1 + .github/workflows/starchart-refresh.yml | 376 +++++++++++++++--- REPOSITORY_ONBOARDING.md | 77 +++- 6 files changed, 639 insertions(+), 64 deletions(-) create mode 100644 .github/tests/main_is_released_contract_test.py create mode 100644 .github/workflows/main-is-released.yml diff --git a/.github/tests/main_is_released_contract_test.py b/.github/tests/main_is_released_contract_test.py new file mode 100644 index 0000000..c9368b2 --- /dev/null +++ b/.github/tests/main_is_released_contract_test.py @@ -0,0 +1,93 @@ +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/main-is-released.yml" + + +class MainIsReleasedContractTest(unittest.TestCase): + def test_reusable_workflow_shape_and_read_only_permissions(self): + workflow = self.read_workflow() + + for expected in ( + " workflow_call:\n", + "permissions: {}", + " runs-on: ubuntu-24.04", + "uses: step-security/harden-runner@", + "egress-policy: block", + "github.com:443", + " contents: read", + ): + self.assertIn(expected, workflow) + + # This check only ever reads. Any write scope here would be a + # scheduled job with credentials to change the thing it audits. + self.assertNotIn(": write", workflow) + self.assertNotIn("persist-credentials: true", workflow) + + def test_the_invariant_is_an_exact_tag_match(self): + workflow = self.read_workflow() + + self.assertIn("git describe --exact-match --tags HEAD", workflow) + # --abbrev=0 alone answers "what tag is nearest", which is true of a + # drifted main too. It may only be used to report inside the failure + # branch, never in the condition that decides pass or fail — so the + # decisive slice is the condition line, not the whole if-block. + decisive = workflow.split("if ! tag=", 1)[1].split("\n", 1)[0] + self.assertIn("--exact-match", decisive) + self.assertNotIn("--abbrev=0", decisive) + + def test_no_tags_is_reported_as_unevaluable_not_as_drift(self): + """A repo with zero tags produces the same 'not tagged' as one that + drifted. Those need different answers, so the measurement proves it + could have worked before its result is trusted.""" + workflow = self.read_workflow() + + self.assertIn('if [ -z "$(git tag)" ]; then', workflow) + self.assertLess( + workflow.index('if [ -z "$(git tag)" ]'), + workflow.index("git describe --exact-match"), + ) + + def test_shallow_checkout_would_break_the_measurement(self): + """describe needs tags and history; a shallow clone fails for the + wrong reason and reads as real drift.""" + workflow = self.read_workflow() + self.assertIn("fetch-depth: 0", workflow) + + def test_a_prerelease_on_main_fails_by_default(self): + """A release candidate on the default branch is the exact drift this + exists to catch: drydock's main sat on v1.7.0-rc.2.""" + workflow = self.read_workflow() + + self.assertIn(" default: false\n", workflow) + self.assertIn('if [ "$ALLOW_PRERELEASE" != "true" ]', workflow) + self.assertIn("ALLOW_PRERELEASE: ${{ inputs.allow-prerelease }}", workflow) + + def test_failures_say_what_to_do_next(self): + workflow = self.read_workflow() + + for expected in ("::error::", "cut a release", "dev branch"): + self.assertIn(expected, workflow) + self.assertIn("set -euo pipefail", workflow) + + def test_workflow_pins_actions_and_is_run_by_standards_validation(self): + workflow = self.read_workflow() + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + validation = (ROOT / ".github/workflows/standards-validation.yml").read_text() + self.assertIn("main_is_released_contract_test.py", validation) + + def read_workflow(self): + self.assertTrue(WORKFLOW.is_file(), f"missing workflow: {WORKFLOW}") + return WORKFLOW.read_text() + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/tests/starchart_refresh_contract_test.py b/.github/tests/starchart_refresh_contract_test.py index 1454c19..8fb137f 100644 --- a/.github/tests/starchart_refresh_contract_test.py +++ b/.github/tests/starchart_refresh_contract_test.py @@ -20,6 +20,7 @@ def test_reusable_workflow_shape_and_narrow_permissions(self): " required: true\n", " output-path:\n", " default: docs/assets/star-history.svg\n", + " accent:\n", " max-pages:\n", " type: number\n", "permissions: {}", @@ -62,10 +63,12 @@ def test_untrusted_input_is_read_from_the_environment(self): for expected in ( "TARGET_REPO: ${{ github.repository }}", "OUTPUT_PATH: ${{ inputs.output-path }}", + "ACCENT: ${{ inputs.accent }}", "MAX_PAGES: ${{ inputs.max-pages }}", "TARGET_BRANCH: ${{ inputs.branch }}", "const repo = process.env.TARGET_REPO", "const out = process.env.OUTPUT_PATH", + "const accent = process.env.ACCENT", ): self.assertIn(expected, workflow) @@ -78,8 +81,13 @@ def test_chart_is_self_contained_with_no_external_references(self): workflow = self.read_workflow() self.assertIn("-embedded SVG, so a self-theming file shows a + # white card to anyone reading GitHub dark with a light OS. Two files + # and a README is the mechanism that does follow the toggle. + self.assertNotIn("prefers-color-scheme", workflow) for forbidden in (" that gained a fresh light chart and kept a stale dark + one shows two different histories depending on who is looking, and + nothing reports it as wrong.""" + workflow = self.read_workflow() + + self.assertIn("writeFileSync(target, light)", workflow) + self.assertIn("writeFileSync(darkTarget, dark)", workflow) + self.assertIn('DARK_PATH="${OUTPUT_PATH%.svg}-dark.svg"', workflow) + self.assertIn('git add -- "$OUTPUT_PATH" "$DARK_PATH"', workflow) + self.assertIn('git status --porcelain -- "$OUTPUT_PATH" "$DARK_PATH"', workflow) + + # Both derivations strip a .svg suffix, so the input has to have one. + self.assertIn("!out.endsWith('.svg')", workflow) + + def test_the_documented_trigger_is_the_release_cut_not_a_cron(self): + """A committed artifact refreshed on a schedule mutates underneath a + tag, which is what 'main is the released version' forbids.""" + workflow = self.read_workflow() + + example = workflow.split("# on:\n", 1)[1].split("# permissions:", 1)[0] + self.assertIn("release:", example) + self.assertIn("types: [published]", example) + self.assertIn('# accent: "#49bcfb"', workflow) + self.assertNotIn("cron", example) + self.assertNotIn("schedule:", example) + + def test_the_embedded_renderer_names_its_source(self): + """The same renderer exists here and in ops. Hand-copying is how they + drift, so the block is generated and says so.""" + workflow = self.read_workflow() + + self.assertIn("// BEGIN GENERATED FROM ops scripts/starchart/render-chart.mjs", workflow) + self.assertIn("// END GENERATED", workflow) + self.assertIn("splice-into-workflow.mjs", workflow) + self.assertLess( + workflow.index("// BEGIN GENERATED"), + workflow.index("// END GENERATED"), + ) + def test_too_few_stars_is_a_clean_exit_not_a_failure(self): """A young repo having one star is a real state, not a broken build. Reporting red there trains people to ignore the signal.""" diff --git a/.github/workflows/main-is-released.yml b/.github/workflows/main-is-released.yml new file mode 100644 index 0000000..c4bb47e --- /dev/null +++ b/.github/workflows/main-is-released.yml @@ -0,0 +1,92 @@ +name: Main Is Released + +# Asserts the one invariant behind "main is the released version, not the +# newest work": every commit on main is a tagged release, so an untagged main +# head is itself the alarm (REPOSITORY_ONBOARDING.md section 3). +# +# Callers declare their own triggers and pin this file by full commit SHA: +# +# on: +# schedule: [{cron: "23 7 * * *"}] +# push: +# branches: [main] +# workflow_dispatch: +# permissions: {} +# jobs: +# released: +# uses: CodesWhat/.github/.github/workflows/main-is-released.yml@ +# +# Continuously deployed repositories that never tag (a website, a meta repo) +# should not call this at all rather than calling it with a carve-out input. +# For them "main equals production" is enforced by the deploy, not by a tag. + +on: + workflow_call: + inputs: + allow-prerelease: + description: > + Accept a prerelease tag (-rc.N, -beta.N) as satisfying the invariant. + Defaults false: a release candidate on main is the exact drift this + check exists to catch. + required: false + default: false + type: boolean + +permissions: {} + +jobs: + released: + name: Main Is Released + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read # Read main and its tags. + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: > + github.com:443 + + - name: Check out main with tags + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + # Tags only travel with full history; a shallow clone would make + # describe fail for the wrong reason and read as real drift. + fetch-depth: 0 + persist-credentials: false + + - name: Assert main points at a release tag + env: + ALLOW_PRERELEASE: ${{ inputs.allow-prerelease }} + run: | + set -euo pipefail + + # Prove the measurement can work before trusting its result. A repo + # with no tags at all reports the same "not tagged" as one that + # drifted, and those need different answers. + if [ -z "$(git tag)" ]; then + echo "::error::no tags in this repository, so the invariant cannot be evaluated; a repository that never releases should not call this workflow" >&2 + exit 1 + fi + + if ! tag="$(git describe --exact-match --tags HEAD 2>/dev/null)"; then + latest="$(git describe --tags --abbrev=0 HEAD 2>/dev/null || echo '')" + ahead="$(git rev-list --count "${latest}..HEAD" 2>/dev/null || echo '?')" + echo "::error::main is not a tagged release. Newest reachable tag is ${latest}, and main is ${ahead} commit(s) past it. Either cut a release or move the unshipped work to a dev branch." >&2 + exit 1 + fi + + case "$tag" in + *-*) + if [ "$ALLOW_PRERELEASE" != "true" ]; then + echo "::error::main points at prerelease ${tag}. Prereleases belong on the dev branch; main carries what users actually run." >&2 + exit 1 + fi + echo "::warning::main points at prerelease ${tag}, accepted because allow-prerelease is set" ;; + esac + + echo "main is released at ${tag}" diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index 3966ebe..425ef4e 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -54,6 +54,7 @@ jobs: python3 .github/tests/quality_report_contract_test.py python3 .github/tests/reusable_ci_contract_test.py python3 .github/tests/starchart_refresh_contract_test.py + python3 .github/tests/main_is_released_contract_test.py - name: Lint Markdown run: | diff --git a/.github/workflows/starchart-refresh.yml b/.github/workflows/starchart-refresh.yml index 8517969..3caaf04 100644 --- a/.github/workflows/starchart-refresh.yml +++ b/.github/workflows/starchart-refresh.yml @@ -8,7 +8,8 @@ name: Star Chart Refresh # Callers declare their own triggers and pin this file by full commit SHA: # # on: -# schedule: [{cron: "17 6 * * 1"}] +# release: +# types: [published] # workflow_dispatch: # permissions: {} # jobs: @@ -18,10 +19,19 @@ name: Star Chart Refresh # uses: CodesWhat/.github/.github/workflows/starchart-refresh.yml@ # with: # branch: dev/v1.7 +# accent: "#49bcfb" +# +# The trigger is the release cut, not a cron. A committed artifact refreshed +# on a schedule mutates underneath a tag, which is exactly what "main is the +# released version" forbids. Regenerating at the cut means the chart in a +# released README is as of that release. # # The generator is embedded rather than checked out from a second repository # so that the caller's SHA pin covers every line of behaviour, with nothing -# resolved at run time. +# resolved at run time. It is written by ops +# scripts/starchart/splice-into-workflow.mjs from that repo's +# render-chart.mjs; edit there and re-splice rather than editing the block +# below by hand. on: workflow_call: @@ -31,10 +41,20 @@ on: required: true type: string output-path: - description: Path the SVG is written to, relative to the repository root. + description: > + Path the light SVG is written to, relative to the repository root. + Must end in .svg; the dark sibling is written alongside it with a + -dark suffix. required: false default: docs/assets/star-history.svg type: string + accent: + description: > + The repository's logo colour as #rrggbb. The dark-theme variant is + derived from it, so there is only one value to keep in sync. The + registry is in ops standards/readme-shape.md. + required: true + type: string max-pages: description: Safety cap on stargazer pages fetched (100 stars per page). required: false @@ -87,11 +107,12 @@ jobs: # commit-back, so the pushing credential has to survive checkout. persist-credentials: true # zizmor: ignore[artipacked] - - name: Generate the star-history SVG + - name: Generate the star-history SVGs env: # Read via the environment, never interpolated into the script body. TARGET_REPO: ${{ github.repository }} OUTPUT_PATH: ${{ inputs.output-path }} + ACCENT: ${{ inputs.accent }} MAX_PAGES: ${{ inputs.max-pages }} GITHUB_API_TOKEN: ${{ github.token }} run: | @@ -101,6 +122,7 @@ jobs: const repo = process.env.TARGET_REPO const out = process.env.OUTPUT_PATH + const accent = process.env.ACCENT const maxPages = Number(process.env.MAX_PAGES) const auth = process.env.GITHUB_API_TOKEN const workspace = process.env.GITHUB_WORKSPACE @@ -112,6 +134,17 @@ jobs: if (isAbsolute(out) || relative(workspace, target).startsWith('..')) { throw new Error(`output-path must stay inside the repository: ${out}`) } + // The dark sibling is derived from this path and the commit step + // derives it again in shell. Both derivations assume the suffix. + if (!out.endsWith('.svg')) { + throw new Error(`output-path must end in .svg: ${out}`) + } + // An accent that isn't a colour renders a chart with no line rather + // than failing, which is the silent-success shape this file exists + // to avoid. Reject it here instead. + if (!/^#[0-9a-fA-F]{6}$/.test(accent ?? '')) { + throw new Error(`accent must be a #rrggbb colour, got: ${accent}`) + } if (!Number.isInteger(maxPages) || maxPages < 1) { throw new Error(`max-pages must be a positive integer, got: ${process.env.MAX_PAGES}`) } @@ -159,58 +192,285 @@ jobs: process.exit(0) } - const W = 800, H = 400, P = { t: 30, r: 30, b: 45, l: 60 } - const [t0, t1] = [stars[0], stars.at(-1)] - const x = (t) => P.l + ((t - t0) / (t1 - t0 || 1)) * (W - P.l - P.r) - const y = (n) => H - P.b - (n / stars.length) * (H - P.t - P.b) - - const pts = stars.map((t, i) => `${x(t).toFixed(1)},${y(i + 1).toFixed(1)}`) - const line = `M ${x(t0).toFixed(1)},${y(0).toFixed(1)} L ${pts.join(' L ')}` - const area = `${line} L ${x(t1).toFixed(1)},${(H - P.b).toFixed(1)} L ${x(t0).toFixed(1)},${(H - P.b).toFixed(1)} Z` - - const fmt = (ms) => new Date(ms).toISOString().slice(0, 7) - const ticks = 4 - const xLabels = Array.from({ length: ticks + 1 }, (_, i) => { - const t = t0 + ((t1 - t0) * i) / ticks - // Anchor the end labels inward so they can't clip the viewBox edges. - const anchor = i === 0 ? 'start' : i === ticks ? 'end' : 'middle' - return `${fmt(t)}` - }).join('\n ') - const yLabels = Array.from({ length: ticks + 1 }, (_, i) => { - const n = Math.round((stars.length * i) / ticks) - return ` - ${n}` - }).join('\n ') + // BEGIN GENERATED FROM ops scripts/starchart/render-chart.mjs + // Shared renderer: star timestamps -> self-contained SVG. + // Split out from the fetch layer so the identical code can be exercised + // against fixture data without hitting the API. + // + // The visual shape is fixed (standards/readme-shape.md). The only thing a + // repository varies is `accent`, its logo's main colour. + + const W = 900 + const H = 460 + const PLOT = { l: 62, r: 860, top: 76, base: 408 } + const SAMPLE = 10 // px between resampled points along the curve + + const LIGHT = { + card: '#ffffff', + edge: '#d0d7de', + ink: '#1f2328', + muted: '#656d76', + grid: '#eaeef2', + axis: '#d0d7de', + ylabel: '#8c959f', + } + const DARK = { + card: '#0d1117', + edge: '#30363d', + ink: '#e6edf3', + muted: '#8b949e', + grid: '#21262d', + axis: '#30363d', + ylabel: '#6e7681', + } + + const SANS = "-apple-system,'Segoe UI',Helvetica,Arial,sans-serif" + const MONO = 'ui-monospace,SFMono-Regular,Menlo,monospace' + + const NICE = [1, 2, 2.5, 4, 5, 10] + // Pick a y-axis that reads like a human wrote it (50/100/150/200) AND wastes + // as little of the plot as possible. Rounding the step alone isn't enough: + // 239 stars over 4 ticks rounds to 100, which puts the curve in the bottom + // 60% of a chart whose whole job is showing the curve. So search step-and- + // tick-count pairs and keep the smallest ceiling that clears the total. + const niceAxis = (total) => { + const need = Math.max(total * 1.02, 1) + let best = null + for (let ticks = 3; ticks <= 6; ticks += 1) { + for (let mag = 1; mag <= 10 ** 7; mag *= 10) { + for (const n of NICE) { + const step = n * mag + if (!Number.isInteger(step)) continue + const max = step * ticks + if (max < need) continue + if (!best || max < best.max) best = { step, ticks, max } + break + } + } + } + return best + } + + const MONTHS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] + const esc = (s) => String(s).replace(/&/g, '&').replace(//g, '>') + const num = (n) => Number(n.toFixed(1)) + + // "Feb 10 - Aug 21, 2026" when the whole history sits in one year, and both + // years spelled out when it doesn't. The axis below carries month names only, + // so this line is the sole place the year appears in the common case. + const range = (a, b) => { + const [d0, d1] = [new Date(a), new Date(b)] + const [y0, y1] = [d0.getUTCFullYear(), d1.getUTCFullYear()] + const day = (d) => `${MONTHS[d.getUTCMonth()]} ${d.getUTCDate()}` + return y0 === y1 + ? `${day(d0)} – ${day(d1)}, ${y1}` + : `${day(d0)}, ${y0} – ${day(d1)}, ${y1}` + } + + const srgb = (c) => (c <= 0.03928 ? c / 12.92 : ((c + 0.055) / 1.055) ** 2.4) + const luminance = (hex) => { + const [r, g, b] = [1, 3, 5].map((i) => parseInt(hex.slice(i, i + 2), 16) / 255) + return 0.2126 * srgb(r) + 0.7152 * srgb(g) + 0.0722 * srgb(b) + } + const contrast = (a, b) => { + const [x, y] = [luminance(a), luminance(b)].sort((p, q) => q - p) + return (x + 0.05) / (y + 0.05) + } + + const toHsl = (hex) => { + const [r, g, b] = [1, 3, 5].map((i) => parseInt(hex.slice(i, i + 2), 16) / 255) + const mx = Math.max(r, g, b) + const mn = Math.min(r, g, b) + const l = (mx + mn) / 2 + if (mx === mn) return [0, 0, l] + const d = mx - mn + const s = l > 0.5 ? d / (2 - mx - mn) : d / (mx + mn) + const h = + mx === r ? (g - b) / d + (g < b ? 6 : 0) : mx === g ? (b - r) / d + 2 : (r - g) / d + 4 + return [h / 6, s, l] + } + const channel = (p, q, t0) => { + const t = (t0 + 1) % 1 + if (t < 1 / 6) return p + (q - p) * 6 * t + if (t < 1 / 2) return q + if (t < 2 / 3) return p + (q - p) * (2 / 3 - t) * 6 + return p + } + const toHex = (h, s, l) => { + let rgb = [l, l, l] + if (s) { + const q = l < 0.5 ? l * (1 + s) : l + s - l * s + const p = 2 * l - q + rgb = [channel(p, q, h + 1 / 3), channel(p, q, h), channel(p, q, h - 1 / 3)] + } + return `#${rgb.map((v) => Math.round(v * 255).toString(16).padStart(2, '0')).join('')}` + } + + // A logo colour picked against white can vanish on GitHub's dark surface: + // portwing's #7230d2 scores 2.8:1 there. Rather than make every repository + // record two colours and keep them in sync, derive the dark one by raising + // lightness at a fixed hue and saturation until it clears 6:1 — the band + // drydock's untouched #49bcfb already sits in. Lifting lightness rather than + // blending toward white is what keeps sockguard's orange an orange instead of + // fading it to salmon. + const DARK_TARGET = 6 + const forDark = (accent, surface) => { + const [h, s] = toHsl(accent) + let [, , l] = toHsl(accent) + let out = accent + while (contrast(out, surface) < DARK_TARGET && l < 0.92) { + l += 0.02 + out = toHex(h, s, l) + } + return out + } + + // Resample the step function onto an even x grid, then run a monotone cubic + // (Fritsch-Carlson) through it. A plain cardinal spline overshoots on a curve + // this flat, and an overshoot on a cumulative star count draws a dip that + // never happened. + const smoothPath = (points) => { + const n = points.length + const slope = [] + for (let i = 0; i < n - 1; i += 1) { + slope.push((points[i + 1][1] - points[i][1]) / (points[i + 1][0] - points[i][0])) + } + const tan = [slope[0]] + for (let i = 1; i < n - 1; i += 1) { + tan.push(slope[i - 1] * slope[i] <= 0 ? 0 : (slope[i - 1] + slope[i]) / 2) + } + tan.push(slope[n - 2]) + for (let i = 0; i < n - 1; i += 1) { + if (slope[i] === 0) { + tan[i] = 0 + tan[i + 1] = 0 + continue + } + const a = tan[i] / slope[i] + const b = tan[i + 1] / slope[i] + const s = a * a + b * b + if (s > 9) { + const t = 3 / Math.sqrt(s) + tan[i] = t * a * slope[i] + tan[i + 1] = t * b * slope[i] + } + } + + let d = `M ${num(points[0][0])},${num(points[0][1])}` + for (let i = 0; i < n - 1; i += 1) { + const [x0, y0] = points[i] + const [x1, y1] = points[i + 1] + const h = (x1 - x0) / 3 + d += ` C ${num(x0 + h)},${num(y0 + tan[i] * h)} ${num(x1 - h)},${num(y1 - tan[i + 1] * h)} ${num(x1)},${num(y1)}` + } + return d + } + + const draw = (repo, stars, accent, theme, id) => { + const c = theme === 'dark' ? DARK : LIGHT + const ink = theme === 'dark' ? forDark(accent, c.card) : accent + const total = stars.length + const [t0, t1] = [stars[0], stars.at(-1)] + const { step, ticks, max: yMax } = niceAxis(total) + + const plotW = PLOT.r - PLOT.l + const plotH = PLOT.base - PLOT.top + const x = (t) => PLOT.l + ((t - t0) / (t1 - t0 || 1)) * plotW + const y = (n) => PLOT.base - (n / yMax) * plotH + + // Star i lands at x(stars[i]); between stars the count is flat. Sample that + // step function on an even grid so the spline has uniform spans to work with. + const grid = [] + for (let px = PLOT.l; px < PLOT.r; px += SAMPLE) grid.push(px) + grid.push(PLOT.r) + let seen = 0 + const points = grid.map((px) => { + while (seen < total && x(stars[seen]) <= px) seen += 1 + return [px, y(seen)] + }) + + const linePath = smoothPath(points) + const areaPath = `${linePath} L ${PLOT.r},${PLOT.base} L ${PLOT.l},${PLOT.base} Z` + + const gridLines = Array.from({ length: ticks - 1 }, (_, i) => { + const gy = num(y(step * (i + 1))) + return `` + }).join('') + + const yLabels = Array.from({ length: ticks - 1 }, (_, i) => { + const n = step * (i + 1) + return `${n}` + }).join('') + + const xCount = 5 + const stops = Array.from({ length: xCount }, (_, i) => new Date(t0 + ((t1 - t0) * i) / (xCount - 1))) + // "Feb 26" reads as the 26th of February, and the subtitle already carries + // the years, so month names alone are the default. A history shorter than + // five months puts two stops in one month and prints "Jun Jun Jul Jul Aug", + // so fall back to day precision when the labels collide rather than guess a + // span threshold. Either way a year appears only where one changes. + const name = (d) => MONTHS[d.getUTCMonth()] + const byMonth = stops.map(name) + const withDay = new Set(byMonth).size < byMonth.length + let lastYear = null + const xLabels = stops + .map((d, i) => { + const year = d.getUTCFullYear() + const base = withDay ? `${name(d)} ${d.getUTCDate()}` : name(d) + const label = lastYear !== null && year !== lastYear ? `${base}, ${year}` : base + lastYear = year + const anchor = + i === 0 ? '' : i === xCount - 1 ? ' text-anchor="end"' : ' text-anchor="middle"' + return `${label}` + }) + .join('') + + const [endX, endY] = points.at(-1) + + return ` + Star history for ${esc(repo)} — ${total} stars + + + + + + + + ${esc(repo)} + ${range(t0, t1)} + ${total} + STARS + ${gridLines} + + + + + ${xLabels} + ${yLabels} + + ` + } + + // GitHub's theme toggle does not reach a media query inside an -embedded + // SVG, but it does drive a element in the README. So ship the pair + // and let the markup choose, rather than one file that guesses. + const renderChart = (repo, stars, accent) => ({ + light: draw(repo, stars, accent, 'light', 'sc-light'), + dark: draw(repo, stars, accent, 'dark', 'sc-dark'), + }) + + const darkAccent = (accent) => forDark(accent, DARK.card) + // END GENERATED // Write the path that was validated, not the raw input. In Actions // the working directory is the workspace so they agree, but a // check that doesn't govern the write is decoration. + const darkTarget = target.replace(/\.svg$/, '-dark.svg') + const { light, dark } = renderChart(repo, stars, accent) mkdirSync(dirname(target), { recursive: true }) - writeFileSync(target, ` - Star history for ${repo} - - - ${repo} · ${stars.length} stars - - ${yLabels} - - - - - ${xLabels} - - - `) - console.log(`${repo}: ${stars.length} stars, ${pages} API call(s) -> ${out}`) + writeFileSync(target, light) + writeFileSync(darkTarget, dark) + console.log(`${repo}: ${stars.length} stars, ${pages} API call(s) -> ${out} + dark`) GENERATOR - name: Commit the chart only when it actually changed @@ -219,14 +479,24 @@ jobs: TARGET_BRANCH: ${{ inputs.branch }} run: | set -euo pipefail + DARK_PATH="${OUTPUT_PATH%.svg}-dark.svg" # --porcelain rather than `git diff`, which reports clean for a # path that is new and therefore still untracked on first run. - if [ -z "$(git status --porcelain -- "$OUTPUT_PATH")" ]; then + if [ -z "$(git status --porcelain -- "$OUTPUT_PATH" "$DARK_PATH")" ]; then echo "::notice::chart unchanged; nothing to commit" exit 0 fi + # Both or neither. A README that gained a light chart and + # kept a stale dark one shows two different histories depending on + # who is looking, and neither is flagged as wrong. + for path in "$OUTPUT_PATH" "$DARK_PATH"; do + if [ ! -f "$path" ]; then + echo "::error::expected $path to exist after generation" >&2 + exit 1 + fi + done git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -- "$OUTPUT_PATH" + git add -- "$OUTPUT_PATH" "$DARK_PATH" git commit -m "chore(docs): refresh the star-history chart" git push origin "HEAD:$TARGET_BRANCH" diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index 896dda6..40809ad 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -142,6 +142,44 @@ checks stay non-required, is in the Qlty subsection of section 4. ## 3. Protect `main` +`main` is the released version, not the newest work. It always equals what +users currently have: the GA tag on a versioned product, production on a +continuously deployed site. It is never the GA tag plus merged work that has +not shipped. The next version lives on its own branch, `dev/vX.Y`, which is +the integration target for every feature pull request, and prereleases are +tagged there rather than on `main`. + +The reason is measurement accuracy rather than tidiness. OpenSSF Scorecard, +CodeQL default-branch analysis, Dependabot alerts, README badges, and every +published vulnerability report describe the default branch and nothing else. +Unshipped work on `main` makes all of them describe software no user runs. + +One invariant covers it, and it is mechanical rather than a judgement call: + +```sh +git describe --exact-match origin/main +``` + +Every commit on `main` is a tagged release, so an untagged `main` head is +itself the alarm. Check the tag rather than auditing what a promotion diff +contains. Call this repository's `main-is-released.yml` on a schedule to +enforce it; see section 4. + +`main` advances only through a promotion pull request from `dev/*` or +`maintenance/*` that is tagged on merge. There is no documentation, README, or +generated-asset exception: a second path into `main` is a second thing that +drifts, and an incorrect README is a defect in the released version like any +other. Fix it on a hotfix branch and cut a patch release. For that to stay +honest, a documentation-only release must skip artifact publication — +GoReleaser, signing, deb/rpm, Homebrew, npm — or maintainers will route around +the rule. Generated assets committed to the repository, such as a star-history +chart, regenerate at the release cut rather than on a schedule that commits to +a branch, so they cannot change underneath a tag. + +Adopting this on a repository whose `main` is already ahead of its newest tag +means either cutting a release or resetting `main` to the last released tag. +That is the maintainer's decision, because it changes what every user sees. + Create an active branch ruleset named `Main branch protection`, targeting only the default branch. Its baseline is: @@ -215,19 +253,38 @@ Add these only when the behavior exists: with a clear enforced or advisory threshold. - [ ] Translation synchronization only for a repository with a translation source of truth and configured provider credentials. +- [ ] Release-invariant enforcement for a repository that tags releases. Call + this repository's `main-is-released.yml` at a pinned full commit SHA from a + thin caller on `schedule` plus `push` to `main` plus `workflow_dispatch`. The + called workflow declares `contents: read` itself, and a reusable workflow can + only narrow what the caller grants, so the caller needs no `permissions` + block beyond the top-level `permissions: {}` — a job-level grant would only + widen the ceiling it runs under. It asserts the section 3 invariant: an + untagged `main` head fails, and so does a prerelease tag unless the caller + passes `allow-prerelease: true`. A repository that never tags, such as a + continuously deployed site or a meta repository, should not call this at all + rather than call it with a carve-out; for those, "`main` equals production" + is enforced by the deploy. - [ ] Star-history chart refresh for a public repository whose README carries a Star History section. Call this repository's `starchart-refresh.yml` at a - pinned full commit SHA from a thin caller on `schedule` plus + pinned full commit SHA from a thin caller on `release: [published]` plus `workflow_dispatch`, granting the job `contents: write` and passing the - active integration branch as `branch`. It regenerates a first-party SVG from - GitHub's stargazer timestamps and commits it only when the chart actually - changed. The chart is a committed artifact rather than a live route or a - third-party embed on purpose: it needs no secret and makes no request at - render time, so a stale one is visible and a missing one is a visibly broken - image, where a route that loses its credential serves a plausible placeholder - at HTTP 200 indefinitely. Do not embed `star-history.com` or `warpchart.dev`; - both are retired organization-wide, and adopting the chart means removing - what it replaced in the same change. + active integration branch as `branch` and the repository's logo colour as + `accent`. It regenerates a first-party SVG pair from GitHub's stargazer + timestamps and commits only when the chart actually changed. The trigger is + the release cut rather than a schedule because a committed artifact + refreshed on a cron mutates underneath a tag, which section 3 forbids. The + chart is a committed artifact rather than a live route or a third-party + embed on purpose: it needs no secret and makes no request at render time, so + a stale one is visible and a missing one is a visibly broken image, where a + route that loses its credential serves a plausible placeholder at HTTP 200 + indefinitely. Two files ship, `star-history.svg` and `star-history-dark.svg`, + because GitHub's theme toggle does not reach a media query inside an + ``-embedded SVG but does drive a `` element in the README, so + the markup chooses and the `` stays the fallback for anything that does + not understand ``. Do not embed `star-history.com` or + `warpchart.dev`; both are retired organization-wide, and adopting the chart + means removing what it replaced in the same change. ### Qlty From 58c1fd46ba9122abe426f0b0cfdd261c1653c55e Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:53:31 -0400 Subject: [PATCH 18/18] fix(workflows): close three shared-workflow defects (#36) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(workflows): close three shared-workflow defects starchart-refresh: the documented `release: [published]` trigger never fires. GitHub suppresses workflow runs for events caused by GITHUB_TOKEN, and every consuming repo publishes its release with exactly that — portwing via GoReleaser, drydock via `gh release create`. A caller wired from this file's own example lints clean, reads as correctly configured, and refreshes nothing forever. That's the silent-success shape the committed-SVG rework existed to remove, reintroduced by the instructions for it. Example is now a workflow_dispatch the release cut fires, with the suppression and its two documented exceptions written down so the next person doesn't rederive the broken version. Found by the sockguard lane after three repos had been told to adopt it. main-is-released: an exact tag match alone was never the invariant. Any tag satisfied it, so one named `snapshot` or `latest` parked on a drifted main read as a pass. Now requires a release-shaped version. Prerelease detection moved off `case *-*`, which called `my-tag` a prerelease and would have accepted it under allow-prerelease. main-is-released: a promotion merges before its tag is pushed, so a run in that window reported drift that resolved itself seconds later. Three attempts with a tag refetch between them. It can't mask real drift — an untagged main is still untagged on the last attempt — and a failed refetch warns rather than passing. Verified by extracting the decision block and running it against real repositories: v1.7.4 and 1.7.4 pass, snapshot/latest/my-tag fail as malformed, v1.7.0-rc.2 fails as prerelease and passes under allow-prerelease, and allow-prerelease does not reopen the any-tag hole. 87 contract tests green. * fix(workflows): correct two overstated claims CodeRabbit caught The refetch warning said the verdict uses the refs from checkout. It might not: attempt 1 can succeed and attempt 2 fail, and a failed fetch can leave some refs updated. Now says the refs currently available on the runner, which is what's actually true. 'The two documented exceptions to the suppression' was an overclaim. pull_request with opened/synchronize/reopened is a third — it creates a run in an approval-required state rather than being suppressed. workflow_dispatch and repository_dispatch are the two that fire UNATTENDED, which is the property a release cut actually needs, so the comment now says that instead. --- .../tests/main_is_released_contract_test.py | 34 ++++++++++++- .../tests/starchart_refresh_contract_test.py | 39 ++++++++++++--- .github/workflows/main-is-released.yml | 48 +++++++++++++++---- .github/workflows/starchart-refresh.yml | 43 ++++++++++++++--- 4 files changed, 140 insertions(+), 24 deletions(-) diff --git a/.github/tests/main_is_released_contract_test.py b/.github/tests/main_is_released_contract_test.py index c9368b2..3a6fbc5 100644 --- a/.github/tests/main_is_released_contract_test.py +++ b/.github/tests/main_is_released_contract_test.py @@ -35,7 +35,7 @@ def test_the_invariant_is_an_exact_tag_match(self): # drifted main too. It may only be used to report inside the failure # branch, never in the condition that decides pass or fail — so the # decisive slice is the condition line, not the whole if-block. - decisive = workflow.split("if ! tag=", 1)[1].split("\n", 1)[0] + decisive = workflow.split('if tag="$(', 1)[1].split("\n", 1)[0] self.assertIn("--exact-match", decisive) self.assertNotIn("--abbrev=0", decisive) @@ -57,6 +57,38 @@ def test_shallow_checkout_would_break_the_measurement(self): workflow = self.read_workflow() self.assertIn("fetch-depth: 0", workflow) + def test_an_exact_match_alone_is_not_the_invariant(self): + """Any tag satisfies --exact-match, including a moving or descriptive + one. A tag literally named `snapshot` parked on a drifted main reads + as a clean pass, which is the exact failure this workflow exists to + catch. Found by the sockguard lane, 2026-08-21.""" + workflow = self.read_workflow() + + self.assertIn("[0-9]+\\.[0-9]+\\.[0-9]+", workflow) + self.assertIn("not a release version tag", workflow) + + # Prerelease detection keys on the hyphen AFTER the version, not any + # hyphen anywhere — the old `case $tag in *-*)` called `my-tag` a + # prerelease and would have accepted it under allow-prerelease. + self.assertNotIn('case "$tag" in', workflow) + self.assertIn("^v?[0-9]+\\.[0-9]+\\.[0-9]+-", workflow) + + def test_the_merge_to_tag_window_is_retried_not_reported_as_drift(self): + """A promotion merges before its tag is pushed. A run landing in that + window sees an untagged main and reports drift that resolves itself + seconds later, which trains people to ignore the one check whose job + is being noticed.""" + workflow = self.read_workflow() + + self.assertIn("for attempt in 1 2 3", workflow) + self.assertIn("git fetch --tags --force", workflow) + self.assertIn("sleep 20", workflow) + + # The retry must not become a way to pass. A failed refetch is warned + # about and the loop still decides on the refs it has. + self.assertIn("::warning::could not refetch tags", workflow) + self.assertNotIn("exit 0", workflow) + def test_a_prerelease_on_main_fails_by_default(self): """A release candidate on the default branch is the exact drift this exists to catch: drydock's main sat on v1.7.0-rc.2.""" diff --git a/.github/tests/starchart_refresh_contract_test.py b/.github/tests/starchart_refresh_contract_test.py index 8fb137f..8962cf7 100644 --- a/.github/tests/starchart_refresh_contract_test.py +++ b/.github/tests/starchart_refresh_contract_test.py @@ -173,17 +173,42 @@ def test_both_themes_are_written_and_committed_together(self): # Both derivations strip a .svg suffix, so the input has to have one. self.assertIn("!out.endsWith('.svg')", workflow) - def test_the_documented_trigger_is_the_release_cut_not_a_cron(self): - """A committed artifact refreshed on a schedule mutates underneath a - tag, which is what 'main is the released version' forbids.""" + def test_the_documented_trigger_is_a_dispatch_not_a_cron_or_a_release(self): + """Two ways to get this wrong, and the second one looks right. + + A cron mutates a committed artifact underneath a tag, which 'main is + the released version' forbids. And `release: [published]` never fires + at all: GitHub suppresses workflow runs for events caused by + GITHUB_TOKEN, which is what every repo here publishes releases with, + so a caller wired that way is green everywhere and refreshes nothing. + This file told three repos to do exactly that on 2026-08-21 before the + sockguard lane caught it, so the example is pinned by a test now.""" workflow = self.read_workflow() example = workflow.split("# on:\n", 1)[1].split("# permissions:", 1)[0] - self.assertIn("release:", example) - self.assertIn("types: [published]", example) + self.assertIn("workflow_dispatch:", example) self.assertIn('# accent: "#49bcfb"', workflow) - self.assertNotIn("cron", example) - self.assertNotIn("schedule:", example) + for dead in ("release:", "types: [published]", "cron", "schedule:"): + self.assertNotIn(dead, example) + + def test_the_suppression_trap_is_documented_not_just_avoided(self): + """Removing the bad example only stops it being copied from here. The + reason has to travel with it, or the next person reaches for the + release trigger from first principles and it fails the same silent + way.""" + workflow = self.read_workflow() + + for expected in ( + "GITHUB_TOKEN", + "gh workflow run", + "workflow_dispatch` and", + "repository_dispatch", + ): + self.assertIn(expected, workflow) + + # The failure mode named, so it reads as a trap rather than a + # preference: wired that way it lints clean and never runs. + self.assertIn("refreshes nothing", workflow) def test_the_embedded_renderer_names_its_source(self): """The same renderer exists here and in ops. Hand-copying is how they diff --git a/.github/workflows/main-is-released.yml b/.github/workflows/main-is-released.yml index c4bb47e..361b473 100644 --- a/.github/workflows/main-is-released.yml +++ b/.github/workflows/main-is-released.yml @@ -73,20 +73,50 @@ jobs: exit 1 fi - if ! tag="$(git describe --exact-match --tags HEAD 2>/dev/null)"; then + # A promotion merges before its tag is pushed, so a run landing in + # that window sees an untagged main and reports drift that resolves + # itself seconds later. Refetch and retry before believing it. This + # cannot mask real drift: a genuinely untagged main is still + # untagged on the last attempt. + tag="" + for attempt in 1 2 3; do + if tag="$(git describe --exact-match --tags HEAD 2>/dev/null)"; then + break + fi + tag="" + [ "$attempt" -eq 3 ] && break + echo "main is untagged on attempt ${attempt}; refetching tags in case a promotion is mid-cut" + sleep 20 + # A failed refetch is reported, never swallowed into a pass: the + # loop still decides on whatever refs we actually have. + git fetch --tags --force --quiet origin \ + || echo "::warning::could not refetch tags on attempt ${attempt}; the verdict below uses the refs currently available on the runner" + done + + if [ -z "$tag" ]; then latest="$(git describe --tags --abbrev=0 HEAD 2>/dev/null || echo '')" ahead="$(git rev-list --count "${latest}..HEAD" 2>/dev/null || echo '?')" echo "::error::main is not a tagged release. Newest reachable tag is ${latest}, and main is ${ahead} commit(s) past it. Either cut a release or move the unshipped work to a dev branch." >&2 exit 1 fi - case "$tag" in - *-*) - if [ "$ALLOW_PRERELEASE" != "true" ]; then - echo "::error::main points at prerelease ${tag}. Prereleases belong on the dev branch; main carries what users actually run." >&2 - exit 1 - fi - echo "::warning::main points at prerelease ${tag}, accepted because allow-prerelease is set" ;; - esac + # An exact match alone is not the invariant. Any tag satisfies it, + # including one literally named "snapshot" or "latest" parked on a + # drifted main — which reads as a pass and is the failure this + # workflow exists to catch. Require a release-shaped tag. + if ! printf '%s' "$tag" | grep -Eq '^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$'; then + echo "::error::main points at ${tag}, which is not a release version tag (expected vMAJOR.MINOR.PATCH). A moving or descriptive tag satisfies an exact-match check while main carries unshipped work." >&2 + exit 1 + fi + + # Prerelease is the part AFTER the version, so match on that rather + # than on any hyphen anywhere in the tag. + if printf '%s' "$tag" | grep -Eq '^v?[0-9]+\.[0-9]+\.[0-9]+-'; then + if [ "$ALLOW_PRERELEASE" != "true" ]; then + echo "::error::main points at prerelease ${tag}. Prereleases belong on the dev branch; main carries what users actually run." >&2 + exit 1 + fi + echo "::warning::main points at prerelease ${tag}, accepted because allow-prerelease is set" + fi echo "main is released at ${tag}" diff --git a/.github/workflows/starchart-refresh.yml b/.github/workflows/starchart-refresh.yml index 3caaf04..e593394 100644 --- a/.github/workflows/starchart-refresh.yml +++ b/.github/workflows/starchart-refresh.yml @@ -8,8 +8,6 @@ name: Star Chart Refresh # Callers declare their own triggers and pin this file by full commit SHA: # # on: -# release: -# types: [published] # workflow_dispatch: # permissions: {} # jobs: @@ -18,13 +16,44 @@ name: Star Chart Refresh # contents: write # uses: CodesWhat/.github/.github/workflows/starchart-refresh.yml@ # with: -# branch: dev/v1.7 +# branch: ${{ github.ref_name }} # accent: "#49bcfb" # -# The trigger is the release cut, not a cron. A committed artifact refreshed -# on a schedule mutates underneath a tag, which is exactly what "main is the -# released version" forbids. Regenerating at the cut means the chart in a -# released README is as of that release. +# The refresh belongs to the release cut, not to a cron. A committed artifact +# refreshed on a schedule mutates underneath a tag, which is exactly what +# "main is the released version" forbids. +# +# DO NOT trigger it with `release: [published]`. That looks right and never +# runs. GitHub suppresses workflow runs for events caused by GITHUB_TOKEN, and +# every repo here publishes its release with that credential — GoReleaser with +# `GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}` (portwing release.yml) or +# `gh release create` with `GH_TOKEN: ${{ github.token }}` (drydock +# release-cut.yml). A caller wired that way reads as correctly configured, is +# green in every lint, and refreshes nothing, forever. Found 2026-08-21 by the +# sockguard lane after this file had already told three repos to do it. +# +# The release-cut workflow dispatches this one instead: +# +# - name: Dispatch starchart refresh +# env: +# GH_TOKEN: ${{ github.token }} +# run: gh workflow run starchart.yml --ref "$BRANCH" +# +# That works with no new credential because `workflow_dispatch` and +# `repository_dispatch` are the two dispatch events that always create a run +# even when GITHUB_TOKEN caused them. They are not the only exceptions — +# `pull_request` with opened/synchronize/reopened creates a run in an +# approval-required state rather than being suppressed outright — but they +# are the two that fire unattended, which is what a release cut needs. +# The dispatch step must fail loudly rather than `|| true`: by the time it +# runs the release is already published, so a swallowed error is the same +# silent-success shape this whole workflow exists to remove. +# +# Prefer dispatching BEFORE the tag is cut where the flow allows it, so the +# released README ships the chart it claims to. Dispatching after publish is +# an accepted tradeoff — the chart then lands on the dev branch and main's +# copy is one cut stale — because the requirement is that regeneration is +# tied to the cut rather than to wall-clock time, and that holds either way. # # The generator is embedded rather than checked out from a second repository # so that the caller's SHA pin covers every line of behaviour, with nothing