From 71ba4ae796292423fa330a9baeaf6352c0b3fb81 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:46:26 -0400 Subject: [PATCH 01/14] docs(standards): add organization health defaults Adds organization-wide community health defaults, validation, ownership, contribution guidance, security policy, and hardened workflow checks. --- .github/CODEOWNERS | 1 + .github/ISSUE_TEMPLATE/bug_report.yml | 77 ++++++ .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.yml | 42 ++++ .github/ISSUE_TEMPLATE/question.yml | 36 +++ .github/PULL_REQUEST_TEMPLATE.md | 35 +++ .github/markdownlint-profile.yaml | 3 + .../tests/community_health_contract_test.py | 223 ++++++++++++++++++ .github/workflows/standards-validation.yml | 97 ++++++++ .github/workflows/update-stats.yml | 9 +- CONTRIBUTING.md | 56 +++++ SECURITY.md | 49 ++++ 12 files changed, 631 insertions(+), 2 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/ISSUE_TEMPLATE/question.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/markdownlint-profile.yaml create mode 100644 .github/tests/community_health_contract_test.py create mode 100644 .github/workflows/standards-validation.yml create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..787cb14 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @scttbnsn @ALARGECOMPANY @biggest-littlest diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..0782aa8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,77 @@ +name: Bug report +description: Report a reproducible problem +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for reporting a bug. Do not include secrets or private data, and redact logs and configuration. Email security@codeswhat.com instead of filing a public issue for a suspected vulnerability. + + - type: input + id: version + attributes: + label: Version or commit + description: Provide the release version, image digest, or commit SHA. + placeholder: "e.g. 1.2.3 or abc1234" + validations: + required: true + + - type: textarea + id: description + attributes: + label: What happened? + description: Describe the observed behavior and its impact. + validations: + required: true + + - type: textarea + id: expected + attributes: + label: What did you expect? + description: Describe what should have happened instead. + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: Provide the smallest complete reproduction you can. + placeholder: | + 1. Configure ... + 2. Run ... + 3. Observe ... + validations: + required: true + + - type: textarea + id: environment + attributes: + label: Environment + description: Include relevant OS, architecture, runtime, and install method. + placeholder: | + - OS and architecture: + - Runtime or toolchain: + - Install method: + validations: + required: true + + - type: textarea + id: configuration + attributes: + label: Redacted configuration + description: Include only configuration needed to reproduce the problem. + render: yaml + + - type: textarea + id: logs + attributes: + label: Redacted logs + description: Remove credentials, private paths, hostnames, and user data. + render: text + + - type: textarea + id: additional + attributes: + label: Additional context + description: Add screenshots, links, or other useful context. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..632633e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Report a security vulnerability + url: mailto:security@codeswhat.com + about: Report suspected vulnerabilities privately, not in a public issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..ff53175 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,42 @@ +name: Feature request +description: Propose a focused improvement +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + Search existing issues and pull requests before proposing a feature. Describe the need before the implementation. + + - type: textarea + id: problem + attributes: + label: Problem + description: What use case or limitation should this change address? + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed change + description: Describe the behavior you want and who benefits from it. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Describe current workarounds or other approaches you considered. + + - type: textarea + id: compatibility + attributes: + label: Compatibility and security + description: Note migration, breaking-change, privacy, or security effects. + + - type: textarea + id: additional + attributes: + label: Additional context + description: Add examples, screenshots, or links that clarify the request. diff --git a/.github/ISSUE_TEMPLATE/question.yml b/.github/ISSUE_TEMPLATE/question.yml new file mode 100644 index 0000000..2d697bf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question.yml @@ -0,0 +1,36 @@ +name: Question or help +description: Ask about setup, usage, or project behavior +labels: ["question"] +body: + - type: markdown + attributes: + value: | + Check the README, documentation, and existing issues first. Email security@codeswhat.com instead of asking publicly about a suspected vulnerability. + + - type: textarea + id: question + attributes: + label: Question + description: What are you trying to understand or accomplish? + validations: + required: true + + - type: textarea + id: attempted + attributes: + label: What have you tried? + description: Include the relevant documentation and commands you used. + validations: + required: true + + - type: input + id: version + attributes: + label: Version or commit + description: Provide the release version or commit SHA when relevant. + + - type: textarea + id: context + attributes: + label: Additional context + description: Add redacted configuration, logs, screenshots, or links. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..96f9712 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,35 @@ +# Pull request + +## Summary + + + +Related issue: + +## Changes + + + +## Verification + + + +## Security and compatibility + + + +## Checklist + +- [ ] This pull request targets the repository's documented target branch. +- [ ] The change is focused and follows the repository's contributor + instructions. +- [ ] Tests cover non-trivial behavior, or the verification section explains + why they do not apply. +- [ ] Relevant documentation and changelog entries are updated, or they do not + apply. +- [ ] Required local checks pass. +- [ ] The change contains no secrets, credentials, or private data. +- [ ] All required checks and reviews are complete before merge. diff --git a/.github/markdownlint-profile.yaml b/.github/markdownlint-profile.yaml new file mode 100644 index 0000000..b6a8dcf --- /dev/null +++ b/.github/markdownlint-profile.yaml @@ -0,0 +1,3 @@ +config: + MD033: false + MD041: false diff --git a/.github/tests/community_health_contract_test.py b/.github/tests/community_health_contract_test.py new file mode 100644 index 0000000..9439282 --- /dev/null +++ b/.github/tests/community_health_contract_test.py @@ -0,0 +1,223 @@ +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class CommunityHealthContractTest(unittest.TestCase): + def test_required_default_files_exist(self): + required = ( + "SECURITY.md", + "CONTRIBUTING.md", + ".github/CODEOWNERS", + ".github/markdownlint-profile.yaml", + ".github/ISSUE_TEMPLATE/bug_report.yml", + ".github/ISSUE_TEMPLATE/feature_request.yml", + ".github/ISSUE_TEMPLATE/question.yml", + ".github/ISSUE_TEMPLATE/config.yml", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/workflows/standards-validation.yml", + ) + + missing = [path for path in required if not (ROOT / path).is_file()] + + self.assertEqual([], missing, f"missing community health files: {missing}") + + def test_codeowners_has_org_catch_all(self): + codeowners = self.read_text(".github/CODEOWNERS") + + self.assertEqual( + "* @scttbnsn @ALARGECOMPANY @biggest-littlest\n", + codeowners, + ) + + def test_security_policy_has_org_baseline(self): + policy = self.read_text("SECURITY.md") + + for expected in ( + "## Supported versions", + "## Reporting a vulnerability", + "## Security scope", + "Do not open a public GitHub issue", + "security@codeswhat.com", + "private vulnerability reporting", + "48 hours", + "7 days", + ): + self.assertIn(expected, policy) + + def test_contributor_guide_matches_org_workflow(self): + guide = self.read_text("CONTRIBUTING.md") + + for expected in ( + "AGENTS.md", + "active development or integration branch", + "Conventional Commits", + "feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert", + "without emoji", + "tests", + "documentation", + "required checks and reviews", + "SECURITY.md", + ): + self.assertIn(expected, guide) + + def test_issue_forms_have_unique_ids_and_required_reproduction_fields(self): + bug_report = self.read_text(".github/ISSUE_TEMPLATE/bug_report.yml") + feature_request = self.read_text( + ".github/ISSUE_TEMPLATE/feature_request.yml" + ) + question = self.read_text(".github/ISSUE_TEMPLATE/question.yml") + + self.assertIn('labels: ["bug"]', bug_report) + self.assertIn('labels: ["enhancement"]', feature_request) + self.assertIn('labels: ["question"]', question) + self.assert_form_ids_are_unique(bug_report) + self.assert_form_ids_are_unique(feature_request) + self.assert_form_ids_are_unique(question) + + required_bug_fields = {"version", "description", "expected", "reproduce"} + actual_required = self.required_form_ids(bug_report) + self.assertTrue(required_bug_fields <= actual_required) + + self.assertIn("security@codeswhat.com", bug_report) + self.assertIn("redact", bug_report.lower()) + + def test_issue_template_config_routes_security_reports_privately(self): + config = self.read_text(".github/ISSUE_TEMPLATE/config.yml") + + self.assertIn("blank_issues_enabled: false", config) + self.assertIn("url: mailto:security@codeswhat.com", config) + self.assertIn("privately", config.lower()) + + def test_pull_request_template_requires_verification_and_safe_content(self): + template = self.read_text(".github/PULL_REQUEST_TEMPLATE.md") + + for expected in ( + "## Summary", + "## Changes", + "## Verification", + "## Security and compatibility", + "## Checklist", + "target branch", + "tests", + "documentation", + "required checks and reviews", + "secrets, credentials, or private data", + ): + self.assertIn(expected, template) + + def test_standards_workflow_has_unconditional_stable_jobs(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + + self.assertEqual(2, workflow.count("name: Standards Validation")) + self.assertIn(" name: CodeQL\n", workflow) + self.assertIn(" pull_request:\n", workflow) + self.assertIn(" - main\n", workflow) + self.assertIn(" - dev/repository-standards\n", workflow) + self.assertNotIn("\n push:", workflow) + self.assertIn("permissions: {}", workflow) + self.assertIn("runs-on: ubuntu-24.04", workflow) + self.assertNotIn("matrix:", workflow) + self.assertNotIn("continue-on-error:", workflow) + self.assertNotIn("\n if:", workflow) + + validation_job = workflow.split(" validation:\n", 1)[1].split( + "\n codeql:\n", 1 + )[0] + self.assertIn(" actions: read", validation_job) + self.assertIn(" contents: read", validation_job) + self.assertNotIn("write", validation_job) + + codeql_job = workflow.split("\n codeql:\n", 1)[1] + self.assertIn(" actions: read", codeql_job) + self.assertIn(" contents: read", codeql_job) + self.assertIn(" security-events: write", codeql_job) + + jobs = re.findall( + r"^ ([a-z][a-z0-9-]+):$", + workflow.split("\njobs:\n", 1)[1], + re.MULTILINE, + ) + self.assertEqual(["validation", "codeql"], jobs) + + def test_standards_workflow_pins_actions_and_runs_every_contract(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + for expected in ( + "persist-credentials: false", + "community_health_contract_test.py", + "python3 -m compileall -q .", + "YAML.parse_file", + "json.load", + "markdownlint-cli2@0.23.2", + "markdownlint-profile.yaml", + "actionlint", + "zizmorcore/zizmor-action@", + "github/codeql-action/init@", + "languages: python", + "github/codeql-action/analyze@", + ): + self.assertIn(expected, workflow) + + def test_zizmor_action_owns_its_supported_cli_version(self): + workflow = self.read_text(".github/workflows/standards-validation.yml") + zizmor_step = workflow.split( + " - name: Validate workflows with zizmor\n", 1 + )[1].split("\n codeql:\n", 1)[0] + + self.assertNotIn("\n version:", zizmor_step) + + def test_linter_configs_do_not_suppress_workflow_findings(self): + markdown_config = self.read_text(".github/markdownlint-profile.yaml") + + self.assertIn("MD033: false", markdown_config) + self.assertIn("MD041: false", markdown_config) + self.assertNotIn("ignores:", markdown_config) + self.assertEqual(2, markdown_config.count(": false")) + self.assertFalse((ROOT / ".github/zizmor.yml").exists()) + + def test_profile_workflow_pins_actions_and_pushes_without_persisted_credentials(self): + workflow = self.read_text(".github/workflows/update-stats.yml") + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertEqual(2, len(actions)) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + checkout_step = workflow.split(" - name: Checkout\n", 1)[1].split( + "\n - name:", 1 + )[0] + push_step = workflow.split(" - name: Commit and push\n", 1)[1] + + self.assertIn("persist-credentials: false", checkout_step) + self.assertIn("GH_TOKEN: ${{ github.token }}", push_step) + self.assertIn("gh auth setup-git", push_step) + + def read_text(self, relative_path): + path = ROOT / relative_path + self.assertTrue(path.is_file(), f"missing community health file: {relative_path}") + return path.read_text() + + def assert_form_ids_are_unique(self, form): + ids = re.findall(r"^ id: ([a-z][a-z0-9_-]*)$", form, re.MULTILINE) + self.assertEqual(len(ids), len(set(ids))) + + def required_form_ids(self, form): + required = set() + for field in form.split("\n - type:"): + field_id = re.search(r"^ id: ([a-z][a-z0-9_-]*)$", field, re.MULTILINE) + if field_id and " required: true" in field: + required.add(field_id.group(1)) + return required + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml new file mode 100644 index 0000000..edfbdfd --- /dev/null +++ b/.github/workflows/standards-validation.yml @@ -0,0 +1,97 @@ +name: Standards Validation + +on: + pull_request: + branches: + - main + - dev/repository-standards + +permissions: {} + +jobs: + validation: + name: Standards Validation + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 24 + + - name: Validate community health contract + run: python3 .github/tests/community_health_contract_test.py + + - name: Compile Python + run: python3 -m compileall -q . + + - name: Parse YAML and JSON + run: | + set -euo pipefail + ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' + python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' + + - name: Lint Markdown + run: | + set -euo pipefail + npx --yes markdownlint-cli2@0.23.2 "**/*.md" "#profile/README.md" + npx --yes markdownlint-cli2@0.23.2 "profile/README.md" --config .github/markdownlint-profile.yaml + + - name: Validate workflows with actionlint + run: | + set -euo pipefail + bash <(curl -fsSL https://raw.githubusercontent.com/rhysd/actionlint/914e7df21a07ef503a81201c76d2b11c789d3fca/scripts/download-actionlint.bash) 1.7.12 "$RUNNER_TEMP" + "$RUNNER_TEMP/actionlint" -color + + - name: Validate workflows with zizmor + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + with: + advanced-security: false + annotations: true + collect: workflows + inputs: .github/workflows/ + online-audits: false + + codeql: + name: CodeQL + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + languages: python + + - name: Analyze + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + category: /language:python diff --git a/.github/workflows/update-stats.yml b/.github/workflows/update-stats.yml index ba295c7..cb79f7c 100644 --- a/.github/workflows/update-stats.yml +++ b/.github/workflows/update-stats.yml @@ -17,10 +17,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' @@ -31,7 +33,10 @@ jobs: python generate_svg.py - name: Commit and push + env: + GH_TOKEN: ${{ github.token }} run: | + gh auth setup-git git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git add profile/dark_mode.svg profile/light_mode.svg cache/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1d31cae --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contributing to CodesWhat projects + +Bug fixes, focused features, tests, and documentation improvements are welcome. +Repository-specific instructions take precedence over this organization +default. + +## Before starting + +1. Read the repository's `README.md`, `AGENTS.md`, and development or release + documentation. +2. Search existing issues and pull requests for the same problem. +3. Open an issue before starting a large feature, breaking change, architecture + change, or release-process change. +4. From the documented active development or integration branch, create a + focused branch in your fork. + +Do not commit credentials, private data, local planning files, editor state, or +generated artifacts that the repository does not track. + +## Making a change + +- Keep the change focused on one concern. +- Follow the repository's existing language, formatting, and architecture + conventions. +- Add regression tests for bug fixes and coverage for non-trivial behavior. +- Update documentation and changelog entries when public behavior changes. +- Run the exact formatter, lint, test, build, and local hook commands documented + by the repository. + +## Commits + +Use plain Conventional Commits without emoji: + +```text +(): +``` + +Allowed types are +`feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert`. Keep the subject +concise and describe the change in the imperative mood. + +## Pull requests + +- Select the repository's documented target branch. Do not assume the GitHub + default branch is the correct target. +- Explain the problem, the change, and the verification performed. +- Call out security, compatibility, migration, or release effects. +- Include tests and documentation, or explain why they do not apply. +- Wait for all required checks and reviews before merge. + +Maintainers may ask for changes or decline work that does not fit the project's +scope. Contributions are licensed under the license of the repository receiving +them. + +Report suspected vulnerabilities privately as described in +[`SECURITY.md`](SECURITY.md), never in a public issue or pull request. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..5cf164d --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,49 @@ +# Security policy + +## Supported versions + +CodesWhat ships security fixes for the latest maintained release or branch of +each project. A repository-specific policy, release notes, or README may define +a narrower support window and takes precedence over this organization default. + +| Release or branch | Supported | +| --- | --- | +| Latest maintained release or branch | Yes | +| Older or unmaintained releases and branches | No | + +For a project that does not publish versioned releases, support applies to the +latest commit on its documented maintained branch. + +## Reporting a vulnerability + +Do not open a public GitHub issue for a suspected security vulnerability. + +Open this repository's **Security** tab and select **Report a vulnerability** +to use GitHub private vulnerability reporting when it is enabled. Otherwise, +email ****. Include the affected version or commit, +minimal reproduction steps, observed and expected behavior, and your assessment +of the impact. Redact credentials, private data, and identifying environment +details. + +You can expect: + +- acknowledgement within 48 hours; +- a status update within 7 days; and +- a fix or mitigation as soon as feasible, depending on severity and release + safety. + +CodesWhat coordinates disclosure with the reporter and credits reporters in +release notes unless they prefer to remain anonymous. + +## Security scope + +Unless a repository-specific policy says otherwise, the following are in +scope: + +- source code and configuration maintained in the repository; +- artifacts published by CodesWhat from that repository; and +- repository-owned build, test, and release automation. + +Third-party services, dependencies, and deployments are out of scope unless the +repository introduces or amplifies the reported impact. If the boundary is +unclear, report the issue privately and let the maintainers triage it. From 22feaaa1cd49740a453e9cd611745edbb13c6e8d Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:15:14 -0400 Subject: [PATCH 02/14] ci(greptile): require manual review requests (#11) --- .github/tests/greptile_config_contract_test.py | 18 ++++++++++++++++++ .github/workflows/standards-validation.yml | 1 + greptile.json | 3 +++ 3 files changed, 22 insertions(+) create mode 100644 .github/tests/greptile_config_contract_test.py create mode 100644 greptile.json diff --git a/.github/tests/greptile_config_contract_test.py b/.github/tests/greptile_config_contract_test.py new file mode 100644 index 0000000..33f7fb8 --- /dev/null +++ b/.github/tests/greptile_config_contract_test.py @@ -0,0 +1,18 @@ +import json +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class GreptileConfigContractTest(unittest.TestCase): + def test_reviews_are_manual_only(self): + with (ROOT / "greptile.json").open() as config_file: + config = json.load(config_file) + + self.assertEqual({"skipReview": "AUTOMATIC"}, config) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index edfbdfd..d9079a1 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -44,6 +44,7 @@ jobs: set -euo pipefail ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' + python3 .github/tests/greptile_config_contract_test.py - name: Lint Markdown run: | diff --git a/greptile.json b/greptile.json new file mode 100644 index 0000000..34c493d --- /dev/null +++ b/greptile.json @@ -0,0 +1,3 @@ +{ + "skipReview": "AUTOMATIC" +} From bc5ab596a2c715cd8eb7b87230abb5e568c33021 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 10:54:26 -0400 Subject: [PATCH 03/14] ci(workflows): add reusable CI foundation (#13) * ci(workflows): add reusable CI foundation * fix(workflows): harden reusable release contracts --- .github/tests/reusable_ci_contract_test.py | 527 +++++++++++++++++++++ .github/workflows/go-ci.yml | 460 ++++++++++++++++++ .github/workflows/node-ci.yml | 186 ++++++++ .github/workflows/release-gate.yml | 172 +++++++ .github/workflows/standards-validation.yml | 1 + 5 files changed, 1346 insertions(+) create mode 100644 .github/tests/reusable_ci_contract_test.py create mode 100644 .github/workflows/go-ci.yml create mode 100644 .github/workflows/node-ci.yml create mode 100644 .github/workflows/release-gate.yml diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py new file mode 100644 index 0000000..a6ec3f5 --- /dev/null +++ b/.github/tests/reusable_ci_contract_test.py @@ -0,0 +1,527 @@ +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = { + "go": ROOT / ".github/workflows/go-ci.yml", + "node": ROOT / ".github/workflows/node-ci.yml", + "release": ROOT / ".github/workflows/release-gate.yml", +} +TARGET_SHA = "a" * 40 + + +class ReusableCIContractTest(unittest.TestCase): + def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self): + go_inputs = [ + "module-directory", + "go-version-file", + "go-cache-dependency-path", + "lint-check-name", + "test-check-name", + "fuzzers-json", + "run-govulncheck", + "run-qlty", + "run-goreleaser", + "run-workflow-security", + "run-commit-message", + "run-codeql", + ] + self.egress_inputs( + ( + "test", + "lint", + "govulncheck", + "workflow-security", + "commit-message", + "goreleaser", + "codeql", + "qlty", + "fuzz", + ) + ) + node_inputs = [ + "node-version", + "lockfile-path", + "lint-check-name", + "test-check-name", + "build-check-name", + "run-lint", + "run-test", + "run-build", + ] + self.egress_inputs(("lint", "test", "build")) + release_inputs = [ + "target-sha", + "workflow-files-json", + "max-attempts", + "sleep-seconds", + ] + + expected_inputs = { + "go": go_inputs, + "node": node_inputs, + "release": release_inputs, + } + expected_jobs = { + "go": { + "test", + "lint", + "govulncheck", + "workflow-security", + "commit-message", + "goreleaser", + "codeql", + "qlty", + "fuzz", + }, + "node": {"lint", "test", "build"}, + "release": {"gate"}, + } + + for name, path in WORKFLOWS.items(): + workflow = self.read_workflow(path) + header = workflow.split("\npermissions:", 1)[0] + inputs = re.findall(r"^ ([a-z][a-z0-9-]+):$", header, re.MULTILINE) + self.assertEqual(expected_inputs[name], inputs) + self.assertRegex(workflow, r"(?m)^on:\n workflow_call:\s+inputs:") + self.assertNotRegex( + workflow, + r"(?m)^ (push|pull_request|schedule|workflow_dispatch):", + ) + self.assertNotIn("concurrency:", workflow) + self.assertNotIn("outputs:", header) + self.assertNotIn("secrets:", header) + self.assertNotIn("secrets: inherit", workflow) + self.assertIn("permissions: {}", workflow) + + jobs = set( + re.findall( + r"^ ([a-z][a-z0-9-]+):$", + workflow.split("\njobs:\n", 1)[1], + re.MULTILINE, + ) + ) + self.assertEqual(expected_jobs[name], jobs) + + self.assert_input("go", "module-directory", "string", default=".") + self.assert_input("go", "go-version-file", "string", default="go.mod") + self.assert_input("go", "go-cache-dependency-path", "string", default="go.sum") + self.assert_input("go", "lint-check-name", "string", default="Go Lint") + self.assert_input("go", "test-check-name", "string", default="Go Test") + self.assert_input("go", "fuzzers-json", "string", default="[]") + for input_name in ( + "run-govulncheck", + "run-qlty", + "run-goreleaser", + "run-workflow-security", + "run-commit-message", + "run-codeql", + ): + self.assert_input("go", input_name, "boolean", default="false") + + self.assert_input("node", "node-version", "string", default="24") + self.assert_input("node", "lockfile-path", "string", default="package-lock.json") + self.assert_input("node", "lint-check-name", "string", default="Node Lint") + self.assert_input("node", "test-check-name", "string", default="Node Test") + self.assert_input("node", "build-check-name", "string", default="Node Build") + for input_name in ("run-lint", "run-test", "run-build"): + self.assert_input("node", input_name, "boolean", default="false") + + for workflow_name, job_names in ( + ("go", ("test", "lint", "govulncheck", "workflow-security", "commit-message", "goreleaser", "codeql", "qlty", "fuzz")), + ("node", ("lint", "test", "build")), + ): + for job_name in job_names: + self.assert_input(workflow_name, f"{job_name}-egress-policy", "string", default="audit") + self.assert_input(workflow_name, f"{job_name}-allowed-endpoints", "string", default="") + + self.assert_input("release", "target-sha", "string", required=True) + self.assert_input("release", "workflow-files-json", "string", required=True) + self.assert_input("release", "max-attempts", "number", default="12") + self.assert_input("release", "sleep-seconds", "number", default="300") + + def test_central_jobs_have_exact_names_fixed_commands_and_per_job_egress(self): + go = self.read_workflow(WORKFLOWS["go"]) + expected_go_names = { + "test": "${{ inputs.test-check-name }}", + "lint": "${{ inputs.lint-check-name }}", + "govulncheck": "Govulncheck", + "workflow-security": "Workflow Security", + "commit-message": "Commit Message", + "goreleaser": "GoReleaser Config", + "codeql": "CodeQL Analysis", + "qlty": "Qlty Check", + "fuzz": 'Go Fuzz (${{ matrix.fuzzer.name }})', + } + for job, display_name in expected_go_names.items(): + self.assertEqual(display_name, self.job_name(go, job)) + self.assert_job_egress(go, job) + + expected_go_scripts = [ + "./scripts/ci/go-test.sh", + "./scripts/ci/go-lint.sh", + "./scripts/ci/go-govulncheck.sh", + "./scripts/ci/commit-message.sh", + "./scripts/ci/go-release-check.sh", + "./scripts/ci/go-codeql-build.sh", + "./scripts/ci/go-qlty.sh", + "./scripts/ci/go-fuzz.sh", + ] + self.assertEqual(expected_go_scripts, self.fixed_scripts(go)) + self.assertIn("fuzzer: ${{ fromJSON(inputs.fuzzers-json) }}", go) + self.assertIn( + "if: inputs.fuzzers-json != '' && fromJSON(inputs.fuzzers-json)[0] != null", + go, + ) + self.assertIn("go-version-file: ${{ inputs.go-version-file }}", go) + self.assertIn("cache-dependency-path: ${{ inputs.go-cache-dependency-path }}", go) + self.assertIn("MODULE_DIRECTORY: ${{ inputs.module-directory }}", go) + + node = self.read_workflow(WORKFLOWS["node"]) + expected_node_names = { + "lint": "${{ inputs.lint-check-name }}", + "test": "${{ inputs.test-check-name }}", + "build": "${{ inputs.build-check-name }}", + } + for job, display_name in expected_node_names.items(): + self.assertEqual(display_name, self.job_name(node, job)) + self.assert_job_egress(node, job) + self.assertEqual( + [ + "./scripts/ci/node-lint.sh", + "./scripts/ci/node-test.sh", + "./scripts/ci/node-build.sh", + ], + self.fixed_scripts(node), + ) + self.assertIn("node-version: ${{ inputs.node-version }}", node) + self.assertIn("cache-dependency-path: ${{ inputs.lockfile-path }}", node) + + for workflow in (go, node, self.read_workflow(WORKFLOWS["release"])): + self.assertIn("runs-on: ubuntu-24.04", workflow) + for action in re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE): + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + for run_block in re.findall(r"run: \|\n((?: {10}.*\n|\n)+)", workflow): + self.assertNotIn("${{ inputs.", run_block) + + def test_artifact_uploads_are_central_and_fixed(self): + go = self.read_workflow(WORKFLOWS["go"]) + node = self.read_workflow(WORKFLOWS["node"]) + + self.assertEqual( + ["artifacts/go-test/", "artifacts/go-fuzz/"], + re.findall(r"^\s+path: (artifacts/[^\s]+)$", go, re.MULTILINE), + ) + self.assertEqual( + ["artifacts/node-test/", "artifacts/node-build/"], + re.findall(r"^\s+path: (artifacts/[^\s]+)$", node, re.MULTILINE), + ) + for workflow in (go, node): + upload_count = workflow.count("actions/upload-artifact@") + self.assertEqual(upload_count, workflow.count("if-no-files-found: ignore")) + self.assertEqual(upload_count, workflow.count("retention-days: 14")) + self.assertEqual(upload_count, workflow.count("if: always()")) + + def test_release_gate_checks_every_workflow_by_exact_sha_push_and_nonempty_branch(self): + exact_success = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "dev/v1.7", + "status": "completed", + "conclusion": "success", + } + wrong_runs = [ + {**exact_success, "event": "pull_request"}, + {**exact_success, "head_branch": ""}, + {**exact_success, "head_sha": "b" * 40}, + ] + result = self.run_release_gate( + { + "ci-verify.yml": wrong_runs + [exact_success], + "e2e-playwright.yml": [ + {**exact_success, "head_branch": "main"}, + ], + } + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn(f"Found successful CI Verify push run for {TARGET_SHA}.", result.stdout) + self.assertIn(f"Found successful E2E Playwright push run for {TARGET_SHA}.", result.stdout) + + def test_release_gate_keeps_authoritative_client_side_run_filtering(self): + release = self.read_workflow(WORKFLOWS["release"]) + + self.assertIn("/runs?per_page=100\"", release) + self.assertNotRegex(release, r"runs\?[^\"\n]*(?:event|head_sha)=") + self.assertIn("spuriously returning 0 results since 2026-04-27", release) + + def test_release_gate_uses_one_shared_poll_budget_for_every_workflow(self): + in_progress = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "dev/v1.7", + "status": "in_progress", + "conclusion": None, + } + success = {**in_progress, "status": "completed", "conclusion": "success"} + result = self.run_release_gate( + {}, + runs_sequences_by_workflow={ + "ci-verify.yml": [[in_progress], [success]], + "e2e-playwright.yml": [[in_progress], [success]], + }, + max_attempts="2", + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(1, result.stdout.count("FAKE_SLEEP"), result.stdout) + + def test_release_gate_waits_when_an_exact_run_is_still_in_progress(self): + exact_run = { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + } + result = self.run_release_gate( + { + "ci-verify.yml": [ + {**exact_run, "status": "completed", "conclusion": "failure"}, + {**exact_run, "status": "in_progress", "conclusion": None}, + ], + } + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("Attempt 1/1: waiting for CI Verify", result.stdout) + self.assertIn( + f"Timed out waiting for successful CI Verify push run for {TARGET_SHA}.", + result.stdout, + ) + + def test_release_gate_fails_closed_when_any_workflow_lacks_exact_success(self): + result = self.run_release_gate( + { + "ci-verify.yml": [ + { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ], + "e2e-playwright.yml": [ + { + "head_sha": "b" * 40, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ], + } + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn( + f"Timed out waiting for successful E2E Playwright push run for {TARGET_SHA}.", + result.stdout, + ) + + def test_release_gate_rejects_unsafe_inputs_before_api_calls(self): + bad_sha = self.run_release_gate({"ci-verify.yml": []}, target_sha="main") + bad_files = self.run_release_gate( + {"ci-verify.yml": []}, + workflow_files_json='["../../release.yml"]', + ) + bad_attempts = self.run_release_gate( + {"ci-verify.yml": []}, + max_attempts="0", + ) + + self.assertNotEqual(0, bad_sha.returncode) + self.assertIn("target-sha must be a full lowercase commit SHA", bad_sha.stdout) + self.assertNotEqual(0, bad_files.returncode) + self.assertIn("workflow-files-json must be a nonempty JSON array", bad_files.stdout) + self.assertNotEqual(0, bad_attempts.returncode) + self.assertIn("max-attempts must be a positive integer", bad_attempts.stdout) + + def test_release_gate_rejects_non_github_api_hosts_before_api_calls(self): + result = self.run_release_gate( + { + "ci-verify.yml": [ + { + "head_sha": TARGET_SHA, + "event": "push", + "head_branch": "main", + "status": "completed", + "conclusion": "success", + } + ] + }, + api_url="https://github.example/api/v3", + ) + + self.assertNotEqual(0, result.returncode) + self.assertIn("release-gate supports api.github.com only", result.stdout) + + def test_standards_validation_runs_this_contract(self): + workflow = self.read_workflow(ROOT / ".github/workflows/standards-validation.yml") + self.assertEqual(1, workflow.count("python3 .github/tests/reusable_ci_contract_test.py")) + + def egress_inputs(self, job_names): + return [item for job in job_names for item in (f"{job}-egress-policy", f"{job}-allowed-endpoints")] + + def read_workflow(self, path): + self.assertTrue(path.is_file(), f"missing reusable workflow: {path.name}") + return path.read_text() + + def input_header(self, workflow_name): + workflow = self.read_workflow(WORKFLOWS[workflow_name]) + return workflow.split("\npermissions:", 1)[0] + + def assert_input(self, workflow_name, input_name, input_type, default=None, required=False): + header = self.input_header(workflow_name) + block = header.split(f" {input_name}:\n", 1)[1] + next_input = re.search(r"^ [a-z][a-z0-9-]+:$", block, re.MULTILINE) + if next_input: + block = block[: next_input.start()] + self.assertIn(f" type: {input_type}\n", block) + if default is not None: + expected = ( + f' default: "{default}"\n' + if default in {"", "[]"} + else f" default: {default}\n" + ) + self.assertIn(expected, block) + if required: + self.assertIn(" required: true\n", block) + + def job_section(self, workflow, job_name): + jobs = workflow.split("\njobs:\n", 1)[1] + section = jobs.split(f" {job_name}:\n", 1)[1] + next_job = re.search(r"^ [a-z][a-z0-9-]+:$", section, re.MULTILINE) + return section[: next_job.start()] if next_job else section + + def job_name(self, workflow, job_name): + section = self.job_section(workflow, job_name) + return re.search(r"^ name: (.+)$", section, re.MULTILINE).group(1).strip('"') + + def assert_job_egress(self, workflow, job_name): + section = self.job_section(workflow, job_name) + self.assertIn(f"egress-policy: ${{{{ inputs.{job_name}-egress-policy }}}}", section) + self.assertIn(f"allowed-endpoints: ${{{{ inputs.{job_name}-allowed-endpoints }}}}", section) + + def fixed_scripts(self, workflow): + return re.findall(r"^\s+run: (\./scripts/ci/[^\s]+)$", workflow, re.MULTILINE) + + def release_gate_script(self): + workflow = self.read_workflow(WORKFLOWS["release"]) + marker = " - name: Verify required CI workflows\n" + self.assertIn(marker, workflow) + step = workflow.split(marker, 1)[1] + block = step.split(" run: |\n", 1)[1] + lines = [] + for line in block.splitlines(): + if line.startswith(" "): + lines.append(line[10:]) + elif not line: + lines.append("") + else: + break + self.assertTrue(lines, "release gate shell step is empty") + return "\n".join(lines) + + def run_release_gate( + self, + runs_by_workflow, + target_sha=TARGET_SHA, + workflow_files_json=None, + max_attempts="1", + api_url="https://api.github.com", + runs_sequences_by_workflow=None, + ): + with tempfile.TemporaryDirectory() as temp_dir: + temp = Path(temp_dir) + fake_bin = temp / "bin" + fake_bin.mkdir() + fixtures = temp / "fixtures" + fixtures.mkdir() + + metadata = { + "ci-verify.yml": (123, "CI Verify"), + "e2e-playwright.yml": (456, "E2E Playwright"), + } + sequences = { + workflow_file: [runs] + for workflow_file, runs in runs_by_workflow.items() + } + sequences.update(runs_sequences_by_workflow or {}) + for workflow_file, runs_sequence in sequences.items(): + workflow_id = metadata[workflow_file][0] + for index, runs in enumerate(runs_sequence, start=1): + (fixtures / f"{workflow_id}.{index}.json").write_text( + json.dumps({"workflow_runs": runs}) + ) + (fixtures / f"{workflow_id}.last").write_text(str(len(runs_sequence))) + + curl = fake_bin / "curl" + curl.write_text( + "#!/usr/bin/env bash\n" + "for argument in \"$@\"; do url=\"$argument\"; done\n" + "case \"$url\" in\n" + " */actions/workflows/ci-verify.yml) printf '%s' '{\"id\":123,\"name\":\"CI Verify\",\"path\":\".github/workflows/ci-verify.yml\"}' ;;\n" + " */actions/workflows/e2e-playwright.yml) printf '%s' '{\"id\":456,\"name\":\"E2E Playwright\",\"path\":\".github/workflows/e2e-playwright.yml\"}' ;;\n" + " */actions/workflows/123/runs?per_page=100) workflow_id=123 ;;\n" + " */actions/workflows/456/runs?per_page=100) workflow_id=456 ;;\n" + " *) printf 'unexpected URL: %s\\n' \"$url\" >&2; exit 22 ;;\n" + "esac\n" + "if [ -n \"${workflow_id:-}\" ]; then\n" + " count_file=\"$RUNS_FIXTURES/$workflow_id.count\"\n" + " count=0\n" + " [ ! -f \"$count_file\" ] || count=$(cat \"$count_file\")\n" + " count=$((count + 1))\n" + " printf '%s' \"$count\" > \"$count_file\"\n" + " last=$(cat \"$RUNS_FIXTURES/$workflow_id.last\")\n" + " [ \"$count\" -le \"$last\" ] || count=\"$last\"\n" + " cat \"$RUNS_FIXTURES/$workflow_id.$count.json\"\n" + "fi\n" + ) + curl.chmod(0o755) + + sleep = fake_bin / "sleep" + sleep.write_text("#!/usr/bin/env bash\nprintf 'FAKE_SLEEP %s\\n' \"$1\"\n") + sleep.chmod(0o755) + + if workflow_files_json is None: + workflow_files_json = json.dumps(list(sequences)) + env = os.environ.copy() + env.update( + { + "GH_TOKEN": "test-token", + "GITHUB_API_URL": api_url, + "GITHUB_REPOSITORY": "CodesWhat/example", + "MAX_ATTEMPTS": max_attempts, + "PATH": f"{fake_bin}:{env['PATH']}", + "RUNS_FIXTURES": str(fixtures), + "SLEEP_SECONDS": "0", + "TARGET_SHA": target_sha, + "WORKFLOW_FILES_JSON": workflow_files_json, + } + ) + return subprocess.run( + ["bash", "-c", self.release_gate_script()], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + check=False, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml new file mode 100644 index 0000000..46601d4 --- /dev/null +++ b/.github/workflows/go-ci.yml @@ -0,0 +1,460 @@ +name: Reusable Go CI + +on: + workflow_call: + inputs: + module-directory: + description: Repository-relative Go module directory + required: false + type: string + default: . + go-version-file: + description: Repository-relative Go version file + required: false + type: string + default: go.mod + go-cache-dependency-path: + description: Repository-relative Go cache dependency path + required: false + type: string + default: go.sum + lint-check-name: + description: Exact display name for the Go lint job + required: false + type: string + default: Go Lint + test-check-name: + description: Exact display name for the Go test job + required: false + type: string + default: Go Test + fuzzers-json: + description: JSON array of caller-owned fuzzer name and pkg objects + required: false + type: string + default: "[]" + run-govulncheck: + description: Run the fixed repository govulncheck gate + required: false + type: boolean + default: false + run-qlty: + description: Run the fixed repository Qlty gate + required: false + type: boolean + default: false + run-goreleaser: + description: Run the fixed repository GoReleaser gate + required: false + type: boolean + default: false + run-workflow-security: + description: Run the central workflow security gate + required: false + type: boolean + default: false + run-commit-message: + description: Run the fixed repository commit-message gate on pull requests + required: false + type: boolean + default: false + run-codeql: + description: Run CodeQL for Actions and Go + required: false + type: boolean + default: false + test-egress-policy: + description: Harden Runner egress policy for Go test + required: false + type: string + default: audit + test-allowed-endpoints: + description: Harden Runner allowlist for Go test + required: false + type: string + default: "" + lint-egress-policy: + description: Harden Runner egress policy for Go lint + required: false + type: string + default: audit + lint-allowed-endpoints: + description: Harden Runner allowlist for Go lint + required: false + type: string + default: "" + govulncheck-egress-policy: + description: Harden Runner egress policy for govulncheck + required: false + type: string + default: audit + govulncheck-allowed-endpoints: + description: Harden Runner allowlist for govulncheck + required: false + type: string + default: "" + workflow-security-egress-policy: + description: Harden Runner egress policy for workflow security + required: false + type: string + default: audit + workflow-security-allowed-endpoints: + description: Harden Runner allowlist for workflow security + required: false + type: string + default: "" + commit-message-egress-policy: + description: Harden Runner egress policy for commit-message validation + required: false + type: string + default: audit + commit-message-allowed-endpoints: + description: Harden Runner allowlist for commit-message validation + required: false + type: string + default: "" + goreleaser-egress-policy: + description: Harden Runner egress policy for GoReleaser + required: false + type: string + default: audit + goreleaser-allowed-endpoints: + description: Harden Runner allowlist for GoReleaser + required: false + type: string + default: "" + codeql-egress-policy: + description: Harden Runner egress policy for CodeQL + required: false + type: string + default: audit + codeql-allowed-endpoints: + description: Harden Runner allowlist for CodeQL + required: false + type: string + default: "" + qlty-egress-policy: + description: Harden Runner egress policy for Qlty + required: false + type: string + default: audit + qlty-allowed-endpoints: + description: Harden Runner allowlist for Qlty + required: false + type: string + default: "" + fuzz-egress-policy: + description: Harden Runner egress policy for Go fuzzing + required: false + type: string + default: audit + fuzz-allowed-endpoints: + description: Harden Runner allowlist for Go fuzzing + required: false + type: string + default: "" + +permissions: {} + +jobs: + test: + name: ${{ inputs.test-check-name }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.test-egress-policy }} + allowed-endpoints: ${{ inputs.test-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go test gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-test.sh + + - name: Upload Go test artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: go-test-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/go-test/ + if-no-files-found: ignore + retention-days: 14 + + lint: + name: ${{ inputs.lint-check-name }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.lint-egress-policy }} + allowed-endpoints: ${{ inputs.lint-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go lint gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-lint.sh + + govulncheck: + name: Govulncheck + if: inputs.run-govulncheck + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.govulncheck-egress-policy }} + allowed-endpoints: ${{ inputs.govulncheck-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository govulncheck gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-govulncheck.sh + + workflow-security: + name: Workflow Security + if: inputs.run-workflow-security + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.workflow-security-egress-policy }} + allowed-endpoints: ${{ inputs.workflow-security-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Run zizmor + uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + with: + token: ${{ github.token }} + advanced-security: false + online-audits: false + + commit-message: + name: Commit Message + if: inputs.run-commit-message && github.event_name == 'pull_request' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.commit-message-egress-policy }} + allowed-endpoints: ${{ inputs.commit-message-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Run repository commit-message gate + env: + BASE_REF: ${{ github.base_ref }} + HEAD_REF: ${{ github.head_ref }} + run: ./scripts/ci/commit-message.sh + + goreleaser: + name: GoReleaser Config + if: inputs.run-goreleaser + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.goreleaser-egress-policy }} + allowed-endpoints: ${{ inputs.goreleaser-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository GoReleaser gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-release-check.sh + + codeql: + name: CodeQL Analysis + if: inputs.run-codeql + runs-on: ubuntu-24.04 + timeout-minutes: 60 + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.codeql-egress-policy }} + allowed-endpoints: ${{ inputs.codeql-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + with: + languages: actions, go + queries: security-and-quality + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Build for CodeQL + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-codeql-build.sh + + - name: Analyze with CodeQL + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 + + qlty: + name: Qlty Check + if: inputs.run-qlty + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.qlty-egress-policy }} + allowed-endpoints: ${{ inputs.qlty-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Qlty + uses: qltysh/qlty-action/install@08a0a862c159eae9b9003081da6663d96efef637 # v2.3.0 + + - name: Run repository Qlty gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} + run: ./scripts/ci/go-qlty.sh + + fuzz: + name: "Go Fuzz (${{ matrix.fuzzer.name }})" + if: inputs.fuzzers-json != '' && fromJSON(inputs.fuzzers-json)[0] != null + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + fuzzer: ${{ fromJSON(inputs.fuzzers-json) }} + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.fuzz-egress-policy }} + allowed-endpoints: ${{ inputs.fuzz-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: ${{ inputs.go-version-file }} + cache-dependency-path: ${{ inputs.go-cache-dependency-path }} + + - name: Run repository Go fuzzer + env: + FUZZER: ${{ matrix.fuzzer.name }} + MODULE_DIRECTORY: ${{ inputs.module-directory }} + PKG: ${{ matrix.fuzzer.pkg }} + run: ./scripts/ci/go-fuzz.sh + + - name: Upload Go fuzz artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: go-fuzz-${{ matrix.fuzzer.name }}-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/go-fuzz/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml new file mode 100644 index 0000000..e699a9b --- /dev/null +++ b/.github/workflows/node-ci.yml @@ -0,0 +1,186 @@ +name: Reusable Node CI + +on: + workflow_call: + inputs: + node-version: + description: Node.js version used by enabled jobs + required: false + type: string + default: 24 + lockfile-path: + description: Repository-relative npm lockfile path or multiline paths + required: false + type: string + default: package-lock.json + lint-check-name: + description: Exact display name for the Node lint job + required: false + type: string + default: Node Lint + test-check-name: + description: Exact display name for the Node test job + required: false + type: string + default: Node Test + build-check-name: + description: Exact display name for the Node build job + required: false + type: string + default: Node Build + run-lint: + description: Run the fixed repository Node lint gate + required: false + type: boolean + default: false + run-test: + description: Run the fixed repository Node test gate + required: false + type: boolean + default: false + run-build: + description: Run the fixed repository Node build gate + required: false + type: boolean + default: false + lint-egress-policy: + description: Harden Runner egress policy for Node lint + required: false + type: string + default: audit + lint-allowed-endpoints: + description: Harden Runner allowlist for Node lint + required: false + type: string + default: "" + test-egress-policy: + description: Harden Runner egress policy for Node test + required: false + type: string + default: audit + test-allowed-endpoints: + description: Harden Runner allowlist for Node test + required: false + type: string + default: "" + build-egress-policy: + description: Harden Runner egress policy for Node build + required: false + type: string + default: audit + build-allowed-endpoints: + description: Harden Runner allowlist for Node build + required: false + type: string + default: "" + +permissions: {} + +jobs: + lint: + name: ${{ inputs.lint-check-name }} + if: inputs.run-lint + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.lint-egress-policy }} + allowed-endpoints: ${{ inputs.lint-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node lint gate + run: ./scripts/ci/node-lint.sh + + test: + name: ${{ inputs.test-check-name }} + if: inputs.run-test + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.test-egress-policy }} + allowed-endpoints: ${{ inputs.test-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node test gate + run: ./scripts/ci/node-test.sh + + - name: Upload Node test artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-test-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/node-test/ + if-no-files-found: ignore + retention-days: 14 + + build: + name: ${{ inputs.build-check-name }} + if: inputs.run-build + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: ${{ inputs.build-egress-policy }} + allowed-endpoints: ${{ inputs.build-allowed-endpoints }} + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + cache: npm + cache-dependency-path: ${{ inputs.lockfile-path }} + + - name: Run repository Node build gate + run: ./scripts/ci/node-build.sh + + - name: Upload Node build artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: node-build-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/node-build/ + if-no-files-found: ignore + retention-days: 14 diff --git a/.github/workflows/release-gate.yml b/.github/workflows/release-gate.yml new file mode 100644 index 0000000..f86ff64 --- /dev/null +++ b/.github/workflows/release-gate.yml @@ -0,0 +1,172 @@ +name: Reusable Release Gate + +on: + workflow_call: + inputs: + target-sha: + description: Full commit SHA whose branch push runs must have succeeded + required: true + type: string + workflow-files-json: + description: Nonempty JSON array of workflow filenames to verify + required: true + type: string + max-attempts: + description: Maximum polls for each required workflow + required: false + type: number + default: 12 + sleep-seconds: + description: Seconds between polls + required: false + type: number + default: 300 + +permissions: {} + +jobs: + gate: + name: Release Gate + runs-on: ubuntu-24.04 + timeout-minutes: 360 + permissions: + actions: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: api.github.com:443 + + - name: Verify required CI workflows + env: + GH_TOKEN: ${{ github.token }} + MAX_ATTEMPTS: ${{ inputs.max-attempts }} + SLEEP_SECONDS: ${{ inputs.sleep-seconds }} + TARGET_SHA: ${{ inputs.target-sha }} + WORKFLOW_FILES_JSON: ${{ inputs.workflow-files-json }} + run: | + set -euo pipefail + + if [ "${GITHUB_API_URL}" != "https://api.github.com" ]; then + echo "::error::release-gate supports api.github.com only so its blocked egress allowlist matches the API host." + exit 1 + fi + if ! [[ "${TARGET_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::target-sha must be a full lowercase commit SHA." + exit 1 + fi + if ! [[ "${MAX_ATTEMPTS}" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::max-attempts must be a positive integer." + exit 1 + fi + if ! [[ "${SLEEP_SECONDS}" =~ ^[0-9]+$ ]]; then + echo "::error::sleep-seconds must be a nonnegative integer." + exit 1 + fi + if ! jq -e ' + type == "array" + and length > 0 + and all(.[]; + type == "string" + and test("^[A-Za-z0-9][A-Za-z0-9._-]*[.]ya?ml$") + ) + ' <<< "${WORKFLOW_FILES_JSON}" >/dev/null; then + echo "::error::workflow-files-json must be a nonempty JSON array of workflow filenames." + exit 1 + fi + + workflow_files=() + while IFS= read -r workflow_file; do + workflow_files+=("${workflow_file}") + done < <(jq -r '.[]' <<< "${WORKFLOW_FILES_JSON}") + + workflow_ids=() + workflow_names=() + for workflow_file in "${workflow_files[@]}"; do + workflow_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow_file}" + workflow_json="$(curl --proto '=https' --tlsv1.2 --retry 3 --fail --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${workflow_url}")" + workflow_id="$(echo "${workflow_json}" | jq -r '.id // empty')" + workflow_name="$(echo "${workflow_json}" | jq -r '.name // empty')" + workflow_path="$(echo "${workflow_json}" | jq -r '.path // empty')" + expected_path=".github/workflows/${workflow_file}" + if [ -z "${workflow_id}" ] || [ -z "${workflow_name}" ] || [ "${workflow_path}" != "${expected_path}" ]; then + echo "::error::Failed to resolve workflow metadata for ${workflow_file}." + exit 1 + fi + workflow_ids+=("${workflow_id}") + workflow_names+=("${workflow_name}") + done + + workflow_succeeded=() + for index in "${!workflow_ids[@]}"; do + workflow_succeeded["${index}"]=false + done + + for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do + waiting=false + for index in "${!workflow_ids[@]}"; do + if [ "${workflow_succeeded[${index}]}" = "true" ]; then + continue + fi + + workflow_id="${workflow_ids[${index}]}" + workflow_name="${workflow_names[${index}]}" + # GitHub's `?head_sha=` and `?event=` query filters have been spuriously returning 0 results since 2026-04-27. + # Filter the latest 100 runs client-side instead. + runs_url="${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/workflows/${workflow_id}/runs?per_page=100" + runs_json="$(curl --proto '=https' --tlsv1.2 --retry 3 --fail --silent --show-error \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/vnd.github+json" \ + "${runs_url}")" + filtered="$(echo "${runs_json}" | jq --arg sha "${TARGET_SHA}" '{ + workflow_runs: [.workflow_runs[] + | select( + .head_sha == $sha + and .event == "push" + and ((.head_branch // "") | length > 0) + )] + }')" + + success_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.conclusion == "success")] | length')" + in_progress_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.status != "completed")] | length')" + completed_count="$(echo "${filtered}" | jq '[.workflow_runs[] | select(.status == "completed")] | length')" + + if [ "${success_count}" -gt 0 ]; then + echo "Found successful ${workflow_name} push run for ${TARGET_SHA}." + workflow_succeeded["${index}"]=true + continue + fi + + if [ "${completed_count}" -gt 0 ] && [ "${in_progress_count}" -eq 0 ]; then + echo "::error::${workflow_name} push runs for ${TARGET_SHA} completed without success." + echo "${filtered}" | jq '.workflow_runs[] | {id, status, conclusion, head_branch, html_url}' + exit 1 + fi + + echo "Attempt ${attempt}/${MAX_ATTEMPTS}: waiting for ${workflow_name} push run for ${TARGET_SHA}." + waiting=true + done + + if [ "${waiting}" != "true" ]; then + break + fi + if [ "${attempt}" -lt "${MAX_ATTEMPTS}" ]; then + sleep "${SLEEP_SECONDS}" + fi + done + + timed_out=false + for index in "${!workflow_ids[@]}"; do + if [ "${workflow_succeeded[${index}]}" != "true" ]; then + echo "::error::Timed out waiting for successful ${workflow_names[${index}]} push run for ${TARGET_SHA}." + timed_out=true + fi + done + if [ "${timed_out}" = "true" ]; then + exit 1 + fi diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index d9079a1..192e5bd 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -45,6 +45,7 @@ jobs: ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' python3 .github/tests/greptile_config_contract_test.py + python3 .github/tests/reusable_ci_contract_test.py - name: Lint Markdown run: | From 16a6680ee0bdeb32f5a768557413503adc4c36c8 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:35:36 -0400 Subject: [PATCH 04/14] feat(quality): standardize long-run reporting (#15) * feat(quality): add normalized reporting foundation * test(quality): run reporting contracts in standards validation * fix(quality): align report validator with schema * test(quality): verify GitHub integration outputs --- .../quality-result-config/target-result.json | 22 + .../quality-result-filter/target-result.json | 22 + .../quality-result-stats/target-result.json | 16 + .../quality-result-verify/target-result.json | 16 + .../quality-result-stats/target-result.json | 15 + .../quality-result-verify/target-result.json | 16 + .../quality-result-ops-1/target-result.json | 22 + .../quality-result-ops-1/target-result.json | 22 + .../quality-result-ops-2/target-result.json | 22 + .../quality-result-config/target-result.json | 22 + .../target-result.invalid | 21 + .github/tests/quality_report_contract_test.py | 447 ++++++++++ .../workflows/quality-report-aggregate.yml | 157 ++++ .github/workflows/standards-validation.yml | 1 + decisions/quality-reporting.md | 162 ++++ quality-report/v1/quality_report.py | 765 ++++++++++++++++++ quality-report/v1/schema.json | 496 ++++++++++++ 17 files changed, 2244 insertions(+) create mode 100644 .github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json create mode 100644 .github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid create mode 100644 .github/tests/quality_report_contract_test.py create mode 100644 .github/workflows/quality-report-aggregate.yml create mode 100644 decisions/quality-reporting.md create mode 100644 quality-report/v1/quality_report.py create mode 100644 quality-report/v1/schema.json diff --git a/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json new file mode 100644 index 0000000..d4f7466 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-config/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/config", + "outcome": "passed", + "metrics": { + "killed": 40, + "timeout": 10, + "survived": 5, + "no_coverage": 5, + "invalid": 2, + "ignored": 3, + "tool_score_numerator": 40, + "tool_score_denominator": 45, + "tool_score_pct": 88.89, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json new file mode 100644 index 0000000..a4b3164 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/canonical-score/quality-result-filter/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/filter", + "outcome": "passed", + "metrics": { + "killed": 30, + "timeout": 0, + "survived": 0, + "no_coverage": 10, + "invalid": 1, + "ignored": 2, + "tool_score_numerator": 30, + "tool_score_denominator": 30, + "tool_score_pct": 100.0, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json new file mode 100644 index 0000000..627a14e --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/docker/FuzzDecodeStats", + "outcome": "passed", + "metrics": { + "budget_seconds": 120, + "elapsed_seconds": 111.25, + "executions": 650000, + "new_interesting_inputs": 2 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json new file mode 100644 index 0000000..472e276 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 288.25, + "executions": 1250000, + "new_interesting_inputs": 3 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json new file mode 100644 index 0000000..0db4a95 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json @@ -0,0 +1,15 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/docker/FuzzDecodeStats", + "outcome": "crashed", + "metrics": null, + "diagnostic": "tool process exited 2", + "reproduction": { + "seed": "-test.fuzzcachedir=/tmp/fuzz-cache" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json new file mode 100644 index 0000000..c2f7e89 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-verify/target-result.json @@ -0,0 +1,16 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "fuzz", + "tool": "go-fuzz", + "target": { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 300.0, + "executions": 990000, + "new_interesting_inputs": 0 + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json b/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json new file mode 100644 index 0000000..4a5d472 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/incomplete/quality-result-ops-1/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-1", + "outcome": "passed", + "metrics": { + "killed": 20, + "timeout": 2, + "survived": 3, + "no_coverage": 5, + "invalid": 1, + "ignored": 2, + "tool_score_numerator": 22, + "tool_score_denominator": 25, + "tool_score_pct": 88.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json new file mode 100644 index 0000000..f79465e --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-1/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-1", + "outcome": "passed", + "metrics": { + "killed": 3, + "timeout": 1, + "survived": 1, + "no_coverage": 1, + "invalid": 0, + "ignored": 0, + "tool_score_numerator": 4, + "tool_score_denominator": 5, + "tool_score_pct": 80.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json new file mode 100644 index 0000000..403a77f --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/native-score/quality-result-ops-2/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "stryker", + "target": { + "name": "apps-api-ops-2", + "outcome": "passed", + "metrics": { + "killed": 2, + "timeout": 0, + "survived": 2, + "no_coverage": 0, + "invalid": 1, + "ignored": 1, + "tool_score_numerator": 2, + "tool_score_denominator": 4, + "tool_score_pct": 50.0, + "tool_score_definition": "detected / mutants with test coverage" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json new file mode 100644 index 0000000..d1f3655 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-config/target-result.json @@ -0,0 +1,22 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/config", + "outcome": "passed", + "metrics": { + "killed": 12, + "timeout": 1, + "survived": 2, + "no_coverage": 1, + "invalid": 0, + "ignored": 1, + "tool_score_numerator": 12, + "tool_score_denominator": 14, + "tool_score_pct": 85.71, + "tool_score_definition": "killed / (killed + survived)" + } + } +} diff --git a/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid new file mode 100644 index 0000000..783dac7 --- /dev/null +++ b/.github/tests/fixtures/quality-report/v1/parse-error/quality-result-filter/target-result.invalid @@ -0,0 +1,21 @@ +{ + "schema_version": "quality-report/v1", + "kind": "target-result", + "track": "mutation", + "tool": "gremlins", + "target": { + "name": "./internal/filter", + "outcome": "passed", + "metrics": { + "killed": 31, + "timeout": 4, + "survived": 2, + "no_coverage": 3, + "invalid": 1, + "ignored": 0, + "tool_score_numerator": 31, + "tool_score_denominator": 33, + "tool_score_pct": 93.94, + "tool_score_definition": "killed / (killed + survived)" + } + } diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py new file mode 100644 index 0000000..b456ce7 --- /dev/null +++ b/.github/tests/quality_report_contract_test.py @@ -0,0 +1,447 @@ +import json +from pathlib import Path +import runpy +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +CONTRACT_ROOT = ROOT / "quality-report" / "v1" +FIXTURES = ROOT / ".github" / "tests" / "fixtures" / "quality-report" / "v1" +SCRIPT = CONTRACT_ROOT / "quality_report.py" +SCHEMA = CONTRACT_ROOT / "schema.json" +WORKFLOW = ROOT / ".github" / "workflows" / "quality-report-aggregate.yml" +DECISION = ROOT / "decisions" / "quality-reporting.md" +SHA = "0123456789abcdef0123456789abcdef01234567" +QUALITY_REPORT = runpy.run_path(str(SCRIPT)) + + +class QualityReportContractTest(unittest.TestCase): + def test_fixture_inventory_covers_failure_and_score_boundaries(self): + self.assertEqual( + { + "canonical-score", + "complete", + "crash", + "incomplete", + "native-score", + "parse-error", + }, + {path.name for path in FIXTURES.iterdir() if path.is_dir()}, + ) + + def test_complete_fuzz_fixture_emits_a_valid_report_and_summary(self): + result, report, summary = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual("quality-report/v1", report["schema_version"]) + self.assertEqual("report", report["kind"]) + self.assertEqual( + {"expected": 2, "reported": 2, "complete": True}, + report["completeness"], + ) + self.assertEqual("passed", report["outcome"]) + self.assertEqual(420, report["metrics"]["declared_budget_seconds"]) + self.assertEqual(399.5, report["metrics"]["elapsed_seconds"]) + self.assertEqual(1900000, report["metrics"]["executions"]) + self.assertEqual(5, report["metrics"]["new_interesting_inputs"]) + self.assertNotIn("canonical_score_pct", report["metrics"]) + self.assertIn("complete 2/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_incomplete_fixture_fails_closed_without_a_subset_score(self): + result, report, summary = self.aggregate( + "incomplete", + track="mutation", + tool="stryker", + expected=["apps-api-ops-1", "apps-api-ops-2"], + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual( + {"expected": 2, "reported": 1, "complete": False}, + report["completeness"], + ) + self.assertEqual("error", report["outcome"]) + self.assertIsNone(report["metrics"]) + self.assertIn("missing target: apps-api-ops-2", report["errors"]) + self.assertIn("partial 1/2", summary) + self.assertNotIn("Canonical score", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_crash_fixture_fails_even_when_every_target_reports(self): + result, report, summary = self.aggregate( + "crash", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + + self.assertNotEqual(0, result.returncode) + self.assertTrue(report["completeness"]["complete"]) + self.assertEqual("crashed", report["outcome"]) + self.assertIsNone(report["metrics"]) + self.assertIn("tool process exited 2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_parse_error_fixture_still_emits_an_error_report(self): + result, report, summary = self.aggregate( + "parse-error", + track="mutation", + tool="gremlins", + expected=["./internal/config", "./internal/filter"], + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual("error", report["outcome"]) + self.assertFalse(report["completeness"]["complete"]) + self.assertIsNone(report["metrics"]) + self.assertTrue( + any("invalid JSON" in error for error in report["errors"]), + report["errors"], + ) + self.assertIn("partial 1/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + + def test_canonical_score_counts_timeout_and_no_coverage(self): + result, report, summary = self.aggregate( + "canonical-score", + track="mutation", + tool="gremlins", + expected=["./internal/config", "./internal/filter"], + ) + + self.assertEqual(0, result.returncode, result.stderr) + metrics = report["metrics"] + self.assertEqual(70, metrics["killed"]) + self.assertEqual(10, metrics["timeout"]) + self.assertEqual(5, metrics["survived"]) + self.assertEqual(15, metrics["no_coverage"]) + self.assertEqual(80, metrics["detected"]) + self.assertEqual(20, metrics["missed"]) + self.assertEqual(80.0, metrics["canonical_score_pct"]) + self.assertEqual(93.33, metrics["tool_score_pct"]) + self.assertIn("Canonical score: **80.00%**", summary) + + def test_decision_defines_canonical_score_as_a_percentage(self): + decision = DECISION.read_text() + + self.assertIn( + "canonical_score_pct = 100 * detected / (detected + missed)", + decision, + ) + + def test_decision_documents_track_and_tool_pairings(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn("`mutation` permits `gremlins` and `stryker`", decision) + self.assertIn("`fuzz` permits `fast-check` and `go-fuzz`", decision) + + def test_native_score_is_weighted_and_keeps_its_definition(self): + result, report, summary = self.aggregate( + "native-score", + track="mutation", + tool="stryker", + expected=["apps-api-ops-1", "apps-api-ops-2"], + ) + + self.assertEqual(0, result.returncode, result.stderr) + metrics = report["metrics"] + self.assertEqual(60.0, metrics["canonical_score_pct"]) + self.assertEqual(66.67, metrics["tool_score_pct"]) + self.assertEqual(6, metrics["tool_score_numerator"]) + self.assertEqual(9, metrics["tool_score_denominator"]) + self.assertEqual( + "detected / mutants with test coverage", + metrics["tool_score_definition"], + ) + self.assertIn("Native score: **66.67%**", summary) + + def test_validator_rejects_unknown_report_fields(self): + result, report, _ = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + self.assertEqual(0, result.returncode, result.stderr) + report["unreviewed"] = True + + validation = self.validate_report(report) + + self.assertNotEqual(0, validation.returncode) + self.assertIn("unknown field", validation.stderr) + + def test_validator_rejects_missing_start_time_from_a_success_report(self): + result, report, _ = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + ) + self.assertEqual(0, result.returncode, result.stderr) + report["run"]["started_at"] = None + + validation = self.validate_report(report) + + self.assertNotEqual(0, validation.returncode) + self.assertIn("started_at", validation.stderr) + + def test_validator_rejects_target_names_beyond_the_schema_limit(self): + target = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + target["target"]["name"] = "x" * 257 + + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], "256"): + QUALITY_REPORT["validate_target_document"](target) + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], "256"): + QUALITY_REPORT["parse_expected_targets"]( + json.dumps([target["target"]["name"]]) + ) + + def test_validator_requires_a_positive_declared_fuzz_budget(self): + metrics = { + "declared_budget_seconds": 0, + "elapsed_seconds": 0, + "executions": 0, + "new_interesting_inputs": 0, + } + + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], ">= 1"): + QUALITY_REPORT["validate_fuzz_report_metrics"](metrics, [], "$.metrics") + + def test_validator_requires_time_and_offset_in_timestamps(self): + for timestamp in ("2026-08-14", "2026-08-14T15:00:00"): + with self.subTest(timestamp=timestamp): + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], "RFC 3339" + ): + QUALITY_REPORT["require_timestamp"](timestamp, "$.run.started_at") + + def test_schema_is_versioned_and_fail_closed(self): + with SCHEMA.open() as schema_file: + schema = json.load(schema_file) + + self.assertEqual( + "https://json-schema.org/draft/2020-12/schema", + schema["$schema"], + ) + self.assertEqual( + "https://codeswhat.com/schemas/quality-report/v1/schema.json", + schema["$id"], + ) + self.assertEqual("quality-report/v1", schema["$defs"]["version"]["const"]) + self.assertNotIn("additionalProperties", schema) + self.assertTrue(self.all_objects_are_closed(schema)) + + def test_reusable_workflow_matches_the_public_contract(self): + workflow = WORKFLOW.read_text() + + self.assertIn(" workflow_call:", workflow) + for input_name in ( + "track", + "tool", + "policy", + "expected_targets", + "result_artifact_pattern", + "report_name", + "retention_days", + "fail_on_incomplete", + ): + self.assertIn(f" {input_name}:\n", workflow) + for output_name in ( + "report_artifact_name", + "completeness", + "expected_targets", + "reported_targets", + "outcome", + "canonical_score_pct", + ): + self.assertIn(f" {output_name}:\n", workflow) + + self.assertIn(" default: quality-result-*", workflow) + self.assertIn(" default: 90", workflow) + self.assertIn(" default: true", workflow) + self.assertIn(" if: always()", workflow) + self.assertIn( + "repository: ${{ fromJSON(toJSON(job)).workflow_repository }}", workflow + ) + self.assertIn("ref: ${{ fromJSON(toJSON(job)).workflow_sha }}", workflow) + self.assertIn( + "name: quality-report-${{ inputs.track }}-${{ github.run_id }}-${{ github.run_attempt }}", + workflow, + ) + self.assertIn("retention-days: ${{ inputs.retention_days }}", workflow) + self.assertIn("if: always()", workflow) + + actions = [ + line.split("uses: ", 1)[1].split()[0] + for line in workflow.splitlines() + if "uses: " in line + ] + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + for forbidden in ("git add", "git commit", "git push", "badges/"): + self.assertNotIn(forbidden, workflow) + + def test_public_decision_records_fail_closed_and_no_source_badges(self): + decision = DECISION.read_text() + + for expected in ( + "quality-report/v1", + "target-result.json", + "if: always()", + "quality-report---", + "90 days", + "partial N/M", + "Do not commit badge JSON to `main`", + "Native percentages are not comparable", + ): + self.assertIn(expected, decision) + + def test_standards_validation_runs_this_contract(self): + workflow = (ROOT / ".github/workflows/standards-validation.yml").read_text() + self.assertEqual( + 1, + workflow.count("python3 .github/tests/quality_report_contract_test.py"), + ) + + def aggregate(self, fixture, *, track, tool, expected): + self.assertTrue(SCRIPT.is_file(), f"missing aggregator: {SCRIPT}") + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) / "input" + shutil.copytree(FIXTURES / fixture, input_dir) + for invalid_fixture in input_dir.rglob("target-result.invalid"): + invalid_fixture.rename(invalid_fixture.with_name("target-result.json")) + output_dir = Path(temp_dir) / "output" + github_output = Path(temp_dir) / "github-output" + summary_path = Path(temp_dir) / "step-summary" + command = [ + "python3", + str(SCRIPT), + "aggregate", + "--input-dir", + str(input_dir), + "--output-dir", + str(output_dir), + "--repository", + "CodesWhat/example", + "--ref", + "refs/heads/dev/example", + "--sha", + SHA, + "--run-id", + "1234567890", + "--run-attempt", + "2", + "--event", + "workflow_dispatch", + "--run-url", + "https://github.com/CodesWhat/example/actions/runs/1234567890", + "--started-at", + "2026-08-13T12:34:56Z", + "--track", + track, + "--tool", + tool, + "--policy", + "advisory", + "--expected-targets", + json.dumps(expected), + "--report-name", + "Fixture quality report", + "--fail-on-incomplete", + "true", + "--github-output", + str(github_output), + "--github-step-summary", + str(summary_path), + ] + result = subprocess.run(command, capture_output=True, text=True) + report_path = output_dir / "report.json" + summary_file = output_dir / "summary.md" + self.assertTrue(report_path.is_file(), result.stderr) + self.assertTrue(summary_file.is_file(), result.stderr) + report = json.loads(report_path.read_text()) + summary = summary_file.read_text() + self.assertTrue(github_output.is_file(), result.stderr) + self.assertTrue(summary_path.is_file(), result.stderr) + github_outputs = dict( + line.split("=", 1) for line in github_output.read_text().splitlines() + ) + canonical_score = "" + if ( + report["track"] == "mutation" + and report["metrics"] is not None + and report["metrics"]["canonical_score_pct"] is not None + ): + canonical_score = "{0:.2f}".format( + report["metrics"]["canonical_score_pct"] + ) + self.assertEqual( + { + "report_artifact_name": "quality-report-{0}-1234567890-2".format( + track + ), + "completeness": str(report["completeness"]["complete"]).lower(), + "expected_targets": str(report["completeness"]["expected"]), + "reported_targets": str(report["completeness"]["reported"]), + "outcome": report["outcome"], + "canonical_score_pct": canonical_score, + }, + github_outputs, + ) + self.assertEqual(summary, summary_path.read_text()) + return result, report, summary + + def validate_report(self, report): + with tempfile.TemporaryDirectory() as temp_dir: + report_path = Path(temp_dir) / "report.json" + report_path.write_text(json.dumps(report)) + return subprocess.run( + ["python3", str(SCRIPT), "validate", str(report_path)], + capture_output=True, + text=True, + ) + + def all_objects_are_closed(self, value): + if isinstance(value, dict): + if ( + value.get("type") == "object" + and value.get("additionalProperties") is not False + ): + return False + return all(self.all_objects_are_closed(item) for item in value.values()) + if isinstance(value, list): + return all(self.all_objects_are_closed(item) for item in value) + return True + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/quality-report-aggregate.yml b/.github/workflows/quality-report-aggregate.yml new file mode 100644 index 0000000..86ef420 --- /dev/null +++ b/.github/workflows/quality-report-aggregate.yml @@ -0,0 +1,157 @@ +name: Quality Report Aggregate + +on: + workflow_call: + inputs: + track: + description: Quality track, mutation or fuzz + required: true + type: string + tool: + description: Native tool that produced the normalized target results + required: true + type: string + policy: + description: Reporting policy, advisory or signal + required: true + type: string + expected_targets: + description: Non-empty JSON array of unique target names + required: true + type: string + result_artifact_pattern: + description: Artifact-name pattern containing target-result.json files + required: false + type: string + default: quality-result-* + report_name: + description: Human-readable report name + required: true + type: string + retention_days: + description: Aggregate artifact retention in days + required: false + type: number + default: 90 + fail_on_incomplete: + description: Must remain true for CodesWhat telemetry + required: false + type: boolean + default: true + outputs: + report_artifact_name: + description: Fixed aggregate artifact name + value: ${{ jobs.aggregate.outputs.report_artifact_name }} + completeness: + description: Whether every expected target reported valid JSON + value: ${{ jobs.aggregate.outputs.completeness }} + expected_targets: + description: Number of expected targets + value: ${{ jobs.aggregate.outputs.expected_targets }} + reported_targets: + description: Number of valid expected target results + value: ${{ jobs.aggregate.outputs.reported_targets }} + outcome: + description: Aggregate outcome + value: ${{ jobs.aggregate.outputs.outcome }} + canonical_score_pct: + description: Canonical mutation score, empty for fuzz or partial reports + value: ${{ jobs.aggregate.outputs.canonical_score_pct }} + +permissions: {} + +jobs: + aggregate: + name: Quality Report Aggregate + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + actions: read + contents: read + outputs: + report_artifact_name: ${{ steps.aggregate.outputs.report_artifact_name }} + completeness: ${{ steps.aggregate.outputs.completeness }} + expected_targets: ${{ steps.aggregate.outputs.expected_targets }} + reported_targets: ${{ steps.aggregate.outputs.reported_targets }} + outcome: ${{ steps.aggregate.outputs.outcome }} + canonical_score_pct: ${{ steps.aggregate.outputs.canonical_score_pct }} + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Check out the called workflow contract + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # actionlint 1.7.12 does not model these current reusable-workflow fields. + repository: ${{ fromJSON(toJSON(job)).workflow_repository }} + ref: ${{ fromJSON(toJSON(job)).workflow_sha }} + path: quality-report-contract + persist-credentials: false + + - name: Read workflow run metadata + id: metadata + if: always() + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + started_at="$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq .run_started_at)" + test -n "$started_at" + echo "started_at=${started_at}" >> "$GITHUB_OUTPUT" + + - name: Download normalized target results + id: download + if: always() + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: ${{ inputs.result_artifact_pattern }} + path: quality-result-artifacts + + - name: Validate and aggregate target results + id: aggregate + if: always() + env: + EXPECTED_TARGETS: ${{ inputs.expected_targets }} + FAIL_ON_INCOMPLETE: ${{ inputs.fail_on_incomplete }} + POLICY: ${{ inputs.policy }} + REPORT_NAME: ${{ inputs.report_name }} + RUN_STARTED_AT: ${{ steps.metadata.outputs.started_at }} + TOOL: ${{ inputs.tool }} + TRACK: ${{ inputs.track }} + run: | + set -euo pipefail + python3 quality-report-contract/quality-report/v1/quality_report.py aggregate \ + --input-dir quality-result-artifacts \ + --output-dir quality-report-output \ + --repository "$GITHUB_REPOSITORY" \ + --ref "$GITHUB_REF" \ + --sha "$GITHUB_SHA" \ + --run-id "$GITHUB_RUN_ID" \ + --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --event "$GITHUB_EVENT_NAME" \ + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ + --started-at "$RUN_STARTED_AT" \ + --track "$TRACK" \ + --tool "$TOOL" \ + --policy "$POLICY" \ + --expected-targets "$EXPECTED_TARGETS" \ + --report-name "$REPORT_NAME" \ + --fail-on-incomplete "$FAIL_ON_INCOMPLETE" \ + --github-output "$GITHUB_OUTPUT" \ + --github-step-summary "$GITHUB_STEP_SUMMARY" + + - name: Upload normalized quality report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: quality-report-${{ inputs.track }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + quality-report-output/report.json + quality-report-output/summary.md + if-no-files-found: error + retention-days: ${{ inputs.retention_days }} diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index 192e5bd..ceb4606 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -45,6 +45,7 @@ jobs: ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' python3 .github/tests/greptile_config_contract_test.py + python3 .github/tests/quality_report_contract_test.py python3 .github/tests/reusable_ci_contract_test.py - name: Lint Markdown diff --git a/decisions/quality-reporting.md b/decisions/quality-reporting.md new file mode 100644 index 0000000..6182c97 --- /dev/null +++ b/decisions/quality-reporting.md @@ -0,0 +1,162 @@ +# Quality Reporting + +Last updated: 2026-08-13 + +## Decision + +Keep each repository's native test tool. Normalize the result envelope, not the +runner: + +- Drydock keeps Stryker and fast-check. +- Sockguard keeps Gremlins and Go fuzzing. +- Portwing keeps Gremlins and Go fuzzing. + +Mutation stays advisory and absent from required PR checks. A scheduled or +manual mutation workflow must still fail when its telemetry is incomplete or a +tool crashes. Non-blocking means it does not gate a PR. It does not mean the +workflow may report success after losing results. + +Long fuzzing succeeds only when every declared target completes its budget. A +crash, repeated boundary flake, setup failure, or missing target result makes +the workflow red. + +## Evidence behind the decision + +The 2026-08-01 Drydock mutation run completed at the workflow level while 13 of +27 Stryker shards failed. Its aggregate used `--allow-missing` and published an +81.79% score from the remaining 14 shards. The workflow status and numeric +badge therefore described a partial result as success. + +Sockguard's existing 97.8% badge uses `killed / (killed + lived)`. Counting +timeout as detected and no-coverage as missed gives 93.91% for the same run. +Portwing has no aggregate badge; applying the same canonical definition to its +latest complete Gremlins run gives 77.31%, versus 81.98% under Gremlins' +efficacy definition. Native percentages are not comparable until their +denominators are explicit. + +Sockguard inventories all 48 in-tree fuzzers and covers every target in at +least one tier. Portwing's `FuzzVerifyRequest` runs in primary CI and scheduled +long tiers. Drydock's five fast-check fuzz tests run as ordinary Vitest tests +with default run counts and have no scheduled long-fuzz workflow. + +## Normalized envelope + +Every matrix leg writes one `target-result.json`. An always-running aggregate +job validates the set and publishes `quality-report/v1` with: + +- `repository`, `ref`, `sha` +- `run`: `id`, `attempt`, `event`, `url`, `started_at` +- `track`: `mutation` or `fuzz` +- `tool`: `mutation` permits `gremlins` and `stryker`; `fuzz` permits + `fast-check` and `go-fuzz` +- `policy`: `advisory` or `signal` +- `completeness`: `expected`, `reported`, `complete` +- `outcome`: `passed`, `failed`, `crashed`, `flaked`, `cancelled`, `error` +- `targets`: the declared target names and their individual results +- track-specific `metrics` + +The schema and dependency-free validator live under [`quality-report/v1`](../quality-report/v1). +Both target-result fragments and aggregate reports carry the exact +`quality-report/v1` version. Objects reject unknown fields, JSON parsing rejects +duplicate fields and non-finite numbers, and semantic validation recomputes +completeness, outcomes, and scores. Callers pass `expected_targets` as a +non-empty JSON array of unique target names. + +Mutation metrics are: + +- `killed`, `timeout`, `survived`, `no_coverage`, `invalid`, `ignored` +- `detected = killed + timeout` +- `missed = survived + no_coverage` +- `canonical_score_pct = 100 * detected / (detected + missed)`, rounded + half-up to two decimals, or `null` when the denominator is zero +- `tool_score_pct` and `tool_score_definition` + +Each target also supplies the native numerator and denominator. The aggregate +weights those counts instead of averaging percentages, and rejects mixed native +score definitions. The native score remains available with its definition. +Only the canonical score is comparable across mutation tools. + +Fuzz metrics include the declared budget, elapsed time, and executions or new +interesting inputs when the runner exposes them. Do not compare execution +counts between runners or tools. A failed, crashed, or flaked fuzz result must +retain a seed, path, or corpus reference for reproduction. + +## Reusable workflow contract + +Inputs: + +- `track` +- `tool` +- `policy` +- `expected_targets` +- `result_artifact_pattern`, default `quality-result-*` +- `report_name` +- `retention_days`, default `90` +- `fail_on_incomplete`, always `true` for CodesWhat telemetry + +Outputs: + +- `report_artifact_name` +- `completeness` +- `expected_targets` +- `reported_targets` +- `outcome` +- `canonical_score_pct` for mutation reports + +The aggregate artifact is named +`quality-report---` and contains `report.json` plus +`summary.md`. Native reports and crash corpora remain separate artifacts. +Target result JSON is always uploaded; crash forensics remain failure-only. +Normalized aggregate artifacts use at least 90 days of retention. + +The reusable workflow checks out its own contract at the exact called-workflow +commit via `job.workflow_repository` and `job.workflow_sha`. Product workflows +remain responsible for native tool adapters and uploading their target-result +artifacts. + +## Status and badge semantics + +- The aggregate job runs under `if: always()` and fails on an incomplete set, + tool crash, parse error, invalid contract, or policy failure. +- Numeric mutation badges publish only a complete canonical score. An + incomplete run displays `partial N/M`, never a subset score or stale score. +- Do not commit badge JSON to `main`. Bot commits bypass the dev-to-main flow + and make telemetry part of source history. +- Use the native workflow-status badge as the stable shared badge. Put numeric + scores and native dashboard links in the run summary and retained artifact + until there is a non-source telemetry endpoint. +- Long-fuzz failures must retain the runner seed, path, or corpus needed to + reproduce the failure. + +## Migration order + +1. Add and test the schema, validator, and reusable aggregate in + `CodesWhat/.github`. +2. Canary Portwing. It has the smallest Gremlins and fuzz matrices. Keep + `FuzzVerifyRequest` in a long tier and preserve its source-to-tier inventory + contract. +3. Migrate Sockguard's Gremlins aggregation, retire its direct badge commit, + and preserve its existing fuzz inventory contract. +4. Migrate Drydock's Stryker aggregate, remove partial score publication, then + add a scheduled fast-check workflow with explicit run count and seed/path + reproduction. +5. Manually dispatch each workflow on the exact active dev head before syncing + it to `main`. Scheduled workflows exercise `main`, not the active dev branch. + +Native runner adapters and product callers are intentionally outside this +foundation change. Each migration supplies and tests its own mapping from +native output into the target-result contract. + +## Risks and controls + +- The three first-of-month mutation runs compete for organization runner + capacity. Stagger their schedules. +- Gremlins text parsing is version-sensitive. Pin the tool and test fixtures + from real output. +- Stryker reports compile/runtime-invalid mutants and may lose shards. Preserve + native categories and fail closed on a missing shard. +- Fast-check failures are not reproducible without seed and path. Make both + mandatory failure fields. +- Artifacts expire. Ninety days is the minimum retention for normalized + reports; durable trend storage is a separate decision. +- `continue-on-error` on a matrix leg must not make the final aggregate green. diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py new file mode 100644 index 0000000..8b8c93a --- /dev/null +++ b/quality-report/v1/quality_report.py @@ -0,0 +1,765 @@ +#!/usr/bin/env python3 +"""Validate and aggregate CodesWhat quality-report/v1 result envelopes.""" + +import argparse +from datetime import datetime +from decimal import Decimal, ROUND_HALF_UP +import json +import math +from pathlib import Path +import re +import sys +from urllib.parse import urlparse + + +VERSION = "quality-report/v1" +KINDS = {"target-result", "report"} +TRACK_TOOLS = { + "mutation": {"gremlins", "stryker"}, + "fuzz": {"fast-check", "go-fuzz"}, +} +POLICIES = {"advisory", "signal"} +OUTCOMES = {"passed", "failed", "crashed", "flaked", "cancelled", "error"} +OUTCOME_PRIORITY = ("error", "crashed", "cancelled", "flaked", "failed") +MUTATION_COUNT_FIELDS = ( + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", +) +TARGET_MUTATION_FIELDS = set(MUTATION_COUNT_FIELDS) | { + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition", +} +REPORT_MUTATION_FIELDS = TARGET_MUTATION_FIELDS | { + "detected", + "missed", + "canonical_score_pct", +} +TARGET_FUZZ_REQUIRED = {"budget_seconds", "elapsed_seconds"} +TARGET_FUZZ_OPTIONAL = {"executions", "new_interesting_inputs"} +REPORT_FUZZ_FIELDS = { + "declared_budget_seconds", + "elapsed_seconds", + "executions", + "new_interesting_inputs", +} +SHA_PATTERN = re.compile(r"^[0-9a-f]{40}$") +RFC3339_PATTERN = re.compile( + r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})$" +) + + +class ContractError(ValueError): + pass + + +def reject_constant(value): + raise ContractError("non-finite JSON number: {0}".format(value)) + + +def reject_duplicate_keys(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ContractError("duplicate JSON field: {0}".format(key)) + result[key] = value + return result + + +def load_json_text(text, source): + try: + return json.loads( + text, + object_pairs_hook=reject_duplicate_keys, + parse_constant=reject_constant, + ) + except (json.JSONDecodeError, ContractError) as error: + raise ContractError("{0}: invalid JSON: {1}".format(source, error)) + + +def load_json_file(path): + try: + return load_json_text(path.read_text(), str(path)) + except OSError as error: + raise ContractError("{0}: cannot read JSON: {1}".format(path, error)) + + +def require_object(value, path, required, optional=()): + if not isinstance(value, dict): + raise ContractError("{0}: expected object".format(path)) + required = set(required) + allowed = required | set(optional) + missing = sorted(required - set(value)) + unknown = sorted(set(value) - allowed) + if missing: + raise ContractError( + "{0}: missing field(s): {1}".format(path, ", ".join(missing)) + ) + if unknown: + raise ContractError( + "{0}: unknown field(s): {1}".format(path, ", ".join(unknown)) + ) + return value + + +def require_string(value, path, *, pattern=None, max_length=None): + if not isinstance(value, str) or not value or "\n" in value or "\r" in value: + raise ContractError("{0}: expected non-empty single-line string".format(path)) + if max_length is not None and len(value) > max_length: + raise ContractError( + "{0}: expected at most {1} characters".format(path, max_length) + ) + if pattern is not None and pattern.fullmatch(value) is None: + raise ContractError("{0}: invalid value".format(path)) + return value + + +def require_enum(value, path, choices): + require_string(value, path) + if value not in choices: + raise ContractError( + "{0}: expected one of {1}".format(path, ", ".join(sorted(choices))) + ) + return value + + +def require_integer(value, path, *, minimum=0): + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ContractError("{0}: expected integer >= {1}".format(path, minimum)) + return value + + +def require_number(value, path, *, minimum=0): + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ContractError("{0}: expected number >= {1}".format(path, minimum)) + if not math.isfinite(value) or value < minimum: + raise ContractError("{0}: expected finite number >= {1}".format(path, minimum)) + return value + + +def require_nullable_integer(value, path): + if value is not None: + require_integer(value, path) + + +def require_percentage(value, path, *, nullable=False): + if value is None and nullable: + return + require_number(value, path) + if value > 100: + raise ContractError("{0}: expected percentage <= 100".format(path)) + + +def require_url(value, path): + require_string(value, path) + parsed = urlparse(value) + if parsed.scheme != "https" or not parsed.netloc: + raise ContractError("{0}: expected HTTPS URL".format(path)) + + +def require_timestamp(value, path): + require_string(value, path) + try: + if RFC3339_PATTERN.fullmatch(value) is None: + raise ValueError + datetime.fromisoformat(value.replace("Z", "+00:00").replace("z", "+00:00")) + except ValueError: + raise ContractError("{0}: expected RFC 3339 timestamp".format(path)) + + +def percentage(numerator, denominator): + if denominator == 0: + return None + value = Decimal(numerator) * Decimal(100) / Decimal(denominator) + return float(value.quantize(Decimal("0.01"), rounding=ROUND_HALF_UP)) + + +def validate_mutation_target_metrics(metrics, path): + require_object(metrics, path, TARGET_MUTATION_FIELDS) + for field in MUTATION_COUNT_FIELDS: + require_integer(metrics[field], "{0}.{1}".format(path, field)) + numerator = require_integer( + metrics["tool_score_numerator"], path + ".tool_score_numerator" + ) + denominator = require_integer( + metrics["tool_score_denominator"], path + ".tool_score_denominator" + ) + require_percentage( + metrics["tool_score_pct"], path + ".tool_score_pct", nullable=True + ) + require_string(metrics["tool_score_definition"], path + ".tool_score_definition") + expected = percentage(numerator, denominator) + if metrics["tool_score_pct"] != expected: + raise ContractError( + "{0}.tool_score_pct: expected {1} from numerator/denominator".format( + path, expected + ) + ) + + +def validate_fuzz_target_metrics(metrics, path): + require_object(metrics, path, TARGET_FUZZ_REQUIRED, TARGET_FUZZ_OPTIONAL) + require_integer(metrics["budget_seconds"], path + ".budget_seconds", minimum=1) + require_number(metrics["elapsed_seconds"], path + ".elapsed_seconds") + for field in TARGET_FUZZ_OPTIONAL: + if field in metrics: + require_integer(metrics[field], "{0}.{1}".format(path, field)) + + +def validate_reproduction(value, path): + require_object(value, path, (), {"seed", "path", "corpus"}) + if not value: + raise ContractError("{0}: expected seed, path, or corpus".format(path)) + for field, item in value.items(): + require_string(item, "{0}.{1}".format(path, field)) + + +def validate_target(target, track, path): + require_object( + target, + path, + {"name", "outcome", "metrics"}, + {"diagnostic", "reproduction"}, + ) + require_string(target["name"], path + ".name", max_length=256) + outcome = require_enum(target["outcome"], path + ".outcome", OUTCOMES) + metrics = target["metrics"] + if metrics is None: + if outcome == "passed": + raise ContractError( + "{0}.metrics: passed target requires metrics".format(path) + ) + elif track == "mutation": + validate_mutation_target_metrics(metrics, path + ".metrics") + else: + validate_fuzz_target_metrics(metrics, path + ".metrics") + + if outcome == "passed": + if "diagnostic" in target or "reproduction" in target: + raise ContractError( + "{0}: passed target cannot contain diagnostic or reproduction".format( + path + ) + ) + else: + if "diagnostic" not in target: + raise ContractError( + "{0}: non-passed target requires diagnostic".format(path) + ) + require_string(target["diagnostic"], path + ".diagnostic") + if track == "fuzz" and outcome in {"failed", "crashed", "flaked"}: + if "reproduction" not in target: + raise ContractError( + "{0}: failed fuzz target requires reproduction".format(path) + ) + if "reproduction" in target: + validate_reproduction(target["reproduction"], path + ".reproduction") + + +def validate_target_document(document, path="$"): + require_object( + document, + path, + {"schema_version", "kind", "track", "tool", "target"}, + ) + if document["schema_version"] != VERSION: + raise ContractError("{0}.schema_version: expected {1}".format(path, VERSION)) + if document["kind"] != "target-result": + raise ContractError("{0}.kind: expected target-result".format(path)) + track = require_enum(document["track"], path + ".track", set(TRACK_TOOLS)) + tool = require_enum( + document["tool"], path + ".tool", set().union(*TRACK_TOOLS.values()) + ) + if tool not in TRACK_TOOLS[track]: + raise ContractError("{0}.tool: {1} is not a {2} tool".format(path, tool, track)) + validate_target(document["target"], track, path + ".target") + return document + + +def aggregate_mutation_metrics(targets): + metrics = {field: 0 for field in MUTATION_COUNT_FIELDS} + definitions = set() + for target in targets: + target_metrics = target["metrics"] + for field in MUTATION_COUNT_FIELDS: + metrics[field] += target_metrics[field] + metrics.setdefault("tool_score_numerator", 0) + metrics.setdefault("tool_score_denominator", 0) + metrics["tool_score_numerator"] += target_metrics["tool_score_numerator"] + metrics["tool_score_denominator"] += target_metrics["tool_score_denominator"] + definitions.add(target_metrics["tool_score_definition"]) + if len(definitions) != 1: + raise ContractError("target results use inconsistent native score definitions") + metrics["detected"] = metrics["killed"] + metrics["timeout"] + metrics["missed"] = metrics["survived"] + metrics["no_coverage"] + metrics["canonical_score_pct"] = percentage( + metrics["detected"], metrics["detected"] + metrics["missed"] + ) + metrics["tool_score_pct"] = percentage( + metrics["tool_score_numerator"], metrics["tool_score_denominator"] + ) + metrics["tool_score_definition"] = definitions.pop() + return metrics + + +def aggregate_fuzz_metrics(targets): + metrics = { + "declared_budget_seconds": sum( + target["metrics"]["budget_seconds"] for target in targets + ), + "elapsed_seconds": sum( + target["metrics"]["elapsed_seconds"] for target in targets + ), + } + for field in ("executions", "new_interesting_inputs"): + if all(field in target["metrics"] for target in targets): + metrics[field] = sum(target["metrics"][field] for target in targets) + else: + metrics[field] = None + return metrics + + +def derive_outcome(targets, errors): + if errors: + return "error" + outcomes = {target["outcome"] for target in targets} + for outcome in OUTCOME_PRIORITY: + if outcome in outcomes: + return outcome + return "passed" + + +def validate_mutation_report_metrics(metrics, targets, path): + require_object(metrics, path, REPORT_MUTATION_FIELDS) + for field in MUTATION_COUNT_FIELDS + ( + "detected", + "missed", + "tool_score_numerator", + "tool_score_denominator", + ): + require_integer(metrics[field], "{0}.{1}".format(path, field)) + require_percentage( + metrics["canonical_score_pct"], path + ".canonical_score_pct", nullable=True + ) + require_percentage( + metrics["tool_score_pct"], path + ".tool_score_pct", nullable=True + ) + require_string(metrics["tool_score_definition"], path + ".tool_score_definition") + expected = aggregate_mutation_metrics(targets) + if metrics != expected: + raise ContractError( + "{0}: aggregate mutation metrics do not match targets".format(path) + ) + + +def validate_fuzz_report_metrics(metrics, targets, path): + require_object(metrics, path, REPORT_FUZZ_FIELDS) + require_integer( + metrics["declared_budget_seconds"], + path + ".declared_budget_seconds", + minimum=1, + ) + require_number(metrics["elapsed_seconds"], path + ".elapsed_seconds") + require_nullable_integer(metrics["executions"], path + ".executions") + require_nullable_integer( + metrics["new_interesting_inputs"], path + ".new_interesting_inputs" + ) + expected = aggregate_fuzz_metrics(targets) + if metrics != expected: + raise ContractError( + "{0}: aggregate fuzz metrics do not match targets".format(path) + ) + + +def validate_report(document, path="$"): + require_object( + document, + path, + { + "schema_version", + "kind", + "repository", + "ref", + "sha", + "run", + "report_name", + "track", + "tool", + "policy", + "completeness", + "outcome", + "targets", + "metrics", + "errors", + }, + ) + if document["schema_version"] != VERSION: + raise ContractError("{0}.schema_version: expected {1}".format(path, VERSION)) + if document["kind"] != "report": + raise ContractError("{0}.kind: expected report".format(path)) + require_string(document["repository"], path + ".repository") + require_string(document["ref"], path + ".ref") + require_string(document["sha"], path + ".sha", pattern=SHA_PATTERN) + require_string(document["report_name"], path + ".report_name") + track = require_enum(document["track"], path + ".track", set(TRACK_TOOLS)) + tool = require_enum( + document["tool"], path + ".tool", set().union(*TRACK_TOOLS.values()) + ) + if tool not in TRACK_TOOLS[track]: + raise ContractError("{0}.tool: {1} is not a {2} tool".format(path, tool, track)) + require_enum(document["policy"], path + ".policy", POLICIES) + + run = require_object( + document["run"], + path + ".run", + {"id", "attempt", "event", "url", "started_at"}, + ) + require_string(run["id"], path + ".run.id") + require_integer(run["attempt"], path + ".run.attempt", minimum=1) + require_string(run["event"], path + ".run.event") + require_url(run["url"], path + ".run.url") + if run["started_at"] is not None: + require_timestamp(run["started_at"], path + ".run.started_at") + + completeness = require_object( + document["completeness"], + path + ".completeness", + {"expected", "reported", "complete"}, + ) + require_integer( + completeness["expected"], path + ".completeness.expected", minimum=1 + ) + require_integer(completeness["reported"], path + ".completeness.reported") + if not isinstance(completeness["complete"], bool): + raise ContractError(path + ".completeness.complete: expected boolean") + + targets = document["targets"] + if not isinstance(targets, list): + raise ContractError(path + ".targets: expected array") + names = [] + for index, target in enumerate(targets): + validate_target(target, track, "{0}.targets[{1}]".format(path, index)) + names.append(target["name"]) + if len(names) != len(set(names)): + raise ContractError(path + ".targets: duplicate target name") + if names != sorted(names): + raise ContractError(path + ".targets: targets must be sorted by name") + if completeness["reported"] != len(targets): + raise ContractError(path + ".completeness.reported: does not match targets") + + errors = document["errors"] + if not isinstance(errors, list) or any( + not isinstance(error, str) or not error for error in errors + ): + raise ContractError(path + ".errors: expected non-empty strings") + if errors != sorted(set(errors)): + raise ContractError(path + ".errors: errors must be sorted and unique") + if run["started_at"] is None and not errors: + raise ContractError(path + ".run.started_at: required for an error-free report") + expected_complete = ( + completeness["expected"] == completeness["reported"] and not errors + ) + if completeness["complete"] != expected_complete: + raise ContractError(path + ".completeness.complete: inconsistent value") + + outcome = require_enum(document["outcome"], path + ".outcome", OUTCOMES) + if outcome != derive_outcome(targets, errors): + raise ContractError(path + ".outcome: does not match target results") + + metrics = document["metrics"] + metrics_available = ( + completeness["complete"] + and bool(targets) + and all(target["metrics"] is not None for target in targets) + ) + if not metrics_available: + if metrics is not None: + raise ContractError(path + ".metrics: must be null for partial results") + elif track == "mutation": + validate_mutation_report_metrics(metrics, targets, path + ".metrics") + else: + validate_fuzz_report_metrics(metrics, targets, path + ".metrics") + return document + + +def validate_document(document): + if not isinstance(document, dict): + raise ContractError("$: expected object") + kind = document.get("kind") + if kind not in KINDS: + raise ContractError("$.kind: expected one of report, target-result") + if kind == "target-result": + return validate_target_document(document) + return validate_report(document) + + +def parse_expected_targets(raw): + value = load_json_text(raw, "expected_targets") + if not isinstance(value, list) or not value: + raise ContractError("expected_targets: expected a non-empty JSON array") + targets = [] + for index, target in enumerate(value): + targets.append( + require_string( + target, + "expected_targets[{0}]".format(index), + max_length=256, + ) + ) + if len(targets) != len(set(targets)): + raise ContractError("expected_targets: duplicate target") + return targets + + +def relative_label(path, root): + try: + return str(path.relative_to(root)) + except ValueError: + return path.name + + +def collect_targets(input_dir, expected_targets, track, tool): + errors = [] + reported = {} + paths = sorted(input_dir.rglob("target-result.json")) if input_dir.is_dir() else [] + if not paths: + errors.append("no target-result.json files found") + for path in paths: + label = relative_label(path, input_dir) + try: + document = load_json_file(path) + validate_target_document(document, label) + if document["track"] != track: + raise ContractError( + "{0}.track: expected {1}, got {2}".format( + label, track, document["track"] + ) + ) + if document["tool"] != tool: + raise ContractError( + "{0}.tool: expected {1}, got {2}".format( + label, tool, document["tool"] + ) + ) + target = document["target"] + name = target["name"] + if name not in expected_targets: + raise ContractError("{0}: unexpected target: {1}".format(label, name)) + if name in reported: + raise ContractError("{0}: duplicate target: {1}".format(label, name)) + reported[name] = target + except ContractError as error: + errors.append(str(error)) + for target in expected_targets: + if target not in reported: + errors.append("missing target: {0}".format(target)) + return [reported[name] for name in sorted(reported)], sorted(set(errors)) + + +def build_summary(report): + completeness = report["completeness"] + if completeness["complete"]: + completeness_text = "complete {0}/{1}".format( + completeness["reported"], completeness["expected"] + ) + else: + completeness_text = "partial {0}/{1}".format( + completeness["reported"], completeness["expected"] + ) + lines = [ + "# {0}".format(report["report_name"]), + "", + "- Track: `{0}` via `{1}`".format(report["track"], report["tool"]), + "- Policy: `{0}`".format(report["policy"]), + "- Completeness: **{0}**".format(completeness_text), + "- Outcome: **{0}**".format(report["outcome"]), + ] + metrics = report["metrics"] + if report["track"] == "mutation" and metrics is not None: + canonical = metrics["canonical_score_pct"] + native = metrics["tool_score_pct"] + if canonical is not None: + lines.append("- Canonical score: **{0:.2f}%**".format(canonical)) + if native is not None: + lines.append( + "- Native score: **{0:.2f}%** ({1})".format( + native, metrics["tool_score_definition"] + ) + ) + lines.extend(("", "## Targets", "")) + for target in report["targets"]: + detail = "" + if "diagnostic" in target: + detail = ": {0}".format(target["diagnostic"]) + lines.append( + "- `{0}`: {1}{2}".format(target["name"], target["outcome"], detail) + ) + if report["errors"]: + lines.extend(("", "## Contract errors", "")) + lines.extend("- {0}".format(error) for error in report["errors"]) + return "\n".join(lines) + "\n" + + +def append_lines(path, values): + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as output: + for key, value in values.items(): + output.write("{0}={1}\n".format(key, value)) + + +def command_aggregate(args): + errors = [] + try: + expected_targets = parse_expected_targets(args.expected_targets) + except ContractError as error: + expected_targets = ["invalid-expected-targets"] + errors.append(str(error)) + if args.fail_on_incomplete != "true": + errors.append("fail_on_incomplete must be true for CodesWhat telemetry") + try: + require_timestamp(args.started_at, "started_at") + started_at = args.started_at + except ContractError as error: + started_at = None + errors.append(str(error)) + + targets, collection_errors = collect_targets( + args.input_dir, expected_targets, args.track, args.tool + ) + errors.extend(collection_errors) + errors = sorted(set(errors)) + completeness = { + "expected": len(expected_targets), + "reported": len(targets), + "complete": len(targets) == len(expected_targets) and not errors, + } + metrics = None + if completeness["complete"] and all( + target["metrics"] is not None for target in targets + ): + try: + if args.track == "mutation": + metrics = aggregate_mutation_metrics(targets) + else: + metrics = aggregate_fuzz_metrics(targets) + except ContractError as error: + errors.append(str(error)) + errors = sorted(set(errors)) + completeness["complete"] = False + + report = { + "schema_version": VERSION, + "kind": "report", + "repository": args.repository, + "ref": args.ref, + "sha": args.sha, + "run": { + "id": args.run_id, + "attempt": args.run_attempt, + "event": args.event, + "url": args.run_url, + "started_at": started_at, + }, + "report_name": args.report_name, + "track": args.track, + "tool": args.tool, + "policy": args.policy, + "completeness": completeness, + "outcome": derive_outcome(targets, errors), + "targets": targets, + "metrics": metrics, + "errors": errors, + } + validate_report(report) + + args.output_dir.mkdir(parents=True, exist_ok=True) + summary = build_summary(report) + (args.output_dir / "report.json").write_text( + json.dumps(report, indent=2, allow_nan=False) + "\n" + ) + (args.output_dir / "summary.md").write_text(summary) + append_lines( + args.github_output, + { + "report_artifact_name": "quality-report-{0}-{1}-{2}".format( + args.track, args.run_id, args.run_attempt + ), + "completeness": str(completeness["complete"]).lower(), + "expected_targets": completeness["expected"], + "reported_targets": completeness["reported"], + "outcome": report["outcome"], + "canonical_score_pct": ( + "" + if metrics is None + or args.track != "mutation" + or metrics["canonical_score_pct"] is None + else "{0:.2f}".format(metrics["canonical_score_pct"]) + ), + }, + ) + if args.github_step_summary is not None: + args.github_step_summary.parent.mkdir(parents=True, exist_ok=True) + with args.github_step_summary.open("a") as step_summary: + step_summary.write(summary) + return 0 if completeness["complete"] and report["outcome"] == "passed" else 1 + + +def command_validate(args): + document = load_json_file(args.document) + validate_document(document) + return 0 + + +def build_parser(): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate_parser = subparsers.add_parser("validate") + validate_parser.add_argument("document", type=Path) + validate_parser.set_defaults(handler=command_validate) + + aggregate_parser = subparsers.add_parser("aggregate") + aggregate_parser.add_argument("--input-dir", type=Path, required=True) + aggregate_parser.add_argument("--output-dir", type=Path, required=True) + aggregate_parser.add_argument("--repository", required=True) + aggregate_parser.add_argument("--ref", required=True) + aggregate_parser.add_argument("--sha", required=True) + aggregate_parser.add_argument("--run-id", required=True) + aggregate_parser.add_argument("--run-attempt", type=int, required=True) + aggregate_parser.add_argument("--event", required=True) + aggregate_parser.add_argument("--run-url", required=True) + aggregate_parser.add_argument("--started-at", required=True) + aggregate_parser.add_argument("--track", choices=sorted(TRACK_TOOLS), required=True) + aggregate_parser.add_argument( + "--tool", choices=sorted(set().union(*TRACK_TOOLS.values())), required=True + ) + aggregate_parser.add_argument("--policy", choices=sorted(POLICIES), required=True) + aggregate_parser.add_argument("--expected-targets", required=True) + aggregate_parser.add_argument("--report-name", required=True) + aggregate_parser.add_argument( + "--fail-on-incomplete", choices=("true", "false"), required=True + ) + aggregate_parser.add_argument("--github-output", type=Path) + aggregate_parser.add_argument("--github-step-summary", type=Path) + aggregate_parser.set_defaults(handler=command_aggregate) + return parser + + +def main(): + args = build_parser().parse_args() + try: + return args.handler(args) + except ContractError as error: + print("quality-report: {0}".format(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/quality-report/v1/schema.json b/quality-report/v1/schema.json new file mode 100644 index 0000000..46c38f1 --- /dev/null +++ b/quality-report/v1/schema.json @@ -0,0 +1,496 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://codeswhat.com/schemas/quality-report/v1/schema.json", + "title": "CodesWhat quality-report/v1", + "description": "Normalized target results and aggregate quality reports.", + "oneOf": [ + { + "$ref": "#/$defs/targetResult" + }, + { + "$ref": "#/$defs/report" + } + ], + "$defs": { + "version": { + "type": "string", + "const": "quality-report/v1" + }, + "outcome": { + "type": "string", + "enum": [ + "passed", + "failed", + "crashed", + "flaked", + "cancelled", + "error" + ] + }, + "reproduction": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "seed": { + "type": "string", + "minLength": 1 + }, + "path": { + "type": "string", + "minLength": 1 + }, + "corpus": { + "type": "string", + "minLength": 1 + } + } + }, + "mutationTargetMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition" + ], + "properties": { + "killed": { + "type": "integer", + "minimum": 0 + }, + "timeout": { + "type": "integer", + "minimum": 0 + }, + "survived": { + "type": "integer", + "minimum": 0 + }, + "no_coverage": { + "type": "integer", + "minimum": 0 + }, + "invalid": { + "type": "integer", + "minimum": 0 + }, + "ignored": { + "type": "integer", + "minimum": 0 + }, + "tool_score_numerator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_denominator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_definition": { + "type": "string", + "minLength": 1 + } + } + }, + "fuzzTargetMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "budget_seconds", + "elapsed_seconds" + ], + "properties": { + "budget_seconds": { + "type": "integer", + "minimum": 1 + }, + "elapsed_seconds": { + "type": "number", + "minimum": 0 + }, + "executions": { + "type": "integer", + "minimum": 0 + }, + "new_interesting_inputs": { + "type": "integer", + "minimum": 0 + } + } + }, + "target": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "outcome", + "metrics" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[^\\r\\n]+$" + }, + "outcome": { + "$ref": "#/$defs/outcome" + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationTargetMetrics" + }, + { + "$ref": "#/$defs/fuzzTargetMetrics" + }, + { + "type": "null" + } + ] + }, + "diagnostic": { + "type": "string", + "minLength": 1 + }, + "reproduction": { + "$ref": "#/$defs/reproduction" + } + } + }, + "targetResult": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "track", + "tool", + "target" + ], + "properties": { + "schema_version": { + "$ref": "#/$defs/version" + }, + "kind": { + "type": "string", + "const": "target-result" + }, + "track": { + "type": "string", + "enum": [ + "mutation", + "fuzz" + ] + }, + "tool": { + "type": "string", + "enum": [ + "stryker", + "gremlins", + "go-fuzz", + "fast-check" + ] + }, + "target": { + "$ref": "#/$defs/target" + } + } + }, + "mutationReportMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "killed", + "timeout", + "survived", + "no_coverage", + "invalid", + "ignored", + "detected", + "missed", + "canonical_score_pct", + "tool_score_numerator", + "tool_score_denominator", + "tool_score_pct", + "tool_score_definition" + ], + "properties": { + "killed": { + "type": "integer", + "minimum": 0 + }, + "timeout": { + "type": "integer", + "minimum": 0 + }, + "survived": { + "type": "integer", + "minimum": 0 + }, + "no_coverage": { + "type": "integer", + "minimum": 0 + }, + "invalid": { + "type": "integer", + "minimum": 0 + }, + "ignored": { + "type": "integer", + "minimum": 0 + }, + "detected": { + "type": "integer", + "minimum": 0 + }, + "missed": { + "type": "integer", + "minimum": 0 + }, + "canonical_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_numerator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_denominator": { + "type": "integer", + "minimum": 0 + }, + "tool_score_pct": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 100 + }, + "tool_score_definition": { + "type": "string", + "minLength": 1 + } + } + }, + "fuzzReportMetrics": { + "type": "object", + "additionalProperties": false, + "required": [ + "declared_budget_seconds", + "elapsed_seconds", + "executions", + "new_interesting_inputs" + ], + "properties": { + "declared_budget_seconds": { + "type": "integer", + "minimum": 1 + }, + "elapsed_seconds": { + "type": "number", + "minimum": 0 + }, + "executions": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "new_interesting_inputs": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + } + } + }, + "run": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "attempt", + "event", + "url", + "started_at" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "attempt": { + "type": "integer", + "minimum": 1 + }, + "event": { + "type": "string", + "minLength": 1 + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https://" + }, + "started_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + } + } + }, + "completeness": { + "type": "object", + "additionalProperties": false, + "required": [ + "expected", + "reported", + "complete" + ], + "properties": { + "expected": { + "type": "integer", + "minimum": 1 + }, + "reported": { + "type": "integer", + "minimum": 0 + }, + "complete": { + "type": "boolean" + } + } + }, + "report": { + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "kind", + "repository", + "ref", + "sha", + "run", + "report_name", + "track", + "tool", + "policy", + "completeness", + "outcome", + "targets", + "metrics", + "errors" + ], + "properties": { + "schema_version": { + "$ref": "#/$defs/version" + }, + "kind": { + "type": "string", + "const": "report" + }, + "repository": { + "type": "string", + "minLength": 1 + }, + "ref": { + "type": "string", + "minLength": 1 + }, + "sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "run": { + "$ref": "#/$defs/run" + }, + "report_name": { + "type": "string", + "minLength": 1 + }, + "track": { + "type": "string", + "enum": [ + "mutation", + "fuzz" + ] + }, + "tool": { + "type": "string", + "enum": [ + "stryker", + "gremlins", + "go-fuzz", + "fast-check" + ] + }, + "policy": { + "type": "string", + "enum": [ + "advisory", + "signal" + ] + }, + "completeness": { + "$ref": "#/$defs/completeness" + }, + "outcome": { + "$ref": "#/$defs/outcome" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/$defs/target" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationReportMetrics" + }, + { + "$ref": "#/$defs/fuzzReportMetrics" + }, + { + "type": "null" + } + ] + }, + "errors": { + "type": "array", + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1 + } + } + } + } + } +} From 99a7a997425bec33f51328d10a784f4612a4d3d3 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 15:50:56 -0400 Subject: [PATCH 05/14] fix(quality): enforce report contract boundaries --- .../quality-result-stats/target-result.json | 2 +- .../quality-result-verify/target-result.json | 2 +- .../quality-result-stats/target-result.json | 2 +- .github/tests/quality_report_contract_test.py | 259 +++++++++++++++++- decisions/quality-reporting.md | 7 + quality-report/v1/quality_report.py | 23 +- quality-report/v1/schema.json | 216 ++++++++++++++- 7 files changed, 495 insertions(+), 16 deletions(-) diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json index 627a14e..1f79ab0 100644 --- a/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-stats/target-result.json @@ -8,7 +8,7 @@ "outcome": "passed", "metrics": { "budget_seconds": 120, - "elapsed_seconds": 111.25, + "elapsed_seconds": 120.0, "executions": 650000, "new_interesting_inputs": 2 } diff --git a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json index 472e276..3e0c633 100644 --- a/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/complete/quality-result-verify/target-result.json @@ -8,7 +8,7 @@ "outcome": "passed", "metrics": { "budget_seconds": 300, - "elapsed_seconds": 288.25, + "elapsed_seconds": 300.0, "executions": 1250000, "new_interesting_inputs": 3 } diff --git a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json index 0db4a95..4530c33 100644 --- a/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json +++ b/.github/tests/fixtures/quality-report/v1/crash/quality-result-stats/target-result.json @@ -9,7 +9,7 @@ "metrics": null, "diagnostic": "tool process exited 2", "reproduction": { - "seed": "-test.fuzzcachedir=/tmp/fuzz-cache" + "path": "internal/docker/testdata/fuzz/FuzzDecodeStats/6f2c1b8d9a4e5c37" } } } diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index b456ce7..6c65a24 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -3,6 +3,7 @@ import runpy import shutil import subprocess +import sys import tempfile import unittest @@ -52,7 +53,7 @@ def test_complete_fuzz_fixture_emits_a_valid_report_and_summary(self): ) self.assertEqual("passed", report["outcome"]) self.assertEqual(420, report["metrics"]["declared_budget_seconds"]) - self.assertEqual(399.5, report["metrics"]["elapsed_seconds"]) + self.assertEqual(420.0, report["metrics"]["elapsed_seconds"]) self.assertEqual(1900000, report["metrics"]["executions"]) self.assertEqual(5, report["metrics"]["new_interesting_inputs"]) self.assertNotIn("canonical_score_pct", report["metrics"]) @@ -97,6 +98,23 @@ def test_crash_fixture_fails_even_when_every_target_reports(self): self.assertIn("tool process exited 2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_crash_fixture_uses_a_native_go_crasher_path(self): + result = json.loads( + ( + FIXTURES + / "crash" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + + self.assertEqual( + { + "path": "internal/docker/testdata/fuzz/FuzzDecodeStats/6f2c1b8d9a4e5c37" + }, + result["target"]["reproduction"], + ) + def test_parse_error_fixture_still_emits_an_error_report(self): result, report, summary = self.aggregate( "parse-error", @@ -116,6 +134,33 @@ def test_parse_error_fixture_still_emits_an_error_report(self): self.assertIn("partial 1/2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_non_utf8_target_still_emits_an_error_report(self): + def corrupt_target(input_dir): + target = next(input_dir.rglob("target-result.json")) + target.write_bytes(b"\xff") + + result, report, summary = self.aggregate( + "complete", + track="fuzz", + tool="go-fuzz", + expected=[ + "internal/auth/FuzzVerifyRequest", + "internal/docker/FuzzDecodeStats", + ], + input_mutator=corrupt_target, + ) + + self.assertNotEqual(0, result.returncode) + self.assertEqual("error", report["outcome"]) + self.assertFalse(report["completeness"]["complete"]) + self.assertIsNone(report["metrics"]) + self.assertTrue( + any("cannot read JSON" in error for error in report["errors"]), + report["errors"], + ) + self.assertIn("partial 1/2", summary) + self.assertEqual(0, self.validate_report(report).returncode) + def test_canonical_score_counts_timeout_and_no_coverage(self): result, report, summary = self.aggregate( "canonical-score", @@ -150,6 +195,14 @@ def test_decision_documents_track_and_tool_pairings(self): self.assertIn("`mutation` permits `gremlins` and `stryker`", decision) self.assertIn("`fuzz` permits `fast-check` and `go-fuzz`", decision) + def test_decision_defines_go_fuzz_as_the_native_go_engine(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn( + "`go-fuzz` identifies Go's native `go test -fuzz` engine, not the legacy third-party go-fuzz tool.", + decision, + ) + def test_native_score_is_weighted_and_keeps_its_definition(self): result, report, summary = self.aggregate( "native-score", @@ -235,6 +288,21 @@ def test_validator_requires_a_positive_declared_fuzz_budget(self): with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], ">= 1"): QUALITY_REPORT["validate_fuzz_report_metrics"](metrics, [], "$.metrics") + def test_passed_fuzz_target_requires_its_full_declared_budget(self): + target = { + "name": "internal/auth/FuzzVerifyRequest", + "outcome": "passed", + "metrics": { + "budget_seconds": 300, + "elapsed_seconds": 299.99, + }, + } + + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], "completed fuzz budget" + ): + QUALITY_REPORT["validate_target"](target, "fuzz", "$.target") + def test_validator_requires_time_and_offset_in_timestamps(self): for timestamp in ("2026-08-14", "2026-08-14T15:00:00"): with self.subTest(timestamp=timestamp): @@ -243,6 +311,91 @@ def test_validator_requires_time_and_offset_in_timestamps(self): ): QUALITY_REPORT["require_timestamp"](timestamp, "$.run.started_at") + def test_invalid_json_contract_error_retains_its_cause(self): + with self.assertRaises(QUALITY_REPORT["ContractError"]) as raised: + QUALITY_REPORT["load_json_text"]("{", "target-result.json") + + self.assertIsInstance(raised.exception.__cause__, json.JSONDecodeError) + + def test_invalid_timestamp_contract_error_retains_its_cause(self): + with self.assertRaises(QUALITY_REPORT["ContractError"]) as raised: + QUALITY_REPORT["require_timestamp"]("2026-08-14", "$.run.started_at") + + self.assertIsInstance(raised.exception.__cause__, ValueError) + + def test_json_parser_rejects_duplicate_fields_and_non_finite_numbers(self): + cases = ( + ('{"value": 1, "value": 2}', "duplicate JSON field"), + ('{"value": NaN}', "non-finite JSON number"), + ('{"value": Infinity}', "non-finite JSON number"), + ) + + for document, message in cases: + with self.subTest(document=document): + with self.assertRaisesRegex(QUALITY_REPORT["ContractError"], message): + QUALITY_REPORT["load_json_text"](document, "target-result.json") + + def test_mutation_aggregate_rejects_mixed_native_score_definitions(self): + targets = [] + for definition in ("detected / covered", "killed / lived"): + targets.append( + { + "metrics": { + "killed": 1, + "timeout": 0, + "survived": 0, + "no_coverage": 0, + "invalid": 0, + "ignored": 0, + "tool_score_numerator": 1, + "tool_score_denominator": 1, + "tool_score_pct": 100.0, + "tool_score_definition": definition, + } + } + ) + + with self.assertRaisesRegex( + QUALITY_REPORT["ContractError"], + "inconsistent native score definitions", + ): + QUALITY_REPORT["aggregate_mutation_metrics"](targets) + + def test_target_collection_rejects_unexpected_and_duplicate_names(self): + document = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + target_name = document["target"]["name"] + + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) + (input_dir / "unexpected").mkdir() + (input_dir / "unexpected" / "target-result.json").write_text( + json.dumps(document) + ) + targets, errors = QUALITY_REPORT["collect_targets"]( + input_dir, ["expected-target"], "fuzz", "go-fuzz" + ) + self.assertEqual([], targets) + self.assertTrue(any("unexpected target" in error for error in errors)) + + with tempfile.TemporaryDirectory() as temp_dir: + input_dir = Path(temp_dir) + for directory in ("first", "second"): + path = input_dir / directory + path.mkdir() + (path / "target-result.json").write_text(json.dumps(document)) + targets, errors = QUALITY_REPORT["collect_targets"]( + input_dir, [target_name], "fuzz", "go-fuzz" + ) + self.assertEqual([target_name], [target["name"] for target in targets]) + self.assertTrue(any("duplicate target" in error for error in errors)) + def test_schema_is_versioned_and_fail_closed(self): with SCHEMA.open() as schema_file: schema = json.load(schema_file) @@ -259,6 +412,97 @@ def test_schema_is_versioned_and_fail_closed(self): self.assertNotIn("additionalProperties", schema) self.assertTrue(self.all_objects_are_closed(schema)) + def test_schema_encodes_python_target_and_track_guards(self): + schema = json.loads(SCHEMA.read_text()) + definitions = schema["$defs"] + + self.assertEqual( + [ + { + "if": { + "properties": {"outcome": {"const": "passed"}}, + "required": ["outcome"], + }, + "then": { + "properties": {"metrics": {"not": {"type": "null"}}}, + "not": { + "anyOf": [ + {"required": ["diagnostic"]}, + {"required": ["reproduction"]}, + ] + }, + }, + "else": {"required": ["diagnostic"]}, + } + ], + definitions["target"]["allOf"], + ) + self.assertEqual( + {"gremlins", "stryker"}, + set( + definitions["targetResult"]["allOf"][0]["then"]["properties"][ + "tool" + ]["enum"] + ), + ) + self.assertEqual( + {"fast-check", "go-fuzz"}, + set( + definitions["targetResult"]["allOf"][0]["else"]["properties"][ + "tool" + ]["enum"] + ), + ) + self.assertEqual( + "#/$defs/mutationTargetMetrics", + definitions["mutationTarget"]["allOf"][1]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzTargetMetrics", + definitions["fuzzTarget"]["allOf"][1]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + ["reproduction"], + definitions["fuzzTarget"]["allOf"][2]["then"]["required"], + ) + self.assertEqual( + "#/$defs/mutationTarget", + definitions["report"]["allOf"][0]["then"]["properties"]["targets"][ + "items" + ]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzTarget", + definitions["report"]["allOf"][0]["else"]["properties"]["targets"][ + "items" + ]["$ref"], + ) + self.assertEqual( + "#/$defs/mutationReportMetrics", + definitions["report"]["allOf"][0]["then"]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertEqual( + "#/$defs/fuzzReportMetrics", + definitions["report"]["allOf"][0]["else"]["properties"]["metrics"][ + "oneOf" + ][0]["$ref"], + ) + self.assertTrue(definitions["report"]["properties"]["targets"]["uniqueItems"]) + + def test_decision_names_python_as_the_cross_field_semantic_authority(self): + decision = " ".join(DECISION.read_text().split()) + + self.assertIn( + "The Python validator is authoritative for cross-field semantics that JSON Schema cannot express, including unique and sorted target names, recomputed completeness, outcomes, and aggregate metrics.", + decision, + ) + def test_reusable_workflow_matches_the_public_contract(self): workflow = WORKFLOW.read_text() @@ -332,18 +576,20 @@ def test_standards_validation_runs_this_contract(self): workflow.count("python3 .github/tests/quality_report_contract_test.py"), ) - def aggregate(self, fixture, *, track, tool, expected): + def aggregate(self, fixture, *, track, tool, expected, input_mutator=None): self.assertTrue(SCRIPT.is_file(), f"missing aggregator: {SCRIPT}") with tempfile.TemporaryDirectory() as temp_dir: input_dir = Path(temp_dir) / "input" shutil.copytree(FIXTURES / fixture, input_dir) + if input_mutator is not None: + input_mutator(input_dir) for invalid_fixture in input_dir.rglob("target-result.invalid"): invalid_fixture.rename(invalid_fixture.with_name("target-result.json")) output_dir = Path(temp_dir) / "output" github_output = Path(temp_dir) / "github-output" summary_path = Path(temp_dir) / "step-summary" command = [ - "python3", + sys.executable, str(SCRIPT), "aggregate", "--input-dir", @@ -383,7 +629,9 @@ def aggregate(self, fixture, *, track, tool, expected): "--github-step-summary", str(summary_path), ] - result = subprocess.run(command, capture_output=True, text=True) + result = subprocess.run( + command, capture_output=True, text=True, timeout=30 + ) report_path = output_dir / "report.json" summary_file = output_dir / "summary.md" self.assertTrue(report_path.is_file(), result.stderr) @@ -425,9 +673,10 @@ def validate_report(self, report): report_path = Path(temp_dir) / "report.json" report_path.write_text(json.dumps(report)) return subprocess.run( - ["python3", str(SCRIPT), "validate", str(report_path)], + [sys.executable, str(SCRIPT), "validate", str(report_path)], capture_output=True, text=True, + timeout=30, ) def all_objects_are_closed(self, value): diff --git a/decisions/quality-reporting.md b/decisions/quality-reporting.md index 6182c97..b826c22 100644 --- a/decisions/quality-reporting.md +++ b/decisions/quality-reporting.md @@ -55,6 +55,9 @@ job validates the set and publishes `quality-report/v1` with: - `targets`: the declared target names and their individual results - track-specific `metrics` +`go-fuzz` identifies Go's native `go test -fuzz` engine, not the legacy +third-party go-fuzz tool. + The schema and dependency-free validator live under [`quality-report/v1`](../quality-report/v1). Both target-result fragments and aggregate reports carry the exact `quality-report/v1` version. Objects reject unknown fields, JSON parsing rejects @@ -62,6 +65,10 @@ duplicate fields and non-finite numbers, and semantic validation recomputes completeness, outcomes, and scores. Callers pass `expected_targets` as a non-empty JSON array of unique target names. +The Python validator is authoritative for cross-field semantics that JSON +Schema cannot express, including unique and sorted target names, recomputed +completeness, outcomes, and aggregate metrics. + Mutation metrics are: - `killed`, `timeout`, `survived`, `no_coverage`, `invalid`, `ignored` diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py index 8b8c93a..274308f 100644 --- a/quality-report/v1/quality_report.py +++ b/quality-report/v1/quality_report.py @@ -79,14 +79,16 @@ def load_json_text(text, source): parse_constant=reject_constant, ) except (json.JSONDecodeError, ContractError) as error: - raise ContractError("{0}: invalid JSON: {1}".format(source, error)) + raise ContractError("{0}: invalid JSON: {1}".format(source, error)) from error def load_json_file(path): try: return load_json_text(path.read_text(), str(path)) - except OSError as error: - raise ContractError("{0}: cannot read JSON: {1}".format(path, error)) + except (OSError, UnicodeDecodeError) as error: + raise ContractError( + "{0}: cannot read JSON: {1}".format(path, error) + ) from error def require_object(value, path, required, optional=()): @@ -168,8 +170,10 @@ def require_timestamp(value, path): if RFC3339_PATTERN.fullmatch(value) is None: raise ValueError datetime.fromisoformat(value.replace("Z", "+00:00").replace("z", "+00:00")) - except ValueError: - raise ContractError("{0}: expected RFC 3339 timestamp".format(path)) + except ValueError as error: + raise ContractError( + "{0}: expected RFC 3339 timestamp".format(path) + ) from error def percentage(numerator, denominator): @@ -238,6 +242,15 @@ def validate_target(target, track, path): validate_mutation_target_metrics(metrics, path + ".metrics") else: validate_fuzz_target_metrics(metrics, path + ".metrics") + if ( + outcome == "passed" + and metrics["elapsed_seconds"] < metrics["budget_seconds"] + ): + raise ContractError( + "{0}.metrics.elapsed_seconds: passed target has not completed fuzz budget".format( + path + ) + ) if outcome == "passed": if "diagnostic" in target or "reproduction" in target: diff --git a/quality-report/v1/schema.json b/quality-report/v1/schema.json index 46c38f1..2ce2acc 100644 --- a/quality-report/v1/schema.json +++ b/quality-report/v1/schema.json @@ -172,7 +172,112 @@ "reproduction": { "$ref": "#/$defs/reproduction" } - } + }, + "allOf": [ + { + "if": { + "properties": { + "outcome": { + "const": "passed" + } + }, + "required": [ + "outcome" + ] + }, + "then": { + "properties": { + "metrics": { + "not": { + "type": "null" + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "diagnostic" + ] + }, + { + "required": [ + "reproduction" + ] + } + ] + } + }, + "else": { + "required": [ + "diagnostic" + ] + } + } + ] + }, + "mutationTarget": { + "allOf": [ + { + "$ref": "#/$defs/target" + }, + { + "properties": { + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationTargetMetrics" + }, + { + "type": "null" + } + ] + } + } + } + ] + }, + "fuzzTarget": { + "allOf": [ + { + "$ref": "#/$defs/target" + }, + { + "properties": { + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/fuzzTargetMetrics" + }, + { + "type": "null" + } + ] + } + } + }, + { + "if": { + "properties": { + "outcome": { + "enum": [ + "failed", + "crashed", + "flaked" + ] + } + }, + "required": [ + "outcome" + ] + }, + "then": { + "required": [ + "reproduction" + ] + } + } + ] }, "targetResult": { "type": "object", @@ -211,7 +316,47 @@ "target": { "$ref": "#/$defs/target" } - } + }, + "allOf": [ + { + "if": { + "properties": { + "track": { + "const": "mutation" + } + }, + "required": [ + "track" + ] + }, + "then": { + "properties": { + "tool": { + "enum": [ + "gremlins", + "stryker" + ] + }, + "target": { + "$ref": "#/$defs/mutationTarget" + } + } + }, + "else": { + "properties": { + "tool": { + "enum": [ + "fast-check", + "go-fuzz" + ] + }, + "target": { + "$ref": "#/$defs/fuzzTarget" + } + } + } + } + ] }, "mutationReportMetrics": { "type": "object", @@ -465,6 +610,7 @@ }, "targets": { "type": "array", + "uniqueItems": true, "items": { "$ref": "#/$defs/target" } @@ -490,7 +636,71 @@ "minLength": 1 } } - } + }, + "allOf": [ + { + "if": { + "properties": { + "track": { + "const": "mutation" + } + }, + "required": [ + "track" + ] + }, + "then": { + "properties": { + "tool": { + "enum": [ + "gremlins", + "stryker" + ] + }, + "targets": { + "items": { + "$ref": "#/$defs/mutationTarget" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/mutationReportMetrics" + }, + { + "type": "null" + } + ] + } + } + }, + "else": { + "properties": { + "tool": { + "enum": [ + "fast-check", + "go-fuzz" + ] + }, + "targets": { + "items": { + "$ref": "#/$defs/fuzzTarget" + } + }, + "metrics": { + "oneOf": [ + { + "$ref": "#/$defs/fuzzReportMetrics" + }, + { + "type": "null" + } + ] + } + } + } + } + ] } } } From 9a5362a6666d6ee42a3c5e47bf78b88eb21021f6 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:08:46 -0400 Subject: [PATCH 06/14] fix(quality): decode reports as utf-8 --- .github/tests/quality_report_contract_test.py | 30 +++++++++++++++++++ quality-report/v1/quality_report.py | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index 6c65a24..858507c 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -6,6 +6,7 @@ import sys import tempfile import unittest +from unittest import mock ROOT = Path(__file__).resolve().parents[2] @@ -161,6 +162,35 @@ def corrupt_target(input_dir): self.assertIn("partial 1/2", summary) self.assertEqual(0, self.validate_report(report).returncode) + def test_utf8_target_load_does_not_depend_on_the_locale_encoding(self): + document = json.loads( + ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ).read_text() + ) + document["target"]["name"] = "internal/docker/FuzzDécodage" + + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "target-result.json" + path.write_bytes(json.dumps(document, ensure_ascii=False).encode("utf-8")) + + def read_with_ascii_default(*, encoding=None, errors=None): + return path.read_bytes().decode( + encoding or "ascii", errors or "strict" + ) + + with mock.patch.object( + Path, "read_text", side_effect=read_with_ascii_default + ): + loaded = QUALITY_REPORT["load_json_file"](path) + + self.assertEqual( + "internal/docker/FuzzDécodage", loaded["target"]["name"] + ) + def test_canonical_score_counts_timeout_and_no_coverage(self): result, report, summary = self.aggregate( "canonical-score", diff --git a/quality-report/v1/quality_report.py b/quality-report/v1/quality_report.py index 274308f..1c4a4c2 100644 --- a/quality-report/v1/quality_report.py +++ b/quality-report/v1/quality_report.py @@ -84,7 +84,7 @@ def load_json_text(text, source): def load_json_file(path): try: - return load_json_text(path.read_text(), str(path)) + return load_json_text(path.read_text(encoding="utf-8"), str(path)) except (OSError, UnicodeDecodeError) as error: raise ContractError( "{0}: cannot read JSON: {1}".format(path, error) From a39c5bd4697a7e513694c4a227dc8b15099f12a8 Mon Sep 17 00:00:00 2001 From: scttbnsn <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 16:38:16 -0400 Subject: [PATCH 07/14] test(quality): pin fixture encoding --- .github/tests/quality_report_contract_test.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/tests/quality_report_contract_test.py b/.github/tests/quality_report_contract_test.py index 858507c..6fabc70 100644 --- a/.github/tests/quality_report_contract_test.py +++ b/.github/tests/quality_report_contract_test.py @@ -163,13 +163,19 @@ def corrupt_target(input_dir): self.assertEqual(0, self.validate_report(report).returncode) def test_utf8_target_load_does_not_depend_on_the_locale_encoding(self): - document = json.loads( - ( - FIXTURES - / "complete" - / "quality-result-stats" - / "target-result.json" - ).read_text() + fixture = ( + FIXTURES + / "complete" + / "quality-result-stats" + / "target-result.json" + ) + original_read_text = Path.read_text + with mock.patch.object( + Path, "read_text", autospec=True, side_effect=original_read_text + ) as fixture_read: + document = json.loads(fixture.read_text(encoding="utf-8")) + self.assertEqual( + mock.call(fixture, encoding="utf-8"), fixture_read.call_args ) document["target"]["name"] = "internal/docker/FuzzDécodage" From 7be30f375a4303edd1f135e919d02b9ac3cfe875 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:23:49 -0400 Subject: [PATCH 08/14] ci(profile): make asset generation read-only (#10) * ci(profile): make asset generation read-only * fix(profile): restrict asset validation egress --- .../tests/community_health_contract_test.py | 78 ++++++++++++++++--- .github/workflows/profile-assets.yml | 60 ++++++++++++++ .github/workflows/standards-validation.yml | 14 ++-- .github/workflows/update-stats.yml | 44 ----------- .gitignore | 1 + cache/.gitignore | 2 - cache/stats.json | 22 ------ .../generate_profile_svg.py | 6 +- 8 files changed, 140 insertions(+), 87 deletions(-) create mode 100644 .github/workflows/profile-assets.yml delete mode 100644 .github/workflows/update-stats.yml delete mode 100644 cache/.gitignore delete mode 100644 cache/stats.json rename generate_svg.py => scripts/generate_profile_svg.py (98%) diff --git a/.github/tests/community_health_contract_test.py b/.github/tests/community_health_contract_test.py index 9439282..7d36a25 100644 --- a/.github/tests/community_health_contract_test.py +++ b/.github/tests/community_health_contract_test.py @@ -1,5 +1,7 @@ from pathlib import Path import re +import subprocess +import sys import unittest @@ -119,6 +121,9 @@ def test_standards_workflow_has_unconditional_stable_jobs(self): self.assertIn(" - dev/repository-standards\n", workflow) self.assertNotIn("\n push:", workflow) self.assertIn("permissions: {}", workflow) + self.assertIn("concurrency:", workflow) + self.assertIn("group: standards-validation-", workflow) + self.assertIn("cancel-in-progress: true", workflow) self.assertIn("runs-on: ubuntu-24.04", workflow) self.assertNotIn("matrix:", workflow) self.assertNotIn("continue-on-error:", workflow) @@ -127,14 +132,14 @@ def test_standards_workflow_has_unconditional_stable_jobs(self): validation_job = workflow.split(" validation:\n", 1)[1].split( "\n codeql:\n", 1 )[0] - self.assertIn(" actions: read", validation_job) - self.assertIn(" contents: read", validation_job) + self.assertIn(" actions: read # Read workflow metadata", validation_job) + self.assertIn(" contents: read # Check out and validate repository files", validation_job) self.assertNotIn("write", validation_job) codeql_job = workflow.split("\n codeql:\n", 1)[1] - self.assertIn(" actions: read", codeql_job) - self.assertIn(" contents: read", codeql_job) - self.assertIn(" security-events: write", codeql_job) + self.assertIn(" actions: read # Read workflow metadata", codeql_job) + self.assertIn(" contents: read # Check out Python sources", codeql_job) + self.assertIn(" security-events: write # Upload CodeQL analysis", codeql_job) jobs = re.findall( r"^ ([a-z][a-z0-9-]+):$", @@ -184,22 +189,73 @@ def test_linter_configs_do_not_suppress_workflow_findings(self): self.assertEqual(2, markdown_config.count(": false")) self.assertFalse((ROOT / ".github/zizmor.yml").exists()) - def test_profile_workflow_pins_actions_and_pushes_without_persisted_credentials(self): - workflow = self.read_text(".github/workflows/update-stats.yml") + def test_profile_generator_has_one_canonical_location(self): + self.assertFalse((ROOT / "generate_svg.py").exists()) + self.assertTrue((ROOT / "scripts/generate_profile_svg.py").is_file()) + + def test_dead_profile_cache_is_removed(self): + self.assertFalse((ROOT / "cache").exists()) + + def test_font_reference_is_local_only(self): + gitignore = self.read_text(".gitignore") + self.assertIn("profile/font_reference.svg", gitignore.splitlines()) + + def test_profile_generator_reproduces_committed_assets(self): + generator = ROOT / "scripts/generate_profile_svg.py" + self.assertTrue(generator.is_file(), f"missing generator: {generator}") + outputs = ( + ROOT / "profile/dark_mode.svg", + ROOT / "profile/light_mode.svg", + ) + before = {path: path.read_bytes() for path in outputs} + + subprocess.run( + [sys.executable, generator], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + after = {path: path.read_bytes() for path in outputs} + self.assertEqual(before, after) + + def test_profile_workflow_is_read_only_bounded_and_pinned(self): + workflow = self.read_text(".github/workflows/profile-assets.yml") actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) self.assertEqual(2, len(actions)) for action in actions: self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") - checkout_step = workflow.split(" - name: Checkout\n", 1)[1].split( + harden_marker = " - name: Harden runner\n" + self.assertIn(harden_marker, workflow) + harden_step = workflow.split(harden_marker, 1)[1].split("\n - name:", 1)[0] + + self.assertIn("egress-policy: block", harden_step) + self.assertIn("allowed-endpoints: github.com:443", harden_step) + + checkout_marker = " - name: Check out repository\n" + self.assertIn(checkout_marker, workflow) + checkout_step = workflow.split(checkout_marker, 1)[1].split( "\n - name:", 1 )[0] - push_step = workflow.split(" - name: Commit and push\n", 1)[1] self.assertIn("persist-credentials: false", checkout_step) - self.assertIn("GH_TOKEN: ${{ github.token }}", push_step) - self.assertIn("gh auth setup-git", push_step) + self.assertIn("permissions: {}", workflow) + self.assertIn("name: Profile Assets", workflow) + self.assertIn("contents: read", workflow) + self.assertIn("timeout-minutes:", workflow) + self.assertIn("concurrency:", workflow) + self.assertIn("step-security/harden-runner@", workflow) + self.assertIn("python3 scripts/generate_profile_svg.py", workflow) + self.assertIn("git diff --exit-code -- profile/", workflow) + self.assertIn("git status --porcelain --untracked-files=all -- profile/", workflow) + self.assertNotIn("contents: write", workflow) + self.assertNotIn("git push", workflow) + self.assertNotIn("GH_TOKEN", workflow) + self.assertNotIn("schedule:", workflow) + self.assertFalse((ROOT / ".github/workflows/update-stats.yml").exists()) def read_text(self, relative_path): path = ROOT / relative_path diff --git a/.github/workflows/profile-assets.yml b/.github/workflows/profile-assets.yml new file mode 100644 index 0000000..0fd6154 --- /dev/null +++ b/.github/workflows/profile-assets.yml @@ -0,0 +1,60 @@ +name: Profile Assets + +on: + pull_request: + branches: + - main + - dev/repository-standards + paths: + - .github/tests/community_health_contract_test.py + - .github/workflows/profile-assets.yml + - .gitignore + - profile/*.svg + - scripts/generate_profile_svg.py + push: + branches: + - main + paths: + - .github/tests/community_health_contract_test.py + - .github/workflows/profile-assets.yml + - .gitignore + - profile/*.svg + - scripts/generate_profile_svg.py + workflow_dispatch: + +permissions: {} + +concurrency: + group: profile-assets-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + validate: + name: Profile Assets + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: github.com:443 + + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Verify generated profile assets + run: | + set -euo pipefail + python3 scripts/generate_profile_svg.py + git diff --exit-code -- profile/ + changes="$(git status --porcelain --untracked-files=all -- profile/)" + if [[ -n "$changes" ]]; then + printf '%s\n' "$changes" + exit 1 + fi diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index ceb4606..30ea7f4 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -8,14 +8,18 @@ on: permissions: {} +concurrency: + group: standards-validation-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: validation: name: Standards Validation runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: - actions: read - contents: read + actions: read # Read workflow metadata + contents: read # Check out and validate repository files steps: - name: Harden runner @@ -74,9 +78,9 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: - actions: read - contents: read - security-events: write + actions: read # Read workflow metadata + contents: read # Check out Python sources + security-events: write # Upload CodeQL analysis steps: - name: Harden runner diff --git a/.github/workflows/update-stats.yml b/.github/workflows/update-stats.yml deleted file mode 100644 index cb79f7c..0000000 --- a/.github/workflows/update-stats.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Update Org Profile Stats - -on: - push: - branches: - - main - schedule: - - cron: '0 0 * * *' - workflow_dispatch: - -permissions: - contents: write - -jobs: - update: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.11' - - - name: Generate SVGs - env: - GH_TOKEN: ${{ secrets.ACCESS_TOKEN }} - run: | - python generate_svg.py - - - name: Commit and push - env: - GH_TOKEN: ${{ github.token }} - run: | - gh auth setup-git - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - git add profile/dark_mode.svg profile/light_mode.svg cache/ - git diff --staged --quiet || git commit -m "Update stats" - git push diff --git a/.gitignore b/.gitignore index c54ade8..d66d040 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ # Local planning/scratch workspace — not for commit .planning/ +profile/font_reference.svg diff --git a/cache/.gitignore b/cache/.gitignore deleted file mode 100644 index 688f1fa..0000000 --- a/cache/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Track cache files (needed for CI) -!*.json diff --git a/cache/stats.json b/cache/stats.json deleted file mode 100644 index 42f10e4..0000000 --- a/cache/stats.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "timestamp": "2026-02-09T23:00:00.795596", - "org": { - "public_repos": 2, - "private_repos": 3, - "total_repos": 5, - "total_stars": 0, - "total_forks": 0, - "total_commits": 398, - "total_prs": 11, - "total_issues": 1, - "members": 0, - "loc_added": 354732, - "loc_deleted": 283673, - "loc_total": 71059, - "languages": [ - "Python", - "Swift", - "TypeScript" - ] - } -} \ No newline at end of file diff --git a/generate_svg.py b/scripts/generate_profile_svg.py similarity index 98% rename from generate_svg.py rename to scripts/generate_profile_svg.py index 02f2b1b..436b7c6 100644 --- a/generate_svg.py +++ b/scripts/generate_profile_svg.py @@ -211,16 +211,16 @@ def art_color(ch): def main(): """Generate both dark and light mode SVGs.""" - script_dir = Path(__file__).parent + repo_root = Path(__file__).resolve().parents[1] dark_svg = generate_svg("dark") - dark_path = script_dir / "profile" / "dark_mode.svg" + dark_path = repo_root / "profile" / "dark_mode.svg" with open(dark_path, 'w', encoding='utf-8') as f: f.write(dark_svg) print(f"Generated: {dark_path}") light_svg = generate_svg("light") - light_path = script_dir / "profile" / "light_mode.svg" + light_path = repo_root / "profile" / "light_mode.svg" with open(light_path, 'w', encoding='utf-8') as f: f.write(light_svg) print(f"Generated: {light_path}") From 55c58e856c5f2a680b8e616158b76328831fad76 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Fri, 14 Aug 2026 19:31:52 -0400 Subject: [PATCH 09/14] ci(review): add deduplicated Greptile summon (#9) * ci(review): add deduplicated Greptile summon * fix(review): serialize exact-head Greptile summons * test(review): lock Greptile security controls --- .../tests/greptile_summon_contract_test.py | 92 ++++++++++++++ .github/workflows/greptile-summon.yml | 120 ++++++++++++++++++ .github/workflows/standards-validation.yml | 4 +- 3 files changed, 215 insertions(+), 1 deletion(-) create mode 100644 .github/tests/greptile_summon_contract_test.py create mode 100644 .github/workflows/greptile-summon.yml diff --git a/.github/tests/greptile_summon_contract_test.py b/.github/tests/greptile_summon_contract_test.py new file mode 100644 index 0000000..7584077 --- /dev/null +++ b/.github/tests/greptile_summon_contract_test.py @@ -0,0 +1,92 @@ +from pathlib import Path +import re +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/greptile-summon.yml" + + +class GreptileSummonContractTest(unittest.TestCase): + def test_reusable_workflow_has_exact_head_inputs_and_narrow_permissions(self): + workflow = self.read_workflow() + + for expected in ( + " workflow_call:\n", + " pr_number:\n", + " type: number\n", + " head_sha:\n", + " type: string\n", + " focus:\n", + "permissions: {}", + " runs-on: ubuntu-24.04", + " timeout-minutes: 5", + " pull-requests: write", + "uses: step-security/harden-runner@", + "egress-policy: block", + "api.github.com:443", + "github-token: ${{ github.token }}", + ): + self.assertIn(expected, workflow) + + self.assertNotIn("actions/checkout", workflow) + + def test_workflow_verifies_head_and_deduplicates_before_commenting(self): + workflow = self.read_workflow() + + for expected in ( + "github.rest.pulls.get", + 'pull.state !== "open"', + "pull.head.sha !== headSha", + "github.paginate", + "github.rest.issues.listComments", + "greptile-summon:${headSha}", + 'comment.user?.login === "github-actions[bot]"', + "github.rest.issues.createComment", + "@greptileai", + ): + self.assertIn(expected, workflow) + + self.assertIn( + "greptile-summon-${{ github.repository }}-${{ inputs.pr_number }}-${{ inputs.head_sha }}", + workflow, + ) + self.assertIn(" cancel-in-progress: false", workflow) + self.assertGreaterEqual(workflow.count("github.rest.pulls.get"), 2) + self.assertIn("currentPull.head.sha !== headSha", workflow) + self.assertIn("HEAD_SHA: ${{ inputs.head_sha }}", workflow) + self.assertIn("REVIEW_FOCUS: ${{ inputs.focus }}", workflow) + self.assertNotIn("const headSha = '${{ inputs.head_sha }}'", workflow) + self.assertNotIn("const focus = '${{ inputs.focus }}'", workflow) + + def test_workflow_rejects_partial_pull_request_numbers(self): + workflow = self.read_workflow() + + self.assertIn("const prNumberText = process.env.PR_NUMBER", workflow) + self.assertIn("const prNumber = Number(prNumberText)", workflow) + self.assertIn("!/^[1-9]\\d*$/.test(prNumberText)", workflow) + self.assertIn("PR_NUMBER: ${{ inputs.pr_number }}", workflow) + self.assertIn("if (!/^[0-9a-f]{40}$/.test(headSha))", workflow) + self.assertIn("if (focus.length === 0 || focus.length > 500)", workflow) + self.assertNotIn("Number.parseInt(process.env.PR_NUMBER", workflow) + + def test_workflow_pins_actions_and_is_run_by_standards_validation(self): + workflow = self.read_workflow() + actions = re.findall(r"^\s+uses: ([^\s#]+)", workflow, re.MULTILINE) + + self.assertTrue(actions) + for action in actions: + self.assertRegex(action, r"^[^@]+@[0-9a-f]{40}$") + + validation = ( + ROOT / ".github/workflows/standards-validation.yml" + ).read_text() + self.assertIn("greptile_summon_contract_test.py", validation) + + def read_workflow(self): + self.assertTrue(WORKFLOW.is_file(), f"missing workflow: {WORKFLOW}") + return WORKFLOW.read_text() + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/greptile-summon.yml b/.github/workflows/greptile-summon.yml new file mode 100644 index 0000000..3484f9b --- /dev/null +++ b/.github/workflows/greptile-summon.yml @@ -0,0 +1,120 @@ +name: Summon Greptile + +on: + workflow_call: + inputs: + pr_number: + description: Pull request number in the caller repository + required: true + type: number + head_sha: + description: Exact pull request head to review + required: true + type: string + focus: + description: Short review focus based on the risk in the diff + required: false + default: Review for correctness, security issues, and cross-file regressions. + type: string + +permissions: {} + +jobs: + summon: + name: Summon Greptile + runs-on: ubuntu-24.04 + timeout-minutes: 5 + concurrency: + group: greptile-summon-${{ github.repository }}-${{ inputs.pr_number }}-${{ inputs.head_sha }} + cancel-in-progress: false + permissions: + pull-requests: write # Create the intentional Greptile summon comment. + + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: block + allowed-endpoints: > + api.github.com:443 + + - name: Comment once on the exact pull request head + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + PR_NUMBER: ${{ inputs.pr_number }} + HEAD_SHA: ${{ inputs.head_sha }} + REVIEW_FOCUS: ${{ inputs.focus }} + with: + github-token: ${{ github.token }} + script: | + const prNumberText = process.env.PR_NUMBER; + const prNumber = Number(prNumberText); + const headSha = process.env.HEAD_SHA; + const focus = process.env.REVIEW_FOCUS.trim(); + const { owner, repo } = context.repo; + + if (!/^[1-9]\d*$/.test(prNumberText) || !Number.isSafeInteger(prNumber)) { + core.setFailed(`Invalid pull request number: ${prNumberText}`); + return; + } + if (!/^[0-9a-f]{40}$/.test(headSha)) { + core.setFailed(`Invalid full head SHA: ${headSha}`); + return; + } + if (focus.length === 0 || focus.length > 500) { + core.setFailed("Review focus must contain 1 to 500 characters"); + return; + } + + const { data: pull } = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber, + }); + if (pull.state !== "open") { + core.setFailed(`Pull request #${prNumber} is ${pull.state}`); + return; + } + if (pull.head.sha !== headSha) { + core.setFailed( + `Pull request head changed: expected ${headSha}, got ${pull.head.sha}`, + ); + return; + } + + const marker = ``; + const comments = await github.paginate( + github.rest.issues.listComments, + { owner, repo, issue_number: prNumber, per_page: 100 }, + ); + const alreadySummoned = comments.some( + (comment) => comment.user?.login === "github-actions[bot]" + && comment.body?.includes(marker), + ); + if (alreadySummoned) { + core.notice(`Greptile already summoned for ${headSha}`); + return; + } + + const { data: currentPull } = await github.rest.pulls.get({ + owner, + repo, + pull_number: prNumber, + }); + if (currentPull.state !== "open") { + core.setFailed(`Pull request #${prNumber} is ${currentPull.state}`); + return; + } + if (currentPull.head.sha !== headSha) { + core.setFailed( + `Pull request head changed: expected ${headSha}, got ${currentPull.head.sha}`, + ); + return; + } + + await github.rest.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body: `@greptileai Review exact head \`${headSha}\`. ${focus}\n\n${marker}`, + }); diff --git a/.github/workflows/standards-validation.yml b/.github/workflows/standards-validation.yml index 30ea7f4..0ea5ad0 100644 --- a/.github/workflows/standards-validation.yml +++ b/.github/workflows/standards-validation.yml @@ -38,7 +38,9 @@ jobs: node-version: 24 - name: Validate community health contract - run: python3 .github/tests/community_health_contract_test.py + run: | + python3 .github/tests/community_health_contract_test.py + python3 .github/tests/greptile_summon_contract_test.py - name: Compile Python run: python3 -m compileall -q . From 6e7a78dc13d099cb27669c0a0f8a75836de97f03 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 08:24:16 -0400 Subject: [PATCH 10/14] ci(workflows): add run-test and run-lint toggles to go-ci (#19) go-ci.yml's test and lint jobs ran unconditionally, so a Go-less repo that only wants the language-agnostic workflow-security (zizmor) job couldn't call it. Add run-test/run-lint boolean inputs, mirroring the existing run-govulncheck/run-workflow-security/etc. toggle pattern, defaulting to true so existing callers see no behavior change. Fixes: #18 --- .github/tests/reusable_ci_contract_test.py | 4 ++++ .github/workflows/go-ci.yml | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py index a6ec3f5..b958eb6 100644 --- a/.github/tests/reusable_ci_contract_test.py +++ b/.github/tests/reusable_ci_contract_test.py @@ -25,6 +25,8 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) "lint-check-name", "test-check-name", "fuzzers-json", + "run-test", + "run-lint", "run-govulncheck", "run-qlty", "run-goreleaser", @@ -113,6 +115,8 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) self.assert_input("go", "lint-check-name", "string", default="Go Lint") self.assert_input("go", "test-check-name", "string", default="Go Test") self.assert_input("go", "fuzzers-json", "string", default="[]") + for input_name in ("run-test", "run-lint"): + self.assert_input("go", input_name, "boolean", default="true") for input_name in ( "run-govulncheck", "run-qlty", diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 46601d4..8e14b1c 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -33,6 +33,16 @@ on: required: false type: string default: "[]" + run-test: + description: Run the fixed repository Go test gate + required: false + type: boolean + default: true + run-lint: + description: Run the fixed repository Go lint gate + required: false + type: boolean + default: true run-govulncheck: description: Run the fixed repository govulncheck gate required: false @@ -159,6 +169,7 @@ permissions: {} jobs: test: name: ${{ inputs.test-check-name }} + if: inputs.run-test runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: @@ -198,6 +209,7 @@ jobs: lint: name: ${{ inputs.lint-check-name }} + if: inputs.run-lint runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: From 04ad251abff43346cd46b836f323421a0f99bd00 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:05:50 -0400 Subject: [PATCH 11/14] ci(workflows): add module-directory input to node-ci (#22) * ci(workflows): add module-directory input to node-ci Mirrors go-ci's module-directory idiom: a string input defaulting to "." threaded into each fixed script's env as MODULE_DIRECTORY, so a repo with several independently-gated Node projects can call node-ci once per project. The default preserves current behavior for existing callers. Extends the reusable CI contract test to assert the new input and its threading, matching how run-test/run-lint were added for go-ci in #19. * test(workflows): assert module-directory threads into all three node jobs --- .github/tests/reusable_ci_contract_test.py | 5 +++++ .github/workflows/node-ci.yml | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/tests/reusable_ci_contract_test.py b/.github/tests/reusable_ci_contract_test.py index b958eb6..b4fc0fa 100644 --- a/.github/tests/reusable_ci_contract_test.py +++ b/.github/tests/reusable_ci_contract_test.py @@ -47,6 +47,7 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) ) ) node_inputs = [ + "module-directory", "node-version", "lockfile-path", "lint-check-name", @@ -127,6 +128,7 @@ def test_workflow_call_interfaces_are_typed_and_have_no_outputs_or_secrets(self) ): self.assert_input("go", input_name, "boolean", default="false") + self.assert_input("node", "module-directory", "string", default=".") self.assert_input("node", "node-version", "string", default="24") self.assert_input("node", "lockfile-path", "string", default="package-lock.json") self.assert_input("node", "lint-check-name", "string", default="Node Lint") @@ -204,6 +206,9 @@ def test_central_jobs_have_exact_names_fixed_commands_and_per_job_egress(self): ) self.assertIn("node-version: ${{ inputs.node-version }}", node) self.assertIn("cache-dependency-path: ${{ inputs.lockfile-path }}", node) + self.assertEqual( + 3, node.count("MODULE_DIRECTORY: ${{ inputs.module-directory }}") + ) for workflow in (go, node, self.read_workflow(WORKFLOWS["release"])): self.assertIn("runs-on: ubuntu-24.04", workflow) diff --git a/.github/workflows/node-ci.yml b/.github/workflows/node-ci.yml index e699a9b..c953151 100644 --- a/.github/workflows/node-ci.yml +++ b/.github/workflows/node-ci.yml @@ -3,6 +3,11 @@ name: Reusable Node CI on: workflow_call: inputs: + module-directory: + description: Repository-relative Node module directory + required: false + type: string + default: . node-version: description: Node.js version used by enabled jobs required: false @@ -105,6 +110,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node lint gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-lint.sh test: @@ -135,6 +142,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node test gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-test.sh - name: Upload Node test artifacts @@ -174,6 +183,8 @@ jobs: cache-dependency-path: ${{ inputs.lockfile-path }} - name: Run repository Node build gate + env: + MODULE_DIRECTORY: ${{ inputs.module-directory }} run: ./scripts/ci/node-build.sh - name: Upload Node build artifacts From 7f9be8b854dd345814537fd0d7f24b2509760f96 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:30:27 -0400 Subject: [PATCH 12/14] docs(onboarding): record the qlty alignment baseline (#24) --- REPOSITORY_ONBOARDING.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index a06bb7a..7510744 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -115,10 +115,8 @@ repositories. Language-specific tests do not. - [ ] Add language formatters to `pre-commit` only when they can safely operate on staged files. Verify the hook does not rewrite unrelated work. -Qlty Cloud enrollment and usage data are not an organization-wide prerequisite. -Do not make a Qlty upload or check required until the repository is enrolled and -the check has proved stable. Local Qlty checks may still be stricter than this -baseline. +Qlty spans local and CI gates; the full baseline, including why Qlty Cloud +checks stay non-required, is in the Qlty subsection of section 4. ## 3. Protect `main` @@ -194,6 +192,37 @@ Add these only when the behavior exists: - [ ] Translation synchronization only for a repository with a translation source of truth and configured provider credentials. +### Qlty + +Qlty appears in three places. The committed configuration and the repository-run +gates are the alignment surface; the Qlty Cloud GitHub App is not. + +- [ ] Commit `.qlty/qlty.toml` (`config_version = "0"`) with the organization + exclude baseline for generated, minified, and vendored paths. Drydock's file + is the reference posture and Portwing's is its Go-repo mirror; start from the + closer of the two. The same file drives the local CLI and Qlty Cloud, so + changes to it are quality-gate changes, not formatting. +- [ ] Add the repository's Qlty gate script (`scripts/qlty-check-gate.sh all`) + to `pre-push`, fail-fast like every other local gate. An advisory smells + gate may run alongside it (Drydock pattern) but must not mask the gating + script's failure. +- [ ] Gate Qlty in CI with the repository-run check, which needs no Qlty Cloud + account. Go repositories call `go-ci.yml` with `run-qlty` enabled, + `qlty-egress-policy: block`, and the proven endpoint allowlist — copy the + allowlist from Portwing's `ci-verify.yml` rather than re-deriving it, and do + not widen it. Node repositories run the SHA-pinned + `qltysh/qlty-action/install` plus the same gate script inside their own + workflow (Drydock pattern). + +The Qlty Cloud GitHub App's statuses (`qlty check`, `qlty coverage`, +`qlty coverage diff`) currently error organization-wide with out-of-minutes +billing failures. Treat them as non-gating: never add them to required status +checks, and do not chase these failures on PRs — the repository-run gate above +is the enforced check. Qlty Cloud enrollment and usage data are not an +organization-wide prerequisite; do not make a Cloud upload or check required +until the repository is enrolled and the check has proved stable. Local Qlty +checks may still be stricter than this baseline. + ## 5. Add delivery controls only for shipped artifacts - [ ] Keep release construction separate from ordinary CI. A release-cut From f4c1d509fe98551d6e6d6c819d2bb618711e460f Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 18:34:58 -0400 Subject: [PATCH 13/14] docs(onboarding): align with the codified standards registry (#26) * docs(onboarding): align with the codified standards registry - docs(onboarding): name Codecov as the coverage cloud; Qlty Cloud App and maintainability badge stay, checks stay non-required - docs(onboarding): trivy deprecated in favor of Grype, including the qlty plugin blocks in the two reference configs (drydock#753, portwing#135) - docs(onboarding): CodeRabbit free Pro is public-only; private repos use cross-account human review - docs(onboarding): add the greptile.json contract and the label-gated second-opinion caller * docs(onboarding): reword the CodeRabbit private-repo claim as org policy - docs(onboarding): free-plan private-repo reviews exist but are rate-limited and never fired here; the skip is policy, not a plan fact - docs(onboarding): pair the Greptile caller with auto-applied CodeRabbit labeling so the second-opinion label is criteria-driven --- REPOSITORY_ONBOARDING.md | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/REPOSITORY_ONBOARDING.md b/REPOSITORY_ONBOARDING.md index 7510744..8929480 100644 --- a/REPOSITORY_ONBOARDING.md +++ b/REPOSITORY_ONBOARDING.md @@ -100,6 +100,20 @@ repositories. Language-specific tests do not. request an explicit CodeRabbit review on the first PR to prove it works. - [ ] Confirm CodeRabbit can access the repository and posts an actual review comment. A green check without review output is not enough for this test. + Automated Pro-level reviews are free on public repositories only; on + private repositories the free plan rate-limits automated reviews, and they + have not fired for this organization in practice. Organization policy: on a + private repository skip both CodeRabbit items, use cross-account human + review, and add the config when the repository goes public. +- [ ] Keep `greptile.json` at exactly `{"skipReview": "AUTOMATIC"}` + (contract-tested in this repository) so Greptile never reviews unbidden. A + repository that wants opt-in second opinions adds a label-gated caller + workflow (`.github/workflows/greptile.yml`, firing on the `second-opinion` + label) that calls this repository's `greptile-summon.yml` at a pinned full + commit SHA. Pair the caller with a CodeRabbit `labeling_instructions` entry + for `second-opinion` and `auto_apply_labels` enabled, so applying the label + is criteria-driven rather than left to memory. The label is Greptile's only + trigger; never wire it to review every PR directly. ### Local gates @@ -156,7 +170,8 @@ must push, and use `step-security/harden-runner` with the smallest practical egress allowlist. Every repository with executable code needs the applicable format or lint and -test gates. Require coverage when the language and test tooling measure it, a +test gates. Require coverage when the language and test tooling measure it +(uploaded to Codecov, not Qlty Cloud), a production build when the repository ships a buildable artifact, dependency review when it has dependencies, workflow validation when it has GitHub Actions, and CodeQL when its language is supported. Add the matching language or artifact @@ -174,8 +189,9 @@ checks below: - **GitHub Actions:** actionlint and zizmor on every PR, plus tests for non-trivial embedded shell and workflow contracts. - **Container image:** build the real Dockerfile, smoke-test the image, and scan - the built image and locked dependencies with Grype or the current organization - scanner. + the built image and locked dependencies with Grype. Trivy is deprecated + organization-wide in favor of Grype; do not adopt it, including as a Qlty + plugin. - **Public repository:** OpenSSF Scorecard on its supported triggers and dependency review on PRs. @@ -200,8 +216,11 @@ gates are the alignment surface; the Qlty Cloud GitHub App is not. - [ ] Commit `.qlty/qlty.toml` (`config_version = "0"`) with the organization exclude baseline for generated, minified, and vendored paths. Drydock's file is the reference posture and Portwing's is its Go-repo mirror; start from the - closer of the two. The same file drives the local CLI and Qlty Cloud, so - changes to it are quality-gate changes, not formatting. + closer of the two. Do not copy the trivy plugin block from either reference: + trivy is deprecated in favor of Grype and its removal from both files is + tracked (drydock#753, portwing#135). The trufflehog plugin stays. The same + file drives the local CLI and Qlty Cloud, so changes to it are quality-gate + changes, not formatting. - [ ] Add the repository's Qlty gate script (`scripts/qlty-check-gate.sh all`) to `pre-push`, fail-fast like every other local gate. An advisory smells gate may run alongside it (Drydock pattern) but must not mask the gating @@ -218,8 +237,12 @@ The Qlty Cloud GitHub App's statuses (`qlty check`, `qlty coverage`, `qlty coverage diff`) currently error organization-wide with out-of-minutes billing failures. Treat them as non-gating: never add them to required status checks, and do not chase these failures on PRs — the repository-run gate above -is the enforced check. Qlty Cloud enrollment and usage data are not an -organization-wide prerequisite; do not make a Cloud upload or check required +is the enforced check. Coverage reporting is Codecov's job (decided +2026-08-16): upload coverage to Codecov and carry its badge in the README. +The Qlty Cloud App and its maintainability badge stay installed, but its +coverage upload is not the coverage system and its checks stay non-required. +Qlty Cloud enrollment and usage data are not an organization-wide +prerequisite; do not make a Cloud upload or check required until the repository is enrolled and the check has proved stable. Local Qlty checks may still be stricter than this baseline. From 5ac2e3fa208ef8724e163230adf8ed18ff607424 Mon Sep 17 00:00:00 2001 From: Scott Benson <80784472+scttbnsn@users.noreply.github.com> Date: Sun, 16 Aug 2026 19:41:50 -0400 Subject: [PATCH 14/14] chore(repo): meet our own onboarding checklist (#28) * chore(repo): meet our own onboarding checklist - chore(repo): MIT LICENSE (infrastructure repos are MIT; products AGPL) - docs(repo): root AGENTS.md with repo-specific rules and validation - build(hooks): lefthook with commit-msg + pre-push mirroring CI via scripts/validate.sh * fix(hooks): tighten the commit-msg exemptions and mirror zizmor's CI flags - fix(hooks): merge/revert exemptions match git's generated subjects only, so a hand-typed 'Merge ...' subject no longer bypasses the check - fix(hooks): require a non-whitespace character after the colon - fix(hooks): zizmor runs --no-online-audits locally, matching CI's online-audits: false for local/CI parity * fix(hooks): exempt only git-generated merge and revert subjects --- AGENTS.md | 50 +++++++++++++++++++++++++++++++++++++ LICENSE | 21 ++++++++++++++++ lefthook.yml | 12 +++++++++ scripts/check-commit-msg.sh | 25 +++++++++++++++++++ scripts/validate.sh | 45 +++++++++++++++++++++++++++++++++ 5 files changed, 153 insertions(+) create mode 100644 AGENTS.md create mode 100644 LICENSE create mode 100644 lefthook.yml create mode 100755 scripts/check-commit-msg.sh create mode 100755 scripts/validate.sh diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..0d27337 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,50 @@ +# Agent instructions for CodesWhat/.github + +This repository is the organization's meta layer: community health defaults, +the repository onboarding checklist (`REPOSITORY_ONBOARDING.md`), reusable CI +workflows, the organization profile README and its generator, and the +contract tests that pin the shapes other repositories depend on. + +## What lives here + +- `.github/workflows/go-ci.yml`, `node-ci.yml`, `release-gate.yml`, + `greptile-summon.yml`, `quality-report-aggregate.yml` — reusable workflows + consumed by other CodesWhat repositories. +- `.github/tests/` — contract tests. These are the public API of the org's + shared configs (the `greptile.json` shape, the quality-report v1 contract, + the reusable CI inputs). Change a contract test and its consumers together, + never one side alone. +- `profile/` — the organization profile README plus generated SVG assets + from `scripts/generate_profile_svg.py`. The untracked + `profile/font_reference.svg` is user-owned; leave it alone. + +## Rules that are specific to this repository + +- Consumers pin the reusable workflows to frozen full commit SHAs. Treat + every workflow change as breaking for pinned consumers: add new inputs + with safe defaults, never repurpose an existing input, and let consumers + adopt by rolling their pin deliberately. +- PRs target `dev/repository-standards`. `main` advances only through + promotion PRs. Reconcile before promoting: + `git merge -s ours origin/main -m "chore(sync): reconcile main before promotion"`. + Verify promotions with tree equality + (`git diff --quiet origin/main origin/dev/repository-standards`), never + commit ancestry. +- CodeRabbit auto-reviews PRs against `dev/*` branches only (scoped in + `.coderabbit.yaml`); summon it explicitly anywhere else and read its + inline comments before merging. +- Commits are plain Conventional Commits, `(scope): `, + no emoji, no AI attribution trailers. +- Never weaken branch protection or rulesets to land a change. + +## Validation + +Run before pushing (the lefthook pre-push hook runs the same set): + +```bash +bash scripts/validate.sh +``` + +That script mirrors the Standards Validation CI job: contract tests, Python +compile, YAML/JSON parse, markdownlint (with the separate profile config for +`profile/README.md`), and actionlint/zizmor when installed locally. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..af599c9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 CodesWhat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lefthook.yml b/lefthook.yml new file mode 100644 index 0000000..3872369 --- /dev/null +++ b/lefthook.yml @@ -0,0 +1,12 @@ +# Local git hooks. Install once with: npx --yes lefthook install +# pre-push mirrors the Standards Validation CI job via scripts/validate.sh. + +commit-msg: + commands: + conventional-commits: + run: bash scripts/check-commit-msg.sh {1} + +pre-push: + commands: + validate: + run: bash scripts/validate.sh diff --git a/scripts/check-commit-msg.sh b/scripts/check-commit-msg.sh new file mode 100755 index 0000000..3b3320e --- /dev/null +++ b/scripts/check-commit-msg.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# commit-msg hook: enforce plain Conventional Commits on the subject line. +set -euo pipefail + +msg_file="$1" +subject="$(head -n 1 "$msg_file")" + +case "$subject" in + "Merge branch '"*|"Merge pull request #"*|"Merge remote-tracking branch '"*|fixup!\ *|squash!\ *) exit 0 ;; +esac + +if [[ "$subject" =~ ^Revert\ \".+\"$ ]]; then + exit 0 +fi + +pattern='^(feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)(\([a-z0-9./-]+\))?!?: [^[:space:]]' +if ! [[ "$subject" =~ $pattern ]]; then + { + echo "commit subject must be plain Conventional Commits:" + echo " (scope): " + echo "allowed types: feat fix docs style refactor perf test build ci chore revert" + echo "got: $subject" + } >&2 + exit 1 +fi diff --git a/scripts/validate.sh b/scripts/validate.sh new file mode 100755 index 0000000..5d02c10 --- /dev/null +++ b/scripts/validate.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Local mirror of the Standards Validation CI job. Run before pushing; +# lefthook pre-push runs this same script. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +echo "==> clean tree" +if ! git diff --quiet || ! git diff --cached --quiet; then + echo "commit or stash local changes before pushing" >&2 + exit 1 +fi + +echo "==> contract tests" +python3 .github/tests/community_health_contract_test.py +python3 .github/tests/greptile_summon_contract_test.py +python3 .github/tests/greptile_config_contract_test.py +python3 .github/tests/quality_report_contract_test.py +python3 .github/tests/reusable_ci_contract_test.py + +echo "==> compile python" +python3 -m compileall -q . + +echo "==> parse yaml and json" +ruby -e 'require "yaml"; Dir.glob("**/*.{yml,yaml}", File::FNM_DOTMATCH).sort.each { |path| YAML.parse_file(path) }' +python3 -c 'import json; from pathlib import Path; [json.load(path.open()) for path in Path(".").rglob("*.json")]' + +echo "==> markdownlint" +npx --yes markdownlint-cli2@0.23.2 "**/*.md" "#profile/README.md" +npx --yes markdownlint-cli2@0.23.2 "profile/README.md" --config .github/markdownlint-profile.yaml + +echo "==> actionlint" +if command -v actionlint >/dev/null 2>&1; then + actionlint -color +else + echo "actionlint not installed locally; CI will run it" +fi + +echo "==> zizmor" +if command -v zizmor >/dev/null 2>&1; then + zizmor --no-online-audits .github/workflows/ +else + echo "zizmor not installed locally; CI will run it" +fi + +echo "==> ok"